diff options
Diffstat (limited to 'src/lib')
42 files changed, 13911 insertions, 1387 deletions
diff --git a/src/lib/CMakeLists.txt b/src/lib/CMakeLists.txt index 6cd3a8a4..48064ce4 100644 --- a/src/lib/CMakeLists.txt +++ b/src/lib/CMakeLists.txt @@ -22,6 +22,7 @@ set(SOURCE_FILES_COMMON crc/crc32.c crc/crc64.c crypt.c + crypt/keyrot.c hash.c lockfile.c logs.c @@ -92,6 +93,13 @@ if(HAVE_FUSE) target_link_libraries(ouroboros-common PRIVATE Fuse::Fuse) endif() +if(HAVE_LIBURCU) + target_link_libraries(ouroboros-common PRIVATE Urcu::Urcu) + # urcu headers require C99; override the global -std=c89 for these TUs. + set_source_files_properties(crypt/keyrot.c dev.c poa/addr.c poa/eth.c + poa/udp.c PROPERTIES COMPILE_OPTIONS "-std=gnu99") +endif() + install(TARGETS ouroboros-common EXPORT OuroborosTargets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) @@ -99,8 +107,14 @@ install(TARGETS ouroboros-common set(SOURCE_FILES_DEV cep.c dev.c + poa/addr.c + poa/udp.c ) +if(HAVE_ETH) + list(APPEND SOURCE_FILES_DEV poa/eth.c) +endif() + add_library(ouroboros-dev SHARED ${SOURCE_FILES_DEV} ${CEP_PROTO_SRCS}) @@ -123,6 +137,11 @@ target_include_directories(ouroboros-dev target_link_libraries(ouroboros-dev PUBLIC ouroboros-common) +if(HAVE_LIBURCU) + # dev.c calls urcu directly; do not rely on transitive linkage. + target_link_libraries(ouroboros-dev PRIVATE Urcu::Urcu) +endif() + install(TARGETS ouroboros-dev EXPORT OuroborosTargets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) diff --git a/src/lib/cap.c b/src/lib/cap.c new file mode 100644 index 00000000..f116bfb0 --- /dev/null +++ b/src/lib/cap.c @@ -0,0 +1,187 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Link capacity estimation + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * version 2.1 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +/* + * Link-capacity estimation by watching the egress queue drain. + * + * A saturated link drains its queue at exactly its capacity, so we + * estimate capacity by measuring the drain rate of the transmit + * queue toward an n-1 flow (the flow to the layer below) while that + * queue is backlogged. + * + * Sampling is lock-free and off the fast path: the queue depth is + * read only at enqueue time, concurrently by many sender threads. + * Each enqueue bumps relaxed counters (packets, bytes, empty-queue + * hits). At most once per CAP_T_MIN, one thread wins a try-lock and + * closes a measurement window. + * + * Over a window, byte conservation gives the bytes that drained: + * drained = queue at start (q0) + enqueued - queue now (q1) + * A window stays open until CAP_N_MIN packets' worth has drained, so + * its length self-scales with the link rate (~1 ms at 1 Gbit, ~19 ms + * at 10 Mbit). CAP_T_MAX discards a window that spanned a traffic gap. + * + * Only a backlogged link measures its own capacity, so a window + * whose ring ran mostly idle is discarded (a few empty samples, as + * from a token-bucket shaper, are tolerated). The drain rate feeds a + * max filter that jumps up at once but decays slowly, converging on + * the capacity from below. A window that touched an empty ring at + * either edge may have drained into downstream buffers faster than + * the wire, so it may only lower the estimate, never raise it. + */ + +#if defined(__linux__) || defined(__CYGWIN__) +#ifndef _DEFAULT_SOURCE +#define _DEFAULT_SOURCE +#endif +#else +#ifndef _POSIX_C_SOURCE +#define _POSIX_C_SOURCE 200809L +#endif +#endif + +#include "config.h" + +#include <ouroboros/atomics.h> +#include <ouroboros/time.h> + +#include "cap.h" + +#include <string.h> + +#define CAP_T_MIN (BILLION / 1000) /* min close spacing ~1 ms */ +#define CAP_T_MAX (1ULL << 27) /* voiding traffic gap ~134 ms */ +#define CAP_N_MIN 16 /* drained packets to close */ +#define CAP_DEC_SHFT 4 /* max-filter decay 1/16 */ +#define CAP_IDL_SHFT 3 /* idle tolerance 1/8 */ + +/* Busy-flag try-lock: test-and-set acquire, store release. */ +#define CAP_TAS(p) __atomic_exchange_n(p, 1, __ATOMIC_ACQUIRE) +#define CAP_REL(p) (__atomic_store_n(p, 0, __ATOMIC_RELEASE)) + +void cap_clear(struct cap_est * e) +{ + memset(e, 0, sizeof(*e)); +} + +uint64_t cap_rate(const struct cap_est * e) +{ + return LOAD_RELAXED(&e->est); +} + +/* Busy flag held; q1 is the caller's pre-write ring sample. */ +static void cap_close(struct cap_est * e, + uint64_t q1, + uint64_t now, + uint64_t gap) +{ + uint64_t pkt; /* current c_pkt snapshot */ + uint64_t byt; /* current c_byt snapshot */ + uint64_t idl; /* current c_idl snapshot */ + uint64_t dt; /* window duration (ns) */ + uint64_t enq; /* packets enqueued in window */ + uint64_t avg; /* mean packet size (bytes) */ + uint64_t r; /* window drain rate (bytes/s) */ + int64_t drained; /* bytes drained over window */ + + pkt = LOAD_RELAXED(&e->c_pkt); + byt = LOAD_RELAXED(&e->c_byt); + idl = LOAD_RELAXED(&e->c_idl); + + dt = now - e->t0; + enq = pkt - e->pkt0; + + drained = (int64_t) (e->q0 + (byt - e->byt0) - q1); + + if (e->t0 == 0 || enq == 0) + goto reopen; + + if (gap > CAP_T_MAX) + goto reopen; /* traffic stopped: window void */ + + avg = (byt - e->byt0) / enq; + if (drained < (int64_t) (CAP_N_MIN * avg)) + return; /* extend the window until enough drains */ + + if ((idl - e->idl0) << CAP_IDL_SHFT > enq) + goto reopen; /* mostly idle ring: not saturated */ + + r = (uint64_t) drained * MILLION / (dt / 1000); + if (r >= e->rate) { + if (e->q0 > 0 && q1 > 0) /* empty edge drains below */ + e->rate = r; + } else { + e->rate -= (e->rate - r) >> CAP_DEC_SHFT; + } + + STORE_RELAXED(&e->est, e->rate); + reopen: + e->t0 = now; + e->q0 = q1; + e->pkt0 = pkt; + e->byt0 = byt; + e->idl0 = idl; +} + +void cap_update_at(struct cap_est * e, + size_t qlen, + size_t len, + uint64_t now) +{ + uint64_t prev; + + FETCH_ADD_RELAXED(&e->c_pkt, 1); + FETCH_ADD_RELAXED(&e->c_byt, len); + + if (qlen == 0) + FETCH_ADD_RELAXED(&e->c_idl, 1); + + prev = LOAD_RELAXED(&e->t_last); + if (prev > now) + prev = now; /* a racing writer stamped ahead */ + + STORE_RELAXED(&e->t_last, now); + + if (now - LOAD_RELAXED(&e->t_gate) < CAP_T_MIN) + return; + + if (CAP_TAS(&e->busy) != 0) + return; + + if (now - e->t_gate >= CAP_T_MIN) { + cap_close(e, qlen, now, now - prev); + STORE_RELAXED(&e->t_gate, now); + } + + CAP_REL(&e->busy); +} + +void cap_update(struct cap_est * e, + size_t qlen, + size_t len) +{ + struct timespec now; + + clock_gettime(PTHREAD_COND_CLOCK, &now); + + cap_update_at(e, qlen, len, TS_TO_UINT64(now)); +} diff --git a/src/lib/cap.h b/src/lib/cap.h new file mode 100644 index 00000000..3d94d9a3 --- /dev/null +++ b/src/lib/cap.h @@ -0,0 +1,63 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Link capacity estimation + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * version 2.1 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#ifndef OUROBOROS_LIB_CAP_H +#define OUROBOROS_LIB_CAP_H + +#include <stddef.h> +#include <stdint.h> + +#define CAP_ALIGN 64 + +struct cap_est { + uint64_t c_pkt; /* total packets enqueued (relaxed) */ + uint64_t c_byt; /* total bytes enqueued (relaxed) */ + uint64_t c_idl; /* times ring seen empty (relaxed) */ + + uint64_t t_gate; /* last window close (ns) */ + uint64_t t_last; /* last update, to spot a gap (ns) */ + uint8_t busy; /* close in progress (try-lock) */ + + uint64_t t0; /* window start (ns), 0 = no window */ + uint64_t q0; /* ring occupancy at window start */ + uint64_t pkt0; /* c_pkt snapshot at window start */ + uint64_t byt0; /* c_byt snapshot at window start */ + uint64_t idl0; /* c_idl snapshot at window start */ + uint64_t rate; /* filtered drain rate (bytes/s) */ + + uint64_t est; /* published estimate (bytes/s) */ +} __attribute__((aligned(CAP_ALIGN))); + +void cap_clear(struct cap_est * e); + +void cap_update(struct cap_est * e, + size_t qlen, + size_t len); + +void cap_update_at(struct cap_est * e, + size_t qlen, + size_t len, + uint64_t now); + +uint64_t cap_rate(const struct cap_est * e); + +#endif /* OUROBOROS_LIB_CAP_H */ diff --git a/src/lib/config.h.in b/src/lib/config.h.in index 7124a974..38d6f768 100644 --- a/src/lib/config.h.in +++ b/src/lib/config.h.in @@ -33,11 +33,11 @@ #cmakedefine HAVE_LIBGCRYPT #cmakedefine HAVE_OPENSSL #ifdef HAVE_OPENSSL -#cmakedefine HAVE_OPENSSL_ML_KEM -#cmakedefine HAVE_OPENSSL_ML_DSA -#cmakedefine HAVE_OPENSSL_SLH_DSA +#cmakedefine HAVE_ML +#cmakedefine HAVE_SLH +#cmakedefine HAVE_OPENSSL_4_1 #define HAVE_ENCRYPTION -#define SECMEM_GUARD @SECMEM_GUARD@ +#define SECMEM_MINSIZE @SECMEM_MINSIZE@ #endif #define PROC_SECMEM_MAX @PROC_SECMEM_MAX@ @@ -49,6 +49,7 @@ #cmakedefine HAVE_PMULL #define SHM_LOCKFILE_NAME "@SHM_LOCKFILE_NAME@" +#define SSM_RBUFF_TXQ_DELAY @SSM_RBUFF_TXQ_DELAY@ /* ms */ #define FLOW_ALLOC_TIMEOUT @FLOW_ALLOC_TIMEOUT@ #define TPM_DEBUG_REPORT_INTERVAL @TPM_DEBUG_REPORT_INTERVAL@ @@ -70,6 +71,8 @@ #cmakedefine PROC_FLOW_STATS #endif +#cmakedefine HAVE_LIBURCU + #cmakedefine FRCT_DEBUG_STDOUT #define PTHREAD_COND_CLOCK @PTHREAD_COND_CLOCK@ @@ -78,6 +81,25 @@ #define PROC_RES_FDS @PROC_RES_FDS@ #define PROC_MAX_FQUEUES @PROC_MAX_FQUEUES@ +/* Flow endpoints */ +#define POA_MGMT_FRAME_SIZE @POA_MGMT_FRAME_SIZE@ +#define POA_MGMT_SND_TIMEO @POA_MGMT_SND_TIMEO@ +#define POA_MAX_POAS @POA_MAX_POAS@ +#define POA_UDP_MPL @POA_UDP_MPL@ +#define POA_UDP4_MTU @POA_UDP4_MTU@ +#define POA_UDP6_MTU @POA_UDP6_MTU@ +#define POA_UDP_RD_BUF @POA_UDP_RD_BUF@ +#cmakedefine HAVE_ETH +#cmakedefine HAVE_RAW_SOCKETS +#cmakedefine HAVE_BPF +#cmakedefine HAVE_NETMAP +#cmakedefine POA_ETH_QDISC_BYPASS +#define POA_ETH_MPL @POA_ETH_MPL@ +#define POA_ETH_SNDBUF @POA_ETH_SNDBUF@ +#define POA_ETH_RCVBUF @POA_ETH_RCVBUF@ +#define POA_ETH_LO_MTU @POA_ETH_LO_MTU@ +#define POA_ETH_RD_BUF @POA_ETH_RD_BUF@ + /* Default Delta-t parameters */ #cmakedefine FRCT_LINUX_RTT_ESTIMATOR #define DELT_A (@DELTA_T_ACK@) /* ms */ @@ -100,4 +122,9 @@ #define ACKQ_SLOTS (@ACK_WHEEL_SLOTS@) #define ACKQ_RES (@ACK_WHEEL_RESOLUTION@) /* 2^N ns */ -#define KEY_ROTATION_BIT (@KEY_ROTATION_BIT@) /* Bit for key rotation */ +#define KEY_LEAF_BITS (@KEY_LEAF_BITS@) /* pkts/leaf-key = 2^n */ +#define KEY_NODE_BITS (@KEY_NODE_BITS@) /* leaf-keys/node = 2^n */ +#define KEY_NODE_COUNT (@KEY_NODE_COUNT@) /* node keys/batch N */ +#define KEY_REKEY_WATERMARK (@KEY_REKEY_WATERMARK@) /* node-keys-left trig */ +#define KEY_REPLAY_WINDOW (@KEY_REPLAY_WINDOW@) /* rx replay win pkts */ +#define FLOW_WM_CHECK (1u << @KEY_REKEY_WM_CHECK_BITS@) /* wm chk/n wr */ diff --git a/src/lib/crypt.c b/src/lib/crypt.c index 71197f6e..8ca7084f 100644 --- a/src/lib/crypt.c +++ b/src/lib/crypt.c @@ -27,10 +27,14 @@ #include <config.h> #include <ouroboros/errno.h> +#include <ouroboros/pthread.h> #include <ouroboros/random.h> #include <ouroboros/crypt.h> +#include "crypt/keyrot.h" + #ifdef HAVE_OPENSSL +#include <openssl/crypto.h> #include <openssl/evp.h> #include "crypt/openssl.h" #endif @@ -50,18 +54,12 @@ static const struct nid_map cipher_nid_map[] = { {NID_aes_192_gcm, "aes-192-gcm"}, {NID_aes_256_gcm, "aes-256-gcm"}, {NID_chacha20_poly1305, "chacha20-poly1305"}, - {NID_aes_128_ctr, "aes-128-ctr"}, - {NID_aes_192_ctr, "aes-192-ctr"}, - {NID_aes_256_ctr, "aes-256-ctr"}, {NID_undef, NULL} }; /* Ordered in strength preference, lowest first */ const uint16_t crypt_supported_nids[] = { #ifdef HAVE_OPENSSL - NID_aes_128_ctr, - NID_aes_192_ctr, - NID_aes_256_ctr, NID_aes_128_gcm, NID_aes_192_gcm, NID_aes_256_gcm, @@ -71,39 +69,43 @@ const uint16_t crypt_supported_nids[] = { }; static const struct nid_map kex_nid_map[] = { - {NID_X9_62_prime256v1, "prime256v1"}, - {NID_secp384r1, "secp384r1"}, - {NID_secp521r1, "secp521r1"}, - {NID_X25519, "X25519"}, - {NID_X448, "X448"}, - {NID_ffdhe2048, "ffdhe2048"}, - {NID_ffdhe3072, "ffdhe3072"}, - {NID_ffdhe4096, "ffdhe4096"}, - {NID_MLKEM512, "ML-KEM-512"}, - {NID_MLKEM768, "ML-KEM-768"}, - {NID_MLKEM1024, "ML-KEM-1024"}, - {NID_X25519MLKEM768, "X25519MLKEM768"}, - {NID_X448MLKEM1024, "X448MLKEM1024"}, - {NID_undef, NULL} + {NID_X9_62_prime256v1, "prime256v1"}, + {NID_secp384r1, "secp384r1"}, + {NID_secp521r1, "secp521r1"}, + {NID_X25519, "X25519"}, + {NID_X448, "X448"}, + {NID_ffdhe2048, "ffdhe2048"}, + {NID_ffdhe3072, "ffdhe3072"}, + {NID_ffdhe4096, "ffdhe4096"}, + {NID_MLKEM512, "ML-KEM-512"}, + {NID_MLKEM768, "ML-KEM-768"}, + {NID_MLKEM1024, "ML-KEM-1024"}, + {NID_X25519MLKEM768, "X25519MLKEM768"}, + {NID_X448MLKEM1024, "X448MLKEM1024"}, + {NID_SecP256r1MLKEM768, "SecP256r1MLKEM768"}, + {NID_SecP384r1MLKEM1024, "SecP384r1MLKEM1024"}, + {NID_undef, NULL} }; -/* Ordered in strength preference, lowest first */ +/* Ordered in strength preference, lowest first (NIST SP 800-57 levels) */ const uint16_t kex_supported_nids[] = { #ifdef HAVE_OPENSSL - NID_ffdhe2048, - NID_X9_62_prime256v1, - NID_X25519, - NID_ffdhe3072, - NID_secp384r1, - NID_ffdhe4096, - NID_X448, - NID_secp521r1, -#ifdef HAVE_OPENSSL_ML_KEM - NID_MLKEM512, - NID_MLKEM768, - NID_MLKEM1024, - NID_X25519MLKEM768, - NID_X448MLKEM1024, + NID_ffdhe2048, /* FFDHE-2048, ~112-bit */ + NID_X9_62_prime256v1, /* ECDH P-256, 128-bit */ + NID_X25519, /* ECDH X25519, 128-bit */ + NID_ffdhe3072, /* FFDHE-3072, ~128-bit */ + NID_ffdhe4096, /* FFDHE-4096, ~152-bit */ + NID_secp384r1, /* ECDH P-384, 192-bit */ + NID_X448, /* ECDH X448, 224-bit */ + NID_secp521r1, /* ECDH P-521, 256-bit */ +#ifdef HAVE_ML + NID_MLKEM512, /* ML-KEM-512, PQC L1 (~AES-128) */ + NID_MLKEM768, /* ML-KEM-768, PQC L3 (~AES-192) */ + NID_MLKEM1024, /* ML-KEM-1024, PQC L5 (~AES-256) */ + NID_SecP256r1MLKEM768, /* P-256 + ML-KEM-768, PQC L3 */ + NID_X25519MLKEM768, /* X25519 + ML-KEM-768, PQC L3 */ + NID_SecP384r1MLKEM1024, /* P-384 + ML-KEM-1024, PQC L5 */ + NID_X448MLKEM1024, /* X448 + ML-KEM-1024, PQC L5 */ #endif #endif NID_undef @@ -137,11 +139,13 @@ const uint16_t md_supported_nids[] = { }; struct crypt_ctx { - void * ctx; /* Encryption context */ + struct keyrot * kr; /* backend-independent key rotation */ + void * cipher; /* backend AEAD cipher context */ }; struct auth_ctx { - void * store; + void * store; /* trusted anchors */ + void * chain; /* untrusted build-only interm */ }; static int parse_kex_value(const char * value, @@ -162,6 +166,7 @@ int parse_sec_config(struct sec_config * cfg, char * equals; char * key; char * value; + bool no_enc = false; assert(cfg != NULL); assert(fp != NULL); @@ -172,6 +177,7 @@ int parse_sec_config(struct sec_config * cfg, SET_KEX_KDF_NID(cfg, NID_sha256); SET_KEX_CIPHER_NID(cfg, NID_aes_256_gcm); SET_KEX_DIGEST_NID(cfg, NID_sha256); + /* a.req is seeded per-role by the caller; only auth= overrides it */ while (fgets(line, sizeof(line), fp) != NULL) { char * trimmed; @@ -180,12 +186,10 @@ int parse_sec_config(struct sec_config * cfg, if (line[0] == '#' || line[0] == '\n') continue; - /* Check for 'none' keyword */ + /* Bare 'none' keyword replaced by encryption=none */ trimmed = trim_whitespace(line); - if (strcmp(trimmed, "none") == 0) { - memset(cfg, 0, sizeof(*cfg)); - return 0; - } + if (strcmp(trimmed, "none") == 0) + return -EINVAL; /* Find the = separator */ equals = strchr(line, '='); @@ -221,12 +225,54 @@ int parse_sec_config(struct sec_config * cfg, } else { return -EINVAL; } + } else if (strcmp(key, "auth") == 0) { + if (strcmp(value, "required") == 0) { + cfg->a.req = true; + } else if (strcmp(value, "optional") == 0) { + cfg->a.req = false; + } else { + return -EINVAL; + } + } else if (strcmp(key, "cacert") == 0) { + if (strlen(value) >= sizeof(cfg->a.cacert)) + return -EINVAL; + strcpy(cfg->a.cacert, value); + } else if (strcmp(key, "encryption") == 0) { + if (strcmp(value, "none") != 0) + return -EINVAL; + no_enc = true; + } else { + return -EINVAL; } } + if (no_enc) { + /* Digest stays: it belongs to the auth axis */ + CLEAR_KEX_ALGO(cfg); + CLEAR_KEX_KDF(cfg); + CLEAR_KEX_CIPHER(cfg); + } + return 0; } +/* + * Not in header, but non-static for unit testing. Without a crypto + * backend a present security config is refused. + */ +int crypt_load_sec_config(struct sec_config * cfg, + FILE * fp) +{ + assert(cfg != NULL); + assert(fp != NULL); + +#ifndef HAVE_OPENSSL + return -ENOTSUP; +#endif + + return parse_sec_config(cfg, fp); +} + /* Parse key exchange config from file */ int load_sec_config_file(struct sec_config * cfg, const char * path) @@ -239,12 +285,19 @@ int load_sec_config_file(struct sec_config * cfg, fp = fopen(path, "r"); if (fp == NULL) { - /* File doesn't exist - disable encryption */ - CLEAR_KEX_ALGO(cfg); - return 0; + /* Absent config disables encryption; other errors fail */ + if (errno == ENOENT) { + CLEAR_KEX_ALGO(cfg); + return 0; + } + return -errno; } - ret = parse_sec_config(cfg, fp); + pthread_cleanup_push(__cleanup_fclose, fp); + + ret = crypt_load_sec_config(cfg, fp); + + pthread_cleanup_pop(0); fclose(fp); @@ -329,14 +382,16 @@ ssize_t kex_kem_encap(buffer_t pk, #endif } -ssize_t kex_kem_encap_raw(buffer_t pk, - uint8_t * ct, - int kdf, - uint8_t * s) +ssize_t kex_kem_encap_raw(const char * algo, + buffer_t pk, + uint8_t * ct, + int kdf, + uint8_t * s) { #ifdef HAVE_OPENSSL - return openssl_kem_encap_raw(pk, ct, kdf, s); + return openssl_kem_encap_raw(algo, pk, ct, kdf, s); #else + (void) algo; (void) pk; (void) ct; (void) kdf; @@ -378,19 +433,6 @@ int kex_get_algo_from_pk_der(buffer_t pk, #endif } -int kex_get_algo_from_pk_raw(buffer_t pk, - char * algo) -{ -#ifdef HAVE_OPENSSL - return openssl_get_algo_from_pk_raw(pk, algo); -#else - (void) pk; - algo[0] = '\0'; - - return -ECRYPT; -#endif -} - int kex_validate_algo(const char * algo) { if (algo == NULL) @@ -498,6 +540,11 @@ int kex_validate_nid(int nid) return -ENOTSUP; } +bool kex_nid_is_hybrid(uint16_t nid) +{ + return nid >= NID_HYBRID_KEM_MIN && nid <= NID_HYBRID_KEM_MAX; +} + const char * md_nid_to_str(uint16_t nid) { const struct nid_map * p; @@ -592,19 +639,71 @@ int crypt_kex_rank(int nid) return -1; } -/* Hash length now returned by md_digest() */ +/* AEAD primitive: 1:1 backend wrappers used by the data path below. */ +static int crypt_seal(void * cipher, + const uint8_t * key, + const uint8_t * nonce, + buffer_t aad, + buffer_t in, + uint8_t * out, + uint8_t * tag) +{ +#ifdef HAVE_OPENSSL + return openssl_seal(cipher, key, nonce, aad, in, out, tag); +#else + (void) cipher; + (void) key; + (void) nonce; + (void) aad; + (void) in; + (void) out; + (void) tag; -int crypt_encrypt(struct crypt_ctx * ctx, - buffer_t in, - buffer_t * out) + return -ECRYPT; +#endif +} + +static int crypt_open(void * cipher, + const uint8_t * key, + const uint8_t * nonce, + buffer_t aad, + buffer_t in, + const uint8_t * tag, + buffer_t * out) { - assert(ctx != NULL); - assert(ctx->ctx != NULL); +#ifdef HAVE_OPENSSL + return openssl_open(cipher, key, nonce, aad, in, tag, out); +#else + (void) cipher; + (void) key; + (void) nonce; + (void) aad; + (void) in; + (void) tag; + (void) out; + + return -ECRYPT; +#endif +} + +int crypt_oneshot_seal(int nid, + const uint8_t * key, + const uint8_t * nonce, + buffer_t aad, + buffer_t in, + buffer_t * out) +{ + assert(key != NULL); + assert(nonce != NULL); + assert(out != NULL); #ifdef HAVE_OPENSSL - return openssl_encrypt(ctx->ctx, in, out); + return openssl_oneshot_seal(nid, key, nonce, aad, in, out); #else - (void) ctx; + (void) nid; + (void) key; + (void) nonce; + (void) aad; (void) in; (void) out; @@ -612,17 +711,24 @@ int crypt_encrypt(struct crypt_ctx * ctx, #endif } -int crypt_decrypt(struct crypt_ctx * ctx, - buffer_t in, - buffer_t * out) +int crypt_oneshot_open(int nid, + const uint8_t * key, + const uint8_t * nonce, + buffer_t aad, + buffer_t in, + buffer_t * out) { - assert(ctx != NULL); - assert(ctx->ctx != NULL); + assert(key != NULL); + assert(nonce != NULL); + assert(out != NULL); #ifdef HAVE_OPENSSL - return openssl_decrypt(ctx->ctx, in, out); + return openssl_oneshot_open(nid, key, nonce, aad, in, out); #else - (void) ctx; + (void) nid; + (void) key; + (void) nonce; + (void) aad; (void) in; (void) out; @@ -630,8 +736,122 @@ int crypt_decrypt(struct crypt_ctx * ctx, #endif } +/* + * Data-path encrypt: rotate the key, frame selector ‖ ct ‖ tag, seal. + * Backend-agnostic: composed from keyrot_*, crypt_seal and crypt_get_tagsz. + */ +int crypt_encrypt(struct crypt_ctx * ctx, + buffer_t in, + buffer_t * out) +{ + uint8_t nonce[KR_NONCE_LEN]; + const uint8_t * key; + uint8_t * ct; + buffer_t aad; + int tagsz; + int out_sz; + + assert(ctx != NULL); + assert(ctx->kr != NULL); + + tagsz = crypt_get_tagsz(ctx); + if (tagsz < 0) + return -ECRYPT; + + out->data = malloc(KR_SELECTOR_LEN + in.len + (size_t) tagsz); + if (out->data == NULL) + goto fail_malloc; + + ct = out->data + KR_SELECTOR_LEN; + + /* keyrot writes the selector into the wire header (== AAD). */ + if (keyrot_tx_next(ctx->kr, out->data, &key, nonce) != 0) + goto fail_encrypt; + + aad.data = out->data; + aad.len = KR_SELECTOR_LEN; + + out_sz = crypt_seal(ctx->cipher, key, nonce, aad, in, ct, ct + in.len); + if (out_sz < 0) + goto fail_encrypt; + + out->len = KR_SELECTOR_LEN + (size_t) out_sz + (size_t) tagsz; + + return 0; + fail_encrypt: + free(out->data); + fail_malloc: + clrbuf(*out); + return -ECRYPT; +} + +/* + * Data-path decrypt: look up the rotated key from the selector, open, and + * commit the replay window only after the tag verifies. + */ +int crypt_decrypt(struct crypt_ctx * ctx, + buffer_t in, + buffer_t * out) +{ + uint8_t nonce[KR_NONCE_LEN]; + const uint8_t * key; + const uint8_t * tag; + struct kr_rx rx; + buffer_t aad; + buffer_t ct; + int tagsz; + int in_sz; + + assert(ctx != NULL); + assert(ctx->kr != NULL); + + tagsz = crypt_get_tagsz(ctx); + if (tagsz < 0) + return -ECRYPT; + + if (in.len < (size_t) (KR_SELECTOR_LEN + tagsz)) + return -ECRYPT; + + if (keyrot_rx_lookup(ctx->kr, in.data, &key, nonce, &rx) != 0) + return -ECRYPT; + + in_sz = (int) in.len - KR_SELECTOR_LEN - tagsz; + + /* +1 keeps malloc(0) defined for an empty (zero-length) frame. */ + out->data = malloc((size_t) in_sz + 1); + if (out->data == NULL) + goto fail_malloc; + + aad.data = in.data; + aad.len = KR_SELECTOR_LEN; + + ct.data = in.data + KR_SELECTOR_LEN; + ct.len = (size_t) in_sz; + + tag = in.data + KR_SELECTOR_LEN + in_sz; + + if (crypt_open(ctx->cipher, key, nonce, aad, ct, tag, out) < 0) + goto fail_decrypt; + + /* Commit replay state only after the tag verifies. */ + if (keyrot_rx_commit(ctx->kr, &rx) != 0) + goto fail_decrypt; + + return 0; + fail_decrypt: + free(out->data); + fail_malloc: + clrbuf(*out); + return -ECRYPT; +} + struct crypt_ctx * crypt_create_ctx(struct crypt_sk * sk) { +#ifndef HAVE_OPENSSL + (void) sk; + + return NULL; /* nothing to seal with */ +#else struct crypt_ctx * crypt; if (crypt_validate_nid(sk->nid) != 0) @@ -643,18 +863,23 @@ struct crypt_ctx * crypt_create_ctx(struct crypt_sk * sk) memset(crypt, 0, sizeof(*crypt)); -#ifdef HAVE_OPENSSL - crypt->ctx = openssl_crypt_create_ctx(sk); - if (crypt->ctx == NULL) - goto fail_ctx; -#endif + crypt->kr = keyrot_create(sk->key, sk->epoch, sk->role); + if (crypt->kr == NULL) + goto fail_kr; + + crypt->cipher = openssl_crypt_create_ctx(sk); + if (crypt->cipher == NULL) + goto fail_cipher; + return crypt; -#ifdef HAVE_OPENSSL - fail_ctx: + + fail_cipher: + keyrot_destroy(crypt->kr); + fail_kr: free(crypt); -#endif fail_crypt: return NULL; +#endif } void crypt_destroy_ctx(struct crypt_ctx * crypt) @@ -662,43 +887,76 @@ void crypt_destroy_ctx(struct crypt_ctx * crypt) if (crypt == NULL) return; + keyrot_destroy(crypt->kr); #ifdef HAVE_OPENSSL - assert(crypt->ctx != NULL); - openssl_crypt_destroy_ctx(crypt->ctx); -#else - assert(crypt->ctx == NULL); + openssl_crypt_destroy_ctx(crypt->cipher); #endif free(crypt); } -int crypt_get_ivsz(struct crypt_ctx * ctx) +int crypt_get_headsz(struct crypt_ctx * ctx) { - if (ctx == NULL) - return -EINVAL; + assert(ctx != NULL); + assert(ctx->kr != NULL); -#ifdef HAVE_OPENSSL - assert(ctx->ctx != NULL); - return openssl_crypt_get_ivsz(ctx->ctx); -#else - assert(ctx->ctx == NULL); - return -ENOTSUP; -#endif + (void) ctx; /* validated only; header size is a constant */ + + return KR_SELECTOR_LEN; +} + +int crypt_rekey(struct crypt_ctx * ctx, + struct crypt_sk * sk) +{ + int ret; + + assert(ctx != NULL); + assert(sk != NULL); + assert(ctx->kr != NULL); + + ret = keyrot_rekey(ctx->kr, sk->key, sk->epoch); + if (ret == -EREPLAY) + return -EREPLAY; + + return ret == 0 ? 0 : -ECRYPT; } int crypt_get_tagsz(struct crypt_ctx * ctx) { - if (ctx == NULL) - return -EINVAL; + assert(ctx != NULL); + assert(ctx->cipher != NULL); #ifdef HAVE_OPENSSL - assert(ctx->ctx != NULL); - return openssl_crypt_get_tagsz(ctx->ctx); + return openssl_crypt_get_tagsz(ctx->cipher); #else - assert(ctx->ctx == NULL); + (void) ctx; return -ENOTSUP; #endif } +int crypt_nodes_left(struct crypt_ctx * ctx) +{ + assert(ctx != NULL); + assert(ctx->kr != NULL); + + return (int) keyrot_tx_nodes_left(ctx->kr); +} + +int crypt_peer_synced(struct crypt_ctx * ctx) +{ + assert(ctx != NULL); + assert(ctx->kr != NULL); + + return keyrot_peer_switched(ctx->kr) ? 1 : 0; +} + +void crypt_tx_promote(struct crypt_ctx * ctx) +{ + assert(ctx != NULL); + assert(ctx->kr != NULL); + + keyrot_tx_promote(ctx->kr); +} + int crypt_load_privkey_file(const char * path, void ** key) { @@ -709,7 +967,7 @@ int crypt_load_privkey_file(const char * path, #else (void) path; - return 0; + return -ENOTSUP; #endif } @@ -723,7 +981,7 @@ int crypt_load_privkey_str(const char * str, #else (void) str; - return 0; + return -ENOTSUP; #endif } @@ -737,7 +995,7 @@ int crypt_load_pubkey_str(const char * str, #else (void) str; - return 0; + return -ENOTSUP; #endif } @@ -751,7 +1009,7 @@ int crypt_load_pubkey_file(const char * path, #else (void) path; - return 0; + return -ENOTSUP; #endif } @@ -772,14 +1030,16 @@ int crypt_load_pubkey_file_to_der(const char * path, } int crypt_load_pubkey_raw_file(const char * path, + const char * algo, buffer_t * buf) { assert(buf != NULL); #ifdef HAVE_OPENSSL - return openssl_load_pubkey_raw_file(path, buf); + return openssl_load_pubkey_raw_file(path, algo, buf); #else (void) path; + (void) algo; buf->data = NULL; buf->len = 0; @@ -788,19 +1048,40 @@ int crypt_load_pubkey_raw_file(const char * path, } int crypt_load_privkey_raw_file(const char * path, + const char * algo, void ** key) { *key = NULL; #ifdef HAVE_OPENSSL - return openssl_load_privkey_raw_file(path, key); + return openssl_load_privkey_raw_file(path, algo, key); #else (void) path; + (void) algo; return 0; #endif } +int crypt_ct_cmp(const void * a, + const void * b, + size_t len) +{ +#ifdef HAVE_OPENSSL + return CRYPTO_memcmp(a, b, len); +#else + const volatile uint8_t * pa = a; + const volatile uint8_t * pb = b; + uint8_t d = 0; + size_t i; + + for (i = 0; i < len; i++) + d |= pa[i] ^ pb[i]; + + return d != 0; +#endif +} + int crypt_cmp_key(const void * key1, const void * key2) { @@ -937,7 +1218,7 @@ int crypt_check_crt_name(void * crt, (void) crt; (void) name; - return 0; + return -ENOTSUP; #endif } @@ -967,9 +1248,15 @@ struct auth_ctx * auth_create_ctx(void) ctx->store = openssl_auth_create_store(); if (ctx->store == NULL) goto fail_store; + + ctx->chain = openssl_auth_create_chain(); + if (ctx->chain == NULL) + goto fail_chain; #endif return ctx; #ifdef HAVE_OPENSSL + fail_chain: + openssl_auth_destroy_store(ctx->store); fail_store: free(ctx); #endif @@ -982,6 +1269,7 @@ void auth_destroy_ctx(struct auth_ctx * ctx) if (ctx == NULL) return; #ifdef HAVE_OPENSSL + openssl_auth_destroy_chain(ctx->chain); openssl_auth_destroy_store(ctx->store); #endif free(ctx); @@ -1003,16 +1291,58 @@ int auth_add_crt_to_store(struct auth_ctx * ctx, #endif } +int auth_add_crt_to_chain(struct auth_ctx * ctx, + void * crt) +{ + assert(ctx != NULL); + assert(crt != NULL); + +#ifdef HAVE_OPENSSL + return openssl_auth_add_crt_to_chain(ctx->chain, crt); +#else + (void) ctx; + (void) crt; + + return 0; +#endif +} + int auth_verify_crt(struct auth_ctx * ctx, void * crt) { #ifdef HAVE_OPENSSL - return openssl_verify_crt(ctx->store, crt); + return openssl_verify_crt(ctx->store, ctx->chain, crt); #else (void) ctx; (void) crt; - return 0; + return -ENOTSUP; +#endif +} + +int auth_verify_crt_pin(struct auth_ctx * ctx, + void * crt, + void * pin) +{ +#ifdef HAVE_OPENSSL + return openssl_verify_crt_pin(ctx->store, ctx->chain, crt, pin); +#else + (void) ctx; + (void) crt; + (void) pin; + + return -ENOTSUP; +#endif +} + +bool crypt_pk_requires_md(const void * pk) +{ +#ifdef HAVE_OPENSSL + return openssl_pk_requires_md((const EVP_PKEY *) pk); +#else + (void) pk; + + return false; #endif } @@ -1048,7 +1378,7 @@ int auth_verify_sig(void * pk, (void) msg; (void) sig; - return 0; + return -ENOTSUP; #endif } @@ -1077,10 +1407,25 @@ ssize_t md_len(int md_nid) #endif } +int crypt_hkdf_expand(buffer_t key, + buffer_t info, + buffer_t out) +{ +#ifdef HAVE_OPENSSL + return openssl_hkdf_expand(key, info, out) == 0 ? 0 : -ECRYPT; +#else + (void) key; + (void) info; + (void) out; + + return -ECRYPT; +#endif +} + int crypt_secure_malloc_init(size_t max) { #ifdef HAVE_OPENSSL - return openssl_secure_malloc_init(max, SECMEM_GUARD); + return openssl_secure_malloc_init(max, SECMEM_MINSIZE); #else (void) max; return 0; diff --git a/src/lib/crypt/keyrot.c b/src/lib/crypt/keyrot.c new file mode 100644 index 00000000..e98df356 --- /dev/null +++ b/src/lib/crypt/keyrot.c @@ -0,0 +1,775 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Data-plane key-rotation schedule (node/leaf keys, selector) + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * version 2.1 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#define _POSIX_C_SOURCE 200809L + +#include <config.h> + +#include <ouroboros/atomics.h> +#include <ouroboros/crypt.h> +#include <ouroboros/errno.h> +#include <ouroboros/pthread.h> +#include <ouroboros/rcu.h> + +#include "crypt/keyrot.h" + +#include <assert.h> +#include <stdbool.h> +#include <stdlib.h> +#include <string.h> + +/* + * Per-flow keys are addressed by (epoch, node, leaf) and derived as: + * root = per-batch HKDF PRK from the OAP exchange, wiped once expanded + * nodes = HKDF-Expand(root, "o7s-keyrot-node") -> KEY_NODE_COUNT keys + * leaf = HKDF-Expand(node, "o7s-keyrot-leaf"|dir|leaf) -> AEAD key + * The epoch is a small wrapping counter, carried in the selector, that picks + * the live batch; a Tier-2 OAP re-key advances it. The "dir" byte forks the + * leaf keys per direction. + * + * Concurrency: cur/prev batch pointers are published by a re-key and read on + * the data path under an rcu_guard (lock-free RCU with liburcu, else a per- + * keyrot rwlock). The per-batch TX counter is atomic, so the (epoch, counter) + * nonce is unique without serialising TX. Leaf caches are THREAD-LOCAL (an app + * writer and the FRCT retransmit timer never share cache state), keyed on a + * global batch id and direct-mapped. + */ + +#define KR_WITHIN_BITS (KEY_LEAF_BITS + KEY_NODE_BITS) +#define KR_WITHIN_MASK (((uint64_t) 1 << KR_WITHIN_BITS) - 1) +#define KR_N (KEY_NODE_COUNT) +#define KR_LEAVES (1u << KEY_NODE_BITS) +#define KR_BATCH_MAX ((uint64_t) KR_N << KR_WITHIN_BITS) +#define KR_NODES_SZ ((size_t) KR_N * SYMMKEYSZ) +#define KR_TCACHE_WAYS 16 /* per-thread cache slots per direction (pow2) */ +#define KR_EPOCHS 16 /* 4-bit wire epoch: gens before wrap */ + +#define KR_RP_WORDS (KEY_REPLAY_WINDOW / 64) /* pow2; RFC 6479 bitmap */ +#define KR_RP_SHIFT 6 +#define KR_RP_MASK 63 +#define KR_RP_WINDOW (KEY_REPLAY_WINDOW - 64) /* reserve 1 slack word */ + +static const char kr_node_label[] = "o7s-keyrot-node"; +static const char kr_leaf_label[] = "o7s-keyrot-leaf"; + +struct kr_batch { + uint64_t id; /* process-global, unique; cache key (no ABA) */ + uint8_t epoch; /* 4-bit wire selector */ + uint8_t * nodes; /* KR_NODES_SZ in secure heap; NULL if empty */ + uint64_t tx_ctr; /* atomic; per-batch so nonces never collide */ + + struct { /* RFC 6479-like anti-replay window */ + uint64_t last; /* highest accepted ctr + 1 */ + uint64_t bits[KR_RP_WORDS]; + pthread_mutex_t mtx; + } rp; +}; + +struct kr_keycache { + uint8_t * key; /* SYMMKEYSZ, points into the per-thread slab */ + uint64_t id; /* batch the cached key belongs to */ + uint16_t node; + uint8_t leaf; + uint8_t dir; + bool valid; +}; + +struct keyrot { + struct kr_batch * cur; /* published; read on data path */ + struct kr_batch * prev; /* NULL = none */ + struct rcu_guard guard; /* re-key vs readers */ + uint8_t role; + uint8_t tx_epoch; /* epoch TX currently stamps */ + bool peer_switched; /* peer is on the cur epoch */ +}; + +/* Per-thread leaf-key caches, freed by the thread-exit destructor. */ +struct kr_tcache { + struct kr_keycache tx[KR_TCACHE_WAYS]; + struct kr_keycache rx[KR_TCACHE_WAYS]; + uint8_t * slab; /* 2*KR_TCACHE_WAYS*SYMMKEYSZ secure heap */ +}; + +static struct { + uint64_t next_id; /* batch-id allocator (atomic) */ + pthread_key_t tcache_key; /* per-thread leaf-key caches */ + pthread_once_t tcache_once; +} kr_g = { 0, 0, PTHREAD_ONCE_INIT }; + +static void kr_tcache_free(void * p) +{ + struct kr_tcache * t = p; + + if (t == NULL) + return; + + crypt_secure_free(t->slab, 2 * KR_TCACHE_WAYS * SYMMKEYSZ); + free(t); +} + +static void kr_tcache_init(void) +{ + pthread_key_create(&kr_g.tcache_key, kr_tcache_free); +} + +static struct kr_tcache * kr_tcache_get(void) +{ + struct kr_tcache * t; + size_t i; + + pthread_once(&kr_g.tcache_once, kr_tcache_init); + + t = pthread_getspecific(kr_g.tcache_key); + if (t != NULL) + return t; + + t = malloc(sizeof(*t)); + if (t == NULL) + goto fail_alloc; + + memset(t, 0, sizeof(*t)); + + t->slab = crypt_secure_malloc(2 * KR_TCACHE_WAYS * SYMMKEYSZ); + if (t->slab == NULL) + goto fail_slab; + + for (i = 0; i < KR_TCACHE_WAYS; i++) { + t->tx[i].key = t->slab + i * SYMMKEYSZ; + t->rx[i].key = t->slab + (KR_TCACHE_WAYS + i) * SYMMKEYSZ; + } + + if (pthread_setspecific(kr_g.tcache_key, t) != 0) + goto fail_set; + + return t; + + fail_set: + crypt_secure_free(t->slab, 2 * KR_TCACHE_WAYS * SYMMKEYSZ); + fail_slab: + free(t); + fail_alloc: + return NULL; +} + +static uint8_t * kr_expand_nodes(const uint8_t * root) +{ + uint8_t * nodes; + buffer_t prk; + buffer_t info; + buffer_t okm; + + nodes = crypt_secure_malloc(KR_NODES_SZ); + if (nodes == NULL) + return NULL; + + prk.len = SYMMKEYSZ; + prk.data = (uint8_t *) root; + info.len = sizeof(kr_node_label) - 1; + info.data = (uint8_t *) kr_node_label; + okm.len = KR_NODES_SZ; + okm.data = nodes; + + if (crypt_hkdf_expand(prk, info, okm) != 0) + goto fail_expand; + + return nodes; + + fail_expand: + crypt_secure_free(nodes, KR_NODES_SZ); + return NULL; +} + +static int kr_leaf_key(const uint8_t * node, + uint8_t leaf, + uint8_t dir, + uint8_t * out) +{ + uint8_t info_buf[sizeof(kr_leaf_label) - 1 + 2]; + buffer_t prk; + buffer_t info; + buffer_t okm; + size_t n = sizeof(kr_leaf_label) - 1; + + memcpy(info_buf, kr_leaf_label, n); + info_buf[n] = dir; + info_buf[n + 1] = leaf; + + prk.len = SYMMKEYSZ; + prk.data = (uint8_t *) node; + info.len = n + 2; + info.data = info_buf; + okm.len = SYMMKEYSZ; + okm.data = out; + + return crypt_hkdf_expand(prk, info, okm); +} + +static __inline__ bool kr_kc_hit(const struct kr_keycache * kc, + const struct kr_batch * b, + uint16_t node, + uint8_t leaf, + uint8_t dir) +{ + if (!kc->valid) + return false; + + if (kc->id != b->id) + return false; + + if (kc->node != node) + return false; + + if (kc->leaf != leaf) + return false; + + return kc->dir == dir; +} + +/* Fetch the leaf key; derive into the (direct-mapped) slot on a miss. */ +static const uint8_t * kr_kc_get(struct kr_keycache * cache, + const struct kr_batch * b, + uint16_t node, + uint8_t leaf, + uint8_t dir) +{ + struct kr_keycache * kc; + uint8_t * nkey; + + kc = &cache[b->id & (KR_TCACHE_WAYS - 1)]; + + if (kr_kc_hit(kc, b, node, leaf, dir)) + return kc->key; + + nkey = b->nodes + (size_t) node * SYMMKEYSZ; + if (kr_leaf_key(nkey, leaf, dir, kc->key) != 0) + return NULL; + + kc->valid = true; + kc->id = b->id; + kc->node = node; + kc->leaf = leaf; + kc->dir = dir; + + return kc->key; +} + +static void kr_sel_enc(uint8_t epoch, + uint16_t node, + uint32_t seq, + uint8_t sel[KR_SELECTOR_LEN]) +{ + sel[0] = (uint8_t) ((epoch << 4) | ((node >> 8) & 0x0F)); + sel[1] = (uint8_t) (node & 0xFF); + sel[2] = (uint8_t) (seq >> 24); + sel[3] = (uint8_t) (seq >> 16); + sel[4] = (uint8_t) (seq >> 8); + sel[5] = (uint8_t) (seq); +} + +static void kr_sel_dec(const uint8_t sel[KR_SELECTOR_LEN], + uint8_t * epoch, + uint16_t * node, + uint32_t * seq) +{ + *epoch = (uint8_t) (sel[0] >> 4); + *node = (uint16_t) (((sel[0] & 0x0F) << 8) | sel[1]); + *seq = ((uint32_t) sel[2] << 24) | ((uint32_t) sel[3] << 16) | + ((uint32_t) sel[4] << 8) | (uint32_t) sel[5]; +} + +static uint64_t kr_ctr(uint16_t node, + uint32_t seq) +{ + return ((uint64_t) node << KR_WITHIN_BITS) | + ((uint64_t) seq & KR_WITHIN_MASK); +} + +static void kr_nonce(uint64_t ctr, + uint8_t * nonce) +{ + size_t i; + + memset(nonce, 0, KR_NONCE_LEN); + + /* ctr big-endian in the low 8 bytes; high bytes stay zero */ + for (i = 0; i < 8; i++) + nonce[i] = (uint8_t) (ctr >> (56 - 8 * i)); +} + +static struct kr_batch * kr_batch_create(uint8_t epoch, + const uint8_t * root) +{ + struct kr_batch * b; + + b = malloc(sizeof(*b)); + if (b == NULL) + goto fail_alloc; + + b->nodes = kr_expand_nodes(root); + if (b->nodes == NULL) + goto fail_nodes; + + b->id = FETCH_ADD_RELAXED(&kr_g.next_id, 1); + b->epoch = epoch; + b->tx_ctr = 0; + if (pthread_mutex_init(&b->rp.mtx, NULL) != 0) + goto fail_lock; + + b->rp.last = 0; + memset(b->rp.bits, 0, sizeof(b->rp.bits)); + + return b; + + fail_lock: + crypt_secure_free(b->nodes, KR_NODES_SZ); + free(b); + return NULL; + fail_nodes: + free(b); + fail_alloc: + return NULL; +} + +static void kr_batch_free(struct kr_batch * b) +{ + if (b == NULL) + return; + + pthread_mutex_destroy(&b->rp.mtx); + crypt_secure_free(b->nodes, KR_NODES_SZ); + free(b); +} + +/* + * RFC 6479 anti-replay window keyed on the per-batch counter, with + * seq = ctr + 1 so 0 means "nothing accepted yet". Returns 0 if the + * packet is fresh (and records it), -1 on a replay or a too-old ctr. + */ +static int kr_rp_commit(struct kr_batch * b, + uint64_t ctr) +{ + uint64_t seq; + uint64_t idx; + uint64_t cur; + uint64_t diff; + + seq = ctr + 1; + + pthread_mutex_lock(&b->rp.mtx); + + if (seq > b->rp.last) { + idx = seq >> KR_RP_SHIFT; + cur = b->rp.last >> KR_RP_SHIFT; + diff = idx - cur; + if (diff > KR_RP_WORDS) + diff = KR_RP_WORDS; + + while (diff-- > 0) { + cur++; + b->rp.bits[cur & (KR_RP_WORDS - 1)] = 0; + } + + b->rp.bits[idx & (KR_RP_WORDS - 1)] |= + (uint64_t) 1 << (seq & KR_RP_MASK); + b->rp.last = seq; + goto finish; + } + + if (b->rp.last - seq >= KR_RP_WINDOW) + goto fail; + + idx = seq >> KR_RP_SHIFT; + if (b->rp.bits[idx & (KR_RP_WORDS - 1)] + & ((uint64_t) 1 << (seq & KR_RP_MASK))) + goto fail; + + b->rp.bits[idx & (KR_RP_WORDS - 1)] |= + (uint64_t) 1 << (seq & KR_RP_MASK); + finish: + pthread_mutex_unlock(&b->rp.mtx); + + return 0; + fail: + pthread_mutex_unlock(&b->rp.mtx); + + return -1; +} + +struct keyrot * keyrot_create(const uint8_t * root, + uint8_t epoch, + uint8_t role) +{ + struct keyrot * kr; + + assert(root != NULL); + assert(role <= 1); + + if (epoch >= KR_EPOCHS) + goto fail_kr; + + kr = malloc(sizeof(*kr)); + if (kr == NULL) + goto fail_kr; + + memset(kr, 0, sizeof(*kr)); + + kr->role = role; + kr->tx_epoch = epoch; + kr->peer_switched = true; + kr->prev = NULL; + + kr->cur = kr_batch_create(epoch, root); + if (kr->cur == NULL) + goto fail_cur; + + if (rcu_guard_init(&kr->guard)) + goto fail_guard; + + return kr; + + fail_guard: + kr_batch_free(kr->cur); + fail_cur: + free(kr); + fail_kr: + return NULL; +} + +void keyrot_destroy(struct keyrot * kr) +{ + if (kr == NULL) + return; + + /* Wait out any in-flight reader before freeing batches. */ + rcu_drain(&kr->guard); + + kr_batch_free(kr->cur); + kr_batch_free(kr->prev); + + rcu_guard_fini(&kr->guard); + + free(kr); +} + +/* A dup live epoch shadows straggler RX; epoch is peer-driven. */ +static struct kr_batch * kr_live_batch(struct kr_batch * cur, + struct kr_batch * prev, + uint8_t epoch) +{ + if (epoch == cur->epoch) + return cur; + + if (prev != NULL && epoch == prev->epoch) + return prev; + + return NULL; +} + +int keyrot_rekey(struct keyrot * kr, + const uint8_t * root, + uint8_t epoch) +{ + struct kr_batch * nb; + struct kr_batch * old_prev; + struct kr_batch * cur; + struct kr_batch * prev; + struct kr_batch * live; + int ret; + + assert(kr != NULL); + assert(root != NULL); + + if (epoch >= KR_EPOCHS) + return -1; + + nb = kr_batch_create(epoch, root); + if (nb == NULL) + return -1; + + rcu_wrlock(&kr->guard); + + cur = rcu_deref(kr->cur); + prev = rcu_deref(kr->prev); + + live = kr_live_batch(cur, prev, epoch); + if (live != NULL) { + /* The first node key identifies the root. */ + if (crypt_ct_cmp(live->nodes, nb->nodes, SYMMKEYSZ) == 0) + ret = -EREPLAY; + else + ret = -1; + rcu_wrunlock(&kr->guard); + kr_batch_free(nb); + return ret; + } + + old_prev = kr->prev; + rcu_assign(kr->prev, kr->cur); + rcu_publish(nb); + rcu_assign(kr->cur, nb); + + /* TX keeps the old epoch until the peer is seen on the new one. */ + STORE_RELEASE(&kr->peer_switched, false); + + rcu_wrunlock(&kr->guard); + + /* old_prev is unreachable now; reclaim past any live reader. */ + rcu_reclaim(&kr->guard); + kr_batch_free(old_prev); + + return 0; +} + +void keyrot_tx_promote(struct keyrot * kr) +{ + assert(kr != NULL); + + /* Serialise with keyrot_rekey so tx_epoch tracks a consistent cur. */ + rcu_wrlock(&kr->guard); + STORE_RELAXED(&kr->tx_epoch, rcu_deref(kr->cur)->epoch); + rcu_wrunlock(&kr->guard); +} + +int keyrot_tx_next(struct keyrot * kr, + uint8_t sel[KR_SELECTOR_LEN], + const uint8_t ** key, + uint8_t nonce[KR_NONCE_LEN]) +{ + struct kr_tcache * tc; + struct kr_batch * cur; + struct kr_batch * prev; + struct kr_batch * b; + uint64_t ctr; + uint16_t node; + uint8_t leaf; + uint8_t txe; + uint8_t epoch; + uint32_t seq; + const uint8_t * k; + + assert(kr != NULL); + assert(key != NULL); + + tc = kr_tcache_get(); + if (tc == NULL) + return -1; + + rcu_rdlock(&kr->guard); + + cur = rcu_deref(kr->cur); + prev = rcu_deref(kr->prev); + rcu_consume(cur); + rcu_consume(prev); + txe = LOAD_RELAXED(&kr->tx_epoch); + + if (cur->epoch == txe) + b = cur; + else if (prev != NULL && prev->epoch == txe) + b = prev; + else + b = NULL; + + if (b == NULL) { + rcu_rdunlock(&kr->guard); + return -1; /* tx_epoch batch gone; next promote resyncs */ + } + + /* Slot reserved even if exhausted; tx_nodes_left clamps the count. */ + ctr = FETCH_ADD_RELAXED(&b->tx_ctr, 1); + if (ctr >= KR_BATCH_MAX) { + rcu_rdunlock(&kr->guard); + return -1; /* batch exhausted */ + } + + node = (uint16_t) (ctr >> KR_WITHIN_BITS); + leaf = (uint8_t) ((ctr >> KEY_LEAF_BITS) & (KR_LEAVES - 1)); + seq = (uint32_t) (ctr & KR_WITHIN_MASK); + epoch = b->epoch; + + k = kr_kc_get(tc->tx, b, node, leaf, kr->role); + + rcu_rdunlock(&kr->guard); + + if (k == NULL) + return -1; + + kr_sel_enc(epoch, node, seq, sel); + kr_nonce(ctr, nonce); + + *key = k; + + return 0; +} + +int keyrot_rx_lookup(struct keyrot * kr, + const uint8_t sel[KR_SELECTOR_LEN], + const uint8_t ** key, + uint8_t nonce[KR_NONCE_LEN], + struct kr_rx * rx) +{ + struct kr_tcache * tc; + struct kr_batch * cur; + struct kr_batch * prev; + struct kr_batch * b; + uint8_t epoch; + uint16_t node; + uint32_t seq; + uint64_t ctr; + uint8_t leaf; + const uint8_t * k; + + assert(kr != NULL); + assert(key != NULL); + + kr_sel_dec(sel, &epoch, &node, &seq); + + if (node >= KR_N) + return -1; + + tc = kr_tcache_get(); + if (tc == NULL) + return -1; + + rcu_rdlock(&kr->guard); + + cur = rcu_deref(kr->cur); + prev = rcu_deref(kr->prev); + rcu_consume(cur); + rcu_consume(prev); + + if (epoch == cur->epoch) { + b = cur; + } else if (prev != NULL && epoch == prev->epoch) { + b = prev; + } else { + rcu_rdunlock(&kr->guard); + return -1; /* unknown epoch */ + } + + ctr = kr_ctr(node, seq); + leaf = (uint8_t) ((ctr >> KEY_LEAF_BITS) & (KR_LEAVES - 1)); + + /* peer's tx direction */ + k = kr_kc_get(tc->rx, b, node, leaf, (uint8_t) (kr->role ^ 1)); + + rx->id = b->id; + rx->ctr = ctr; + + rcu_rdunlock(&kr->guard); + + if (k == NULL) + return -1; + + kr_nonce(ctr, nonce); + + *key = k; + + return 0; +} + +/* + * Commit a packet that authenticated under the batch keyrot_rx_lookup + * selected. Re-finds that batch by id (epoch may have advanced) and, + * if still resident, advances the replay window and records that the + * peer is on the current batch. Runs only post-AEAD so a forged or + * replayed packet can mutate no receiver state. Returns -1 on replay. + */ +int keyrot_rx_commit(struct keyrot * kr, + const struct kr_rx * rx) +{ + struct kr_batch * cur; + struct kr_batch * prev; + struct kr_batch * b; + int rc; + + assert(kr != NULL); + assert(rx != NULL); + + rcu_rdlock(&kr->guard); + + cur = rcu_deref(kr->cur); + prev = rcu_deref(kr->prev); + rcu_consume(cur); + rcu_consume(prev); + + if (cur->id == rx->id) + b = cur; + else if (prev != NULL && prev->id == rx->id) + b = prev; + else + b = NULL; + + if (b == NULL) { + rcu_rdunlock(&kr->guard); + return 0; /* batch evicted post-auth; nothing to protect */ + } + + rc = kr_rp_commit(b, rx->ctr); + if (rc == 0 && b == cur) + STORE_RELEASE(&kr->peer_switched, true); + + rcu_rdunlock(&kr->guard); + + return rc; +} + +bool keyrot_peer_switched(const struct keyrot * kr) +{ + assert(kr != NULL); + + return LOAD_ACQUIRE(&kr->peer_switched); +} + +unsigned keyrot_tx_nodes_left(struct keyrot * kr) +{ + struct kr_batch * cur; + struct kr_batch * prev; + struct kr_batch * b; + uint64_t ctr; + unsigned used; + uint8_t txe; + + assert(kr != NULL); + + rcu_rdlock(&kr->guard); + cur = rcu_deref(kr->cur); + prev = rcu_deref(kr->prev); + rcu_consume(cur); + rcu_consume(prev); + txe = LOAD_RELAXED(&kr->tx_epoch); + + if (cur->epoch == txe) + b = cur; + else if (prev != NULL && prev->epoch == txe) + b = prev; + else + b = NULL; + + ctr = b != NULL ? LOAD_RELAXED(&b->tx_ctr) : KR_BATCH_MAX; + rcu_rdunlock(&kr->guard); + + used = (unsigned) (ctr >> KR_WITHIN_BITS); + if (used >= KR_N) + return 0; + + return KR_N - used; +} diff --git a/src/lib/crypt/keyrot.h b/src/lib/crypt/keyrot.h new file mode 100644 index 00000000..6a598f76 --- /dev/null +++ b/src/lib/crypt/keyrot.h @@ -0,0 +1,74 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Data-plane key-rotation schedule (node/leaf keys, selector) + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * version 2.1 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#ifndef OUROBOROS_LIB_CRYPT_KEYROT_H +#define OUROBOROS_LIB_CRYPT_KEYROT_H + +#include <ouroboros/crypt.h> /* SYMMKEYSZ, NONCESZ */ + +#include <stdbool.h> +#include <stdint.h> + +#define KR_SELECTOR_LEN 6 +#define KR_NONCE_LEN NONCESZ + +struct keyrot; + +struct kr_rx { + uint64_t id; /* batch id of the matched epoch */ + uint64_t ctr; /* packet counter for replay check */ +}; + +struct keyrot * keyrot_create(const uint8_t * root, + uint8_t epoch, + uint8_t role); + +void keyrot_destroy(struct keyrot * kr); + +int keyrot_rekey(struct keyrot * kr, + const uint8_t * root, + uint8_t epoch); + +/* Promote TX to the installed (new) batch once the peer is on it. */ +void keyrot_tx_promote(struct keyrot * kr); + +int keyrot_tx_next(struct keyrot * kr, + uint8_t sel[KR_SELECTOR_LEN], + const uint8_t ** key, + uint8_t nonce[KR_NONCE_LEN]); + +int keyrot_rx_lookup(struct keyrot * kr, + const uint8_t sel[KR_SELECTOR_LEN], + const uint8_t ** key, + uint8_t nonce[KR_NONCE_LEN], + struct kr_rx * rx); + +/* Commit an authenticated packet: replay window + peer-switched. */ +int keyrot_rx_commit(struct keyrot * kr, + const struct kr_rx * rx); + +/* True once an RX packet under the current batch has been observed. */ +bool keyrot_peer_switched(const struct keyrot * kr); + +unsigned keyrot_tx_nodes_left(struct keyrot * kr); + +#endif /* OUROBOROS_LIB_CRYPT_KEYROT_H */ diff --git a/src/lib/crypt/openssl.c b/src/lib/crypt/openssl.c index 5916e3cb..9c488b9d 100644 --- a/src/lib/crypt/openssl.c +++ b/src/lib/crypt/openssl.c @@ -30,11 +30,14 @@ #include <ouroboros/errno.h> #include <ouroboros/crypt.h> #include <ouroboros/hash.h> +#include <ouroboros/name.h> +#include <ouroboros/pthread.h> #include <ouroboros/random.h> #include <ouroboros/utils.h> #include <openssl/evp.h> #include <openssl/bio.h> +#include <openssl/crypto.h> #include <openssl/ec.h> #include <openssl/err.h> #include <openssl/kdf.h> @@ -45,6 +48,7 @@ #include <openssl/x509_vfy.h> #include <assert.h> +#include <limits.h> #include <stdio.h> #define IS_EC_GROUP(str) (strcmp(str, "EC") == 0) @@ -52,29 +56,43 @@ #define HKDF_INFO_DHE "o7s-ossl-dhe" #define HKDF_INFO_ENCAP "o7s-ossl-encap" -#define HKDF_INFO_ROTATION "o7s-key-rotation" #define HKDF_SALT_LEN 32 /* SHA-256 output size */ +#define AEAD_NONCE_LEN 12 /* 96-bit deterministic IV (SP 800-38D) */ +#define AEAD_TAG_LEN 16 /* 128-bit AEAD authentication tag */ +/* + * Holds only the cipher identity, which is fixed at creation. A flow's + * context is sealed and opened by several threads at once, so nothing + * here may carry per-packet state. + */ struct ossl_crypt_ctx { - EVP_CIPHER_CTX * evp_ctx; const EVP_CIPHER * cipher; - int ivsz; int tagsz; +}; + +/* + * Per-thread AEAD context. A key covers 2^KEY_LEAF_BITS packets, so + * the key schedule is worth keeping between them; only the nonce + * changes. Thread-local, so concurrent sealers share nothing, and a + * miss costs no more than a full install. + */ +struct ossl_aead { + EVP_CIPHER_CTX * evp; + const EVP_CIPHER * cipher; /* NULL when the state is unusable */ + uint8_t key[SYMMKEYSZ]; + size_t keylen; +}; - struct { - uint8_t * cur; /* current key */ - uint8_t * prv; /* rotated key */ - } keys; - - struct { - uint32_t cntr; /* counter */ - uint32_t mask; /* phase mask */ - uint32_t age; /* counter within epoch */ - uint8_t phase; /* current key phase */ - uint8_t salt[HKDF_SALT_LEN]; - } rot; /* rotation logic */ +struct ossl_aead_tls { + struct ossl_aead seal; + struct ossl_aead open; }; +static struct { + pthread_key_t key; + pthread_once_t once; +} aead_g = { 0, PTHREAD_ONCE_INIT }; + struct kdf_info { buffer_t secret; int nid; @@ -83,17 +101,6 @@ struct kdf_info { buffer_t key; }; -/* Key rotation macros */ -#define HAS_PHASE_BIT_TOGGLED(ctx) \ - (((ctx)->rot.cntr & (ctx)->rot.mask) != \ - (((ctx)->rot.cntr - 1) & (ctx)->rot.mask)) - -#define HAS_GRACE_EXPIRED(ctx) \ - ((ctx)->rot.age >= ((ctx)->rot.mask >> 1)) - -#define ROTATION_TOO_RECENT(ctx) \ - ((ctx)->rot.age < ((ctx)->rot.mask - ((ctx)->rot.mask >> 2))) - /* Convert hash NID to OpenSSL digest name string for HKDF */ static const char * hash_nid_to_digest_name(int nid) { @@ -102,11 +109,11 @@ static const char * hash_nid_to_digest_name(int nid) md = EVP_get_digestbynid(nid); if (md == NULL) - return "SHA256"; /* fallback to SHA-256 */ + return NULL; name = EVP_MD_get0_name(md); if (name == NULL) - return "SHA256"; /* fallback to SHA-256 */ + return NULL; return name; } @@ -144,21 +151,20 @@ static int get_pk_bytes_from_key(EVP_PKEY * key, } /* Derive salt from public key bytes by hashing them */ -static int derive_salt_from_pk_bytes(buffer_t pk, - uint8_t * salt, - size_t salt_len) +static int derive_salt_from_pk_bytes(buffer_t pk, + buffer_t salt) { uint8_t hash[EVP_MAX_MD_SIZE]; unsigned hash_len; assert(pk.data != NULL); - assert(salt != NULL); + assert(salt.data != NULL); if (EVP_Digest(pk.data, pk.len, hash, &hash_len, EVP_sha256(), NULL) != 1) goto fail_digest; - memcpy(salt, hash, salt_len < hash_len ? salt_len : hash_len); + memcpy(salt.data, hash, salt.len < hash_len ? salt.len : hash_len); return 0; fail_digest: @@ -166,10 +172,9 @@ static int derive_salt_from_pk_bytes(buffer_t pk, } /* Derive salt from two public key byte buffers (DHE) in canonical order */ -static int derive_salt_from_pk_bytes_dhe(buffer_t local, - buffer_t remote, - uint8_t * salt, - size_t salt_len) +static int derive_salt_from_pk_bytes_dhe(buffer_t local, + buffer_t remote, + buffer_t salt) { uint8_t * concat; size_t concat_len; @@ -180,7 +185,7 @@ static int derive_salt_from_pk_bytes_dhe(buffer_t local, assert(local.data != NULL); assert(remote.data != NULL); - assert(salt != NULL); + assert(salt.data != NULL); concat_len = local.len + remote.len; concat = OPENSSL_malloc(concat_len); @@ -204,7 +209,7 @@ static int derive_salt_from_pk_bytes_dhe(buffer_t local, OPENSSL_free(concat); - memcpy(salt, hash, salt_len < hash_len ? salt_len : hash_len); + memcpy(salt.data, hash, salt.len < hash_len ? salt.len : hash_len); return 0; fail_digest: @@ -225,6 +230,8 @@ static int derive_key_hkdf(struct kdf_info * ki) int idx; digest = hash_nid_to_digest_name(ki->nid); + if (digest == NULL) + goto fail_fetch; kdf = EVP_KDF_fetch(NULL, "HKDF", NULL); if (kdf == NULL) @@ -258,117 +265,258 @@ static int derive_key_hkdf(struct kdf_info * ki) return -ECRYPT; } -/* Key rotation helper functions implementation */ -static int should_rotate_key_rx(struct ossl_crypt_ctx * ctx, - uint8_t rx_phase) +int openssl_hkdf_expand(buffer_t key, + buffer_t info, + buffer_t out) { - assert(ctx != NULL); + EVP_KDF * kdf; + EVP_KDF_CTX * kctx; + OSSL_PARAM params[5]; + int mode = EVP_KDF_HKDF_MODE_EXPAND_ONLY; + int idx = 0; + int ret = -1; + + kdf = EVP_KDF_fetch(NULL, "HKDF", NULL); + if (kdf == NULL) + goto fail_fetch; + + kctx = EVP_KDF_CTX_new(kdf); + if (kctx == NULL) + goto fail_ctx; - /* Phase must have changed */ - if (rx_phase == ctx->rot.phase) - return 0; + params[idx++] = OSSL_PARAM_construct_utf8_string( + "digest", (char *) "SHA256", 0); + params[idx++] = OSSL_PARAM_construct_int("mode", &mode); + params[idx++] = OSSL_PARAM_construct_octet_string( + "key", key.data, key.len); + params[idx++] = OSSL_PARAM_construct_octet_string( + "info", info.data, info.len); + params[idx] = OSSL_PARAM_construct_end(); - if (ROTATION_TOO_RECENT(ctx)) - return 0; + if (EVP_KDF_derive(kctx, out.data, out.len, params) == 1) + ret = 0; - return 1; + EVP_KDF_CTX_free(kctx); + fail_ctx: + EVP_KDF_free(kdf); + fail_fetch: + return ret; } -static int rotate_key(struct ossl_crypt_ctx * ctx) +static void aead_tls_free(void * p) { - struct kdf_info ki; - uint8_t * tmp; + struct ossl_aead_tls * t = p; + if (t == NULL) + return; - assert(ctx != NULL); + EVP_CIPHER_CTX_free(t->seal.evp); + EVP_CIPHER_CTX_free(t->open.evp); - /* Swap keys - move current to prev */ - tmp = ctx->keys.prv; - ctx->keys.prv = ctx->keys.cur; + crypt_secure_clear(t->seal.key, SYMMKEYSZ); + crypt_secure_clear(t->open.key, SYMMKEYSZ); - if (tmp != NULL) { - /* Reuse old prev_key memory for new key */ - ctx->keys.cur = tmp; - } else { - /* First rotation - allocate new memory */ - ctx->keys.cur = OPENSSL_secure_malloc(SYMMKEYSZ); - if (ctx->keys.cur == NULL) - return -ECRYPT; + free(t); +} + +static void aead_tls_init(void) +{ + pthread_key_create(&aead_g.key, aead_tls_free); +} + +static struct ossl_aead_tls * aead_tls_get(void) +{ + struct ossl_aead_tls * t; + + pthread_once(&aead_g.once, aead_tls_init); + + t = pthread_getspecific(aead_g.key); + if (t != NULL) + return t; + + t = malloc(sizeof(*t)); + if (t == NULL) + return NULL; + + memset(t, 0, sizeof(*t)); + + if (pthread_setspecific(aead_g.key, t) != 0) { + free(t); + return NULL; } - /* Derive new key from previous key using HKDF */ - ki.secret.data = ctx->keys.prv; - ki.secret.len = SYMMKEYSZ; - ki.nid = NID_sha256; - ki.salt.data = ctx->rot.salt; - ki.salt.len = HKDF_SALT_LEN; - ki.info.data = (uint8_t *) HKDF_INFO_ROTATION; - ki.info.len = strlen(HKDF_INFO_ROTATION); - ki.key.data = ctx->keys.cur; - ki.key.len = SYMMKEYSZ; + return t; +} - if (derive_key_hkdf(&ki) != 0) - return -ECRYPT; +/* Install cipher and key; the nonce is set per packet by the caller. */ +static int aead_install(EVP_CIPHER_CTX * evp, + const EVP_CIPHER * cipher, + const uint8_t * key, + bool enc) +{ + EVP_CIPHER_CTX_reset(evp); + + if (enc) { + if (EVP_EncryptInit_ex(evp, cipher, NULL, NULL, NULL) != 1) + return -1; + } else { + if (EVP_DecryptInit_ex(evp, cipher, NULL, NULL, NULL) != 1) + return -1; + } - ctx->rot.age = 0; - ctx->rot.phase = !ctx->rot.phase; + /* Pin the AEAD nonce to 96 bits (SP 800-38D deterministic IV). */ + if (EVP_CIPHER_CTX_ctrl(evp, EVP_CTRL_AEAD_SET_IVLEN, + AEAD_NONCE_LEN, NULL) != 1) + return -1; + + if (enc) { + if (EVP_EncryptInit_ex(evp, NULL, NULL, key, NULL) != 1) + return -1; + } else { + if (EVP_DecryptInit_ex(evp, NULL, NULL, key, NULL) != 1) + return -1; + } return 0; } -static void cleanup_old_key(struct ossl_crypt_ctx * ctx) +/* This thread's context for cipher/key, ready to take a nonce. */ +static EVP_CIPHER_CTX * aead_ctx(struct ossl_aead * a, + const EVP_CIPHER * cipher, + const uint8_t * key, + bool enc) +{ + int keylen; + + keylen = EVP_CIPHER_get_key_length(cipher); + if (keylen <= 0 || (size_t) keylen > SYMMKEYSZ) + return NULL; + + /* Compare the bytes: a cache slot can be reused for a new key. */ + if (a->cipher == cipher && a->keylen == (size_t) keylen + && CRYPTO_memcmp(a->key, key, a->keylen) == 0) + return a->evp; + + if (a->evp == NULL) { + a->evp = EVP_CIPHER_CTX_new(); + if (a->evp == NULL) + return NULL; + } + + a->cipher = NULL; + if (aead_install(a->evp, cipher, key, enc) < 0) + return NULL; + + memcpy(a->key, key, (size_t) keylen); + + a->keylen = (size_t) keylen; + a->cipher = cipher; + + return a->evp; +} + +/* AEAD seal: encrypt in with key/nonce, bind aad, append tag */ +int openssl_seal(struct ossl_crypt_ctx * ctx, + const uint8_t * key, + const uint8_t * nonce, + buffer_t aad, + buffer_t in, + uint8_t * out, + uint8_t * tag) { + struct ossl_aead_tls * tls; + EVP_CIPHER_CTX * evp; + int out_sz; + int tmp_sz; + assert(ctx != NULL); + assert(ctx->tagsz > 0); /* AEAD mandated at ctx creation */ - if (ctx->keys.prv == NULL) - return; + tls = aead_tls_get(); + if (tls == NULL) + goto fail; - if (!HAS_GRACE_EXPIRED(ctx)) - return; + evp = aead_ctx(&tls->seal, ctx->cipher, key, true); + if (evp == NULL) + goto fail; + + if (EVP_EncryptInit_ex(evp, NULL, NULL, NULL, nonce) != 1) + goto fail_evp; + + if (EVP_EncryptUpdate(evp, NULL, &tmp_sz, aad.data, (int) aad.len) != 1) + goto fail_evp; + + if (EVP_EncryptUpdate(evp, out, &out_sz, in.data, (int) in.len) != 1) + goto fail_evp; + + if (EVP_EncryptFinal_ex(evp, out + out_sz, &tmp_sz) != 1) + goto fail_evp; + + out_sz += tmp_sz; + + if (EVP_CIPHER_CTX_ctrl(evp, EVP_CTRL_AEAD_GET_TAG, + ctx->tagsz, tag) != 1) + goto fail_evp; - OPENSSL_secure_clear_free(ctx->keys.prv, SYMMKEYSZ); - ctx->keys.prv = NULL; + return out_sz; + fail_evp: + tls->seal.cipher = NULL; /* state unknown; install afresh */ + fail: + return -1; } -static int try_decrypt(struct ossl_crypt_ctx * ctx, - uint8_t * key, - uint8_t * iv, - uint8_t * input, - int in_sz, - uint8_t * out, - int * out_sz) +/* AEAD open: decrypt in with key/nonce, verify aad and tag */ +int openssl_open(struct ossl_crypt_ctx * ctx, + const uint8_t * key, + const uint8_t * nonce, + buffer_t aad, + buffer_t in, + const uint8_t * tag, + buffer_t * out) { - uint8_t * tag; - int tmp_sz; - int ret; + struct ossl_aead_tls * tls; + EVP_CIPHER_CTX * evp; + int out_sz; + int tmp_sz; - tag = input + in_sz; + assert(ctx != NULL); + assert(ctx->tagsz > 0); /* AEAD mandated at ctx creation */ - EVP_CIPHER_CTX_reset(ctx->evp_ctx); + tls = aead_tls_get(); + if (tls == NULL) + goto fail; - ret = EVP_DecryptInit_ex(ctx->evp_ctx, ctx->cipher, NULL, key, iv); - if (ret != 1) - return -1; + evp = aead_ctx(&tls->open, ctx->cipher, key, false); + if (evp == NULL) + goto fail; - if (ctx->tagsz > 0) { - ret = EVP_CIPHER_CTX_ctrl(ctx->evp_ctx, EVP_CTRL_AEAD_SET_TAG, - ctx->tagsz, tag); - if (ret != 1) - return -1; - } + if (EVP_DecryptInit_ex(evp, NULL, NULL, NULL, nonce) != 1) + goto fail_evp; - ret = EVP_DecryptUpdate(ctx->evp_ctx, out, &tmp_sz, input, in_sz); - if (ret != 1) - return -1; + if (EVP_CIPHER_CTX_ctrl(evp, EVP_CTRL_AEAD_SET_TAG, + ctx->tagsz, (void *) tag) != 1) + goto fail_evp; - *out_sz = tmp_sz; + if (EVP_DecryptUpdate(evp, NULL, &tmp_sz, aad.data, (int) aad.len) != 1) + goto fail_evp; - ret = EVP_DecryptFinal_ex(ctx->evp_ctx, out + tmp_sz, &tmp_sz); - if (ret != 1) - return -1; + if (EVP_DecryptUpdate(evp, out->data, &out_sz, + in.data, (int) in.len) != 1) + goto fail_evp; - *out_sz += tmp_sz; + /* A failed verify leaves defined state; keep the key cached. */ + if (EVP_DecryptFinal_ex(evp, out->data + out_sz, &tmp_sz) != 1) + goto fail_verify; - return 0; + out_sz += tmp_sz; + + out->len = (size_t) out_sz; + + return out_sz; + fail_evp: + tls->open.cipher = NULL; /* state unknown; install afresh */ + fail_verify: + fail: + return -1; } /* @@ -396,11 +544,14 @@ static int __openssl_dhe_derive(EVP_PKEY * pkp, ret = i2d_PUBKEY(pkp, &local_pk.data); if (ret <= 0) goto fail_local; + local_pk.len = (size_t) ret; + ki.salt.len = HKDF_SALT_LEN; + ki.salt.data = salt_buf; + /* Derive salt from both public keys */ - if (derive_salt_from_pk_bytes_dhe(local_pk, remote_pk, salt_buf, - HKDF_SALT_LEN) < 0) + if (derive_salt_from_pk_bytes_dhe(local_pk, remote_pk, ki.salt) < 0) goto fail_salt; ctx = EVP_PKEY_CTX_new(pkp, NULL); @@ -437,13 +588,11 @@ static int __openssl_dhe_derive(EVP_PKEY * pkp, ki.info.data = (uint8_t *) HKDF_INFO_DHE; ki.key.len = SYMMKEYSZ; ki.key.data = s; - ki.salt.len = HKDF_SALT_LEN; - ki.salt.data = salt_buf; /* Derive symmetric key from shared secret using HKDF */ ret = derive_key_hkdf(&ki); - OPENSSL_free(secret); + OPENSSL_clear_free(secret, secret_len); EVP_PKEY_CTX_free(ctx); OPENSSL_free(local_pk.data); @@ -452,7 +601,7 @@ static int __openssl_dhe_derive(EVP_PKEY * pkp, return 0; fail_derive: - OPENSSL_free(secret); + OPENSSL_clear_free(secret, secret_len); fail_ctx: EVP_PKEY_CTX_free(ctx); fail_salt: @@ -573,23 +722,6 @@ static int __openssl_kem_gen_key(const char * algo, return -ECRYPT; } -/* Determine hybrid KEM algorithm from raw key/ciphertext length */ -static const char * __openssl_hybrid_algo_from_len(size_t len) -{ - switch(len) { - case X25519MLKEM768_PKSZ: - return "X25519MLKEM768"; - case X25519MLKEM768_CTSZ: - return "X25519MLKEM768"; - case X448MLKEM1024_PKSZ: - return "X448MLKEM1024"; - default: - break; - } - - return NULL; -} - static int __openssl_kex_gen_key(const char * algo, EVP_PKEY ** kp) { @@ -624,14 +756,22 @@ ssize_t openssl_pkp_create(const char * algo, if (raw.len == 0) goto fail_pubkey; + if (raw.len > CRYPT_KEY_BUFSZ) { + OPENSSL_free(raw.data); + goto fail_pubkey; + } + memcpy(pk, raw.data, raw.len); OPENSSL_free(raw.data); return (ssize_t) raw.len; } else { /* DER encode standard algorithms */ + len = i2d_PUBKEY(*pkp, NULL); /* pre-flight length */ + if (len < 0 || len > CRYPT_KEY_BUFSZ) + goto fail_pubkey; + pos = pk; /* i2d_PUBKEY increments the ptr, don't use pk! */ - len = i2d_PUBKEY(*pkp, &pos); - if (len < 0) + if (i2d_PUBKEY(*pkp, &pos) < 0) goto fail_pubkey; return len; @@ -692,7 +832,7 @@ static ssize_t __openssl_kem_encap(EVP_PKEY * pub, /* Derive symmetric key from shared secret using HKDF */ ret = derive_key_hkdf(&ki); - OPENSSL_free(secret); + OPENSSL_clear_free(secret, secret_len); EVP_PKEY_CTX_free(ctx); if (ret != 0) @@ -701,7 +841,7 @@ static ssize_t __openssl_kem_encap(EVP_PKEY * pub, return (ssize_t) ct_len; fail_secret: - OPENSSL_free(secret); + OPENSSL_clear_free(secret, secret_len); fail_encap: EVP_PKEY_CTX_free(ctx); fail_ctx: @@ -717,13 +857,17 @@ ssize_t openssl_kem_encap(buffer_t pk, EVP_PKEY * pub; uint8_t * pos; uint8_t salt[HKDF_SALT_LEN]; + buffer_t salt_b; ssize_t ret; assert(pk.data != NULL); assert(ct != NULL); assert(s != NULL); - if (derive_salt_from_pk_bytes(pk, salt, HKDF_SALT_LEN) < 0) + salt_b.len = HKDF_SALT_LEN; + salt_b.data = salt; + + if (derive_salt_from_pk_bytes(pk, salt_b) < 0) goto fail_salt; pos = pk.data; @@ -740,26 +884,27 @@ ssize_t openssl_kem_encap(buffer_t pk, return -ECRYPT; } -/* Hybrid KEM encapsulation: raw-encoded public key */ -ssize_t openssl_kem_encap_raw(buffer_t pk, - uint8_t * ct, - int kdf, - uint8_t * s) +/* Hybrid KEM encapsulation: NID-tagged raw-encoded public key */ +ssize_t openssl_kem_encap_raw(const char * algo, + buffer_t pk, + uint8_t * ct, + int kdf, + uint8_t * s) { - EVP_PKEY * pub; - const char * algo; - uint8_t salt[HKDF_SALT_LEN]; - ssize_t ret; + EVP_PKEY * pub; + uint8_t salt[HKDF_SALT_LEN]; + buffer_t salt_b; + ssize_t ret; + assert(algo != NULL); assert(pk.data != NULL); assert(ct != NULL); assert(s != NULL); - if (derive_salt_from_pk_bytes(pk, salt, HKDF_SALT_LEN) < 0) - goto fail_salt; + salt_b.len = HKDF_SALT_LEN; + salt_b.data = salt; - algo = __openssl_hybrid_algo_from_len(pk.len); - if (algo == NULL) + if (derive_salt_from_pk_bytes(pk, salt_b) < 0) goto fail_salt; pub = EVP_PKEY_new_raw_public_key_ex(NULL, algo, NULL, @@ -789,12 +934,16 @@ int openssl_kem_decap(EVP_PKEY * priv, size_t secret_len; int ret; uint8_t salt[HKDF_SALT_LEN]; + buffer_t salt_b; /* Extract public key bytes from private key */ if (get_pk_bytes_from_key(priv, &pk) < 0) goto fail_pk; - if (derive_salt_from_pk_bytes(pk, salt, HKDF_SALT_LEN) < 0) + salt_b.len = HKDF_SALT_LEN; + salt_b.data = salt; + + if (derive_salt_from_pk_bytes(pk, salt_b) < 0) goto fail_salt; ctx = EVP_PKEY_CTX_new(priv, NULL); @@ -833,7 +982,7 @@ int openssl_kem_decap(EVP_PKEY * priv, /* Derive symmetric key from shared secret using HKDF */ ret = derive_key_hkdf(&ki); - OPENSSL_free(secret); + OPENSSL_clear_free(secret, secret_len); EVP_PKEY_CTX_free(ctx); OPENSSL_free(pk.data); @@ -843,7 +992,7 @@ int openssl_kem_decap(EVP_PKEY * priv, return 0; fail_secret: - OPENSSL_free(secret); + OPENSSL_clear_free(secret, secret_len); fail_ctx: EVP_PKEY_CTX_free(ctx); fail_salt: @@ -857,13 +1006,14 @@ void openssl_pkp_destroy(EVP_PKEY * pkp) EVP_PKEY_free(pkp); } -int __openssl_get_curve(EVP_PKEY * pub, - char * algo) +static int openssl_get_curve(EVP_PKEY * pub, + char * algo) { int ret; size_t len = KEX_ALGO_BUFSZ; ret = EVP_PKEY_get_utf8_string_param(pub, "group", algo, len, &len); + return ret == 1 ? 0 : -ECRYPT; } @@ -888,9 +1038,10 @@ int openssl_get_algo_from_pk_der(buffer_t pk, strcpy(algo, type_str); - if ((IS_EC_GROUP(algo) || IS_DH_GROUP(algo)) && - __openssl_get_curve(pub, algo) < 0) - goto fail_pub; + if (IS_EC_GROUP(algo) || IS_DH_GROUP(algo)) { + if (openssl_get_curve(pub, algo) < 0) + goto fail_pub; + } EVP_PKEY_free(pub); return 0; @@ -901,30 +1052,14 @@ int openssl_get_algo_from_pk_der(buffer_t pk, return -ECRYPT; } -int openssl_get_algo_from_pk_raw(buffer_t pk, - char * algo) -{ - const char * hybrid_algo; - - assert(pk.data != NULL); - assert(algo != NULL); - - hybrid_algo = __openssl_hybrid_algo_from_len(pk.len); - if (hybrid_algo == NULL) - return -ECRYPT; - - strcpy(algo, hybrid_algo); - - return 0; -} - int openssl_dhe_derive(EVP_PKEY * pkp, buffer_t pk, int kdf, uint8_t * s) { - uint8_t * pos; - EVP_PKEY * pub; + uint8_t * pos; + EVP_PKEY * pub; + const char * name; assert(pkp != NULL); assert(pk.data != NULL); @@ -936,6 +1071,11 @@ int openssl_dhe_derive(EVP_PKEY * pkp, if (pub == NULL) goto fail_decode; + /* A peer key of another type must not reach the derivation */ + name = EVP_PKEY_get0_type_name(pkp); + if (name == NULL || EVP_PKEY_is_a(pub, name) != 1) + goto fail_derive; + if (__openssl_dhe_derive(pkp, pub, pk, kdf, s) < 0) goto fail_derive; @@ -948,141 +1088,110 @@ int openssl_dhe_derive(EVP_PKEY * pkp, return -ECRYPT; } -int openssl_encrypt(struct ossl_crypt_ctx * ctx, - buffer_t in, - buffer_t * out) +/* Set up a fresh AEAD cipher ctx for nid: reject non-AEAD / oversized IV. */ +static int ossl_cipher_ctx_init(struct ossl_crypt_ctx * ctx, + int nid) { - uint8_t * ptr; - uint8_t * iv; - int in_sz; - int out_sz; - int tmp_sz; - int ret; - - assert(ctx != NULL); - - in_sz = (int) in.len; - - out->data = malloc(in.len + EVP_MAX_BLOCK_LENGTH + \ - ctx->ivsz + ctx->tagsz); - if (out->data == NULL) - goto fail_malloc; - - iv = out->data; - ptr = out->data + ctx->ivsz; - - if (random_buffer(iv, ctx->ivsz) < 0) - goto fail_encrypt; - - /* Set IV bit 7 to current key phase (KEY_ROTATION_BIT of counter) */ - if (ctx->rot.cntr & ctx->rot.mask) - iv[0] |= 0x80; - else - iv[0] &= 0x7F; + ctx->cipher = EVP_get_cipherbynid(nid); + if (ctx->cipher == NULL) + return -1; - EVP_CIPHER_CTX_reset(ctx->evp_ctx); + /* IV must fit the NONCESZ nonce buffer. */ + if (EVP_CIPHER_get_iv_length(ctx->cipher) > NONCESZ) + return -1; - ret = EVP_EncryptInit_ex(ctx->evp_ctx, ctx->cipher, NULL, - ctx->keys.cur, iv); - if (ret != 1) - goto fail_encrypt; + /* Authenticated encryption is mandatory; reject non-AEAD ciphers. */ + if ((EVP_CIPHER_flags(ctx->cipher) & EVP_CIPH_FLAG_AEAD_CIPHER) == 0) + return -1; - ret = EVP_EncryptUpdate(ctx->evp_ctx, ptr, &tmp_sz, in.data, in_sz); - if (ret != 1) - goto fail_encrypt; + ctx->tagsz = AEAD_TAG_LEN; - out_sz = tmp_sz; - ret = EVP_EncryptFinal_ex(ctx->evp_ctx, ptr + tmp_sz, &tmp_sz); - if (ret != 1) - goto fail_encrypt; + return 0; +} - out_sz += tmp_sz; +/* One-shot AEAD seal over an explicit key/nonce (no keyrot). out = ct ‖ tag. */ +int openssl_oneshot_seal(int nid, + const uint8_t * key, + const uint8_t * nonce, + buffer_t aad, + buffer_t in, + buffer_t * out) +{ + struct ossl_crypt_ctx ctx; + int out_sz; - /* For AEAD ciphers, get and append the authentication tag */ - if (ctx->tagsz > 0) { - ret = EVP_CIPHER_CTX_ctrl(ctx->evp_ctx, EVP_CTRL_AEAD_GET_TAG, - ctx->tagsz, ptr + out_sz); - if (ret != 1) - goto fail_encrypt; - out_sz += ctx->tagsz; - } + assert(key != NULL); + assert(nonce != NULL); + assert(out != NULL); - assert(out_sz >= in_sz); + memset(&ctx, 0, sizeof(ctx)); - out->len = (size_t) out_sz + ctx->ivsz; + if (ossl_cipher_ctx_init(&ctx, nid) < 0) + goto fail_cipher; - /* Increment packet counter and check for key rotation */ - ctx->rot.cntr++; - ctx->rot.age++; + out->data = malloc(in.len + EVP_MAX_BLOCK_LENGTH + ctx.tagsz); + if (out->data == NULL) + goto fail_cipher; - if (HAS_PHASE_BIT_TOGGLED(ctx)) { - if (rotate_key(ctx) != 0) - goto fail_encrypt; - } + out_sz = openssl_seal(&ctx, key, nonce, aad, in, + out->data, out->data + in.len); + if (out_sz < 0) + goto fail_seal; - cleanup_old_key(ctx); + out->len = (size_t) out_sz + ctx.tagsz; return 0; - fail_encrypt: + + fail_seal: free(out->data); - fail_malloc: + fail_cipher: clrbuf(*out); return -ECRYPT; } -int openssl_decrypt(struct ossl_crypt_ctx * ctx, - buffer_t in, - buffer_t * out) +/* One-shot AEAD open; in = ct ‖ tag, verifies aad and tag. */ +int openssl_oneshot_open(int nid, + const uint8_t * key, + const uint8_t * nonce, + buffer_t aad, + buffer_t in, + buffer_t * out) { - uint8_t * iv; - uint8_t * input; - uint8_t rx_phase; - int out_sz; - int in_sz; - - assert(ctx != NULL); + struct ossl_crypt_ctx ctx; + buffer_t ct; + const uint8_t * tag; + int in_sz; - in_sz = (int) in.len - ctx->ivsz; - if (in_sz < ctx->tagsz) - return -ECRYPT; - - in_sz -= ctx->tagsz; - - out->data = malloc(in_sz + EVP_MAX_BLOCK_LENGTH); - if (out->data == NULL) - goto fail_malloc; + assert(key != NULL); + assert(nonce != NULL); + assert(out != NULL); - iv = in.data; - input = in.data + ctx->ivsz; + memset(&ctx, 0, sizeof(ctx)); - /* Extract phase from IV bit 7 and check for key rotation */ - rx_phase = (iv[0] & 0x80) ? 1 : 0; + if (ossl_cipher_ctx_init(&ctx, nid) < 0) + goto fail_cipher; - if (should_rotate_key_rx(ctx, rx_phase)) { - if (rotate_key(ctx) != 0) - goto fail_decrypt; - } + if (in.len < (size_t) ctx.tagsz) + goto fail_cipher; - ctx->rot.cntr++; - ctx->rot.age++; + in_sz = (int) in.len - ctx.tagsz; - if (try_decrypt(ctx, ctx->keys.cur, iv, input, in_sz, out->data, - &out_sz) != 0) { - if (ctx->keys.prv == NULL) - goto fail_decrypt; - if (try_decrypt(ctx, ctx->keys.prv, iv, input, in_sz, - out->data, &out_sz) != 0) - goto fail_decrypt; - } + out->data = malloc((size_t) in_sz + EVP_MAX_BLOCK_LENGTH); + if (out->data == NULL) + goto fail_cipher; - assert(out_sz <= in_sz); + ct.data = in.data; + ct.len = (size_t) in_sz; + tag = in.data + in_sz; - out->len = (size_t) out_sz; + if (openssl_open(&ctx, key, nonce, aad, ct, tag, out) < 0) + goto fail_open; return 0; - fail_decrypt: + + fail_open: free(out->data); - fail_malloc: + fail_cipher: clrbuf(*out); return -ECRYPT; } @@ -1093,51 +1202,19 @@ struct ossl_crypt_ctx * openssl_crypt_create_ctx(struct crypt_sk * sk) assert(sk != NULL); assert(sk->key != NULL); - assert(sk->rot_bit > 0 && sk->rot_bit < 32); ctx = malloc(sizeof(*ctx)); if (ctx == NULL) - goto fail_malloc; + goto fail_malloc; memset(ctx, 0, sizeof(*ctx)); - ctx->keys.cur = OPENSSL_secure_malloc(SYMMKEYSZ); - if (ctx->keys.cur == NULL) - goto fail_key; - - memcpy(ctx->keys.cur, sk->key, SYMMKEYSZ); - - ctx->keys.prv = NULL; - - /* Derive rotation salt from initial shared secret */ - if (EVP_Digest(sk->key, SYMMKEYSZ, ctx->rot.salt, NULL, - EVP_sha256(), NULL) != 1) - goto fail_cipher; - - ctx->cipher = EVP_get_cipherbynid(sk->nid); - if (ctx->cipher == NULL) - goto fail_cipher; - - ctx->ivsz = EVP_CIPHER_iv_length(ctx->cipher); - - /* Set tag size for AEAD ciphers (GCM, CCM, OCB, ChaCha20-Poly1305) */ - if (EVP_CIPHER_flags(ctx->cipher) & EVP_CIPH_FLAG_AEAD_CIPHER) - ctx->tagsz = 16; /* Standard AEAD tag length (128 bits) */ - - ctx->rot.cntr = 0; - ctx->rot.mask = (1U << sk->rot_bit); - ctx->rot.age = 0; - ctx->rot.phase = 0; - - ctx->evp_ctx = EVP_CIPHER_CTX_new(); - if (ctx->evp_ctx == NULL) + if (ossl_cipher_ctx_init(ctx, sk->nid) < 0) goto fail_cipher; return ctx; fail_cipher: - OPENSSL_secure_clear_free(ctx->keys.cur, SYMMKEYSZ); - fail_key: free(ctx); fail_malloc: return NULL; @@ -1148,23 +1225,9 @@ void openssl_crypt_destroy_ctx(struct ossl_crypt_ctx * ctx) if (ctx == NULL) return; - if (ctx->keys.cur != NULL) - OPENSSL_secure_clear_free(ctx->keys.cur, SYMMKEYSZ); - - if (ctx->keys.prv != NULL) - OPENSSL_secure_clear_free(ctx->keys.prv, SYMMKEYSZ); - - EVP_CIPHER_CTX_free(ctx->evp_ctx); free(ctx); } -int openssl_crypt_get_ivsz(struct ossl_crypt_ctx * ctx) -{ - assert(ctx != NULL); - - return ctx->ivsz; -} - int openssl_crypt_get_tagsz(struct ossl_crypt_ctx * ctx) { assert(ctx != NULL); @@ -1184,7 +1247,12 @@ int openssl_load_crt_file(const char * path, if (fp == NULL) goto fail_file; + pthread_cleanup_push(__cleanup_fclose, fp); + xcrt = PEM_read_X509(fp, NULL, NULL, NULL); + + pthread_cleanup_pop(false); + if (xcrt == NULL) goto fail_crt; @@ -1200,35 +1268,58 @@ int openssl_load_crt_file(const char * path, return -1; } -int openssl_load_crt_str(const char * str, - void ** crt) +static void * rd_crt_bio(BIO * bio) +{ + return PEM_read_bio_X509(bio, NULL, NULL, NULL); +} + +static void * rd_privkey_bio(BIO * bio) +{ + return PEM_read_bio_PrivateKey(bio, NULL, NULL, ""); +} + +static void * rd_pubkey_bio(BIO * bio) +{ + return PEM_read_bio_PUBKEY(bio, NULL, NULL, NULL); +} + +/* Decode a PEM object from an in-memory string via rd. */ +static int load_pem_str(const char * str, + void * (* rd)(BIO *), + void ** out) { BIO * bio; - X509 * xcrt; + void * obj; bio = BIO_new(BIO_s_mem()); if (bio == NULL) goto fail_bio; if (BIO_write(bio, str, strlen(str)) < 0) - goto fail_crt; + goto fail_obj; - xcrt = PEM_read_bio_X509(bio, NULL, NULL, NULL); - if (xcrt == NULL) - goto fail_crt; + obj = rd(bio); + if (obj == NULL) + goto fail_obj; BIO_free(bio); - *crt = (void *) xcrt; + *out = obj; return 0; - fail_crt: + fail_obj: BIO_free(bio); fail_bio: - *crt = NULL; + *out = NULL; return -1; } +int openssl_load_crt_str(const char * str, + void ** crt) +{ + return load_pem_str(str, rd_crt_bio, crt); +} + int openssl_load_crt_der(buffer_t buf, void ** crt) { @@ -1288,7 +1379,12 @@ int openssl_load_privkey_file(const char * path, if (fp == NULL) goto fail_file; + pthread_cleanup_push(__cleanup_fclose, fp); + pkey = PEM_read_PrivateKey(fp, NULL, NULL, ""); + + pthread_cleanup_pop(false); + if (pkey == NULL) goto fail_key; @@ -1307,30 +1403,7 @@ int openssl_load_privkey_file(const char * path, int openssl_load_privkey_str(const char * str, void ** key) { - BIO * bio; - EVP_PKEY * pkey; - - bio = BIO_new(BIO_s_mem()); - if (bio == NULL) - goto fail_bio; - - if (BIO_write(bio, str, strlen(str)) < 0) - goto fail_key; - - pkey = PEM_read_bio_PrivateKey(bio, NULL, NULL, NULL); - if (pkey == NULL) - goto fail_key; - - BIO_free(bio); - - *key = (void *) pkey; - - return 0; - fail_key: - BIO_free(bio); - fail_bio: - *key = NULL; - return -1; + return load_pem_str(str, rd_privkey_bio, key); } int openssl_load_pubkey_file(const char * path, @@ -1343,7 +1416,12 @@ int openssl_load_pubkey_file(const char * path, if (fp == NULL) goto fail_file; + pthread_cleanup_push(__cleanup_fclose, fp); + pkey = PEM_read_PUBKEY(fp, NULL, NULL, NULL); + + pthread_cleanup_pop(false); + if (pkey == NULL) goto fail_key; @@ -1375,7 +1453,12 @@ int openssl_load_pubkey_file_to_der(const char * path, if (fp == NULL) goto fail_file; + pthread_cleanup_push(__cleanup_fclose, fp); + pkey = PEM_read_PUBKEY(fp, NULL, NULL, NULL); + + pthread_cleanup_pop(false); + if (pkey == NULL) goto fail_key; @@ -1402,56 +1485,45 @@ int openssl_load_pubkey_file_to_der(const char * path, int openssl_load_pubkey_str(const char * str, void ** key) { - BIO * bio; - EVP_PKEY * pkey; - - bio = BIO_new(BIO_s_mem()); - if (bio == NULL) - goto fail_bio; - - if (BIO_write(bio, str, strlen(str)) < 0) - goto fail_key; - - pkey = PEM_read_bio_PUBKEY(bio, NULL, NULL, NULL); - if (pkey == NULL) - goto fail_key; - - BIO_free(bio); - - *key = (void *) pkey; - - return 0; - fail_key: - BIO_free(bio); - fail_bio: - *key = NULL; - return -1; + return load_pem_str(str, rd_pubkey_bio, key); } int openssl_load_pubkey_raw_file(const char * path, + const char * algo, buffer_t * buf) { - FILE * fp; - uint8_t tmp_buf[CRYPT_KEY_BUFSZ]; - size_t bytes_read; - const char * algo; + FILE * fp; + uint8_t tmp_buf[CRYPT_KEY_BUFSZ]; + size_t bytes_read; + EVP_PKEY * chk; assert(path != NULL); + assert(algo != NULL); assert(buf != NULL); fp = fopen(path, "rb"); if (fp == NULL) goto fail_file; + pthread_cleanup_push(__cleanup_fclose, fp); + bytes_read = fread(tmp_buf, 1, CRYPT_KEY_BUFSZ, fp); - if (bytes_read == 0) + + pthread_cleanup_pop(false); + + /* A full buffer means the file was truncated */ + if (bytes_read == 0 || bytes_read == CRYPT_KEY_BUFSZ) goto fail_read; - /* Validate that this is a known hybrid KEM format */ - algo = __openssl_hybrid_algo_from_len(bytes_read); - if (algo == NULL) + /* Trial import: reject bad keys at load time */ + chk = EVP_PKEY_new_raw_public_key_ex(NULL, algo, NULL, + tmp_buf, bytes_read); + + if (chk == NULL) goto fail_read; + EVP_PKEY_free(chk); + buf->data = malloc(bytes_read); if (buf->data == NULL) goto fail_malloc; @@ -1470,44 +1542,38 @@ int openssl_load_pubkey_raw_file(const char * path, return -1; } -/* Determine hybrid KEM algorithm from raw private key length */ -static const char * __openssl_hybrid_algo_from_sk_len(size_t len) +/* Wipe the raw-key staging buffer if a cancel aborts the read. */ +static void __cleanse_key_buf(void * o) { - switch(len) { - case X25519MLKEM768_SKSZ: - return "X25519MLKEM768"; - case X448MLKEM1024_SKSZ: - return "X448MLKEM1024"; - default: - break; - } - - return NULL; + OPENSSL_cleanse(o, CRYPT_KEY_BUFSZ); } int openssl_load_privkey_raw_file(const char * path, + const char * algo, void ** key) { - FILE * fp; - uint8_t tmp_buf[4096]; - size_t bytes_read; - const char * algo; - EVP_PKEY * pkey; + FILE * fp; + uint8_t tmp_buf[CRYPT_KEY_BUFSZ]; + size_t bytes_read; + EVP_PKEY * pkey; assert(path != NULL); + assert(algo != NULL); assert(key != NULL); fp = fopen(path, "rb"); if (fp == NULL) goto fail_file; + pthread_cleanup_push(__cleanup_fclose, fp); + pthread_cleanup_push(__cleanse_key_buf, tmp_buf); + bytes_read = fread(tmp_buf, 1, sizeof(tmp_buf), fp); - if (bytes_read == 0) - goto fail_read; - /* Determine algorithm from key size */ - algo = __openssl_hybrid_algo_from_sk_len(bytes_read); - if (algo == NULL) + pthread_cleanup_pop(false); + pthread_cleanup_pop(false); + + if (bytes_read == 0) goto fail_read; pkey = EVP_PKEY_new_raw_private_key_ex(NULL, algo, NULL, @@ -1549,68 +1615,88 @@ void openssl_free_key(EVP_PKEY * key) EVP_PKEY_free(key); } +/* ASN1_STRING_length is deprecated in OpenSSL 4.1, and returns size_t */ +static int ossl_asn1_str_len(const ASN1_STRING * val) +{ +#ifdef HAVE_OPENSSL_4_1 + size_t len; + + len = ASN1_STRING_get_length(val); + + return len > INT_MAX ? -1 : (int) len; +#else + return ASN1_STRING_length(val); +#endif +} + int openssl_check_crt_name(void * crt, const char * name) { - char * subj; - char * cn; - X509 * xcrt; + const unsigned char * cn; + const ASN1_STRING * val; + const X509_NAME * nm; + int idx; + int len; - xcrt = (X509 *) crt; + nm = X509_get_subject_name((X509 *) crt); + if (nm == NULL) + return -1; - subj = X509_NAME_oneline(X509_get_subject_name(xcrt), NULL, 0); - if (subj == NULL) - goto fail_subj; + idx = X509_NAME_get_index_by_NID(nm, NID_commonName, -1); + if (idx < 0) + return -1; - cn = strstr(subj, "CN="); - if (cn == NULL) - goto fail_cn; + val = X509_NAME_ENTRY_get_data(X509_NAME_get_entry(nm, idx)); + cn = ASN1_STRING_get0_data(val); + len = ossl_asn1_str_len(val); - if (strcmp(cn + 3, name) != 0) - goto fail_cn; + if (len < 0 || (size_t) len != strlen(name)) + return -1; - free(subj); + if (memchr(cn, '\0', (size_t) len) != NULL) + return -1; + + if (memcmp(cn, name, (size_t) len) != 0) + return -1; return 0; - fail_cn: - free(subj); - fail_subj: - return -1; } int openssl_get_crt_name(void * crt, char * name) { - char * subj; - char * cn; - char * end; - X509 * xcrt; + const unsigned char * cn; + const ASN1_STRING * val; + const X509_NAME * nm; + int idx; + int len; - xcrt = (X509 *) crt; + nm = X509_get_subject_name((X509 *) crt); + if (nm == NULL) + return -1; + + idx = X509_NAME_get_index_by_NID(nm, NID_commonName, -1); + if (idx < 0) + return -1; - subj = X509_NAME_oneline(X509_get_subject_name(xcrt), NULL, 0); - if (subj == NULL) - goto fail_subj; + val = X509_NAME_ENTRY_get_data(X509_NAME_get_entry(nm, idx)); + cn = ASN1_STRING_get0_data(val); + len = ossl_asn1_str_len(val); - cn = strstr(subj, "CN="); - if (cn == NULL) - goto fail_cn; + if (len < 0) + return -1; - cn += 3; /* Skip "CN=" */ + if ((size_t) len > NAME_SIZE) + return -ENAME; - /* Find end of CN (comma or slash for next field) */ - end = strpbrk(cn, ",/"); - if (end != NULL) - *end = '\0'; + /* Reject an embedded NUL that would truncate the parsed name. */ + if (memchr(cn, '\0', (size_t) len) != NULL) + return -1; - strcpy(name, cn); - free(subj); + memcpy(name, cn, (size_t) len); + name[len] = '\0'; return 0; - fail_cn: - free(subj); - fail_subj: - return -1; } int openssl_crt_str(const void * crt, @@ -1695,12 +1781,43 @@ int openssl_auth_add_crt_to_store(void * store, return ret == 1 ? 0 : -1; } -int openssl_verify_crt(void * store, - void * crt) +void * openssl_auth_create_chain(void) +{ + return sk_X509_new_null(); +} + +void openssl_auth_destroy_chain(void * chain) +{ + sk_X509_pop_free((STACK_OF(X509) *) chain, X509_free); +} + +int openssl_auth_add_crt_to_chain(void * chain, + void * crt) +{ + if (X509_up_ref((X509 *) crt) != 1) + goto fail_ref; + + if (sk_X509_push((STACK_OF(X509) *) chain, (X509 *) crt) == 0) + goto fail_push; + + return 0; + fail_push: + X509_free((X509 *) crt); + fail_ref: + return -1; +} + +int openssl_verify_crt_pin(void * store, + void * untrusted, + void * crt, + void * pin) { X509_STORE_CTX * ctx; X509_STORE * _store; X509* _crt; + STACK_OF(X509) * chain; + int i; + int n; int ret; _store = (X509_STORE *) store; @@ -1710,7 +1827,8 @@ int openssl_verify_crt(void * store, if (ctx == NULL) goto fail_store_ctx; - ret = X509_STORE_CTX_init(ctx, _store, _crt, NULL); + ret = X509_STORE_CTX_init(ctx, _store, _crt, + (STACK_OF(X509) *) untrusted); if (ret != 1) goto fail_ca; @@ -1718,13 +1836,39 @@ int openssl_verify_crt(void * store, if (ret != 1) goto fail_ca; + /* Peer cert only verifies a signature; gate on sig KU, not role. */ + if ((X509_get_key_usage(_crt) & KU_DIGITAL_SIGNATURE) == 0) + goto fail_ca; + + if (pin != NULL) { + chain = X509_STORE_CTX_get0_chain(ctx); + if (chain == NULL) + goto fail_ca; + n = sk_X509_num(chain); + for (i = 1; i < n; i++) /* Skip the leaf */ + if (X509_cmp(sk_X509_value(chain, i), pin) == 0) + break; + if (i == n) + goto fail_pin; + } + X509_STORE_CTX_free(ctx); return 0; + fail_pin: + X509_STORE_CTX_free(ctx); + return -ENOENT; fail_ca: X509_STORE_CTX_free(ctx); fail_store_ctx: - return -1; + return -EAUTH; +} + +int openssl_verify_crt(void * store, + void * untrusted, + void * crt) +{ + return openssl_verify_crt_pin(store, untrusted, crt, NULL); } static const EVP_MD * select_md(EVP_PKEY * pkey, @@ -1739,6 +1883,12 @@ static const EVP_MD * select_md(EVP_PKEY * pkey, return EVP_get_digestbynid(nid); } +bool openssl_pk_requires_md(const EVP_PKEY * pk) +{ + /* Provider-based (PQC) signatures have an intrinsic digest */ + return EVP_PKEY_get_id(pk) >= 0; +} + int openssl_sign(EVP_PKEY * pkp, int nid, buffer_t msg, @@ -1866,9 +2016,10 @@ void * openssl_secure_malloc(size_t size) return OPENSSL_secure_malloc(size); } -void openssl_secure_free(void * ptr) +void openssl_secure_free(void * ptr, + size_t size) { - OPENSSL_secure_free(ptr); + OPENSSL_secure_clear_free(ptr, size); } void openssl_secure_clear(void * ptr, @@ -1876,6 +2027,7 @@ void openssl_secure_clear(void * ptr, { OPENSSL_cleanse(ptr, size); } + void openssl_cleanup(void) { OPENSSL_cleanup(); diff --git a/src/lib/crypt/openssl.h b/src/lib/crypt/openssl.h index af285232..6857e39b 100644 --- a/src/lib/crypt/openssl.h +++ b/src/lib/crypt/openssl.h @@ -28,147 +28,185 @@ struct ossl_crypt_ctx; -ssize_t openssl_pkp_create(const char * algo, - EVP_PKEY ** pkp, - uint8_t * pk); +ssize_t openssl_pkp_create(const char * algo, + EVP_PKEY ** pkp, + uint8_t * pk); -void openssl_pkp_destroy(EVP_PKEY * pkp); +void openssl_pkp_destroy(EVP_PKEY * pkp); -int openssl_dhe_derive(EVP_PKEY * pkp, - buffer_t pk, - int kdf_nid, - uint8_t * s); +int openssl_dhe_derive(EVP_PKEY * pkp, + buffer_t pk, + int kdf_nid, + uint8_t * s); -ssize_t openssl_kem_encap(buffer_t pk, - uint8_t * ct, - int kdf_nid, - uint8_t * s); +ssize_t openssl_kem_encap(buffer_t pk, + uint8_t * ct, + int kdf_nid, + uint8_t * s); /* no X509 DER support yet for DHKEM public keys */ -ssize_t openssl_kem_encap_raw(buffer_t pk, - uint8_t * ct, - int kdf_nid, - uint8_t * s); +ssize_t openssl_kem_encap_raw(const char * algo, + buffer_t pk, + uint8_t * ct, + int kdf_nid, + uint8_t * s); + +int openssl_kem_decap(EVP_PKEY * priv, + buffer_t ct, + int kdf_nid, + uint8_t * s); + +int openssl_get_algo_from_pk_der(buffer_t pk, + char * algo); + +int openssl_seal(struct ossl_crypt_ctx * ctx, + const uint8_t * key, + const uint8_t * nonce, + buffer_t aad, + buffer_t in, + uint8_t * out, + uint8_t * tag); + +int openssl_open(struct ossl_crypt_ctx * ctx, + const uint8_t * key, + const uint8_t * nonce, + buffer_t aad, + buffer_t in, + const uint8_t * tag, + buffer_t * out); + +int openssl_oneshot_seal(int nid, + const uint8_t * key, + const uint8_t * nonce, + buffer_t aad, + buffer_t in, + buffer_t * out); + +int openssl_oneshot_open(int nid, + const uint8_t * key, + const uint8_t * nonce, + buffer_t aad, + buffer_t in, + buffer_t * out); + +int openssl_hkdf_expand(buffer_t key, + buffer_t info, + buffer_t out); -int openssl_kem_decap(EVP_PKEY * priv, - buffer_t ct, - int kdf_nid, - uint8_t * s); +struct ossl_crypt_ctx * openssl_crypt_create_ctx(struct crypt_sk * sk); -int openssl_get_algo_from_pk_der(buffer_t pk, - char * algo); +void openssl_crypt_destroy_ctx(struct ossl_crypt_ctx * ctx); -int openssl_get_algo_from_pk_raw(buffer_t pk, - char * algo); +int openssl_crypt_get_tagsz(struct ossl_crypt_ctx * ctx); -int openssl_encrypt(struct ossl_crypt_ctx * ctx, - buffer_t in, - buffer_t * out); +/* AUTHENTICATION */ +int openssl_load_crt_file(const char * path, + void ** crt); -int openssl_decrypt(struct ossl_crypt_ctx * ctx, - buffer_t in, - buffer_t * out); +int openssl_load_crt_str(const char * str, + void ** crt); -struct ossl_crypt_ctx * openssl_crypt_create_ctx(struct crypt_sk * sk); +int openssl_load_crt_der(buffer_t buf, + void ** crt); -void openssl_crypt_destroy_ctx(struct ossl_crypt_ctx * ctx); +int openssl_get_pubkey_crt(void * crt, + void ** pk); -int openssl_crypt_get_ivsz(struct ossl_crypt_ctx * ctx); +void openssl_free_crt(void * crt); -int openssl_crypt_get_tagsz(struct ossl_crypt_ctx * ctx); +int openssl_load_privkey_file(const char * path, + void ** key); -/* AUTHENTICATION */ +int openssl_load_privkey_str(const char * str, + void ** key); -int openssl_load_crt_file(const char * path, - void ** crt); +int openssl_load_pubkey_file(const char * path, + void ** key); -int openssl_load_crt_str(const char * str, - void ** crt); +int openssl_load_pubkey_str(const char * str, + void ** key); +int openssl_load_pubkey_file_to_der(const char * path, + buffer_t * buf); +int openssl_load_pubkey_raw_file(const char * path, + const char * algo, + buffer_t * buf); -int openssl_load_crt_der(buffer_t buf, - void ** crt); +int openssl_load_privkey_raw_file(const char * path, + const char * algo, + void ** key); -int openssl_get_pubkey_crt(void * crt, - void ** pk); +int openssl_cmp_key(const EVP_PKEY * key1, + const EVP_PKEY * key2); -void openssl_free_crt(void * crt); +void openssl_free_key(EVP_PKEY * key); -int openssl_load_privkey_file(const char * path, - void ** key); +int openssl_check_crt_name(void * crt, + const char * name); -int openssl_load_privkey_str(const char * str, - void ** key); +int openssl_get_crt_name(void * crt, + char * name); -int openssl_load_pubkey_file(const char * path, - void ** key); +int openssl_crt_str(const void * crt, + char * str); -int openssl_load_pubkey_str(const char * str, - void ** key); -int openssl_load_pubkey_file_to_der(const char * path, +int openssl_crt_der(const void * crt, buffer_t * buf); -int openssl_load_pubkey_raw_file(const char * path, - buffer_t * buf); - -int openssl_load_privkey_raw_file(const char * path, - void ** key); - -int openssl_cmp_key(const EVP_PKEY * key1, - const EVP_PKEY * key2); -void openssl_free_key(EVP_PKEY * key); +void * openssl_auth_create_store(void); -int openssl_check_crt_name(void * crt, - const char * name); +void openssl_auth_destroy_store(void * store); -int openssl_get_crt_name(void * crt, - char * name); +int openssl_auth_add_crt_to_store(void * store, + void * crt); -int openssl_crt_str(const void * crt, - char * str); +void * openssl_auth_create_chain(void); -int openssl_crt_der(const void * crt, - buffer_t * buf); +void openssl_auth_destroy_chain(void * chain); -void * openssl_auth_create_store(void); +int openssl_auth_add_crt_to_chain(void * chain, + void * crt); -void openssl_auth_destroy_store(void * store); +int openssl_verify_crt(void * store, + void * untrusted, + void * crt); -int openssl_auth_add_crt_to_store(void * store, - void * crt); +int openssl_verify_crt_pin(void * store, + void * untrusted, + void * crt, + void * pin); -int openssl_verify_crt(void * store, - void * crt); +bool openssl_pk_requires_md(const EVP_PKEY * pk); -int openssl_sign(EVP_PKEY * pkp, - int md_nid, - buffer_t msg, - buffer_t * sig); +int openssl_sign(EVP_PKEY * pkp, + int md_nid, + buffer_t msg, + buffer_t * sig); -int openssl_verify_sig(EVP_PKEY * pk, - int md_nid, - buffer_t msg, - buffer_t sig); +int openssl_verify_sig(EVP_PKEY * pk, + int md_nid, + buffer_t msg, + buffer_t sig); -ssize_t openssl_md_digest(int md_nid, - buffer_t in, - uint8_t * out); +ssize_t openssl_md_digest(int md_nid, + buffer_t in, + uint8_t * out); -ssize_t openssl_md_len(int md_nid); +ssize_t openssl_md_len(int md_nid); /* Secure memory allocation */ -int openssl_secure_malloc_init(size_t max, - size_t guard); +int openssl_secure_malloc_init(size_t max, + size_t guard); -void openssl_secure_malloc_fini(void); +void openssl_secure_malloc_fini(void); -void * openssl_secure_malloc(size_t size); +void * openssl_secure_malloc(size_t size); -void openssl_secure_free(void * ptr, - size_t size); +void openssl_secure_free(void * ptr, + size_t size); -void openssl_secure_clear(void * ptr, - size_t size); +void openssl_secure_clear(void * ptr, + size_t size); -void openssl_cleanup(void); +void openssl_cleanup(void); #endif /* OUROBOROS_LIB_CRYPT_OPENSSL_H */ diff --git a/src/lib/dev.c b/src/lib/dev.c index ae0401b7..3b0363da 100644 --- a/src/lib/dev.c +++ b/src/lib/dev.c @@ -27,7 +27,11 @@ #endif #include "config.h" +#include "cap.h" #include "ssm.h" +#include "poa/poa.h" + +#define OUROBOROS_PREFIX "libouroboros" #include <ouroboros/atomics.h> #include <ouroboros/bitmap.h> @@ -45,6 +49,7 @@ #include <ouroboros/ipcp-dev.h> #include <ouroboros/list.h> #include <ouroboros/local-dev.h> +#include <ouroboros/logs.h> #include <ouroboros/np1_flow.h> #include <ouroboros/pthread.h> #include <ouroboros/random.h> @@ -56,6 +61,7 @@ #include <ouroboros/ssm_flow_set.h> #include <ouroboros/ssm_pool.h> #include <ouroboros/ssm_rbuff.h> +#include <ouroboros/time.h> #include <ouroboros/tw.h> #include <ouroboros/utils.h> @@ -63,8 +69,10 @@ #ifdef HAVE_LIBGCRYPT #include <gcrypt.h> #endif +#include <arpa/inet.h> #include <stdarg.h> #include <stdbool.h> +#include <stddef.h> #include <inttypes.h> #include <stdio.h> #include <stdlib.h> @@ -79,6 +87,7 @@ #define DONE_PART -2 #define CRCLEN (sizeof(uint32_t)) +#define FLOW_AVG_SHIFT 3 #define SECMEMSZ 16384 #define MSGBUFSZ 2048 @@ -98,8 +107,14 @@ struct flow { ssize_t part_idx; struct crypt_ctx * crypt; - int headsz; /* IV */ - int tailsz; /* Tag + CRC */ + int headsz; /* Selector */ + int tailsz; /* Tag + CRC */ + + struct timespec rk_grace; /* TX-promote deadline */ + struct timespec rk_attempt; /* Last re-key attempt */ + bool rk_wm_inflight; /* Re-key trigger in flight */ + uint32_t rk_wm_ctr; /* Throttles the consult */ + bool rk_initiator; /* OAP initiator this re-key */ struct timespec snd_act; struct timespec rcv_act; @@ -110,6 +125,14 @@ struct flow { struct timespec rcv_timeo; struct frcti * frcti; + + /* Mean written packet size (bytes), EWMA over the send path. */ + size_t mean_len; + + struct poa_flow * poa; /* NULL for shared memory flows */ + + /* Egress capacity estimator; armed by the IPCP, else NULL. */ + struct cap_est * cap; }; struct flow_set { @@ -133,6 +156,8 @@ struct { struct flow * flows; struct fmap * id_to_fd; + uint32_t max_rtt; /* IPCPs: declared layer RTT (ms) */ + pthread_mutex_t mtx; pthread_cond_t cond; @@ -261,7 +286,7 @@ static int spb_encrypt(struct flow * flow, in.len = ssm_pk_buff_len(spb); if (crypt_encrypt(flow->crypt, in, &out) < 0) - goto fail_encrypt; + return -ECRYPT; head = ssm_pk_buff_push(spb, flow->headsz); if (head == NULL) @@ -278,7 +303,7 @@ static int spb_encrypt(struct flow * flow, return 0; fail_alloc: freebuf(out); - fail_encrypt: + return -ECRYPT; } @@ -296,8 +321,7 @@ static int spb_decrypt(struct flow * flow, in.len = ssm_pk_buff_len(spb); if (crypt_decrypt(flow->crypt, in, &out) < 0) - return -ENOMEM; - + return -ECRYPT; head = ssm_pk_buff_pop(spb, flow->headsz) + flow->headsz; ssm_pk_buff_pop_tail(spb, flow->tailsz); @@ -342,8 +366,14 @@ static int crc_check(struct ssm_pk_buff * spb, size_t head_skip) { uint32_t crc; - uint8_t * head = ssm_pk_buff_head(spb) + head_skip; - uint8_t * tail = ssm_pk_buff_pop_tail(spb, CRCLEN); + uint8_t * head; + uint8_t * tail; + + if (ssm_pk_buff_len(spb) < head_skip + CRCLEN) + return 1; + + head = ssm_pk_buff_head(spb) + head_skip; + tail = ssm_pk_buff_pop_tail(spb, CRCLEN); mem_hash(HASH_CRC32, &crc, head, tail - head); @@ -353,10 +383,7 @@ static int crc_check(struct ssm_pk_buff * spb, /* FRCT included here so it can use proc and dev.c statics directly. */ #include "frct.c" -/* - * SACK / DATA carry trailer CRC32; HCS protects the headers on every - * FRCT packet. Decrypt before any check so plaintext is authoritative. - */ +/* Decrypt before any check so the plaintext is authoritative. */ static bool invalid_pkt(struct flow * flow, struct ssm_pk_buff * spb) { @@ -438,10 +465,6 @@ static void compute_wait_deadline(const struct timespec * dl, *out = *dl; } -/* - * proc.lock rdlock held across each iteration so flow_fini's wrlock - * waits for us to finish; FLOWDOWN already set means we exit promptly. - */ static void flow_drain_rx_nb(struct flow * flow) { ssize_t idx; @@ -509,10 +532,80 @@ static void flow_drain_rx_nb(struct flow * flow) } } -/* - * Wait clamped by caller deadline, next tw expiry, and TICTIME; - * a clamp-timeout means tw work is due, not caller-deadline. - */ +/* TX-promotion grace when the peer's install latency is unknown (raw). */ +#define REKEY_GRACE_MS 1000 + +/* Last-resort promote within N node-keys of exhaustion (< watermark). */ +#define REKEY_PROMOTE_FLOOR 1 + +/* Throttle re-key retries so a failed attempt can't storm the IRMd. */ +#define REKEY_BACKOFF_NS (250 * MILLION) + +/* proc.lock (rd) only guards teardown; crypt_rekey self-synchronises. */ +static void flow_rekey(struct flow * flow) +{ + struct flow_info info; + struct crypt_sk sk; + struct timespec now; + struct timespec intv; + time_t ms; + uint8_t key[SYMMKEYSZ]; + uint8_t buf[SOCK_BUF_SIZE]; + buffer_t msg = {SOCK_BUF_SIZE, buf}; + bool has_key; + bool initiator = false; + + pthread_rwlock_rdlock(&proc.lock); + if (flow->info.id < 0 || flow->crypt == NULL) { + pthread_rwlock_unlock(&proc.lock); + return; + } + + /* Back off so a failed attempt can't storm the IRMd per syscall. */ + clock_gettime(PTHREAD_COND_CLOCK, &now); + if (ts_diff_ns(&now, &flow->rk_attempt) < REKEY_BACKOFF_NS) { + pthread_rwlock_unlock(&proc.lock); + return; + } + + flow->rk_attempt = now; + info = flow->info; + pthread_rwlock_unlock(&proc.lock); + + if (flow_update__irm_req_ser(&msg, &info, false) < 0) + return; + + if (send_recv_msg(&msg) < 0) + return; + + sk.key = key; + if (flow_rekey__irm_result_des(&msg, &sk, &has_key, &initiator) < 0) + return; + + if (!has_key) + return; + + pthread_rwlock_rdlock(&proc.lock); + if (flow->info.id == info.id && flow->crypt != NULL) { + if (crypt_rekey(flow->crypt, &sk) == 0) { + flow->rk_initiator = initiator; + /* Hold TX on the old epoch until the peer installs. */ + ms = flow->info.mpl > 0 ? flow->info.mpl * 3 + : REKEY_GRACE_MS; + intv.tv_sec = ms / 1000; + intv.tv_nsec = (ms % 1000) * MILLION; + clock_gettime(PTHREAD_COND_CLOCK, &now); + ts_add(&now, &intv, &flow->rk_grace); + } + /* Re-arm the watermark even if the install was a no-op. */ + STORE_RELAXED(&flow->rk_wm_inflight, false); + } + pthread_rwlock_unlock(&proc.lock); + + crypt_secure_clear(key, SYMMKEYSZ); +} + +/* A clamp-timeout means tw work is due, not the caller deadline. */ static int flow_rx_one(struct flow * flow, struct timespec * abs) { @@ -533,7 +626,20 @@ static int flow_rx_one(struct flow * flow, return -EFLOWDOWN; } + /* Pull a parked re-key before re-blocking (idle reader). */ + if (flow->crypt != NULL + && (ssm_rbuff_get_flags(rx_rb) & RB_REKEY)) { + pthread_rwlock_unlock(&proc.lock); + flow_rekey(flow); + continue; + } + + pthread_cleanup_push(__cleanup_rwlock_unlock, &proc.lock); + idx = ssm_rbuff_read_b(rx_rb, &wait_abs); + + pthread_cleanup_pop(false); + if (idx == -ETIMEDOUT) { pthread_rwlock_unlock(&proc.lock); if (deadline_passed(abs)) @@ -592,27 +698,26 @@ static void flow_clear(int fd) proc.flows[fd].info.id = -1; } -/* - * Set ACL_FLOWDOWN on rx/tx so any in-flight blocking reads or writes - * wake up and drop their proc.lock rdlock. Must run BEFORE flow_fini's - * wrlock, else the wrlock blocks on those rdlock holders and the - * in-flight calls never see the FLOWDOWN signal. - */ +/* Order before flow_fini's wrlock, which blocks on rdlock holders. */ static void flow_quiesce(int fd) { struct ssm_rbuff * rx_rb = proc.flows[fd].rx_rb; struct ssm_rbuff * tx_rb = proc.flows[fd].tx_rb; if (rx_rb != NULL) - ssm_rbuff_set_acl(rx_rb, ACL_FLOWDOWN); + ssm_rbuff_set_flags(rx_rb, RB_FLOWDOWN); + if (tx_rb != NULL) - ssm_rbuff_set_acl(tx_rb, ACL_FLOWDOWN); + ssm_rbuff_set_flags(tx_rb, RB_FLOWDOWN); } static void do_flow_fini(int fd) { assert(fd >= 0 && fd < PROC_MAX_FLOWS); + if (proc.flows[fd].poa != NULL) + poa_flow_detach(proc.flows[fd].poa); + if (proc.flows[fd].frcti != NULL) frcti_destroy(proc.flows[fd].frcti); @@ -636,6 +741,8 @@ static void do_flow_fini(int fd) crypt_destroy_ctx(proc.flows[fd].crypt); + free(proc.flows[fd].cap); + flow_clear(fd); } @@ -667,14 +774,21 @@ static __inline__ size_t flow_user_mtu(const struct flow * flow, return raw > hdr ? raw - hdr : 0; } +/* A PoA flow transmits on its own socket; it has no tx ring. */ static int flow_init(struct flow_info * info, struct crypt_sk * sk, - time_t rtt_hint) + time_t rtt_hint, + struct poa_flow * pf) { - struct timespec now; - struct flow * flow; - int fd; - int err = -ENOMEM; + struct timespec now; + struct timespec txq; + struct flow * flow; + struct ssm_rbuff * tx_rb = NULL; + int fd; + int err = -ENOMEM; + + if (info->id < 0 || info->id >= SYS_MAX_FLOWS) + return -EBADF; clock_gettime(PTHREAD_COND_CLOCK, &now); @@ -694,13 +808,17 @@ static int flow_init(struct flow_info * info, if (flow->rx_rb == NULL) goto fail_rx_rb; - flow->tx_rb = ssm_rbuff_open(info->n_1_pid, info->id); - if (flow->tx_rb == NULL) - goto fail_tx_rb; + if (pf == NULL) { + flow->tx_rb = ssm_rbuff_open(info->n_1_pid, info->id); + if (flow->tx_rb == NULL) + goto fail_tx_rb; - flow->set = ssm_flow_set_open(info->n_1_pid); - if (flow->set == NULL) - goto fail_set; + tx_rb = flow->tx_rb; + + flow->set = ssm_flow_set_open(info->n_1_pid); + if (flow->set == NULL) + goto fail_set; + } flow->oflags = FLOWFDEFAULT; flow->part_idx = NO_PART; @@ -709,13 +827,15 @@ static int flow_init(struct flow_info * info, flow->crypt = NULL; flow->headsz = 0; flow->tailsz = 0; + flow->poa = pf; if (IS_ENCRYPTED(sk)) { - sk->rot_bit = KEY_ROTATION_BIT; flow->crypt = crypt_create_ctx(sk); - if (flow->crypt == NULL) + if (flow->crypt == NULL) { + err = -ECRYPT; goto fail_crypt; - flow->headsz = crypt_get_ivsz(flow->crypt); + } + flow->headsz = crypt_get_headsz(flow->crypt); flow->tailsz = crypt_get_tagsz(flow->crypt); } @@ -725,7 +845,7 @@ static int flow_init(struct flow_info * info, uint32_t frct_mtu = flow_user_mtu(flow, info->mtu); flow->frcti = frcti_create(fd, DELT_A, DELT_R, - info->mpl, rtt_hint, + info->mpl, rtt_hint, info->max_rtt, info->qs, frct_mtu); if (flow->frcti == NULL) goto fail_frcti; @@ -733,18 +853,30 @@ static int flow_init(struct flow_info * info, proc.id_to_fd[info->id].fd = fd; + if (pf != NULL) + poa_flow_attach(pf, info->id, flow->rx_rb); + flow_set_state(&proc.id_to_fd[info->id], FLOW_ALLOCATED); pthread_rwlock_unlock(&proc.lock); + if (tx_rb != NULL) { + txq.tv_sec = SSM_RBUFF_TXQ_DELAY / 1000; + txq.tv_nsec = (SSM_RBUFF_TXQ_DELAY % 1000) * MILLION; + + ssm_rbuff_set_txq_target(tx_rb, &txq); + } + return fd; fail_frcti: crypt_destroy_ctx(flow->crypt); fail_crypt: - ssm_flow_set_close(flow->set); + if (flow->set != NULL) + ssm_flow_set_close(flow->set); fail_set: - ssm_rbuff_close(flow->tx_rb); + if (flow->tx_rb != NULL) + ssm_rbuff_close(flow->tx_rb); fail_tx_rb: ssm_rbuff_close(flow->rx_rb); fail_rx_rb: @@ -768,9 +900,10 @@ static void init(int argc, char ** argv, char ** envp) { - struct proc_info info; - char * prog = argv[0]; - int i; + struct proc_info info; + char * prog = argv[0]; + int i; + pthread_rwlockattr_t attr; #ifdef PROC_FLOW_STATS char procstr[32]; #endif @@ -855,7 +988,16 @@ static void init(int argc, goto fail_cond; } - if (pthread_rwlock_init(&proc.lock, NULL) < 0) { + /* Writer-preferred: FRCT readers must not starve flow accept. */ + if (pthread_rwlockattr_init(&attr) != 0) { + fprintf(stderr, "FATAL: Could not init rwlock attributes.\n"); + goto fail_rwlock_attr; + } +#if defined(__GLIBC__) + pthread_rwlockattr_setkind_np( + &attr, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP); +#endif + if (pthread_rwlock_init(&proc.lock, &attr) != 0) { fprintf(stderr, "FATAL: Could not initialize flow lock.\n"); goto fail_flow_lock; } @@ -885,6 +1027,8 @@ static void init(int argc, } } #endif + pthread_rwlockattr_destroy(&attr); + return; #if defined PROC_FLOW_STATS @@ -898,6 +1042,8 @@ static void init(int argc, fail_fqset: pthread_rwlock_destroy(&proc.lock); fail_flow_lock: + pthread_rwlockattr_destroy(&attr); + fail_rwlock_attr: pthread_cond_destroy(&proc.cond); fail_cond: pthread_mutex_destroy(&proc.mtx); @@ -981,14 +1127,20 @@ static void fini(void) __attribute__((section(INIT_SECTION))) __typeof__(init) * __init = init; __attribute__((section(FINI_SECTION))) __typeof__(fini) * __fini = fini; +/* + * A PoA flow is announced by its peer before the accept; from the + * reply on, the peer may transmit, so the flow must be able to + * receive. + */ int flow_accept(qosspec_t * qs, const struct timespec * timeo) { struct flow_info flow; - struct crypt_sk crypt; + struct crypt_sk crypt; uint8_t buf[SOCK_BUF_SIZE]; buffer_t msg = {SOCK_BUF_SIZE, buf}; uint8_t key[SYMMKEYSZ]; + struct poa_flow * pf; int fd; int err; @@ -1012,14 +1164,21 @@ int flow_accept(qosspec_t * qs, if (err < 0) return err; - crypt.key = key; + crypt.key = key; + crypt.epoch = 0; + crypt.role = CRYPT_ROLE_RESP; err = flow__irm_result_des(&msg, &flow, &crypt); if (err < 0) return err; - /* No RTT in accept; rtt_hint=0 bootstraps from first ACK. */ - fd = flow_init(&flow, &crypt, 0); + pf = poa_flow_take_pending(flow.id); + + fd = flow_init(&flow, &crypt, 0, pf); + if (fd >= 0) + poa_flow_ready(pf); + else if (pf != NULL) + poa_flow_detach(pf); crypt_secure_clear(key, SYMMKEYSZ); @@ -1067,13 +1226,15 @@ int flow_alloc(const char * dst, clock_gettime(PTHREAD_COND_CLOCK, &t1); - crypt.key = key; + crypt.key = key; + crypt.epoch = 0; + crypt.role = CRYPT_ROLE_INIT; err = flow__irm_result_des(&msg, &flow, &crypt); if (err < 0) return err; - fd = flow_init(&flow, &crypt, ts_diff_ns(&t1, &t0)); + fd = flow_init(&flow, &crypt, ts_diff_ns(&t1, &t0), NULL); crypt_secure_clear(key, SYMMKEYSZ); @@ -1106,13 +1267,15 @@ int flow_join(const char * dst, if (err < 0) return err; - crypt.key = key; + crypt.key = key; + crypt.epoch = 0; + crypt.role = CRYPT_ROLE_INIT; err = flow__irm_result_des(&msg, &flow, &crypt); if (err < 0) return err; - fd = flow_init(&flow, &crypt, 0); + fd = flow_init(&flow, &crypt, 0, NULL); crypt_secure_clear(key, SYMMKEYSZ); @@ -1152,6 +1315,8 @@ int flow_dealloc(int fd) pthread_rwlock_unlock(&proc.lock); + frcti_set_draining(flow->frcti); + flow_read(fd, buf, SOCK_BUF_SIZE); pthread_rwlock_rdlock(&proc.lock); @@ -1173,7 +1338,8 @@ int flow_dealloc(int fd) pthread_cleanup_push(__cleanup_rwlock_unlock, &proc.lock); - ssm_rbuff_fini(flow->tx_rb); + if (flow->tx_rb != NULL) + ssm_rbuff_fini(flow->tx_rb); pthread_cleanup_pop(true); @@ -1241,6 +1407,18 @@ int ipcp_flow_dealloc(int fd) return err; } +/* A settable delay is a normalised, non-negative timespec. */ +static bool delay_is_valid(const struct timespec * ts) +{ + if (ts->tv_sec < 0 || ts->tv_nsec < 0) + return false; + + if (ts->tv_nsec >= BILLION) + return false; + + return TS_TO_UINT64(*ts) <= SSM_RBUFF_TXQ_MAX_DELAY; +} + int fccntl(int fd, int cmd, ...) @@ -1251,8 +1429,6 @@ int fccntl(int fd, va_list l; struct timespec * timeo; qosspec_t * qs; - uint32_t rx_acl; - uint32_t tx_acl; size_t * qlen; struct flow * flow; uint16_t old_acc; @@ -1265,6 +1441,7 @@ int fccntl(int fd, time_t * rtop; int rc; bool emit_eos = false; + bool set_txq = false; if (fd < 0 || fd >= PROC_MAX_FLOWS) return -EBADF; @@ -1328,7 +1505,11 @@ int fccntl(int fd, break; case FLOWGTXQLEN: qlen = va_arg(l, size_t *); - *qlen = ssm_rbuff_queued(flow->tx_rb); + + if (flow->poa != NULL) + *qlen = poa_flow_qpkts(flow->poa); + else + *qlen = ssm_rbuff_queued(flow->tx_rb); break; case FLOWGMTU: maxp = va_arg(l, size_t *); @@ -1336,6 +1517,28 @@ int fccntl(int fd, goto einval; *maxp = flow_user_mtu(flow, flow->info.mtu); break; + case FLOWSTXQDLY: + timeo = va_arg(l, struct timespec *); + if (timeo == NULL) + goto einval; + + if (flow->tx_rb == NULL) + goto eperm; + + if (!delay_is_valid(timeo)) + goto einval; + + set_txq = true; + break; + case FLOWGTXQDLY: + timeo = va_arg(l, struct timespec *); + if (timeo == NULL) + goto einval; + + if (flow->tx_rb == NULL) + goto eperm; + ssm_rbuff_get_txq_target(flow->tx_rb, timeo); + break; case FLOWSFLAGS: old_acc = flow->oflags & FLOWFACCMODE; flow->oflags = va_arg(l, uint32_t); @@ -1348,31 +1551,28 @@ int fccntl(int fd, && flow->frcti != NULL) emit_eos = true; - rx_acl = ssm_rbuff_get_acl(flow->rx_rb); - tx_acl = ssm_rbuff_get_acl(flow->tx_rb); - /* Our flow write-only -> peer's read-only. */ + /* Our flow write-only -> peer's read-only; restore on RDWR. */ if (flow->oflags & FLOWFWRONLY) - rx_acl |= ACL_RDONLY; - if (flow->oflags & FLOWFRDWR) - rx_acl |= ACL_RDWR; + ssm_rbuff_clr_flags(flow->rx_rb, RB_WR); + else + ssm_rbuff_set_flags(flow->rx_rb, RB_WR); if (flow->oflags & FLOWFDOWN) { - rx_acl |= ACL_FLOWDOWN; - tx_acl |= ACL_FLOWDOWN; - ssm_flow_set_notify(flow->set, - flow->info.id, - FLOW_DOWN); + ssm_rbuff_set_flags(flow->rx_rb, RB_FLOWDOWN); + if (flow->tx_rb != NULL) + ssm_rbuff_set_flags(flow->tx_rb, RB_FLOWDOWN); + if (flow->set != NULL) + ssm_flow_set_notify(flow->set, flow->info.id, + FLOW_DOWN); } else { - rx_acl &= ~ACL_FLOWDOWN; - tx_acl &= ~ACL_FLOWDOWN; - ssm_flow_set_notify(flow->set, - flow->info.id, - FLOW_UP); + ssm_rbuff_clr_flags(flow->rx_rb, RB_FLOWDOWN); + if (flow->tx_rb != NULL) + ssm_rbuff_clr_flags(flow->tx_rb, RB_FLOWDOWN); + if (flow->set != NULL) + ssm_flow_set_notify(flow->set, flow->info.id, + FLOW_UP); } - ssm_rbuff_set_acl(flow->rx_rb, rx_acl); - ssm_rbuff_set_acl(flow->tx_rb, tx_acl); - break; case FLOWGFLAGS: fflags = va_arg(l, uint32_t *); @@ -1459,6 +1659,9 @@ int fccntl(int fd, if (emit_eos) frcti_fin_snd(flow->frcti); + if (set_txq) + ssm_rbuff_set_txq_target(flow->tx_rb, timeo); + va_end(l); return 0; @@ -1473,6 +1676,25 @@ int fccntl(int fd, return -EPERM; } +/* + * The ring counts slots, so the queue is only bytes if we know what a + * packet weighs. Ordered so the unsigned arithmetic cannot wrap. + */ +static void flow_mean_len_update(struct flow * flow, + size_t len) +{ + size_t avg = LOAD_RELAXED(&flow->mean_len); + + if (avg == 0) { + STORE_RELAXED(&flow->mean_len, len); + return; + } + + avg = avg + (len >> FLOW_AVG_SHIFT) - (avg >> FLOW_AVG_SHIFT); + + STORE_RELAXED(&flow->mean_len, avg == 0 ? 1 : avg); +} + static int flow_tx_spb(struct flow * flow, struct ssm_pk_buff * spb, uint16_t flags, @@ -1504,21 +1726,23 @@ static int flow_tx_spb(struct flow * flow, goto enomem; } + if (flow->poa != NULL) + return poa_flow_tx(flow->poa, spb, block, abstime); + + flow_mean_len_update(flow, ssm_pk_buff_len(spb)); + if (!block) ret = ssm_rbuff_write(flow->tx_rb, idx); else ret = ssm_rbuff_write_b(flow->tx_rb, idx, abstime); - if (ret < 0) { - ssm_pool_remove(proc.pool, idx); + if (ret < 0) return ret; - } ssm_flow_set_notify(flow->set, flow->info.id, FLOW_PKT); return 0; enomem: - ssm_pool_remove(proc.pool, idx); return -ENOMEM; } @@ -1527,20 +1751,16 @@ static __inline__ uint16_t flow_frag_role(size_t i, size_t n) { if (n == 1) return FRCT_FR_SOLE; + if (i == 0) return FRCT_FR_FIRST; + if (i + 1 == n) return FRCT_FR_LAST; return FRCT_FR_MID; } -/* - * Stream-mode write: split buf into chunks of - * (frag_mtu - PCI - PCI_STREAM) bytes; each chunk goes through the - * normal tx path. frcti_snd injects the [start,end) extension and - * advances snd_byte_next under its wrlock. No FFGM/LFGM role bits. - */ static ssize_t flow_write_stream(struct flow * flow, const void * buf, size_t count, @@ -1581,8 +1801,10 @@ static ssize_t flow_write_stream(struct flow * flow, memcpy(ptr, src + off, clen); ret = flow_tx_spb(flow, spb, 0, block, dl); - if (ret < 0) + if (ret < 0) { + ssm_pool_remove(proc.pool, idx); return off > 0 ? (ssize_t) off : (ssize_t) ret; + } off += clen; } @@ -1614,6 +1836,7 @@ static ssize_t flow_write_frag(struct flow * flow, /* Guard the ceil-divide against size_t overflow. */ if (count > SIZE_MAX - frag_payload + 1) return -EMSGSIZE; + n = (count + frag_payload - 1) / frag_payload; /* SDU larger than the FC window can ever offer would deadlock. */ @@ -1648,9 +1871,9 @@ static ssize_t flow_write_frag(struct flow * flow, memcpy(ptr, src + off, clen); - ret = flow_tx_spb(flow, spb, flow_frag_role(i, n), - block, dl); + ret = flow_tx_spb(flow, spb, flow_frag_role(i, n), block, dl); if (ret < 0) { + ssm_pool_remove(proc.pool, idx); if (off > 0) STAT_BUMP(flow->frcti, sdu_snd_tx); return off > 0 ? (ssize_t) off : (ssize_t) ret; @@ -1662,6 +1885,91 @@ static ssize_t flow_write_frag(struct flow * flow, return (ssize_t) count; } +/* + * Initiator promotes on the install grace (it holds the key-confirm + * tag); responder waits for peer_synced. The near-exhaustion floor + * backstops both roles: the receiver selects the epoch by the wire + * selector, so promoting beats wedging TX on a spent keyring. + */ +static void flow_tx_promote(struct flow * flow) +{ + struct timespec now; + int nodes_left; + bool promote; + + if (flow->crypt == NULL) + return; + + if (flow->rk_grace.tv_sec == 0 && flow->rk_grace.tv_nsec == 0) + return; + + promote = crypt_peer_synced(flow->crypt); + + if (!promote && flow->rk_initiator) { + clock_gettime(PTHREAD_COND_CLOCK, &now); + promote = ts_diff_ns(&now, &flow->rk_grace) >= 0; + } + + if (!promote) { + nodes_left = crypt_nodes_left(flow->crypt); + promote = nodes_left >= 0 && nodes_left <= REKEY_PROMOTE_FLOOR; + } + + if (!promote) + return; + + crypt_tx_promote(flow->crypt); + flow->rk_grace.tv_sec = 0; + flow->rk_grace.tv_nsec = 0; +} + +/* The reply carries no key; the seed arrives later over RB_REKEY. */ +static int flow_rekey_trigger(struct flow * flow) +{ + struct flow_info info; + uint8_t buf[SOCK_BUF_SIZE]; + buffer_t msg = {SOCK_BUF_SIZE, buf}; + + pthread_rwlock_rdlock(&proc.lock); + if (flow->info.id < 0 || flow->crypt == NULL) { + pthread_rwlock_unlock(&proc.lock); + return -1; + } + info = flow->info; + pthread_rwlock_unlock(&proc.lock); + + if (flow_update__irm_req_ser(&msg, &info, true) < 0) + return -1; + + if (send_recv_msg(&msg) < 0) + return -1; + + return 0; +} + +static bool flow_wm_due(struct flow * flow) +{ + uint32_t tick; + + if (KEY_REKEY_WATERMARK == 0) + return false; + + if (flow->crypt == NULL) + return false; + + if (LOAD_RELAXED(&flow->rk_wm_inflight)) + return false; + + tick = FETCH_ADD_RELAXED(&flow->rk_wm_ctr, 1); + if ((tick & (FLOW_WM_CHECK - 1)) != 0) + return false; + + if (ssm_rbuff_get_flags(flow->rx_rb) & RB_REKEY) + return false; + + return crypt_nodes_left(flow->crypt) <= KEY_REKEY_WATERMARK; +} + ssize_t flow_write(int fd, const void * buf, size_t count) @@ -1705,6 +2013,19 @@ ssize_t flow_write(int fd, if ((flags & FLOWFACCMODE) == FLOWFRDONLY) return -EPERM; + if (flow->crypt != NULL + && (ssm_rbuff_get_flags(flow->rx_rb) & RB_REKEY)) + flow_rekey(flow); + + flow_tx_promote(flow); + + /* Pre-empt TX key exhaustion; the timer is the backstop. */ + if (flow_wm_due(flow)) { + STORE_RELAXED(&flow->rk_wm_inflight, true); + if (flow_rekey_trigger(flow) < 0) + STORE_RELAXED(&flow->rk_wm_inflight, false); + } + tw_move_safe(); if (flow->frcti != NULL) { @@ -1736,8 +2057,12 @@ ssize_t flow_write(int fd, ret = flow_tx_spb(flow, spb, FRCT_FR_SOLE, !(flags & FLOWFWNOBLOCK), dl); + if (ret < 0) { + ssm_pool_remove(proc.pool, idx); + return (ssize_t) ret; + } - return ret < 0 ? (ssize_t) ret : (ssize_t) count; + return (ssize_t) count; } static ssize_t flow_rx_spb(struct flow * flow, @@ -1775,6 +2100,10 @@ static ssize_t raw_flow_read_pkt(struct flow * flow, ssize_t idx; while (true) { + if (flow->crypt != NULL + && (ssm_rbuff_get_flags(flow->rx_rb) & RB_REKEY)) + flow_rekey(flow); + if (!block) { idx = ssm_rbuff_read(flow->rx_rb); if (idx < 0) @@ -1908,6 +2237,13 @@ ssize_t flow_read(int fd, pthread_rwlock_unlock(&proc.lock); + if (flow->crypt != NULL + && (ssm_rbuff_get_flags(flow->rx_rb) & RB_REKEY)) + flow_rekey(flow); + + /* Advance TX off a stale epoch even on recv-mostly (ACK-only) flows. */ + flow_tx_promote(flow); + tw_move_safe(); idx = flow->part_idx; @@ -2092,6 +2428,18 @@ static int fqueue_filter(struct fqueue * fq) pthread_rwlock_rdlock(&proc.lock); while (fq->next < fq->fqsize) { + if (fq->fqueue[fq->next].event == FLOW_UPD) { + /* Re-key doorbell: pull internally, never surface. */ + fd = proc.id_to_fd[fq->fqueue[fq->next].flow_id].fd; + ++fq->next; + if (fd >= 0) { + pthread_rwlock_unlock(&proc.lock); + flow_rekey(&proc.flows[fd]); + pthread_rwlock_rdlock(&proc.lock); + } + continue; + } + if (fq->fqueue[fq->next].event != FLOW_PKT) { ret = 1; goto out; @@ -2224,7 +2572,8 @@ int np1_flow_alloc(pid_t n_pid, int flow_id) { struct flow_info flow; - struct crypt_sk crypt = { .nid = NID_undef, .key = NULL }; + struct crypt_sk crypt = { .nid = NID_undef, .key = NULL, + .epoch = 0, .role = CRYPT_ROLE_INIT }; memset(&flow, 0, sizeof(flow)); @@ -2235,7 +2584,7 @@ int np1_flow_alloc(pid_t n_pid, /* np1 flow: n_1_pid is the upper. */ flow.n_1_pid = n_pid; - return flow_init(&flow, &crypt, 0); + return flow_init(&flow, &crypt, 0, NULL); } int np1_flow_dealloc(int flow_id, @@ -2272,6 +2621,38 @@ int np1_flow_resp(int flow_id, return fd; } +int np1_flow_fd(int flow_id) +{ + int fd; + + if (flow_id < 0 || flow_id >= SYS_MAX_FLOWS) + return -1; + + pthread_rwlock_rdlock(&proc.lock); + + fd = proc.id_to_fd[flow_id].fd; + + pthread_rwlock_unlock(&proc.lock); + + return fd; +} + +int np1_flow_id(int fd) +{ + int flow_id; + + if (fd < 0 || fd >= PROC_MAX_FLOWS) + return -1; + + pthread_rwlock_rdlock(&proc.lock); + + flow_id = proc.flows[fd].info.id; + + pthread_rwlock_unlock(&proc.lock); + + return flow_id; +} + int ipcp_create_r(const struct ipcp_info * info) { uint8_t buf[SOCK_BUF_SIZE]; @@ -2288,6 +2669,12 @@ int ipcp_create_r(const struct ipcp_info * info) return irm__irm_result_des(&msg); } +/* Layer-wide bound for flow_info; set once before flows are served. */ +void ipcp_flow_set_max_rtt(uint32_t max_rtt) +{ + proc.max_rtt = max_rtt; +} + int ipcp_flow_req_arr(const buffer_t * dst, qosspec_t qs, time_t mpl, @@ -2309,6 +2696,7 @@ int ipcp_flow_req_arr(const buffer_t * dst, flow.qs = qs; flow.mpl = mpl; flow.mtu = mtu; + flow.max_rtt = proc.max_rtt; if (ipcp_flow_req_arr__irm_req_ser(&msg, dst, &flow, data) < 0) return -ENOMEM; @@ -2317,7 +2705,9 @@ int ipcp_flow_req_arr(const buffer_t * dst, if (err < 0) return err; - crypt.key = key; + crypt.key = key; + crypt.epoch = 0; + crypt.role = CRYPT_ROLE_INIT; err = flow__irm_result_des(&msg, &flow, &crypt); if (err < 0) @@ -2335,7 +2725,30 @@ int ipcp_flow_req_arr(const buffer_t * dst, crypt.nid = NID_undef; - return flow_init(&flow, &crypt, 0); + return flow_init(&flow, &crypt, 0, NULL); +} + +int ipcp_flow_update_arr(int flow_id, + const buffer_t * data) +{ + struct flow_info flow; + uint8_t buf[SOCK_BUF_SIZE]; + buffer_t msg = {SOCK_BUF_SIZE, buf}; + int err; + + memset(&flow, 0, sizeof(flow)); + + flow.id = flow_id; + flow.n_1_pid = getpid(); + + if (ipcp_flow_update_arr__irm_req_ser(&msg, &flow, data) < 0) + return -ENOMEM; + + err = send_recv_msg(&msg); + if (err < 0) + return err; + + return irm__irm_result_des(&msg); } int ipcp_flow_alloc_reply(int fd, @@ -2359,6 +2772,7 @@ int ipcp_flow_alloc_reply(int fd, flow.mpl = mpl; flow.mtu = mtu; + flow.max_rtt = proc.max_rtt; if (ipcp_flow_alloc_reply__irm_msg_ser(&msg, &flow, response, data) < 0) return -ENOMEM; @@ -2373,8 +2787,14 @@ int ipcp_flow_alloc_reply(int fd, int ipcp_flow_read(int fd, struct ssm_pk_buff ** spb) { - struct flow * flow; - ssize_t idx = -1; + struct flow * flow; + struct ssm_pk_buff * out; + uint8_t * ptr; + ssize_t idx = -1; + ssize_t fret; + size_t len; + size_t nfrags; + int ret; assert(fd >= 0 && fd < PROC_MAX_FLOWS); assert(spb); @@ -2383,37 +2803,101 @@ int ipcp_flow_read(int fd, pthread_rwlock_rdlock(&proc.lock); - assert(flow->info.id >= 0); + if (flow->info.id < 0) { + pthread_rwlock_unlock(&proc.lock); + return -ENOTALLOC; + } + + if (FRCTI_IS_STREAM(flow->frcti)) { + pthread_rwlock_unlock(&proc.lock); + return -ENOTSUP; + } + + pthread_rwlock_unlock(&proc.lock); + + if (flow->crypt != NULL + && (ssm_rbuff_get_flags(flow->rx_rb) & RB_REKEY)) + flow_rekey(flow); + + /* Advance TX off a stale epoch even on recv-mostly flows. */ + flow_tx_promote(flow); + + tw_move_safe(); + + pthread_rwlock_rdlock(&proc.lock); /* Raw flow: deliver the popped pkt directly (no FRCT rq). */ if (flow->frcti == NULL) { - pthread_rwlock_unlock(&proc.lock); idx = flow_rx_spb(flow, spb, false, NULL); + pthread_rwlock_unlock(&proc.lock); return idx < 0 ? (int) idx : 0; } while (!FRCTI_PDU_READY(flow->frcti)) { - pthread_rwlock_unlock(&proc.lock); - idx = flow_rx_spb(flow, spb, false, NULL); - if (idx < 0) + if (idx < 0) { + pthread_rwlock_unlock(&proc.lock); return idx; - - pthread_rwlock_rdlock(&proc.lock); + } FRCTI_RCV(flow->frcti, *spb); } pthread_rwlock_unlock(&proc.lock); + /* + * A hand-back of the fed spb would leave it double-owned by + * the reorder queue; frcti_consume is the only safe way to + * take it. A write can also complete a PDU, so PDU_READY may + * be true with no loop-local spb to fall back on anyway. + */ + + ret = FRCTI_PDU_INFO(flow->frcti, &len, &nfrags); + if (ret < 0) + return ret; + + /* + * Oversize (over frcti's own cap, or too big for any pool + * class): force frcti_consume's total > count drop branch + * now, so the run leaves the delivery edge instead of + * stalling every read after this one. + */ + if (len > frcti_get_max_rcv_sdu(flow->frcti)) { + (void) FRCTI_CONSUME(flow->frcti, NULL, 0); + return -EMSGSIZE; + } + + idx = ssm_pool_alloc_b(proc.pool, len, &ptr, &out, NULL); + if (idx < 0) { + if (idx == -EMSGSIZE) + (void) FRCTI_CONSUME(flow->frcti, NULL, 0); + return (int) idx; + } + + fret = FRCTI_CONSUME(flow->frcti, ptr, len); + if (fret < 0 || (size_t) fret != len) { + ssm_pool_remove(proc.pool, idx); + return fret < 0 ? (int) fret : -EIO; + } + + *spb = out; + return 0; } +/* + * Writes an spb to an IPCP-internal flow, splitting it over multiple + * FRCT fragments when it exceeds the flow's fragment payload cap. + * Consumes spb on success; on failure spb is left to the caller. + */ int ipcp_flow_write(int fd, struct ssm_pk_buff * spb) { - struct flow * flow; - int ret; + struct flow * flow; + int oflags; + size_t len; + ssize_t fret; + int ret; assert(fd >= 0 && fd < PROC_MAX_FLOWS); assert(spb); @@ -2432,8 +2916,54 @@ int ipcp_flow_write(int fd, return -EPERM; } + if (FRCTI_IS_STREAM(flow->frcti)) { + pthread_rwlock_unlock(&proc.lock); + return -ENOTSUP; + } + + oflags = flow->oflags; + pthread_rwlock_unlock(&proc.lock); + if (flow->crypt != NULL + && (ssm_rbuff_get_flags(flow->rx_rb) & RB_REKEY)) + flow_rekey(flow); + + flow_tx_promote(flow); + + /* Pre-empt TX key exhaustion; the timer is the backstop. */ + if (flow_wm_due(flow)) { + STORE_RELAXED(&flow->rk_wm_inflight, true); + + if (flow_rekey_trigger(flow) < 0) + STORE_RELAXED(&flow->rk_wm_inflight, false); + } + + tw_move_safe(); + + len = ssm_pk_buff_len(spb); + if (FRCTI_NEEDS_FRAG(flow->frcti, len)) { + fret = flow_write_frag(flow, ssm_pk_buff_head(spb), len, + oflags, NULL); + + if (fret < 0) + return (int) fret; + + /* Partial: flow_write_frag swallowed the real cause. */ + if (fret != (ssize_t) len) { + /* PoA flows have no tx_rb flag to consult. */ + if (flow->tx_rb != NULL + && (ssm_rbuff_get_flags(flow->tx_rb) + & RB_FLOWDOWN)) + return -EFLOWDOWN; + return -EIO; + } + + ipcp_spb_release(spb); + + return 0; + } + ret = flow_tx_spb(flow, spb, FRCT_FR_SOLE, true, NULL); return ret; @@ -2472,10 +3002,13 @@ int np1_flow_read(int fd, flow = &proc.flows[fd]; - assert(flow->info.id >= 0); - pthread_rwlock_rdlock(&proc.lock); + if (flow->info.id < 0) { + pthread_rwlock_unlock(&proc.lock); + return -ENOTALLOC; + } + off = ssm_rbuff_read(flow->rx_rb); if (off < 0) { pthread_rwlock_unlock(&proc.lock); @@ -2498,6 +3031,11 @@ int np1_flow_read(int fd, return 0; } +/* + * An N-1 flow gets no flow_write to advance its TX epoch off a rotated + * key. Promoting is local; a re-key request here would block on the + * IRMd. + */ int np1_flow_write(int fd, struct ssm_pk_buff * spb, struct ssm_pool * pool) @@ -2576,12 +3114,14 @@ int ipcp_flow_fini(int fd) return -1; } - ssm_rbuff_set_acl(proc.flows[fd].rx_rb, ACL_FLOWDOWN); - ssm_rbuff_set_acl(proc.flows[fd].tx_rb, ACL_FLOWDOWN); + ssm_rbuff_set_flags(proc.flows[fd].rx_rb, RB_FLOWDOWN); + + if (proc.flows[fd].tx_rb != NULL) + ssm_rbuff_set_flags(proc.flows[fd].tx_rb, RB_FLOWDOWN); - ssm_flow_set_notify(proc.flows[fd].set, - proc.flows[fd].info.id, - FLOW_DEALLOC); + if (proc.flows[fd].set != NULL) + ssm_flow_set_notify(proc.flows[fd].set, proc.flows[fd].info.id, + FLOW_DEALLOC); rx_rb = proc.flows[fd].rx_rb; @@ -2610,19 +3150,101 @@ int ipcp_flow_get_qoscube(int fd, return 0; } +/* Not a snapshot: two atomic loads; caller keeps the fd live. */ size_t ipcp_flow_queued(int fd) { - size_t q; + assert(fd >= 0 && fd < PROC_MAX_FLOWS); + assert(proc.flows[fd].info.id >= 0); - pthread_rwlock_rdlock(&proc.lock); + if (proc.flows[fd].poa != NULL) + return poa_flow_qlen(proc.flows[fd].poa); + + return ssm_rbuff_queued(proc.flows[fd].tx_rb) + * LOAD_RELAXED(&proc.flows[fd].mean_len); +} +size_t ipcp_flow_mean_len(int fd) +{ + assert(fd >= 0 && fd < PROC_MAX_FLOWS); assert(proc.flows[fd].info.id >= 0); - q = ssm_rbuff_queued(proc.flows[fd].tx_rb); + if (proc.flows[fd].poa != NULL) + return poa_flow_mean_len(proc.flows[fd].poa); - pthread_rwlock_unlock(&proc.lock); + return LOAD_RELAXED(&proc.flows[fd].mean_len); +} + +/* An update racing the arm seeds one bogus window; the filter absorbs. */ +int ipcp_flow_cap_arm(int fd) +{ + struct flow * flow; + struct cap_est * e; - return q; + assert(fd >= 0 && fd < PROC_MAX_FLOWS); + assert(proc.flows[fd].info.id >= 0); + + flow = &proc.flows[fd]; + if (flow->poa != NULL) { + cap_clear(poa_flow_cap_est(flow->poa)); + return 0; + } + + e = flow->cap; + if (e != NULL) { + cap_clear(e); + return 0; + } + + if (posix_memalign((void **) &e, CAP_ALIGN, sizeof(*e)) != 0) + return -ENOMEM; + + cap_clear(e); + + STORE_RELEASE(&flow->cap, e); + + return 0; +} + +void ipcp_flow_cap_update(int fd, + size_t qlen, + size_t len) +{ + struct flow * flow; + struct cap_est * e; + + assert(fd >= 0 && fd < PROC_MAX_FLOWS); + assert(proc.flows[fd].info.id >= 0); + + flow = &proc.flows[fd]; + if (flow->poa != NULL) { + cap_update(poa_flow_cap_est(flow->poa), qlen, len); + return; + } + + e = LOAD_ACQUIRE(&flow->cap); + if (e == NULL) + return; + + cap_update(e, qlen, len); +} + +uint64_t ipcp_flow_cap(int fd) +{ + struct flow * flow; + struct cap_est * e; + + assert(fd >= 0 && fd < PROC_MAX_FLOWS); + assert(proc.flows[fd].info.id >= 0); + + flow = &proc.flows[fd]; + if (flow->poa != NULL) + return cap_rate(poa_flow_cap_est(flow->poa)); + + e = LOAD_ACQUIRE(&flow->cap); + if (e == NULL) + return 0; + + return cap_rate(e); } int local_flow_transfer(int src_fd, @@ -2690,3 +3312,6 @@ int local_flow_transfer(int src_fd, return ret; } + +#include "cap.c" +#include "poa/poa.c" diff --git a/src/lib/frct.c b/src/lib/frct.c index 2e8955e3..ecec2543 100644 --- a/src/lib/frct.c +++ b/src/lib/frct.c @@ -25,16 +25,18 @@ #define DELT_RDV (100 * MILLION) /* ns */ #define MAX_RDV (1 * BILLION) /* ns */ -#define MAX_RTO_MUL 8 /* caps the RTO backoff shift */ +#define RXM_TRIES_SHIFT 5 /* >= 32 HoL tries within t_r */ +#define MAX_RTO_MUL 16 /* guard; rxm_backoff clamps */ #define MAX_TLP_PER_EP 2 /* RFC 8985 §7.3: up to 2 TLPs */ -#define INITIAL_RTO (1 * BILLION) /* RFC 6298 §2.1: 1 s default */ #define RTT_BOOT_NS (10 * MILLION) /* rtt_hint floor + initial mdev */ #define SRTT_FLOOR_NS 1000L /* 1 us; smoothed RTT floor */ #define MDEV_FLOOR_NS 100L /* 100 ns; mdev sanity floor */ #define RTT_CLAMP_MUL 16 /* probe sample cap = N * srtt */ #define MIN_RTT_WIN_NS (300ULL * BILLION) /* 5 min, Linux tcp default */ +#define MIN_RTT_SLOTS 3 /* windowed-min sample slots */ #define NACK_COOLDOWN_NS (100 * MILLION) /* pre-DRF NACK cooldown */ #define FRCT_TX_TIMEO_NS (250 * 1000) /* tx ring write deadline */ +#define RTT_LOUD_NS (500 * MILLION) /* diagnostic sample threshold */ #define ACK_DELAY_NS (2ULL * TICTIME) /* delayed-ACK fire delay */ #define FRCT "frct" @@ -49,11 +51,13 @@ #define SACK_MIN_GAP_NS (250u * 1000u) /* 250 us SACK gap */ #define MIN_REORDER_NS (250u * 1000u) /* 250 us RACK floor */ #define SACK_RXM_MAX 32 /* Cap on retransmits staged from single SACK.*/ -#define DUP_THRESH 3 /* RFC 8985 §6.2 step 2.2 SACK count gate. */ +#define DUP_THRESH 3 /* RFC 8985 §6.2 step 4 SACK count gate. */ +/* Repair budget: burst cap on SACK-driven retransmits (tokens). */ +#define RXM_BUDGET_MAX (2 * SACK_RXM_MAX) -/* RFC 8985 §7.2 RACK reorder-window scaling cap. */ +/* RFC 8985 §6.2 RACK reorder-window scaling cap. */ #define REO_WND_MULT_MAX 20 -/* RFC 8985 §7.2 step 5: round trips of no DSACK before halving. */ +/* RFC 8985 §6.2: fresh-ACKed seqnos before decaying the scale. */ #define REO_DECAY_PKTS 16 /* DSACK seqno sanity: reject reports older/farther than one rcv window. */ #define MAX_DSACK_LAG RQ_SIZE @@ -186,13 +190,17 @@ struct frcti_stat { size_t rxm_dup_rcv; /* RXM dups (peer already had it) */ size_t rxm_sack; /* SACK-mechanism retransmits */ size_t rxm_rack; /* RACK-driven retransmits */ - size_t rxm_dupthresh; /* DupThresh-driven retransmits */ + size_t rxm_zero_reo; /* repairs at zero reorder wnd */ size_t rxm_nack; /* NACK-pulled retransmits */ size_t rxm_due_count; /* rxm_due entries (pre-bail) */ size_t rxm_due_acked; /* bail: seqno < snd_lwe */ size_t rxm_due_unowned; /* bail: slot.rxm replaced */ size_t rxm_due_aged; /* bail: r->t0 + t_r < now */ size_t rxm_due_defer; /* bail: non-HoL, deferred to HoL */ + size_t rxm_hol_gone; /* defers with no rxm at HoL slot */ + size_t rxm_fast_skip; /* SACK skips: slot has FAST_RXM */ + size_t rxm_fast_stuck; /* those skips with age > rto */ + size_t rxm_no_budget; /* SACKs cut short: no repair token*/ size_t rxm_arm_fail; /* rxm_arm: malloc failed */ size_t rxm_cancel; /* entries cancelled at teardown */ size_t rxm_tx_dead; /* RXM tx into terminal flow */ @@ -302,6 +310,11 @@ struct frct_cr { uint64_t inact; /* Inactivity threshold (ns) */ }; +struct rtt_min { + time_t v; /* measured RTT (ns) */ + uint64_t t; /* when it was measured (ns) */ +}; + struct frcti { /* IMM: set once in frcti_create; read-only thereafter. */ int fd; @@ -322,18 +335,17 @@ struct frcti { struct frct_cr rcv_cr; /* RTT/RACK estimator */ - time_t srtt; /* smoothed RTT */ - time_t mdev; /* mean deviation */ - time_t min_rtt; /* RACK base, ns */ - uint64_t t_min_rtt; /* min_rtt last set */ - time_t rto; /* retransmit TO */ - time_t rto_min; /* RTO floor (ns) */ - uint8_t rto_mul; /* RTO backoff bits */ - uint32_t rtt_lwe; /* RTT-sample fence */ - uint64_t t_rcv_rtt; /* last RTT feed */ - uint64_t t_snd_probe; /* last probe sent */ - uint64_t t_latest_ack; /* RACK.fack snd-ts */ - uint32_t probe_id_next; + time_t srtt; /* smoothed RTT */ + time_t mdev; /* mean deviation */ + struct rtt_min min_rtt[MIN_RTT_SLOTS]; + time_t rto; /* retransmit TO */ + time_t rto_min; /* RTO floor (ns) */ + uint8_t rto_mul; /* RTO backoff bits */ + uint32_t rtt_lwe; /* RTT-sample fence */ + uint64_t t_rcv_rtt; /* last RTT feed */ + uint64_t t_snd_probe; /* last probe sent */ + uint64_t t_latest_ack; /* RACK.fack snd-ts */ + uint32_t probe_id_next; struct { uint32_t id; uint64_t ts; /* ts_to_ns send */ @@ -342,6 +354,7 @@ struct frcti { /* rcv reassembly */ size_t max_rcv_sdu; /* max reasm bytes */ + bool draining; /* dealloc drain */ uint8_t * rcv_ring; /* lazy alloc */ size_t rcv_ring_sz; /* power of 2 */ uint32_t ring_seq_cap; /* ring/per_pkt */ @@ -366,12 +379,13 @@ struct frcti { uint32_t dsack_seqno; bool dsack_valid; - /* RFC 8985 §7.2 RACK reorder-window scaling. */ + /* RFC 8985 §6.2 RACK reorder-window scaling. */ uint8_t reo_wnd_mult; /* REO_WND_MULT_MAX */ uint32_t dsack_lwe_snap; /* lwe @ last DSACK */ uint64_t t_last_reo_widen; /* once-per-RTT */ uint32_t dup_thresh; /* RFC 8985 */ + uint32_t rxm_budget; /* repair tokens */ uint32_t tlp_high_seq; /* §7.3: 0 = none */ uint8_t tlp_count; /* §7.3 per-episode */ uint64_t t_nack; @@ -465,7 +479,7 @@ static int frct_rib_read(const char * path, s.srtt = frcti->srtt; s.mdev = frcti->mdev; s.rto = frcti->rto; - s.min_rtt = frcti->min_rtt; + s.min_rtt = frcti->min_rtt[0].v; s.snd_cr = frcti->snd_cr; s.rcv_cr = frcti->rcv_cr; s.stat = frcti->stat; @@ -494,7 +508,7 @@ static int frct_rib_read(const char * path, " duplicates received: %20zu\n" "RXM (SACK mechanism) sent: %20zu\n" "RXM (RACK-driven) sent: %20zu\n" - "RXM (DupThresh-driven) sent: %20zu\n" + "RXM (zero reorder wnd) sent: %20zu\n" "RXM (NACK-driven) sent: %20zu\n" "ACK packets sent: %20zu\n" "Delayed-ACK timer fires: %20zu\n" @@ -549,6 +563,10 @@ static int frct_rib_read(const char * path, " bail (unowned): %20zu\n" " bail (aged): %20zu\n" " bail (defer): %20zu\n" + " defer, no rxm at HoL: %20zu\n" + " skip (fast-rxm set): %20zu\n" + " skip (stuck past rto): %20zu\n" + " skip (no repair budget): %20zu\n" "RXM-arm malloc failures: %20zu\n" "RXM cancels (teardown): %20zu\n" "RXM tx into dead flow: %20zu\n" @@ -570,7 +588,7 @@ static int frct_rib_read(const char * path, (long long)(now_ns - s.rcv_cr.act), s.rcv_cr.seqno, s.stat.rxm_rto, s.stat.rxm_rcv, s.stat.rxm_dup_rcv, - s.stat.rxm_sack, s.stat.rxm_rack, s.stat.rxm_dupthresh, + s.stat.rxm_sack, s.stat.rxm_rack, s.stat.rxm_zero_reo, s.stat.rxm_nack, s.stat.ack_snd, s.stat.ack_fire, s.stat.ack_supp_seqno, s.stat.ack_supp_inact, @@ -597,6 +615,9 @@ static int frct_rib_read(const char * path, s.stat.rxm_due_count, s.stat.rxm_due_acked, s.stat.rxm_due_unowned, s.stat.rxm_due_aged, s.stat.rxm_due_defer, + s.stat.rxm_hol_gone, + s.stat.rxm_fast_skip, s.stat.rxm_fast_stuck, + s.stat.rxm_no_budget, s.stat.rxm_arm_fail, s.stat.rxm_cancel, s.stat.rxm_tx_dead, s.stat.tx_drop, @@ -689,15 +710,15 @@ static __inline__ bool same_epoch_drf(uint32_t seq, /* * RACK reorder window R (RFC 8985 §6.2): * R = MIN(reo_wnd_mult * RACK.min_RTT / 4, SRTT) - * reo_wnd_mult scales on D-SACK evidence of under-tolerance (§7.2). + * reo_wnd_mult scales on D-SACK evidence of under-tolerance (§6.2). * Fall back to srtt when no min_rtt sample exists yet; MIN_REORDER_NS * floor guards collapse below the timer-tick resolution. */ static __inline__ uint64_t rack_reorder_window(struct frcti * frcti) { uint64_t mult = frcti->reo_wnd_mult > 0 ? frcti->reo_wnd_mult : 1; - uint64_t base = frcti->min_rtt > 0 ? (uint64_t) frcti->min_rtt - : (uint64_t) frcti->srtt; + time_t min = frcti->min_rtt[0].v; + uint64_t base = min > 0 ? (uint64_t) min : (uint64_t) frcti->srtt; uint64_t R = mult * (base / 4); R = MAX(R, (uint64_t) MIN_REORDER_NS); @@ -706,6 +727,24 @@ static __inline__ uint64_t rack_reorder_window(struct frcti * frcti) return R; } +/* + * RFC 8985 §6.2 RACK_update_reo_wnd(): as long as no reordering has + * been observed, a repair episode or DupThresh SACKs above the head + * drop the reordering tolerance to zero. This removes the tolerance + * only; the RACK time test still gates every repair. + */ +static __inline__ uint64_t rack_reo_wnd(struct frcti * frcti, + uint64_t R) +{ + if (frcti->reo_wnd_mult > 1) + return R; + + if (frcti->in_recovery || frcti->dup_thresh >= DUP_THRESH) + return 0; + + return R; +} + static __inline__ int frct_spb_reserve(size_t len, struct ssm_pk_buff ** spb) { @@ -822,7 +861,9 @@ static void frct_tx_drop_bump(struct frcti * frcti, STAT_BUMP(frcti, tx_drop_other); } -static int frct_tx(struct frcti * frcti, struct ssm_pk_buff * spb) +static int frct_tx(struct frcti * frcti, + struct ssm_pk_buff * spb, + bool prio) { struct flow * f = frcti_to_flow(frcti); const struct frct_pci * pci; @@ -849,16 +890,33 @@ static int frct_tx(struct frcti * frcti, struct ssm_pk_buff * spb) if (spb_encrypt(f, spb) < 0) goto fail; - idx = ssm_pk_buff_get_off(spb); - - /* DATA blocks; control times out so a full ring can't stall wheel. */ + /* Control times out so a full queue cannot stall the wheel. */ if (!(flags & FRCT_DATA)) { clock_gettime(PTHREAD_COND_CLOCK, &now); ts_add(&now, &intv, &deadline); + dl = &deadline; } - ret = ssm_rbuff_write_b(f->tx_rb, idx, dl); + if (f->poa != NULL) { + ret = poa_flow_tx(f->poa, spb, true, dl); + if (ret < 0) + goto fail; + + return 0; + } + + idx = ssm_pk_buff_get_off(spb); + + /* + * The peer is already waiting on a retransmission, so it skips + * the occupancy limit and never waits: the timer that sent it + * must not block, and the r-timer retries what does not fit. + */ + if (prio) + ret = ssm_rbuff_write_prio(f->tx_rb, idx); + else + ret = ssm_rbuff_write_b(f->tx_rb, idx, dl); if (ret < 0) goto fail; @@ -878,10 +936,10 @@ static void frct_mark_flow_down(struct frcti * frcti) struct flow * f = frcti_to_flow(frcti); if (f->rx_rb != NULL) - ssm_rbuff_set_acl(f->rx_rb, ACL_FLOWDOWN); + ssm_rbuff_set_flags(f->rx_rb, RB_FLOWDOWN); if (f->tx_rb != NULL) - ssm_rbuff_set_acl(f->tx_rb, ACL_FLOWDOWN); + ssm_rbuff_set_flags(f->tx_rb, RB_FLOWDOWN); } __attribute__((cold)) @@ -890,7 +948,7 @@ static void frct_mark_peer_dead(struct frcti * frcti) struct flow * f = frcti_to_flow(frcti); if (f->rx_rb != NULL) - ssm_rbuff_set_acl(f->rx_rb, ACL_FLOWPEER); + ssm_rbuff_set_flags(f->rx_rb, RB_FLOWPEER); if (proc.fqset != NULL) ssm_flow_set_notify(proc.fqset, f->info.id, FLOW_PEER); @@ -950,14 +1008,30 @@ static void frcti_pkt_snd(struct frcti * frcti, frct_hcs_set(pci, false); - frct_tx(frcti, spb); + frct_tx(frcti, spb, false); +} + +/* Restart the window from a single sample. */ +static __inline__ void min_rtt_seed(struct frcti * frcti, + time_t mrtt, + uint64_t now_ns) +{ + size_t i; + + for (i = 0; i < MIN_RTT_SLOTS; i++) { + frcti->min_rtt[i].v = mrtt; + frcti->min_rtt[i].t = now_ns; + } } /* RTO floor scales with srtt; hard floor rto_min guards sub-ms RTT. */ static void rtt_init(struct frcti * frcti, - time_t rtt_hint) + time_t rtt_hint, + uint32_t max_rtt, + uint64_t now_ns) { time_t floor; + time_t cap; if (rtt_hint > 0) { rtt_hint = MAX(rtt_hint, (time_t) RTT_BOOT_NS); @@ -965,42 +1039,85 @@ static void rtt_init(struct frcti * frcti, frcti->mdev = rtt_hint >> 3; floor = MAX(frcti->rto_min, 2 * frcti->srtt); frcti->rto = MAX(floor, rtt_hint + (frcti->mdev << MDEV_MUL)); - frcti->min_rtt = rtt_hint; + + min_rtt_seed(frcti, rtt_hint, now_ns); } else { - /* Boot from first ACK. */ + /* Boot from first ACK; declared max path RTT caps RTO. */ + cap = (time_t) (frcti->t_r >> RXM_TRIES_SHIFT); + + if (max_rtt > 0) + cap = MIN(cap, (time_t) max_rtt * 2 * MILLION); frcti->srtt = 0; frcti->mdev = RTT_BOOT_NS; - frcti->rto = MAX((time_t) INITIAL_RTO, frcti->rto_min); - frcti->min_rtt = 0; + frcti->rto = MAX(cap, frcti->rto_min); + + min_rtt_seed(frcti, 0, now_ns); } frcti->rto_mul = 0; } -/* RFC 8985 §6.2: replace min_RTT on unset, smaller sample, or expiry. */ -static __inline__ bool min_rtt_stale(struct frcti * frcti, - time_t mrtt, - uint64_t now_ns) +/* Promote the runners-up as the window slides past each slot. */ +static __inline__ void min_rtt_subwin(struct frcti * frcti, + const struct rtt_min * val) { - if (frcti->min_rtt == 0) - return true; + struct rtt_min * s = frcti->min_rtt; + int64_t dt = ts_age_ns(val->t, s[0].t); + int64_t win = (int64_t) MIN_RTT_WIN_NS; - if (mrtt < frcti->min_rtt) - return true; + /* A clock step or an out-of-order stamp: hold the window. */ + if (dt < 0) + return; - return ts_aged_ns(now_ns, frcti->t_min_rtt, MIN_RTT_WIN_NS); + if (dt > win) { + /* Slot 0 fell out; slot 1 may be stale in turn. */ + s[0] = s[1]; + s[1] = s[2]; + s[2] = *val; + if (ts_aged_ns(val->t, s[0].t, MIN_RTT_WIN_NS)) { + s[0] = s[1]; + s[1] = s[2]; + s[2] = *val; + } + } else if (s[1].t == s[0].t && dt > win / 4) { + s[2] = s[1] = *val; + } else if (s[2].t == s[1].t && dt > win / 2) { + s[2] = *val; + } } -/* Linux-style windowed-min refresh of RACK.min_RTT. */ +/* + * Windowed minimum of RACK.min_RTT over MIN_RTT_WIN_NS, after Linux + * lib/minmax.c. Slots 1 and 2 hold minima over the trailing 3/4 and + * 1/2 of the window, so when slot 0 ages out the estimate drops back + * to a true minimum over what remains rather than to a spot sample. + */ static __inline__ void min_rtt_update(struct frcti * frcti, time_t mrtt, uint64_t now_ns) { - if (!min_rtt_stale(frcti, mrtt, now_ns)) + struct rtt_min * s = frcti->min_rtt; + struct rtt_min val; + + if (mrtt <= 0) + return; + + val.v = mrtt; + val.t = now_ns; + + /* New min, unseeded, or nothing left in the window. */ + if (s[0].v == 0 || mrtt <= s[0].v + || ts_aged_ns(now_ns, s[2].t, MIN_RTT_WIN_NS)) { + min_rtt_seed(frcti, mrtt, now_ns); return; + } + + if (mrtt <= s[1].v) + s[1] = s[2] = val; + else if (mrtt <= s[2].v) + s[2] = val; - frcti->min_rtt = mrtt; - frcti->t_min_rtt = now_ns; + min_rtt_subwin(frcti, &val); } static void rtt_update(struct frcti * frcti, @@ -1035,8 +1152,15 @@ static void rtt_update(struct frcti * frcti, floor = MAX(frcti->rto_min, 2 * frcti->srtt); rto = MAX(floor, frcti->srtt + (frcti->mdev << MDEV_MUL)); + /* FIXME: align with t_r; an rto that spans it retries nothing. */ STORE_RELEASE(&frcti->rto, rto); STORE_RELEASE(&frcti->rto_mul, 0); + + /* Diagnostic: a sample this large is not a path RTT. */ + if (mrtt > RTT_LOUD_NS) + log_warn("RTT sample %lld ms, srtt %lld ms on fd %d.", + (long long) mrtt / MILLION, + (long long) frcti->srtt / MILLION, frcti->fd); } /* Fill probes[pos], return new probe_id; 0 on entropy failure. Wrlock. */ @@ -1111,7 +1235,7 @@ static void frcti_rttp_snd(struct frcti * frcti, rttp->echo_id = hton32(echo_id); memcpy(rttp->nonce, nonce, sizeof(rttp->nonce)); - frct_tx(frcti, spb); + frct_tx(frcti, spb, false); } struct rxm_entry { @@ -1124,33 +1248,6 @@ struct rxm_entry { uint8_t pkt[]; /* flexible — sized at alloc time */ }; -static struct rxm_entry * rxm_entry_create(struct frcti * frcti, - uint32_t seqno, - const struct ssm_pk_buff * spb) -{ - struct rxm_entry * r; - struct timespec now; - size_t len = ssm_pk_buff_len(spb); - - r = malloc(sizeof(*r) + len); - if (r == NULL) { - STAT_BUMP(frcti, rxm_arm_fail); - return NULL; - } - - memcpy(r->pkt, ssm_pk_buff_head(spb), len); - r->len = len; - r->frcti = frcti; - r->seqno = seqno; - - clock_gettime(PTHREAD_COND_CLOCK, &now); - r->t0 = TS_TO_UINT64(now); - - tw_init_entry(&r->tw); - - return r; -} - static void rxm_entry_destroy(struct rxm_entry * r) { free(r); @@ -1164,6 +1261,29 @@ static bool rxm_still_owned(struct frcti * frcti, } /* + * Backoff clamped to a fixed fraction of t_r, so the ladder always + * leaves room for 1 << RXM_TRIES_SHIFT tries inside the flow's life + * whatever t_r is. Never returns less than the RTO estimate itself: + * on a path whose RTT is large against t_r that many tries do not + * fit, and retrying faster than the estimate only duplicates. + */ +static uint64_t rxm_backoff(struct frcti * frcti, + time_t rto, + uint8_t rto_mul) +{ + uint64_t cap = frcti->t_r >> RXM_TRIES_SHIFT; + + if (cap < (uint64_t) rto) + return (uint64_t) rto; + + /* Compare before shifting; the product can overflow at large t_r. */ + if (rto_mul >= 64 || (uint64_t) rto > (cap >> rto_mul)) + return cap; + + return (uint64_t) rto << rto_mul; +} + +/* * All in-flight slots share the HoL backoff; otherwise non-HoL timers * cycle at base RTO and storm the wire while HoL is still backing off. */ @@ -1173,7 +1293,7 @@ static uint64_t rxm_next_deadline(struct frcti * frcti, time_t rto = LOAD_RELAXED(&frcti->rto); uint8_t rto_mul = LOAD_RELAXED(&frcti->rto_mul); - return now_ns + ((uint64_t) rto << rto_mul); + return now_ns + rxm_backoff(frcti, rto, rto_mul); } /* Copy pkt, set FRCT_RXM, refresh ackno, re-seal HCS. */ @@ -1238,7 +1358,7 @@ static void rxm_snd(struct frcti * frcti, if (seqno == snd_lwe && frcti->rto_mul < MAX_RTO_MUL) STORE_RELEASE(&frcti->rto_mul, frcti->rto_mul + 1); - /* RFC 8985 §7.2 step 4: RTO on HoL resets RACK reo scaling. */ + /* RFC 8985 §6.3: RTO on HoL resets RACK reo scaling. */ if (seqno == snd_lwe) frcti->reo_wnd_mult = 1; @@ -1251,7 +1371,7 @@ static void rxm_snd(struct frcti * frcti, return; /* ETIMEDOUT/ENOMEM: let r-timer drive teardown. */ - ret = frct_tx(frcti, spb); + ret = frct_tx(frcti, spb, true); if (ret == -EFLOWDOWN || ret == -ENOTALLOC) STAT_BUMP(frcti, rxm_tx_dead); } @@ -1287,6 +1407,24 @@ static void rxm_due(void * arg) /* R-timer expired: peer unreachable. */ if (RXM_AGED_OUT(r->t0, now_ns, frcti->t_r)) { STAT_BUMP(frcti, rxm_due_aged); + log_warn("Flow down: rxm seq=%u aged out (hol=%u) " + "age_ms=%llu t_r_ms=%llu rto_ms=%llu mul=%u " + "ack_age_ms=%lld hol_rxm=%s hol_flags=0x%x " + "budget=%u tlp_hi=%u tlp_n=%u on fd %d.", + r->seqno, snd_lwe, + (unsigned long long)(now_ns - r->t0) / MILLION, + (unsigned long long) frcti->t_r / MILLION, + (unsigned long long) LOAD_RELAXED(&frcti->rto) + / MILLION, + (unsigned) LOAD_RELAXED(&frcti->rto_mul), + (long long)(now_ns - frcti->t_latest_ack) / MILLION, + LOAD_ACQUIRE(&frcti->snd_slots[RQ_SLOT(snd_lwe)].rxm) + == NULL ? "none" : "live", + (unsigned) frcti->snd_slots[RQ_SLOT(snd_lwe)].flags, + (unsigned) frcti->rxm_budget, + frcti->tlp_high_seq, + (unsigned) frcti->tlp_count, + frcti->fd); frct_mark_flow_down(frcti); goto cleanup; } @@ -1294,8 +1432,11 @@ static void rxm_due(void * arg) /* HoL-only retx; defer at base rto so HoL transitions react. */ if (r->seqno != snd_lwe) { STAT_BUMP(frcti, rxm_due_defer); - tw_post(&r->tw, now_ns + LOAD_RELAXED(&frcti->rto), - rxm_due, r); + + if (LOAD_ACQUIRE(&frcti->snd_slots[RQ_SLOT(snd_lwe)].rxm) + == NULL) + STAT_BUMP(frcti, rxm_hol_gone); + tw_post(&r->tw, now_ns + LOAD_RELAXED(&frcti->rto), rxm_due, r); return; } @@ -1324,33 +1465,56 @@ static void rxm_due(void * arg) rxm_entry_destroy(r); } -static int rxm_arm(struct frcti * frcti, - uint32_t seqno, - const struct ssm_pk_buff * spb) +/* Pre-allocate rxm entry so frcti_snd can fail before committing seqno. */ +static struct rxm_entry * rxm_alloc(struct frcti * frcti, + size_t pkt_len) { struct rxm_entry * r; - time_t rto; - uint8_t rto_mul; - uint64_t deadline; - r = rxm_entry_create(frcti, seqno, spb); - if (r == NULL) - return -ENOMEM; + r = malloc(sizeof(*r) + pkt_len); + if (r == NULL) { + STAT_BUMP(frcti, rxm_arm_fail); + return NULL; + } + + r->frcti = frcti; + tw_init_entry(&r->tw); + + return r; +} + +static void rxm_arm(struct frcti * frcti, + uint32_t seqno, + struct rxm_entry * r, + const struct ssm_pk_buff * spb) +{ + struct timespec now; + time_t rto; + uint8_t rto_mul; + uint64_t deadline; + size_t len = ssm_pk_buff_len(spb); + + memcpy(r->pkt, ssm_pk_buff_head(spb), len); + r->len = len; + r->seqno = seqno; + + clock_gettime(PTHREAD_COND_CLOCK, &now); + r->t0 = TS_TO_UINT64(now); rto = LOAD_RELAXED(&frcti->rto); rto_mul = LOAD_RELAXED(&frcti->rto_mul); - deadline = r->t0 + ((uint64_t) rto << rto_mul); + deadline = r->t0 + rxm_backoff(frcti, rto, rto_mul); pthread_rwlock_wrlock(&frcti->lock); + assert(before(seqno, frcti->snd_cr.lwe + RQ_SIZE)); + list_add_tail(&r->next, &frcti->rxm_list); STORE_RELEASE(&frcti->snd_slots[RQ_SLOT(seqno)].rxm, r); pthread_rwlock_unlock(&frcti->lock); tw_post(&r->tw, deadline, rxm_due, r); - - return 0; } static void rxm_cancel_all(struct frcti * frcti) @@ -1475,7 +1639,7 @@ static void frcti_sack_snd(struct frcti * frcti, for (i = 0; i < sa->n; ++i) sack_block_put(buf.data, i, sa->blocks[i][0], sa->blocks[i][1]); - frct_tx(frcti, spb); + frct_tx(frcti, spb, false); } static void ack_snd(struct frcti * frcti, @@ -1652,6 +1816,8 @@ static void ka_snd(struct frcti * frcti) snd_idle = ts_age_ns(now_ns, LOAD_RELAXED(&frcti->snd_cr.act)); if (rcv_idle > timeo_ns) { + log_warn("Peer dead: rcv idle %lld ms on fd %d.", + (long long) rcv_idle / MILLION, frcti->fd); frct_mark_peer_dead(frcti); return; } @@ -1674,7 +1840,7 @@ static void ka_snd(struct frcti * frcti) frct_hcs_set(pci, false); STAT_BUMP(frcti, ka_snd); - frct_tx(frcti, spb); + frct_tx(frcti, spb, false); ka_arm(frcti); } @@ -1806,6 +1972,7 @@ struct frcti * frcti_create(int fd, uint64_t r, uint64_t mpl, time_t rtt_hint, + uint32_t max_rtt, qosspec_t qs, uint32_t mtu) { @@ -1861,6 +2028,7 @@ struct frcti * frcti_create(int fd, / SACK_BLOCK_SIZE; if (bb > SACK_MAX_BLOCKS) bb = SACK_MAX_BLOCKS; + frcti->sack_n_max = (uint16_t) bb; frcti->max_rcv_sdu = FRCT_MAX_SDU; @@ -1874,8 +2042,8 @@ struct frcti * frcti_create(int fd, } frcti->rto_min = (time_t) MAX(RTO_MIN, 1ULL << RXMQ_RES); - rtt_init(frcti, rtt_hint); - frcti->t_min_rtt = now_ns; + + rtt_init(frcti, rtt_hint, max_rtt, now_ns); frcti->probe_id_next = 1; frcti->t_rcv_rtt = now_ns; frcti->t_snd_probe = now_ns; @@ -1892,6 +2060,7 @@ struct frcti * frcti_create(int fd, frcti->in_recovery = false; frcti->recovery_high = 0; frcti->rack_fired_lwe = 0; + frcti->rxm_budget = RXM_BUDGET_MAX; tw_init_entry(&frcti->ack_tw); tw_init_entry(&frcti->ka_tw); @@ -1952,10 +2121,14 @@ void frcti_destroy(struct frcti * frcti) printf("[FRCT teardown] pid=%d fd=%d " "sdu_snd=%zu sdu_reasm=%zu sdu_sole=%zu " "frag_snd=%zu frag_rcv=%zu frag_drop=%zu " - "rxm_rto=%zu rxm_sack=%zu rxm_dup=%zu " + "rxm_rto=%zu rxm_sack=%zu rxm_rack=%zu rxm_zreo=%zu " "rxm_due=%zu acked=%zu unowned=%zu aged=%zu defer=%zu " + "hol_gone=%zu " + "fast_skip=%zu fast_stuck=%zu no_budget=%zu " "cancel=%zu arm_fail=%zu inflight=%u " "nack_snd=%zu nack_rcv=%zu inact_drop=%zu " + "tlp_snd=%zu sack_snd=%zu sack_rcv=%zu ack_supp=%zu " + "out_rcv=%zu rqo_rcv=%zu dup_rcv=%zu rxm_dup_rcv=%zu " "drf_rebase=%zu rq_released=%zu\n", (int) getpid(), frcti->fd, frcti->stat.sdu_snd_frag, frcti->stat.sdu_reasm, @@ -1963,14 +2136,21 @@ void frcti_destroy(struct frcti * frcti) frcti->stat.frag_snd, frcti->stat.frag_rcv, frcti->stat.frag_drop, frcti->stat.rxm_rto, frcti->stat.rxm_sack, - frcti->stat.rxm_dupthresh, + frcti->stat.rxm_rack, frcti->stat.rxm_zero_reo, frcti->stat.rxm_due_count, frcti->stat.rxm_due_acked, frcti->stat.rxm_due_unowned, frcti->stat.rxm_due_aged, frcti->stat.rxm_due_defer, + frcti->stat.rxm_hol_gone, + frcti->stat.rxm_fast_skip, frcti->stat.rxm_fast_stuck, + frcti->stat.rxm_no_budget, frcti->stat.rxm_cancel, frcti->stat.rxm_arm_fail, frcti->snd_cr.seqno - frcti->snd_cr.lwe, frcti->stat.nack_snd, frcti->stat.nack_rcv, frcti->stat.inact_drop, + frcti->stat.tlp_snd, frcti->stat.sack_snd, + frcti->stat.sack_rcv, frcti->stat.ack_supp_seqno, + frcti->stat.out_rcv, frcti->stat.rqo_rcv, + frcti->stat.dup_rcv, frcti->stat.rxm_dup_rcv, frcti->stat.drf_rebase, frcti->stat.rq_released); #endif @@ -2042,6 +2222,19 @@ int frcti_set_max_rcv_sdu(struct frcti * frcti, return 0; } +/* Dealloc drain discards SDUs by design; don't count them as drops. */ +static void frcti_set_draining(struct frcti * frcti) +{ + if (frcti == NULL) + return; + + pthread_rwlock_wrlock(&frcti->lock); + + frcti->draining = true; + + pthread_rwlock_unlock(&frcti->lock); +} + size_t frcti_get_rcv_ring_sz(struct frcti * frcti) { size_t ret; @@ -2066,6 +2259,7 @@ int frcti_set_rcv_ring_sz(struct frcti * frcti, if (!frcti->stream) return -ENOTSUP; + if (!stream_ring_sz_ok(frcti, n)) return -EINVAL; @@ -2135,6 +2329,7 @@ static void sack_rxm_snd(struct frcti * frcti, { struct ssm_pk_buff * spb; const struct frct_pci * pci; + struct rxm_entry * rxm; uint32_t rcv_lwe; uint32_t seqno; int ret; @@ -2148,14 +2343,15 @@ static void sack_rxm_snd(struct frcti * frcti, pci = (const struct frct_pci *) ssm_pk_buff_head(spb); seqno = ntoh32(pci->seqno); - /* Register fresh rxm before send; old entry self-cleans. */ - if (rxm_arm(frcti, seqno, spb) < 0) { + rxm = rxm_alloc(frcti, ssm_pk_buff_len(spb)); + if (rxm == NULL) { frct_spb_release(spb); return; } + rxm_arm(frcti, seqno, rxm, spb); STAT_BUMP(frcti, rxm_sack); - ret = frct_tx(frcti, spb); + ret = frct_tx(frcti, spb, true); if (ret == -EFLOWDOWN || ret == -ENOTALLOC) STAT_BUMP(frcti, rxm_tx_dead); } @@ -2174,7 +2370,7 @@ static int fast_rxm_send(struct frcti * frcti, if (spb == NULL) return 0; - return frct_tx(frcti, spb); + return frct_tx(frcti, spb, true); } /* PCI bytes survive head_release at receive; just rewind the pointer. */ @@ -2629,14 +2825,16 @@ static ssize_t frcti_consume(struct frcti * frcti, goto unlock; } if (st == FRAG_DROP) { - STAT_ADD(frcti, frag_drop, n); + if (!frcti->draining) + STAT_ADD(frcti, frag_drop, n); frag_drop(frcti, n); continue; } /* FRAG_DELIVER */ total = frag_total_len(frcti, n, &overflow); if (overflow || total > frcti->max_rcv_sdu || total > count) { - STAT_ADD(frcti, frag_drop, n); + if (!frcti->draining) + STAT_ADD(frcti, frag_drop, n); frag_drop(frcti, n); ret = -EMSGSIZE; goto unlock; @@ -2685,6 +2883,49 @@ static bool frcti_pdu_ready(struct frcti * frcti) return ready; } +/* + * Size a ready SDU before consuming it: *len is the total byte + * count, *nfrags the fragment count. 0 on success, -EAGAIN if no + * complete SDU is ready (includes the stream and overflow cases). + */ +static int frcti_pdu_info(struct frcti * frcti, + size_t * len, + size_t * nfrags) +{ + size_t count; + bool overflow; + int ret; + + assert(frcti); + + pthread_rwlock_rdlock(&frcti->lock); + + if (frcti->stream) { + ret = -EAGAIN; + goto unlock; + } + + if (frag_run_inspect(frcti, &count) != FRAG_DELIVER) { + ret = -EAGAIN; + goto unlock; + } + + *len = frag_total_len(frcti, count, &overflow); + + if (overflow) { + ret = -EAGAIN; + goto unlock; + } + + *nfrags = count; + ret = 0; + + unlock: + pthread_rwlock_unlock(&frcti->lock); + + return ret; +} + /* No srtt yet: probe at the cold-probe cadence to seed it. */ #define PROBE_DUE_COLD(frcti, now_ns) \ ((now_ns) - (frcti)->t_snd_probe > (uint64_t) RTTP_COLD_NS) @@ -2932,9 +3173,6 @@ static void tlp_due(void * arg) if (frcti->snd_cr.seqno == frcti->snd_cr.lwe) goto unlock; - if (!before(frcti->snd_cr.seqno, frcti->snd_cr.rwe)) - goto unlock; /* FC-blocked: RDV handles it. */ - /* RFC 8985 §7.3: one outstanding probe, MAX_TLP_PER_EP per ep. */ if (frcti->tlp_high_seq != 0) goto unlock; @@ -2957,8 +3195,8 @@ static void tlp_due(void * arg) goto unlock; /* Cap: if HoL RTO is due, let rxm_due fire instead. */ - rto_at = rxm->t0 + ((uint64_t) frcti->rto - << LOAD_RELAXED(&frcti->rto_mul)); + rto_at = rxm->t0 + rxm_backoff(frcti, frcti->rto, + LOAD_RELAXED(&frcti->rto_mul)); if (rto_at <= now_ns) goto unlock; @@ -2967,10 +3205,10 @@ static void tlp_due(void * arg) memcpy(pkt_copy, rxm->pkt, rxm->len); pkt_len = rxm->len; frcti->snd_slots[hp].time = now_ns; - frcti->snd_slots[hp].flags |= SND_TLP | SND_FAST_RXM; + frcti->snd_slots[hp].flags |= SND_TLP; frcti->rtt_lwe = frcti->snd_cr.lwe + 1; - /* §7.3 outstanding-probe marker; ack_rcv/rxm_snd clear. */ - frcti->tlp_high_seq = frcti->snd_cr.seqno; + /* Probe is the HoL: any cum-ACK resolves the episode. */ + frcti->tlp_high_seq = frcti->snd_cr.lwe + 1; frcti->tlp_count++; STAT_BUMP(frcti, tlp_snd); } @@ -3000,8 +3238,10 @@ static int tlp_arm(struct frcti * frcti) /* §7.3: one outstanding probe, MAX_TLP_PER_EP per recovery ep. */ if (LOAD_RELAXED(&frcti->tlp_high_seq) != 0) return 0; + if (LOAD_RELAXED(&frcti->tlp_count) >= MAX_TLP_PER_EP) return 0; + if (__atomic_test_and_set(&frcti->tlp_pending, __ATOMIC_RELAXED)) return 0; @@ -3084,15 +3324,20 @@ static bool rtt_sample_eligible(struct frcti * frcti, { if (flags & FRCT_RXM) return false; + if (frcti->snd_slots[p].flags & (SND_RTX | SND_TLP)) return false; + if (LOAD_ACQUIRE(&frcti->snd_slots[p].rxm) == NULL) return false; + if (before(lwe, frcti->rtt_lwe)) return false; + /* Don't seed srtt from a cum-ACK; let probes seed. */ if (frcti->srtt == 0) return false; + return true; } @@ -3109,19 +3354,21 @@ static void fast_rxm_consider(struct frcti * frcti, struct snd_slot * slot; size_t hp; uint64_t R; - bool rack_ok; + uint64_t reo; + int64_t age; hp = RQ_SLOT(frcti->snd_cr.lwe); slot = &frcti->snd_slots[hp]; rxm = LOAD_ACQUIRE(&slot->rxm); R = rack_reorder_window(frcti); + reo = rack_reo_wnd(frcti, R); if (RXM_SLOT_EMPTY(rxm)) return; - /* RFC 8985 §6.2: time-based RACK OR DupThresh count. */ - rack_ok = (int64_t)(frcti->t_latest_ack - slot->time) > (int64_t) R; - if (!rack_ok && frcti->dup_thresh < DUP_THRESH) + /* RFC 8985 §6.2: last transmission older than the latest ack + reo. */ + age = (int64_t)(frcti->t_latest_ack - slot->time); + if (age <= (int64_t) reo) return; /* HoL aged past t_r; let rxm_due tear the flow down. */ @@ -3142,10 +3389,11 @@ static void fast_rxm_consider(struct frcti * frcti, memcpy(pending->fast_rxm.data, rxm->pkt, rxm->len); slot->flags |= SND_RTX | SND_FAST_RXM; frcti->rtt_lwe = frcti->snd_cr.lwe + 1; - if (rack_ok) + + if (age > (int64_t) R) STAT_BUMP(frcti, rxm_rack); else - STAT_BUMP(frcti, rxm_dupthresh); + STAT_BUMP(frcti, rxm_zero_reo); } /* Caller holds wrlock; RACK fast retransmit queued in pending. */ @@ -3158,6 +3406,7 @@ static void frcti_ack_rcv(struct frcti * frcti, { uint32_t ackno; uint32_t lwe; + uint64_t t_ack; size_t p; size_t fresh; @@ -3182,16 +3431,24 @@ static void frcti_ack_rcv(struct frcti * frcti, STORE_RELEASE(&frcti->snd_cr.lwe, ackno); + /* Packet conservation: one repair token per seqno that left. */ + frcti->rxm_budget += ackno - lwe; + + if (frcti->rxm_budget > RXM_BUDGET_MAX) + frcti->rxm_budget = RXM_BUDGET_MAX; + /* §7.3: cum-ACK past the probed seqno resolves the TLP. */ if (frcti->tlp_high_seq != 0 - && !before(ackno, frcti->tlp_high_seq)) + && !before(ackno, frcti->tlp_high_seq)) { frcti->tlp_high_seq = 0; + frcti->tlp_count = 0; + } /* §7.3: end the probe episode once inflight drains. */ if (ackno == frcti->snd_cr.seqno) frcti->tlp_count = 0; - /* RFC 8985 §7.2: halve mult per REO_DECAY_PKTS fresh-ACK'd seqnos. */ + /* RFC 8985 §6.2: halve mult per REO_DECAY_PKTS fresh-ACK'd seqnos. */ fresh = ackno - frcti->dsack_lwe_snap; if (frcti->reo_wnd_mult > 1 && fresh >= REO_DECAY_PKTS) { uint8_t half = frcti->reo_wnd_mult >> 1; @@ -3199,8 +3456,15 @@ static void frcti_ack_rcv(struct frcti * frcti, frcti->dsack_lwe_snap = ackno; } - /* RFC 8985: latest cum-ACKed send-time (slot of ackno-1). */ - frcti->t_latest_ack = frcti->snd_slots[RQ_SLOT(ackno - 1)].time; + /* + * RFC 8985 §6.2 RACK_sent_after: RACK.xmit_ts only ever moves + * forward. A cum-ACK covers older seqnos than the SACK blocks + * that raised it, so assigning here would drop it back and + * wedge the loss test for every hole above the cum-ACK. + */ + t_ack = frcti->snd_slots[RQ_SLOT(ackno - 1)].time; + if (t_ack > frcti->t_latest_ack) + frcti->t_latest_ack = t_ack; /* RFC 8985: SACK-above-lwe count is per-recovery-episode. */ frcti->dup_thresh = 0; @@ -3228,10 +3492,12 @@ static void frcti_ack_rcv(struct frcti * frcti, static uint32_t sack_mark_blocks(struct frcti * frcti, const uint8_t * payload, uint16_t n, - uint32_t * newly_marked) + uint32_t * newly_marked, + uint64_t now_ns) { uint32_t hi_sacked = frcti->snd_cr.lwe; uint32_t marked = 0; + uint64_t rtt_t = 0; /* freshest send time worth timing */ uint16_t i; for (i = 0; i < n; ++i) { @@ -3254,10 +3520,14 @@ static uint32_t sack_mark_blocks(struct frcti * frcti, for (k = s; before(k, e); ++k) { size_t kp = RQ_SLOT(k); uint64_t t_k; + uint8_t f_k; if (clamped && k == frcti->snd_cr.lwe) continue; if (LOAD_ACQUIRE(&frcti->snd_slots[kp].rxm) == NULL) continue; + + f_k = frcti->snd_slots[kp].flags; + STORE_RELEASE(&frcti->snd_slots[kp].rxm, NULL); frcti->snd_slots[kp].flags = 0; marked++; @@ -3265,12 +3535,38 @@ static uint32_t sack_mark_blocks(struct frcti * frcti, t_k = frcti->snd_slots[kp].time; if (t_k > frcti->t_latest_ack) frcti->t_latest_ack = t_k; + + /* Karn: a retransmitted seqno times nothing. */ + if (f_k & (SND_RTX | SND_TLP | SND_FAST_RXM)) + continue; + + if (before(k, frcti->rtt_lwe)) + continue; + + if (t_k > rtt_t) + rtt_t = t_k; } if (after(e, hi_sacked)) hi_sacked = e; } + /* + * One sample per SACK, off the freshest packet it confirms. + * A hole keeps every seqno out of the cum-ACK path, so this + * is the only estimator input while one is open. Seeding is + * still left to the probes. + */ + if (rtt_t > 0 && frcti->srtt != 0) { + int64_t mrtt = ts_age_ns(now_ns, rtt_t); + + if (mrtt > 0) { + rtt_update(frcti, (time_t) mrtt, now_ns); + + frcti->t_rcv_rtt = now_ns; + } + } + *newly_marked = marked; return hi_sacked; } @@ -3281,9 +3577,9 @@ static void sack_queue_rxm(struct frcti * frcti, uint64_t now_ns, struct pending * pending) { - uint64_t R = rack_reorder_window(frcti); + uint64_t R = rack_reorder_window(frcti); + uint64_t reo = rack_reo_wnd(frcti, R); uint32_t k; - bool rack_ok; for (k = frcti->snd_cr.lwe; before(k, hi_sacked); ++k) { struct rxm_entry * rxm; @@ -3299,22 +3595,40 @@ static void sack_queue_rxm(struct frcti * frcti, if (rxm == NULL) continue; - if (frcti->snd_slots[kp].flags & SND_FAST_RXM) - continue; + /* Repairs are ACK-clocked; RTO/HoL cover a dry bucket. */ + if (frcti->rxm_budget == 0) { + STAT_BUMP(frcti, rxm_no_budget); + break; + } + + /* + * A fast-retx outstanding past the reorder window is + * presumed lost in turn; clear the flag so RACK can + * repair it again. The rack_ok test below still needs + * an ack for a later packet, so this cannot storm. + */ + if (frcti->snd_slots[kp].flags & SND_FAST_RXM) { + if (!ts_aged_ns(now_ns, frcti->snd_slots[kp].time, R)) { + STAT_BUMP(frcti, rxm_fast_skip); + continue; + } + + STAT_BUMP(frcti, rxm_fast_stuck); + frcti->snd_slots[kp].flags &= ~SND_FAST_RXM; + } if (RXM_AGED_OUT(rxm->t0, now_ns, frcti->t_r)) continue; rack_age = frcti->t_latest_ack - frcti->snd_slots[kp].time; - /* RFC 8985 §6.2: time-based RACK OR DupThresh count. */ - rack_ok = (int64_t) rack_age > (int64_t) R; - if (!rack_ok && frcti->dup_thresh < DUP_THRESH) + /* RFC 8985 §6.2: last transmission older than latest + reo. */ + if ((int64_t) rack_age <= (int64_t) reo) continue; - if (rack_ok) + if ((int64_t) rack_age > (int64_t) R) STAT_BUMP(frcti, rxm_rack); else - STAT_BUMP(frcti, rxm_dupthresh); + STAT_BUMP(frcti, rxm_zero_reo); pending->sack_rxm[cnt].data = malloc(rxm->len); if (pending->sack_rxm[cnt].data == NULL) @@ -3323,6 +3637,7 @@ static void sack_queue_rxm(struct frcti * frcti, pending->sack_rxm[cnt].len = rxm->len; memcpy(pending->sack_rxm[cnt].data, rxm->pkt, rxm->len); pending->sack_rxm_cnt++; + frcti->rxm_budget--; /* NULL slot so the original timer self-cleans. */ STORE_RELEASE(&frcti->snd_slots[kp].rxm, NULL); frcti->snd_slots[kp].time = now_ns; @@ -3376,7 +3691,7 @@ static bool sack_is_dsack(struct frcti * frcti, return false; } -/* RFC 8985 §7.2: grow reo_wnd_mult on DSACK; at most once per RTT. */ +/* RFC 8985 §6.2: grow reo_wnd_mult on DSACK; at most once per RTT. */ static __inline__ void reo_wnd_on_dsack(struct frcti * frcti, uint64_t now_ns) { @@ -3433,9 +3748,15 @@ static void frcti_sack_rcv(struct frcti * frcti, recovery_enter(frcti); marked = 0; - hi_sacked = sack_mark_blocks(frcti, pkt.data, n, &marked); + hi_sacked = sack_mark_blocks(frcti, pkt.data, n, &marked, now_ns); frcti->dup_thresh += marked; + /* Packet conservation: a newly SACKed seqno also left the wire. */ + frcti->rxm_budget += marked; + + if (frcti->rxm_budget > RXM_BUDGET_MAX) + frcti->rxm_budget = RXM_BUDGET_MAX; + if (after(hi_sacked, frcti->snd_cr.lwe)) sack_queue_rxm(frcti, hi_sacked, now_ns, pending); } @@ -3476,7 +3797,7 @@ static void frcti_nack_snd(struct frcti * frcti, frct_hcs_set(pci, false); - frct_tx(frcti, spb); + frct_tx(frcti, spb, false); } enum frct_act { @@ -3586,13 +3907,10 @@ static bool sack_check(struct frcti * frcti, n = dsack_consume(frcti, out->blocks); if (n == 1) out->dsack = true; + n += sack_blocks_build(frcti, out->blocks + n, frcti->sack_n_max - n); - if (!out->dsack - && rcv_cr->lwe == frcti->sack_lwe && n == frcti->sack_n) - return false; - out->n = n; out->ack = rcv_cr->lwe; out->rwe = frcti_advert_rwe(frcti); @@ -3648,6 +3966,7 @@ static void seqno_rotate(struct frcti * frcti, if (!ts_aged_ns(now_ns, snd_cr->act, snd_cr->inact)) return; + /* Idle-on-wire ≠idle e2e: don't orphan in-flight rxm. */ if (snd_cr->seqno != snd_cr->lwe) return; @@ -3673,6 +3992,7 @@ static int frcti_snd(struct frcti * frcti, struct timespec now; struct frct_cr * snd_cr; struct frct_cr * rcv_cr; + struct rxm_entry * rxm = NULL; uint32_t seqno; uint16_t pci_flags = 0; bool rtx; @@ -3699,10 +4019,16 @@ static int frcti_snd(struct frcti * frcti, if (pci == NULL) return -ENOMEM; - memset(pci, 0, FRCT_PCILEN); + /* Pre-allocate rxm so alloc fail can't orphan a seqno. */ + if (snd_cr->cflags & FRCTFRTX) { + rxm = rxm_alloc(frcti, ssm_pk_buff_len(spb)); + if (rxm == NULL) { + ssm_pk_buff_pop(spb, frcti_data_hdr_len(frcti)); + return -ENOMEM; + } + } - if (frcti->stream) - spci = FRCT_SPCI(pci); + memset(pci, 0, FRCT_PCILEN); clock_gettime(PTHREAD_COND_CLOCK, &now); now_ns = TS_TO_UINT64(now); @@ -3719,6 +4045,8 @@ static int frcti_snd(struct frcti * frcti, STAT_BUMP(frcti, frag_snd); if (frcti->stream) { + spci = FRCT_SPCI(pci); + if (flags & FRCT_FIN) pci_flags |= FRCT_FIN; @@ -3773,13 +4101,23 @@ static int frcti_snd(struct frcti * frcti, frcti_rttp_snd(frcti, probe_id, 0, probe_nonce); if (rtx) { - rxm_arm(frcti, seqno, spb); + assert(rxm != NULL); + rxm_arm(frcti, seqno, rxm, spb); tlp_arm(frcti); } return 0; } +/* Stream FIN is armed for rxm; needs to be in window. */ +static __inline__ bool stream_fin_blocked(struct frcti * frcti) +{ + if (!frcti->stream) + return false; + + return !before(frcti->snd_cr.seqno, frcti->snd_cr.lwe + RQ_SIZE); +} + /* * Stream: 0-byte FRCT_FIN DATA so peer's flow_read returns 0 at this * byte. Msg: control packet with FRCT_FIN flag, snd_cr.seqno carried @@ -3797,6 +4135,13 @@ static void frcti_fin_snd(struct frcti * frcti) pthread_rwlock_wrlock(&frcti->lock); already = frcti->snd_fin_sent; + + /* Defer before committing snd_fin_sent; linger loop retries. */ + if (!already && stream_fin_blocked(frcti)) { + pthread_rwlock_unlock(&frcti->lock); + return; + } + frcti->snd_fin_sent = true; fin_seqno = frcti->snd_cr.seqno; @@ -3824,7 +4169,7 @@ static void frcti_fin_snd(struct frcti * frcti) return; } - if (frct_tx(frcti, spb) < 0) + if (frct_tx(frcti, spb, false) < 0) return; pthread_rwlock_wrlock(&frcti->lock); @@ -4154,6 +4499,9 @@ static void frcti_rcv(struct frcti * frcti, #define FRCTI_PDU_READY(frcti) \ ((frcti) != NULL && frcti_pdu_ready(frcti)) +#define FRCTI_PDU_INFO(frcti, len, nfrags) \ + ((frcti) == NULL ? -EAGAIN : frcti_pdu_info((frcti), (len), (nfrags))) + #define FRCTI_CONSUME(frcti, buf, count) \ ((frcti) == NULL ? (ssize_t) -EAGAIN \ : (frcti)->stream \ diff --git a/src/lib/hash.c b/src/lib/hash.c index 62bbf2b8..903474df 100644 --- a/src/lib/hash.c +++ b/src/lib/hash.c @@ -74,8 +74,10 @@ uint16_t hash_len(enum hash_algo algo) { if (algo == HASH_CRC8) return CRC8_HASH_LEN; + if (algo == HASH_CRC16) return CRC16_HASH_LEN; + if (algo == HASH_CRC64) return CRC64_HASH_LEN; #ifdef HAVE_LIBGCRYPT @@ -101,6 +103,7 @@ void mem_hash(enum hash_algo algo, *(uint8_t *) dst = crc; return; } + if (algo == HASH_CRC16) { uint16_t crc = 0; @@ -108,6 +111,7 @@ void mem_hash(enum hash_algo algo, *(uint16_t *) dst = htobe16(crc); return; } + if (algo == HASH_CRC64) { uint64_t crc = 0; @@ -162,3 +166,14 @@ void str_hash(enum hash_algo algo, { return mem_hash(algo, dst, (const uint8_t *) str, strlen(str)); } + +uint64_t hash_mix64(uint64_t key) +{ + key ^= key >> 33; + key *= 0xff51afd7ed558ccdULL; + key ^= key >> 33; + key *= 0xc4ceb9fe1a85ec53ULL; + key ^= key >> 33; + + return key; +} diff --git a/src/lib/irm.c b/src/lib/irm.c index c62701aa..5d53841f 100644 --- a/src/lib/irm.c +++ b/src/lib/irm.c @@ -118,6 +118,8 @@ int irm_bootstrap_ipcp(pid_t pid, msg.has_pid = true; msg.pid = pid; msg.conf = ipcp_config_s_to_msg(conf); + if (msg.conf == NULL) + return -ENOMEM; recv_msg = send_recv_irm_msg(&msg); ipcp_config_msg__free_unpacked(msg.conf, NULL); @@ -135,10 +137,11 @@ int irm_bootstrap_ipcp(pid_t pid, return ret; } -int irm_connect_ipcp(pid_t pid, - const char * dst, - const char * component, - qosspec_t qs) +int irm_connect_ipcp(pid_t pid, + const char * dst, + const char * component, + qosspec_t qs, + const struct poa_addr * addr) { irm_msg_t msg = IRM_MSG__INIT; irm_msg_t * recv_msg; @@ -152,7 +155,18 @@ int irm_connect_ipcp(pid_t pid, msg.pid = pid; msg.qosspec = qos_spec_s_to_msg(&qs); + if (addr != NULL) { + msg.peer = poa_addr_s_to_msg(addr); + if (msg.peer == NULL) { + qosspec_msg__free_unpacked(msg.qosspec, NULL); + return -ENOMEM; + } + } + recv_msg = send_recv_irm_msg(&msg); + + if (msg.peer != NULL) + poa_addr_msg__free_unpacked(msg.peer, NULL); qosspec_msg__free_unpacked(msg.qosspec, NULL); if (recv_msg == NULL) @@ -245,8 +259,99 @@ ssize_t irm_list_ipcps(struct ipcp_list_info ** ipcps) return nr; } -int irm_enroll_ipcp(pid_t pid, - const char * dst) +static int irm_attach_detach(pid_t pid, + const struct poa_spec * poa, + int code) +{ + irm_msg_t msg = IRM_MSG__INIT; + irm_msg_t * recv_msg; + int ret; + + if (pid == -1 || poa == NULL) + return -EINVAL; + + msg.code = code; + msg.has_pid = true; + msg.pid = pid; + msg.poa = poa_spec_s_to_msg(poa); + if (msg.poa == NULL) + return -EINVAL; + + recv_msg = send_recv_irm_msg(&msg); + + poa_spec_msg__free_unpacked(msg.poa, NULL); + + if (recv_msg == NULL) + return -EIRMD; + + if (!recv_msg->has_result) { + irm_msg__free_unpacked(recv_msg, NULL); + return -EIRMD; + } + + ret = recv_msg->result; + + irm_msg__free_unpacked(recv_msg, NULL); + + return ret; +} + +ssize_t irm_list_poas(pid_t pid, + struct poa_spec ** poas) +{ + irm_msg_t msg = IRM_MSG__INIT; + irm_msg_t * recv_msg; + size_t nr; + size_t i; + + if (pid == -1 || poas == NULL) + return -EINVAL; + + *poas = NULL; + + msg.code = IRM_MSG_CODE__IRM_LIST_POAS; + msg.has_pid = true; + msg.pid = pid; + + recv_msg = send_recv_irm_msg(&msg); + if (recv_msg == NULL) + return -EIRMD; + + nr = recv_msg->n_poas; + if (nr == 0) { + irm_msg__free_unpacked(recv_msg, NULL); + return 0; + } + + *poas = malloc(nr * sizeof(**poas)); + if (*poas == NULL) { + irm_msg__free_unpacked(recv_msg, NULL); + return -ENOMEM; + } + + for (i = 0; i < nr; i++) + (*poas)[i] = poa_spec_msg_to_s(recv_msg->poas[i]); + + irm_msg__free_unpacked(recv_msg, NULL); + + return (ssize_t) nr; +} + +int irm_attach_ipcp(pid_t pid, + const struct poa_spec * poa) +{ + return irm_attach_detach(pid, poa, IRM_MSG_CODE__IRM_ATTACH_IPCP); +} + +int irm_detach_ipcp(pid_t pid, + const struct poa_spec * poa) +{ + return irm_attach_detach(pid, poa, IRM_MSG_CODE__IRM_DETACH_IPCP); +} + +int irm_enroll_ipcp(pid_t pid, + const char * dst, + const struct poa_addr * addr) { irm_msg_t msg = IRM_MSG__INIT; irm_msg_t * recv_msg = NULL; @@ -260,7 +365,17 @@ int irm_enroll_ipcp(pid_t pid, msg.pid = pid; msg.dst = (char *) dst; + if (addr != NULL) { + msg.peer = poa_addr_s_to_msg(addr); + if (msg.peer == NULL) + return -ENOMEM; + } + recv_msg = send_recv_irm_msg(&msg); + + if (msg.peer != NULL) + poa_addr_msg__free_unpacked(msg.peer, NULL); + if (recv_msg == NULL) return -EIRMD; diff --git a/src/lib/pb/ipcp.proto b/src/lib/pb/ipcp.proto index 406b8d9c..298a15e2 100644 --- a/src/lib/pb/ipcp.proto +++ b/src/lib/pb/ipcp.proto @@ -39,6 +39,10 @@ enum ipcp_msg_code { IPCP_CONNECT = 10; IPCP_DISCONNECT = 11; IPCP_REPLY = 12; + IPCP_FLOW_UPDATE = 13; + IPCP_ATTACH = 14; + IPCP_DETACH = 15; + IPCP_LIST_POAS = 16; } message ipcp_msg { @@ -57,4 +61,8 @@ message ipcp_msg { optional sint32 mpl = 13; /* MPL in ms. */ optional int32 result = 14; optional uint32 uid = 15; /* 0 = GSPP, >0 = PUP uid */ + optional poa_addr_msg peer = 16; /* peer PoA address */ + optional bool is_poa = 17; /* flow rides a PoA */ + optional poa_spec_msg poa = 18; /* PoA to attach */ + repeated poa_spec_msg poas = 19; /* PoAs attached */ } diff --git a/src/lib/pb/ipcp_config.proto b/src/lib/pb/ipcp_config.proto index eac4da37..f6d313ac 100644 --- a/src/lib/pb/ipcp_config.proto +++ b/src/lib/pb/ipcp_config.proto @@ -42,6 +42,7 @@ message dt_config_msg { required uint32 eid_size = 2; required uint32 max_ttl = 3; required routing_config_msg routing = 4; + optional uint32 max_rtt = 5; } message dir_dht_config_msg { @@ -65,29 +66,32 @@ message uni_config_msg { required uint32 cong_avoid = 4; } -message eth_config_msg { +message eth_poa_msg { required string dev = 1; required uint32 ethertype = 2; + optional bytes mac = 3; } -message udp4_config_msg { - required uint32 ip_addr = 1; - required uint32 port = 2; - required uint32 dns_addr = 3; /* set to 0 if unused */ +message udp4_poa_msg { + required uint32 ip_addr = 1; + required uint32 port = 2; } -message udp6_config_msg { - required bytes ip_addr = 1; - required uint32 port = 2; - required bytes dns_addr = 3; /* set to NULL if unused */ +message udp6_poa_msg { + required bytes ip_addr = 1; + required uint32 port = 2; } +/* Exactly one field is set; presence is the PoA type. */ +message poa_spec_msg { + optional udp4_poa_msg udp4 = 1; + optional udp6_poa_msg udp6 = 2; + optional eth_poa_msg eth = 3; +} + message ipcp_config_msg { required layer_info_msg layer_info = 1; required uint32 ipcp_type = 2; optional uni_config_msg unicast = 3; - optional udp4_config_msg udp4 = 4; - optional udp6_config_msg udp6 = 5; - optional eth_config_msg eth = 6; } diff --git a/src/lib/pb/irm.proto b/src/lib/pb/irm.proto index 5de860a5..8f594fa3 100644 --- a/src/lib/pb/irm.proto +++ b/src/lib/pb/irm.proto @@ -53,6 +53,14 @@ enum irm_msg_code { IPCP_FLOW_REQ_ARR = 25; IPCP_FLOW_ALLOC_REPLY = 26; IRM_REPLY = 27; + IRM_FLOW_UPDATE = 28; + IPCP_FLOW_UPDATE_ARR = 29; + IRM_POA_FLOW_ALLOC = 30; + IRM_POA_FLOW_ALLOC_R = 31; + IPCP_POA_FLOW_REQ_ARR = 32; + IRM_ATTACH_IPCP = 33; + IRM_DETACH_IPCP = 34; + IRM_LIST_POAS = 35; } message timespec_msg { @@ -96,4 +104,10 @@ message irm_msg { optional sint32 result = 25; optional bytes sym_key = 26; /* symmetric encryption key */ optional sint32 cipher_nid = 27; /* cipher NID */ + optional uint32 generation = 28; /* re-key batch generation */ + optional bool rekey = 29; /* re-key watermark trigger */ + optional bool rk_initiator = 30; /* re-key proof-holder side */ + optional poa_addr_msg peer = 32; /* peer PoA address */ + optional poa_spec_msg poa = 33; /* PoA to attach */ + repeated poa_spec_msg poas = 34; /* PoAs attached */ } diff --git a/src/lib/pb/model.proto b/src/lib/pb/model.proto index 4c1564a5..f3e85c33 100644 --- a/src/lib/pb/model.proto +++ b/src/lib/pb/model.proto @@ -42,6 +42,7 @@ message flow_info_msg { required qosspec_msg qos = 6; required uint32 uid = 7; required uint32 mtu = 8; /* Layer MTU (bytes). */ + required uint32 max_rtt = 9; /* Max path RTT (ms). */ } message name_info_msg { @@ -58,6 +59,19 @@ message layer_info_msg { required uint32 dir_hash_algo = 2; } +/* Address of a flow PoA peer. */ +message poa_addr_msg { + required uint32 type = 1; /* enum poa_type */ + optional uint32 ip4 = 2; + optional bytes ip6 = 3; + optional uint32 port = 4; + optional bytes mac = 5; + optional uint32 ethertype = 6; + optional string dev = 7; /* our device, eth only */ + optional string hostname = 8; /* IRMd resolves, UDP only */ + optional bytes src_mac = 9; /* our MAC, eth only */ +} + message ipcp_info_msg { required uint32 type = 1; required string name = 2; diff --git a/src/lib/poa/addr.c b/src/lib/poa/addr.c new file mode 100644 index 00000000..d8811244 --- /dev/null +++ b/src/lib/poa/addr.c @@ -0,0 +1,142 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Points of attachment (PoA) - addresses and management messages + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * version 2.1 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#if defined(__linux__) || defined(__CYGWIN__) +#ifndef _DEFAULT_SOURCE /* Test include source */ +#define _DEFAULT_SOURCE +#endif +#endif + +#include "config.h" + +#include <ouroboros/endian.h> +#include <ouroboros/errno.h> + +#include "poa.h" + +#include <arpa/inet.h> +#include <stdio.h> +#include <string.h> + +int poa_addr_cmp(const struct poa_addr * a, + const struct poa_addr * b) +{ + if (a->type != b->type) + return 1; + + switch (a->type) { + case POA_UDP4: + if (a->udp4.port != b->udp4.port) + return 1; + return memcmp(&a->udp4.ip_addr, &b->udp4.ip_addr, + sizeof(a->udp4.ip_addr)); + case POA_UDP6: + if (a->udp6.port != b->udp6.port) + return 1; + return memcmp(&a->udp6.ip_addr, &b->udp6.ip_addr, + sizeof(a->udp6.ip_addr)); + case POA_ETH: + if (a->eth.dst.ethertype != b->eth.dst.ethertype) + return 1; + return memcmp(a->eth.dst.mac, b->eth.dst.mac, POA_MAC_SIZE); + default: + return 1; + } +} + +/* Display/RIB entry name, e.g. "udp4.<ip>.<port>". */ +int poa_addr_name(const struct poa_addr * a, + char * buf, + size_t len) +{ + char ip[INET6_ADDRSTRLEN]; + int ret; + + switch (a->type) { + case POA_UDP4: + if (inet_ntop(AF_INET, &a->udp4.ip_addr, + ip, sizeof(ip)) == NULL) + return -EINVAL; + ret = snprintf(buf, len, "udp4.%s.%u", ip, a->udp4.port); + break; + case POA_UDP6: + if (inet_ntop(AF_INET6, &a->udp6.ip_addr, + ip, sizeof(ip)) == NULL) + return -EINVAL; + ret = snprintf(buf, len, "udp6.%s.%u", ip, a->udp6.port); + break; + case POA_ETH: + ret = snprintf(buf, len, "eth.%s.%04X", + a->eth.src.dev, a->eth.src.ethertype); + break; + default: + return -EINVAL; + } + + if (ret < 0 || (size_t) ret >= len) + return -EMSGSIZE; + + return 0; +} + +void poa_mgmt_msg_ser(struct poa_mgmt_msg * msg, + uint8_t code, + uint32_t s_eid, + uint32_t d_eid, + qosspec_t qs, + int response, + size_t data_len) +{ + memset(msg, 0, sizeof(*msg)); + + msg->code = code; + msg->s_eid = hton32(s_eid); + msg->d_eid = hton32(d_eid); + msg->response = hton32(response); + msg->data_len = hton16((uint16_t) data_len); + + if (code != POA_FLOW_REQ) + return; + + msg->bandwidth = hton64(qs.bandwidth); + msg->delay = hton32(qs.delay); + msg->loss = hton32(qs.loss); + msg->ber = hton32(qs.ber); + msg->max_gap = hton32(qs.max_gap); + msg->timeout = hton32(qs.timeout); + msg->availability = qs.availability; + msg->service = qs.service; +} + +void poa_mgmt_msg_qos(const struct poa_mgmt_msg * msg, + qosspec_t * qs) +{ + qs->bandwidth = ntoh64(msg->bandwidth); + qs->delay = ntoh32(msg->delay); + qs->loss = ntoh32(msg->loss); + qs->ber = ntoh32(msg->ber); + qs->max_gap = ntoh32(msg->max_gap); + qs->timeout = ntoh32(msg->timeout); + qs->availability = msg->availability; + qs->service = msg->service; +} + diff --git a/src/lib/poa/eth.c b/src/lib/poa/eth.c new file mode 100644 index 00000000..93ccbdfe --- /dev/null +++ b/src/lib/poa/eth.c @@ -0,0 +1,2011 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Points of attachment (PoA) - Ethernet transport + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * version 2.1 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#if defined(__APPLE__) +#define _BSD_SOURCE +#define _DARWIN_C_SOURCE +#elif defined(__FreeBSD__) +#define __BSD_VISIBLE 1 +#elif defined(__linux__) || defined(__CYGWIN__) +#ifndef _DEFAULT_SOURCE +#define _DEFAULT_SOURCE +#endif +#else +#ifndef _POSIX_C_SOURCE +#define _POSIX_C_SOURCE 200809L +#endif +#endif + +#include "config.h" + +#define OUROBOROS_PREFIX "poa-eth" + +#include <ouroboros/endian.h> +#include <ouroboros/errno.h> +#include <ouroboros/hash.h> +#include <ouroboros/logs.h> +#include <ouroboros/pthread.h> +#include <ouroboros/random.h> +#include <ouroboros/sockets.h> +#include <ouroboros/time.h> + +#include "poa.h" + +#ifdef HAVE_RAW_SOCKETS +#include <net/if.h> +#include <netinet/in.h> +#include <linux/if_ether.h> +#include <linux/if_packet.h> +#include <linux/netlink.h> +#include <linux/gen_stats.h> +#include <linux/pkt_sched.h> +#include <linux/rtnetlink.h> +#include <linux/sockios.h> +#include <sys/ioctl.h> +#include <sys/socket.h> +#include <sys/uio.h> + +#include <ifaddrs.h> +#elif defined(HAVE_BPF) +#include <net/bpf.h> +#include <net/if.h> +#include <net/if_dl.h> +#include <sys/ioctl.h> +#include <sys/socket.h> +#include <sys/uio.h> + +#include <fcntl.h> +#include <ifaddrs.h> +#elif defined(HAVE_NETMAP) +#define NETMAP_WITH_LIBS +#include <net/netmap_user.h> +#include <net/if.h> +#include <sys/ioctl.h> +#include <sys/socket.h> + +#include <poll.h> +#ifndef __linux__ +#include <net/if_dl.h> +#include <ifaddrs.h> +#endif +#endif + +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +#define ETH_TYPE_LEN_SIZE sizeof(uint16_t) +#define ETH_HDR_SIZE (2 * POA_MAC_SIZE + ETH_TYPE_LEN_SIZE) +#define POA_HDR_SIZE (sizeof(struct poa_hdr)) +#define ETH_HDR_TOT_SIZE (ETH_HDR_SIZE + POA_HDR_SIZE) + +#define BPF_DEV_MAX 256 /* /dev/bpfN units to try */ +/* Wait for the link to come back before reading it again. */ +#define ETH_DOWN_TIMEO 100 /* ms */ +/* Budget for a qdisc dump; the send path must not wait on netlink. */ +#define POA_QDISC_TIMEO 5 /* ms */ + +#ifndef ETH_MAX_MTU /* In if_ether.h as of Linux 4.10. */ +#define ETH_MAX_MTU 0xFFFFU +#endif + +struct poa_hdr { + uint16_t eid; + uint16_t len; + uint8_t hcs; +} __attribute__((packed)); + +struct eth_hdr { + uint8_t dst[POA_MAC_SIZE]; + uint8_t src[POA_MAC_SIZE]; + uint16_t ethertype; + struct poa_hdr poa; +} __attribute__((packed)); + + +struct eth_priv { + struct poa * poa; + + int s_fd; /* raw socket or bpf device */ + uint16_t ethertype; /* network order */ + uint8_t hw_addr[POA_MAC_SIZE]; + uint32_t mtu; /* device payload capacity */ + int ifindex; /* link id; 0 where unknown */ + + pthread_t reader; + bool running; +#ifdef HAVE_RAW_SOCKETS + struct sockaddr_ll device; +#elif defined(HAVE_BPF) + size_t blen; /* bpf device buffer size */ +#elif defined(HAVE_NETMAP) + struct nm_desc * nmd; + struct pollfd poll_in; + struct pollfd poll_out; +#endif + /* The kernel zeroes its counters on read, so accumulate. */ + size_t kern_rcv; + size_t kern_drp; +#ifdef HAVE_RAW_SOCKETS + /* Qdisc depth: netlink descriptor, its try-lock and cache. */ + int nl_fd; + uint8_t nl_busy; + size_t nl_pkt; + size_t nl_byt; + uint64_t nl_time; +#endif +}; + +struct eth_query { + struct list_head next; + uint8_t hash[POA_QUERY_HLEN]; + /* The resolve's constraints; replies must satisfy them. */ + uint16_t ethertype; + char c_dev[DEV_NAME_SIZE + 1]; + uint8_t c_mac[POA_MAC_SIZE]; + bool replied; + char dev[DEV_NAME_SIZE + 1]; + uint8_t src_mac[POA_MAC_SIZE]; + uint8_t mac[POA_MAC_SIZE]; + uint16_t r_ethertype; +}; + +static struct { + pthread_once_t once; + + struct llist pending; + pthread_mutex_t mtx; + pthread_cond_t cond; +} queries = { .once = PTHREAD_ONCE_INIT }; + +static void eth_hdr_ser(const struct eth_priv * priv, + struct eth_hdr * hdr, + const uint8_t * dst, + uint32_t eid, + size_t len) +{ + memcpy(hdr->dst, dst, POA_MAC_SIZE); + memcpy(hdr->src, priv->hw_addr, POA_MAC_SIZE); + + hdr->ethertype = priv->ethertype; + hdr->poa.eid = htons((uint16_t) eid); + hdr->poa.len = htons((uint16_t) len); + mem_hash(HASH_CRC8, &hdr->poa.hcs, + (uint8_t *) &hdr->poa.eid, 2 * sizeof(uint16_t)); +} + +/* Oriented from us to the sender: our PoA, then their MAC. */ +static void frame_to_addr(const struct eth_priv * priv, + const struct eth_hdr * hdr, + struct poa_addr * addr) +{ + memset(addr, 0, sizeof(*addr)); + + addr->type = priv->poa->type; + + addr->eth.src = priv->poa->local.eth.src; + + memcpy(addr->eth.dst.mac, hdr->src, POA_MAC_SIZE); + + addr->eth.dst.ethertype = ntohs(priv->ethertype); +} + +static bool frame_is_for_us(const struct eth_priv * priv, + const uint8_t * dst) +{ + static const uint8_t bc[POA_MAC_SIZE] = + { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; + + if (memcmp(dst, priv->hw_addr, POA_MAC_SIZE) == 0) + return true; + + return memcmp(dst, bc, POA_MAC_SIZE) == 0; +} + +/* + * Parse and validate a frame; on success *eid and *plen locate the + * payload. Errors are silent: anyone can spray an interface. + */ +static int frame_parse(const struct eth_priv * priv, + const uint8_t * buf, + size_t len, + uint32_t * eid, + size_t * plen) +{ + const struct eth_hdr * hdr = (const struct eth_hdr *) buf; + uint8_t hcs; + + if (len < ETH_HDR_TOT_SIZE) + return -1; + + if (!frame_is_for_us(priv, hdr->dst)) + return -1; + + if (hdr->ethertype != priv->ethertype) + return -1; + + mem_hash(HASH_CRC8, &hcs, + (const uint8_t *) &hdr->poa.eid, 2 * sizeof(uint16_t)); + + if (hcs != hdr->poa.hcs) + return -1; + + *eid = ntohs(hdr->poa.eid); + + *plen = ntohs(hdr->poa.len); + if (*plen > len - ETH_HDR_TOT_SIZE) + return -1; + + return 0; +} + +#define ETH_QUERY_TIMEO 1900 /* ms total budget */ +#define ETH_QUERY_RETRIES 3 /* retransmits, 4 attempts total */ + +/* A resolve in progress; a reply fills every matching entry. */ +static void queries_init(void) +{ + pthread_condattr_t cattr; + + llist_init(&queries.pending); + + pthread_mutex_init(&queries.mtx, NULL); + + pthread_condattr_init(&cattr); +#ifndef __APPLE__ + pthread_condattr_setclock(&cattr, PTHREAD_COND_CLOCK); +#endif + pthread_cond_init(&queries.cond, &cattr); + pthread_condattr_destroy(&cattr); +} + +static void eth_query_cleanup(void * o) +{ + struct eth_query * q = (struct eth_query *) o; + + pthread_mutex_lock(&queries.mtx); + llist_del(&q->next, &queries.pending); + pthread_mutex_unlock(&queries.mtx); +} + +static void eth_query_arr(struct poa * poa, + const struct poa_addr * src, + const uint8_t * hash) +{ + uint8_t buf[sizeof(struct poa_mgmt_msg) + + POA_QUERY_HLEN]; + struct poa_mgmt_msg * msg = (struct poa_mgmt_msg *) buf; + + if (!poa_has_name(hash)) + return; + + poa_mgmt_msg_ser(msg, POA_NAME_REPLY, 0, 0, qos_raw, 0, POA_QUERY_HLEN); + + memcpy(buf + sizeof(*msg), hash, POA_QUERY_HLEN); + + if (poa->ops->poa_send_mgmt(poa, src, buf, sizeof(buf)) < 0) + return; /* the requester retransmits */ + + POA_STAT_BUMP(poa, rep_tx); +} + +static bool mac_is_zero(const uint8_t * mac) +{ + static const uint8_t zero[POA_MAC_SIZE] = { 0 }; + + return memcmp(mac, zero, POA_MAC_SIZE) == 0; +} + +/* Assign a random MAC on loopback interfaces (zero MAC). */ +static int eth_dev_mac(uint8_t * mac) +{ + if (!mac_is_zero(mac)) + return 0; + + if (random_buffer(mac, POA_MAC_SIZE) < 0) + return -1; + + mac[0] = (mac[0] | 0x02) & 0xFE; + + return 0; +} + +static bool eth_query_is_match(const struct eth_query * q, + const struct poa * poa) +{ + if (q->ethertype != 0 && + q->ethertype != poa->local.eth.src.ethertype) + return false; + + if (q->c_dev[0] != '\0' && + strcmp(q->c_dev, poa->local.eth.src.dev) != 0) + return false; + + if (mac_is_zero(q->c_mac)) + return true; + + return memcmp(q->c_mac, poa->local.eth.src.mac, POA_MAC_SIZE) == 0; +} + +static void eth_reply_arr(struct poa * poa, + const struct poa_addr * src, + const uint8_t * hash) +{ + struct list_head * p; + + pthread_once(&queries.once, queries_init); + + pthread_mutex_lock(&queries.mtx); + + list_for_each(p, &queries.pending.list) { + struct eth_query * q; + + q = list_entry(p, struct eth_query, next); + if (q->replied || memcmp(q->hash, hash, POA_QUERY_HLEN) != 0) + continue; + + if (!eth_query_is_match(q, poa)) + continue; + + memcpy(q->mac, src->eth.dst.mac, POA_MAC_SIZE); + memcpy(q->src_mac, src->eth.src.mac, POA_MAC_SIZE); + strcpy(q->dev, src->eth.src.dev); + + q->r_ethertype = poa->local.eth.src.ethertype; + + q->replied = true; + } + + pthread_cond_broadcast(&queries.cond); + + pthread_mutex_unlock(&queries.mtx); +} + +/* Name query frames are handled in the transport. */ +static void eth_rx_mgmt(struct poa * poa, + const struct poa_addr * src, + const uint8_t * buf, + size_t len) +{ + const struct poa_mgmt_msg * msg; + const uint8_t * hash; + + msg = (const struct poa_mgmt_msg *) buf; + if (len < sizeof(*msg)) { + poa_rx_mgmt(poa, src, buf, len); + return; + } + + if (msg->code != POA_NAME_QUERY && msg->code != POA_NAME_REPLY) { + poa_rx_mgmt(poa, src, buf, len); + return; + } + + if (ntoh16(msg->data_len) != POA_QUERY_HLEN) + return; /* malformed */ + + if (len < sizeof(*msg) + POA_QUERY_HLEN) + return; /* malformed */ + + hash = buf + sizeof(*msg); + + if (msg->code == POA_NAME_QUERY) { + POA_STAT_BUMP(poa, qry_rx); + eth_query_arr(poa, src, hash); + } else { + POA_STAT_BUMP(poa, rep_rx); + eth_reply_arr(poa, src, hash); + } +} + +static int eth_query_send(const uint8_t * hash, + const struct poa_addr * addr) +{ + uint8_t buf[sizeof(struct poa_mgmt_msg) + + POA_QUERY_HLEN]; + struct poa_mgmt_msg * msg = (struct poa_mgmt_msg *) buf; + struct poa_addr bcast; + + bcast = *addr; + + memset(bcast.eth.dst.mac, 0xff, POA_MAC_SIZE); + + poa_mgmt_msg_ser(msg, POA_NAME_QUERY, 0, 0, qos_raw, 0, POA_QUERY_HLEN); + + memcpy(buf + sizeof(*msg), hash, POA_QUERY_HLEN); + + return poa_bcast_mgmt(&bcast, buf, sizeof(buf)); +} + +/* Complete addr for dst by broadcast query; the poa_query op. */ +static int eth_query(const char * dst, + const struct timespec * timeo, + struct poa_addr * addr) +{ + struct timespec dflt = TIMESPEC_INIT_MS(ETH_QUERY_TIMEO); + struct timespec rintv = TIMESPEC_INIT_MS(ETH_QUERY_TIMEO + / (ETH_QUERY_RETRIES + 1)); + struct eth_query q; + struct timespec abstime; + struct timespec now; + struct timespec dl; + time_t rintv_ns = (time_t) TS_TO_UINT64(rintv); + time_t diff; + uint8_t hash[POA_QUERY_HLEN]; + int n; + int err = -ETIMEDOUT; + + if (strlen(addr->eth.src.dev) > DEV_NAME_SIZE) + return -EINVAL; + + addr->eth.src.ethertype = addr->eth.dst.ethertype; + + /* The destination is set; a zero ethertype cannot be sent. */ + if (!mac_is_zero(addr->eth.dst.mac)) + return addr->eth.dst.ethertype < 0x0600 ? -EINVAL : 0; + + str_hash(HASH_SHA3_256, hash, dst); + + pthread_once(&queries.once, queries_init); + + memset(&q, 0, sizeof(q)); + memcpy(q.hash, hash, POA_QUERY_HLEN); + + q.ethertype = addr->eth.dst.ethertype; + + memcpy(q.c_mac, addr->eth.src.mac, POA_MAC_SIZE); + strcpy(q.c_dev, addr->eth.src.dev); + + pthread_mutex_lock(&queries.mtx); + llist_add(&q.next, &queries.pending); + pthread_mutex_unlock(&queries.mtx); + + pthread_cleanup_push(eth_query_cleanup, &q); + + clock_gettime(PTHREAD_COND_CLOCK, &now); + ts_add(&now, timeo != NULL ? timeo : &dflt, &abstime); + + while (ts_diff_ns(&now, &abstime) < 0) { + n = eth_query_send(hash, addr); + if (n < 0) { + err = n; + break; + } + + if (n == 0) { + err = -EPERM; + break; + } + + ts_add(&now, &rintv, &dl); + + if (ts_diff_ns(&dl, &abstime) > 0) + dl = abstime; + + pthread_mutex_lock(&queries.mtx); + + pthread_cleanup_push(__cleanup_mutex_unlock, &queries.mtx); + + while (!q.replied) { + if (pthread_cond_timedwait(&queries.cond, &queries.mtx, + &dl) == ETIMEDOUT) + break; + } + + if (q.replied) { + memcpy(addr->eth.dst.mac, q.mac, POA_MAC_SIZE); + memcpy(addr->eth.src.mac, q.src_mac, POA_MAC_SIZE); + strcpy(addr->eth.src.dev, q.dev); + + addr->eth.dst.ethertype = q.r_ethertype; + addr->eth.src.ethertype = q.r_ethertype; + + err = 0; + } + + pthread_cleanup_pop(true); + + if (err == 0) + break; + + clock_gettime(PTHREAD_COND_CLOCK, &now); + + diff = ts_diff_ns(&now, &abstime); + if (diff > -rintv_ns) /* skip the runt attempt */ + break; + } + + pthread_cleanup_pop(true); + + return err; +} + +static void eth_spec(const struct poa * poa, + struct poa_spec * spec) +{ + spec->type = poa->type; + spec->eth = poa->local.eth.src; +} + +static bool eth_has_id(const struct poa * poa, + const struct poa_spec * spec) +{ + if (strnlen(spec->eth.dev, sizeof(spec->eth.dev)) > DEV_NAME_SIZE) + return false; + + if (strcmp(poa->local.eth.src.dev, spec->eth.dev) != 0) + return false; + + return poa->local.eth.src.ethertype == spec->eth.ethertype; +} + +/* Our end of the link; an unnamed one matches any. */ +static bool eth_is_src(const struct poa * poa, + const struct eth_poa * src) +{ + if (src->dev[0] != '\0' && + strcmp(poa->local.eth.src.dev, src->dev) != 0) + return false; + + if (mac_is_zero(src->mac)) + return true; + + return memcmp(poa->local.eth.src.mac, src->mac, + POA_MAC_SIZE) == 0; +} + +/* An ethertype of 0 in dst matches any, for broadcast queries. */ +static bool eth_match(const struct poa * poa, + const struct poa_addr * dst) +{ + uint16_t et = dst->eth.dst.ethertype; + + if (et != 0 && et != poa->local.eth.src.ethertype) + return false; + + return eth_is_src(poa, &dst->eth.src); +} + +static bool eth_link_match(const struct poa * poa, + int id) +{ + struct eth_priv * priv = (struct eth_priv *) poa->priv; + + return priv->ifindex == id; +} + +#ifdef HAVE_RAW_SOCKETS + +/* MSG_DONTWAIT: the reader blocks on this socket. */ +static int eth_sendv(struct eth_priv * priv, + const uint8_t * dst, + uint32_t eid, + const uint8_t * body, + size_t len, + bool block, + const struct timespec * abstime) +{ + struct eth_hdr hdr; + struct msghdr msg; + struct iovec iov[2]; + int ret; + + if (len > priv->mtu - POA_HDR_SIZE) + return -EMSGSIZE; + + eth_hdr_ser(priv, &hdr, dst, eid, len); + + iov[0].iov_base = &hdr; + iov[0].iov_len = ETH_HDR_TOT_SIZE; + iov[1].iov_base = (void *) body; + iov[1].iov_len = len; + + memset(&msg, 0, sizeof(msg)); + + msg.msg_name = &priv->device; + msg.msg_namelen = sizeof(priv->device); + msg.msg_iov = iov; + msg.msg_iovlen = len > 0 ? 2 : 1; + while (sendmsg(priv->s_fd, &msg, MSG_DONTWAIT) < 0) { + if (errno != EAGAIN && errno != EWOULDBLOCK) + return -EIO; + + if (!block) + return -EAGAIN; + + ret = poa_wait_out(priv->s_fd, abstime); + if (ret < 0) + return ret; + } + + return 0; +} + +static void * eth_reader(void * o) +{ + struct poa * poa = (struct poa *) o; + struct eth_priv * priv = (struct eth_priv *) poa->priv; + struct timespec down = TIMESPEC_INIT_MS(ETH_DOWN_TIMEO); + uint8_t * buf; + size_t bufsz; + + bufsz = ETH_HDR_SIZE + priv->mtu; + + buf = malloc(bufsz); + if (buf == NULL) + return (void *) -1; + + pthread_cleanup_push(free, buf); + + while (true) { + struct ssm_pk_buff * spb; + struct sockaddr_ll from; + socklen_t flen; + struct poa_addr src; + ssize_t n; + uint32_t eid; + size_t plen; + const uint8_t * body; + + flen = sizeof(from); + + n = recvfrom(priv->s_fd, buf, bufsz, 0, + (struct sockaddr *) &from, &flen); + + if (n < 0) { + if (errno == EINTR) + continue; + + POA_STAT_BUMP(poa, rcv_fail); + + if (errno == ENETDOWN) { + nanosleep(&down, NULL); + continue; + } + + log_err("Reader on %s stopped: %s.", + poa->local.eth.src.dev, + strerror(errno)); + break; + } + + if (from.sll_pkttype == PACKET_OUTGOING) + continue; + + if (frame_parse(priv, buf, (size_t) n, &eid, &plen) < 0) + continue; + + body = buf + ETH_HDR_TOT_SIZE; + + if (eid == POA_MGMT_EID) { + frame_to_addr(priv, (struct eth_hdr *) buf, &src); + eth_rx_mgmt(poa, &src, body, plen); + continue; + } + + if (poa_spb_reserve(&spb, plen) < 0) { + POA_STAT_BUMP(poa, buf_fail); + continue; + } + + memcpy(ssm_pk_buff_head(spb), body, plen); + + poa_rx_pkt(poa, eid, spb); + } + + pthread_cleanup_pop(true); + + return (void *) 0; +} + +/* + * One netlink socket for the whole subsystem: RTMGRP_LINK delivers the + * events of every interface anyway, so a socket per PoA only added + * discards. + */ +int poa_monitor_open(void) +{ + struct sockaddr_nl sa; + int fd; + + memset(&sa, 0, sizeof(sa)); + + sa.nl_family = AF_NETLINK; + sa.nl_groups = RTMGRP_LINK; + + fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE); + if (fd < 0) + return -1; + + if (bind(fd, (struct sockaddr *) &sa, sizeof(sa)) < 0) { + close(fd); + return -1; + } + + return fd; +} + +void poa_monitor_read(int fd) +{ + uint8_t buf[4096]; + struct nlmsghdr * h; + ssize_t n; + + n = recv(fd, buf, sizeof(buf), 0); + if (n < 0) + return; + + for (h = (struct nlmsghdr *) buf; + NLMSG_OK(h, (unsigned int) n); + h = NLMSG_NEXT(h, n)) { + struct ifinfomsg * ifi; + unsigned int usable; + bool up; + size_t cnt; + + if (h->nlmsg_type == NLMSG_DONE) + break; + + if (h->nlmsg_type != RTM_NEWLINK) + continue; + + ifi = NLMSG_DATA(h); + + usable = ifi->ifi_flags & (IFF_UP | IFF_RUNNING); + up = usable == (IFF_UP | IFF_RUNNING); + + cnt = poa_link_updown(ifi->ifi_index, up); + if (cnt > 0) + log_info("Link %d %s, %zu flows.", ifi->ifi_index, + up ? "up" : "down", cnt); + } +} + +static int eth_dev_info(const char * dev, + int * idx, + uint8_t * mac, + uint32_t * mtu) +{ + struct ifreq ifr; + int fd; + + if (strlen(dev) >= IFNAMSIZ) + return -EINVAL; + + *idx = if_nametoindex(dev); + if (*idx == 0) { + log_err("Failed to find device %s.", dev); + return -ENODEV; + } + + fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (fd < 0) + return -EIO; + + memset(&ifr, 0, sizeof(ifr)); + strcpy(ifr.ifr_name, dev); + + if (ioctl(fd, SIOCGIFHWADDR, &ifr) < 0) { + log_err("Failed to get hardware address of %s.", dev); + goto fail_ioctl; + } + + memcpy(mac, ifr.ifr_hwaddr.sa_data, POA_MAC_SIZE); + + if (ioctl(fd, SIOCGIFMTU, &ifr) < 0) { + log_err("Failed to get MTU of %s.", dev); + goto fail_ioctl; + } + + close(fd); + + if (eth_dev_mac(mac) < 0) + return -EIO; + + *mtu = MIN(MIN(ETH_MAX_MTU, POA_ETH_RD_BUF), (uint32_t) ifr.ifr_mtu); + if (memcmp(dev, "lo", 2) == 0 && *mtu > POA_ETH_LO_MTU) + *mtu = POA_ETH_LO_MTU; + + return 0; + + fail_ioctl: + close(fd); + return -EIO; +} + +/* SO_RCVBUFFORCE bypasses rmem_max; SO_RCVBUF is the fallback. */ +static void eth_set_rcvbuf(int fd, + int rcvbuf) +{ + if (setsockopt(fd, SOL_SOCKET, SO_RCVBUFFORCE, + &rcvbuf, sizeof(rcvbuf)) == 0) + return; + + if (setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &rcvbuf, sizeof(rcvbuf)) < 0) + log_info("Failed to set SO_RCVBUF to %d.", rcvbuf); +} + +/* The send buffer holds at least one frame, or sendmsg cannot fit a PDU. */ +static int eth_socket(struct eth_priv * priv, + int idx, + const uint8_t * mac) +{ + int sndbuf; + int rcvbuf; +#ifdef POA_ETH_QDISC_BYPASS + int bypass = 1; +#endif + + memset(&priv->device, 0, sizeof(priv->device)); + + priv->device.sll_ifindex = idx; + priv->device.sll_family = AF_PACKET; + priv->device.sll_halen = POA_MAC_SIZE; + priv->device.sll_protocol = htons(ETH_P_ALL); + + memcpy(priv->device.sll_addr, mac, POA_MAC_SIZE); + memcpy(priv->hw_addr, mac, POA_MAC_SIZE); + + priv->s_fd = socket(AF_PACKET, SOCK_RAW, priv->ethertype); + if (priv->s_fd < 0) { + log_err("Failed to create socket: %s.", strerror(errno)); + return -1; + } + +#ifdef POA_ETH_QDISC_BYPASS + if (setsockopt(priv->s_fd, SOL_PACKET, PACKET_QDISC_BYPASS, + &bypass, sizeof(bypass)) < 0) + log_info("Qdisc bypass not supported."); +#endif + + sndbuf = POA_ETH_SNDBUF; + if (sndbuf > 0) { + sndbuf = MAX(sndbuf, (int) (ETH_HDR_SIZE + priv->mtu)); + + if (setsockopt(priv->s_fd, SOL_SOCKET, SO_SNDBUF, + &sndbuf, sizeof(sndbuf)) < 0) + log_info("Failed to set SO_SNDBUF to %d.", sndbuf); + } + + rcvbuf = POA_ETH_RCVBUF; + if (rcvbuf > 0) + eth_set_rcvbuf(priv->s_fd, rcvbuf); + + if (bind(priv->s_fd, (struct sockaddr *) &priv->device, + sizeof(priv->device)) < 0) { + log_err("Failed to bind socket to %d.", idx); + close(priv->s_fd); + return -1; + } + + return 0; +} + +/* + * Pull qlen and backlog from the nested TCA_STATS2. The top-level + * TCA_STATS shares TCA_STATS_QUEUE's id but carries a wholly + * different struct, so descend first; never match on id alone. + */ +static void eth_qdisc_parse(struct rtattr * rta, + size_t rlen, + size_t * byt, + size_t * pkt) +{ + struct gnet_stats_queue q; + struct rtattr * in; + size_t ilen; + + for (; RTA_OK(rta, rlen); rta = RTA_NEXT(rta, rlen)) { + if (rta->rta_type != TCA_STATS2) + continue; + + in = (struct rtattr *) RTA_DATA(rta); + ilen = RTA_PAYLOAD(rta); + for (; RTA_OK(in, ilen); in = RTA_NEXT(in, ilen)) { + if (in->rta_type != TCA_STATS_QUEUE) + continue; + + if (RTA_PAYLOAD(in) < sizeof(q)) + continue; + + memcpy(&q, RTA_DATA(in), sizeof(q)); + + *byt = q.backlog; + *pkt = q.qlen; + } + } +} + +/* + * Egress backlog of the device's root qdisc, in bytes and packets, + * as the kernel queues them. The caller owns the netlink descriptor + * and serialises the query. An early end of dump reports failure. + */ +static int eth_qdisc_backlog(int fd, + int ifindex, + size_t * byt, + size_t * pkt) +{ + struct { + struct nlmsghdr nh; + struct tcmsg tc; + } req; + struct nlmsghdr * nh; + struct rtattr * rta; + struct tcmsg * tc; + char buf[16384]; + ssize_t len; + int ret = -1; + + if (fd < 0) + goto fail; + + memset(&req, 0, sizeof(req)); + + req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(req.tc)); + req.nh.nlmsg_type = RTM_GETQDISC; + req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP; + req.nh.nlmsg_seq = 1; + req.tc.tcm_family = AF_UNSPEC; + req.tc.tcm_ifindex = ifindex; + if (send(fd, &req, req.nh.nlmsg_len, 0) < 0) + goto fail; + + *byt = 0; + *pkt = 0; + + while ((len = recv(fd, buf, sizeof(buf), 0)) > 0) { + nh = (struct nlmsghdr *) buf; + for (; NLMSG_OK(nh, len); nh = NLMSG_NEXT(nh, len)) { + if (nh->nlmsg_type == NLMSG_DONE) + goto done; + + if (nh->nlmsg_type == NLMSG_ERROR) + goto fail; + + if (nh->nlmsg_type != RTM_NEWQDISC) + continue; + + tc = (struct tcmsg *) NLMSG_DATA(nh); + if (tc->tcm_ifindex != ifindex) + continue; + + if (tc->tcm_parent != TC_H_ROOT) + continue; + + rta = (struct rtattr *) + ((char *) tc + NLMSG_ALIGN(sizeof(*tc))); + + eth_qdisc_parse(rta, nh->nlmsg_len + - NLMSG_LENGTH(sizeof(*tc)), + byt, pkt); + } + } + + goto fail; /* early end: a zero would read as empty */ + done: + ret = 0; + fail: + return ret; +} + +/* + * A netlink descriptor for the qdisc query. SO_RCVTIMEO bounds the + * dump: this is read from the send path, and a reply that never + * arrives must not park a sender thread. + */ +static int eth_qdisc_open(void) +{ + struct sockaddr_nl sa; + struct timeval tv = TIMEVAL_INIT_MS(POA_QDISC_TIMEO); + int fd; + + fd = socket(AF_NETLINK, SOCK_RAW | SOCK_CLOEXEC, NETLINK_ROUTE); + if (fd < 0) + return -1; + + memset(&sa, 0, sizeof(sa)); + + sa.nl_family = AF_NETLINK; + if (bind(fd, (struct sockaddr *) &sa, sizeof(sa)) < 0) + goto fail; + + if (setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)) < 0) + goto fail; + + return fd; + fail: + close(fd); + + return -1; +} + +/* + * Queue depth in packets, straight from the qdisc. Gated like the + * transport's own depth and skipped when another sender is already + * asking, so the send path never waits on netlink. + */ +static int eth_qpkts(struct poa * poa, + size_t * pkts, + size_t * byts) +{ + struct eth_priv * priv = (struct eth_priv *) poa->priv; + struct timespec now; + uint64_t ns; + size_t byt; + size_t pkt; + + if (priv->nl_fd < 0) + return -1; + + clock_gettime(PTHREAD_COND_CLOCK, &now); + + ns = TS_TO_UINT64(now); + if (ns - LOAD_RELAXED(&priv->nl_time) < POA_QLEN_GATE) + goto cached; + + if (__atomic_exchange_n(&priv->nl_busy, 1, __ATOMIC_ACQUIRE) != 0) + goto cached; + + if (eth_qdisc_backlog(priv->nl_fd, priv->ifindex, &byt, &pkt) == 0) { + STORE_RELAXED(&priv->nl_pkt, pkt); + STORE_RELAXED(&priv->nl_byt, byt); + STORE_RELAXED(&priv->nl_time, ns); + } + + __atomic_store_n(&priv->nl_busy, 0, __ATOMIC_RELEASE); + cached: + if (LOAD_RELAXED(&priv->nl_time) == 0) /* nothing measured yet */ + return -1; + + *pkts = LOAD_RELAXED(&priv->nl_pkt); + *byts = LOAD_RELAXED(&priv->nl_byt); + + return 0; +} + +static int eth_attach(struct poa * poa, + const struct poa_spec * spec) +{ + const struct eth_poa * c = &spec->eth; + struct eth_priv * priv; + uint8_t mac[POA_MAC_SIZE]; + uint32_t mtu; + int idx; + int err; + + if (strnlen(c->dev, sizeof(c->dev)) > DEV_NAME_SIZE) + return -EINVAL; + + if (c->ethertype < 0x0600 || c->ethertype == 0xFFFF) { + log_err("Invalid Ethertype 0x%x.", c->ethertype); + return -EINVAL; + } + + priv = malloc(sizeof(*priv)); + if (priv == NULL) + return -ENOMEM; + + memset(priv, 0, sizeof(*priv)); + + priv->poa = poa; + priv->nl_fd = eth_qdisc_open(); /* optional; socket fallback */ + + priv->ethertype = htons(c->ethertype); + + err = eth_dev_info(c->dev, &idx, mac, &mtu); + if (err < 0) + goto fail_conf; + + priv->mtu = mtu; + if (eth_socket(priv, idx, mac) < 0) + goto fail_conf; + + poa->priv = priv; + poa->local.type = poa->type; + poa->local.eth.src.ethertype = c->ethertype; + priv->ifindex = idx; + + memcpy(poa->local.eth.src.mac, mac, POA_MAC_SIZE); + strcpy(poa->local.eth.src.dev, c->dev); + + return 0; + + fail_conf: + if (priv->nl_fd >= 0) + close(priv->nl_fd); + + free(priv); + + return -EIO; +} + +static void eth_detach(struct poa * poa) +{ + struct eth_priv * priv = (struct eth_priv *) poa->priv; + + if (priv == NULL) + return; + + close(priv->s_fd); + + if (priv->nl_fd >= 0) + close(priv->nl_fd); + + free(priv); + + poa->priv = NULL; +} + +static uint32_t eth_mtu(struct poa * poa, + const struct poa_addr * dst) +{ + struct eth_priv * priv = (struct eth_priv *) poa->priv; + + (void) dst; + + return priv->mtu - POA_HDR_SIZE; +} + +/* All flows on the PoA share the socket, so this is aggregate. */ +static size_t eth_qlen(struct poa * poa) +{ +#ifdef SIOCOUTQ + struct eth_priv * priv = (struct eth_priv *) poa->priv; + int qlen; + + qlen = 0; + if (ioctl(priv->s_fd, SIOCOUTQ, &qlen) < 0) + return 0; + + return (size_t) qlen; +#else + (void) poa; + + return 0; +#endif +} + +/* + * PACKET_STATISTICS zeroes the kernel counters on read, so totals + * accumulate here; relaxed atomics allow concurrent RIB reads. + * The qdisc depth rides the sender's guarded path: one dump at a time. + */ +static int eth_rib(struct poa * poa, + char * buf, + size_t len) +{ + struct eth_priv * priv = (struct eth_priv *) poa->priv; + struct tpacket_stats ts; + socklen_t optlen; + size_t sndbuf = 0; + size_t rcvbuf = 0; + size_t qd_byt; + size_t qd_pkt; + int val; + int size; + + optlen = sizeof(val); + if (getsockopt(priv->s_fd, SOL_SOCKET, SO_SNDBUF, &val, &optlen) == 0) + sndbuf = (size_t) val; + + optlen = sizeof(val); + if (getsockopt(priv->s_fd, SOL_SOCKET, SO_RCVBUF, &val, &optlen) == 0) + rcvbuf = (size_t) val; + + optlen = sizeof(ts); + + if (getsockopt(priv->s_fd, SOL_PACKET, PACKET_STATISTICS, + &ts, &optlen) == 0) { + FETCH_ADD_RELAXED(&priv->kern_rcv, ts.tp_packets); + FETCH_ADD_RELAXED(&priv->kern_drp, ts.tp_drops); + } + + if (eth_qpkts(poa, &qd_pkt, &qd_byt) < 0) { + qd_byt = 0; + qd_pkt = 0; + } + + size = snprintf(buf, len, + "Socket sndbuf (bytes): %zu\n" + "Socket rcvbuf (bytes): %zu\n" + "Kernel packets received: %zu\n" + "Kernel packets dropped: %zu\n" + "Qdisc backlog (bytes): %zu\n" + "Qdisc backlog (packets): %zu\n", + sndbuf, rcvbuf, + LOAD_RELAXED(&priv->kern_rcv), + LOAD_RELAXED(&priv->kern_drp), + qd_byt, qd_pkt); + if (size < 0 || (size_t) size >= len) + return -1; + + return size; +} + +#elif defined(HAVE_BPF) + +/* + * BSD and macOS reach the link layer through a cloned /dev/bpf + * device, bound to an interface with BIOCSETIF. One device per PoA. + */ + +static int eth_sendv(struct eth_priv * priv, + const uint8_t * dst, + uint32_t eid, + const uint8_t * body, + size_t len, + bool block, + const struct timespec * abstime) +{ + struct eth_hdr hdr; + struct iovec iov[2]; + int n; + int ret; + + if (len > priv->mtu - POA_HDR_SIZE) + return -EMSGSIZE; + + eth_hdr_ser(priv, &hdr, dst, eid, len); + + iov[0].iov_base = &hdr; + iov[0].iov_len = ETH_HDR_TOT_SIZE; + iov[1].iov_base = (void *) body; + iov[1].iov_len = len; + n = len > 0 ? 2 : 1; + while (writev(priv->s_fd, iov, n) < 0) { + if (errno != EAGAIN && errno != EWOULDBLOCK) + return -EIO; + + if (!block) + return -EAGAIN; + + ret = poa_wait_out(priv->s_fd, abstime); + if (ret < 0) + return ret; + } + + return 0; +} + +/* One read yields a batch of BPF_WORDALIGN'ed frames; walk all of them. */ +static void * eth_reader(void * o) +{ + struct poa * poa = (struct poa *) o; + struct eth_priv * priv = (struct eth_priv *) poa->priv; + uint8_t * buf; + + buf = malloc(priv->blen); + if (buf == NULL) + return (void *) -1; + + pthread_cleanup_push(free, buf); + + while (true) { + uint8_t * p; + uint8_t * end; + ssize_t n; + + n = read(priv->s_fd, buf, priv->blen); + if (n < 0) { + if (errno == EINTR) + continue; + + POA_STAT_BUMP(poa, rcv_fail); + break; + } + + p = buf; + end = buf + n; + while (p + sizeof(struct bpf_hdr) <= end) { + struct bpf_hdr * bh = (struct bpf_hdr *) p; + struct ssm_pk_buff * spb; + struct poa_addr src; + const uint8_t * frame; + const uint8_t * body; + uint32_t eid; + size_t plen; + + frame = p + bh->bh_hdrlen; + if (frame + bh->bh_caplen > end) + break; + + if (frame_parse(priv, frame, bh->bh_caplen, + &eid, &plen) < 0) + goto next; + + body = frame + ETH_HDR_TOT_SIZE; + + if (eid == POA_MGMT_EID) { + frame_to_addr(priv, + (const struct eth_hdr *) frame, + &src); + eth_rx_mgmt(poa, &src, body, plen); + goto next; + } + + if (poa_spb_reserve(&spb, plen) < 0) { + POA_STAT_BUMP(poa, buf_fail); + goto next; + } + + memcpy(ssm_pk_buff_head(spb), body, plen); + + poa_rx_pkt(poa, eid, spb); + next: + p += BPF_WORDALIGN(bh->bh_hdrlen + bh->bh_caplen); + } + } + + pthread_cleanup_pop(true); + + return (void *) 0; +} + +static int eth_dev_info(const char * dev, + uint8_t * mac, + uint32_t * mtu) +{ + struct ifaddrs * ifas; + struct ifaddrs * ifa; + struct ifreq ifr; + int fd; + int found = 0; + + if (getifaddrs(&ifas) < 0) { + log_err("Failed to list interfaces."); + return -1; + } + + for (ifa = ifas; ifa != NULL; ifa = ifa->ifa_next) { + struct sockaddr_dl * dl; + + if (ifa->ifa_addr == NULL) + continue; + + if (ifa->ifa_addr->sa_family != AF_LINK) + continue; + + if (strcmp(ifa->ifa_name, dev) != 0) + continue; + + dl = (struct sockaddr_dl *) ifa->ifa_addr; + if (dl->sdl_alen != POA_MAC_SIZE) + continue; + + memcpy(mac, LLADDR(dl), POA_MAC_SIZE); + + found = 1; + break; + } + + freeifaddrs(ifas); + + if (!found) { + log_err("No such device: %s.", dev); + return -1; + } + + fd = socket(AF_INET, SOCK_DGRAM, 0); + if (fd < 0) + return -1; + + memset(&ifr, 0, sizeof(ifr)); + + strcpy(ifr.ifr_name, dev); + + if (ioctl(fd, SIOCGIFMTU, &ifr) < 0) { + log_err("Failed to get MTU of %s.", dev); + close(fd); + return -1; + } + + close(fd); + + if (eth_dev_mac(mac) < 0) + return -1; + + *mtu = MIN(MIN(ETH_MAX_MTU, POA_ETH_RD_BUF), (uint32_t) ifr.ifr_mtu); + if (memcmp(dev, "lo", 2) == 0 && *mtu > POA_ETH_LO_MTU) + *mtu = POA_ETH_LO_MTU; + + return 0; +} + +static int eth_bpf_open(void) +{ + char dev[32]; + size_t i; + + for (i = 0; i < BPF_DEV_MAX; ++i) { + int fd; + + sprintf(dev, "/dev/bpf%zu", i); + + fd = open(dev, O_RDWR); + if (fd >= 0) + return fd; + } + + return -1; +} + +/* + * BIOCSHDRCMPLT: we fill in the source address ourselves. + * BIOCSSEESENT: our own egress must not come back at us. + * BIOCIMMEDIATE: deliver on arrival, do not wait for a full buffer. + */ +static int eth_bpf_setup(struct eth_priv * priv, + const char * dev) +{ + struct ifreq ifr; + int enable = 1; + int disable = 0; + int blen = 0; + + memset(&ifr, 0, sizeof(ifr)); + + strcpy(ifr.ifr_name, dev); + + if (ioctl(priv->s_fd, BIOCSETIF, &ifr) < 0) { + log_err("Failed to bind bpf device to %s.", dev); + return -1; + } + + if (ioctl(priv->s_fd, BIOCGBLEN, &blen) < 0 || blen <= 0) { + log_err("Failed to get the bpf buffer length."); + return -1; + } + + priv->blen = (size_t) blen; + if (ioctl(priv->s_fd, BIOCSHDRCMPLT, &enable) < 0) { + log_err("Failed to set BIOCSHDRCMPLT."); + return -1; + } + + if (ioctl(priv->s_fd, BIOCSSEESENT, &disable) < 0) { + log_err("Failed to set BIOCSSEESENT."); + return -1; + } + + if (ioctl(priv->s_fd, BIOCIMMEDIATE, &enable) < 0) { + log_err("Failed to set BIOCIMMEDIATE."); + return -1; + } + + return 0; +} + +static int eth_attach(struct poa * poa, + const struct poa_spec * spec) +{ + const struct eth_poa * c = &spec->eth; + struct eth_priv * priv; + uint8_t mac[POA_MAC_SIZE]; + uint32_t mtu; + + if (strnlen(c->dev, sizeof(c->dev)) > DEV_NAME_SIZE) + return -EINVAL; + + if (c->ethertype < 0x0600 || c->ethertype == 0xFFFF) { + log_err("Invalid Ethertype 0x%x.", c->ethertype); + return -EINVAL; + } + + priv = malloc(sizeof(*priv)); + if (priv == NULL) + return -ENOMEM; + + memset(priv, 0, sizeof(*priv)); + + priv->poa = poa; + priv->s_fd = -1; + + priv->ethertype = htons(c->ethertype); + + if (eth_dev_info(c->dev, mac, &mtu) < 0) + goto fail_conf; + + priv->mtu = mtu; + + memcpy(priv->hw_addr, mac, POA_MAC_SIZE); + + priv->s_fd = eth_bpf_open(); + if (priv->s_fd < 0) { + log_err("Failed to open a bpf device."); + goto fail_conf; + } + + if (eth_bpf_setup(priv, c->dev) < 0) + goto fail_device; + + poa->priv = priv; + poa->local.type = poa->type; + poa->local.eth.src.ethertype = c->ethertype; + + memcpy(poa->local.eth.src.mac, mac, POA_MAC_SIZE); + strcpy(poa->local.eth.src.dev, c->dev); + + log_info("Using Berkeley Packet Filter on %s.", c->dev); + + return 0; + + fail_device: + close(priv->s_fd); + fail_conf: + free(priv); + + return -EIO; +} + +static void eth_detach(struct poa * poa) +{ + struct eth_priv * priv = (struct eth_priv *) poa->priv; + + if (priv == NULL) + return; + + close(priv->s_fd); + + free(priv); + + poa->priv = NULL; +} + +static uint32_t eth_mtu(struct poa * poa, + const struct poa_addr * dst) +{ + struct eth_priv * priv = (struct eth_priv *) poa->priv; + + (void) dst; + + return priv->mtu - POA_HDR_SIZE; +} + +/* A bpf device has no send queue to report; mb-ECN cannot mark here. */ +static size_t eth_qlen(struct poa * poa) +{ + (void) poa; + + return 0; +} + +/* The bpf device buffer is all the receive queue there is. */ +static int eth_rib(struct poa * poa, + char * buf, + size_t len) +{ + struct eth_priv * priv = (struct eth_priv *) poa->priv; + struct bpf_stat bs; + int size; + + if (ioctl(priv->s_fd, BIOCGSTATS, &bs) == 0) { + FETCH_ADD_RELAXED(&priv->kern_rcv, bs.bs_recv); + FETCH_ADD_RELAXED(&priv->kern_drp, bs.bs_drop); + } + + size = snprintf(buf, len, + "Socket rcvbuf (bytes): %zu\n" + "Kernel packets received: %zu\n" + "Kernel packets dropped: %zu\n", + priv->blen, + LOAD_RELAXED(&priv->kern_rcv), + LOAD_RELAXED(&priv->kern_drp)); + if (size < 0 || (size_t) size >= len) + return -1; + + return size; +} + +#elif defined(HAVE_NETMAP) + +/* + * netmap gives one PoA the whole port: reads copy out of the NIC + * ring, writes inject into it. + */ + +/* nm_inject takes one contiguous frame, so the header is copied in. */ +static int eth_sendv(struct eth_priv * priv, + const uint8_t * dst, + uint32_t eid, + const uint8_t * body, + size_t len, + bool block, + const struct timespec * abstime) +{ + uint8_t * frame; + size_t flen; + int ret; + + if (len > priv->mtu - POA_HDR_SIZE) + return -EMSGSIZE; + + flen = ETH_HDR_TOT_SIZE + len; + + frame = malloc(flen); + if (frame == NULL) + return -ENOMEM; + + eth_hdr_ser(priv, (struct eth_hdr *) frame, dst, eid, len); + + if (len > 0) + memcpy(frame + ETH_HDR_TOT_SIZE, body, len); + + if (block) + ret = poa_wait_out(priv->poll_out.fd, abstime); + else + ret = poll(&priv->poll_out, 1, 0) > 0 ? 0 : -EAGAIN; + + if (ret < 0) + goto fail; + + ret = nm_inject(priv->nmd, frame, flen) == (int) flen ? 0 : -EIO; + fail: + free(frame); + + return ret; +} + +/* A slot stays owned by the ring, so each frame is copied out. */ +static void * eth_reader(void * o) +{ + struct poa * poa = (struct poa *) o; + struct eth_priv * priv = (struct eth_priv *) poa->priv; + + while (true) { + struct ssm_pk_buff * spb; + struct poa_addr src; + struct nm_pkthdr hdr; + const uint8_t * frame; + const uint8_t * body; + uint32_t eid; + size_t plen; + + if (poll(&priv->poll_in, 1, -1) < 0) { + if (errno == EINTR) + continue; + + POA_STAT_BUMP(poa, rcv_fail); + break; + } + + if (priv->poll_in.revents == 0) + continue; + + frame = nm_nextpkt(priv->nmd, &hdr); + if (frame == NULL) + continue; + + if (frame_parse(priv, frame, hdr.len, &eid, &plen) < 0) + continue; + + body = frame + ETH_HDR_TOT_SIZE; + + if (eid == POA_MGMT_EID) { + frame_to_addr(priv, (const struct eth_hdr *) frame, + &src); + eth_rx_mgmt(poa, &src, body, plen); + continue; + } + + if (poa_spb_reserve(&spb, plen) < 0) { + POA_STAT_BUMP(poa, buf_fail); + continue; + } + + memcpy(ssm_pk_buff_head(spb), body, plen); + + poa_rx_pkt(poa, eid, spb); + } + + return (void *) 0; +} + +static int eth_dev_info(const char * dev, + uint8_t * mac, + uint32_t * mtu) +{ + struct ifreq ifr; +#ifndef __linux__ + struct ifaddrs * ifas; + struct ifaddrs * ifa; + int found = 0; +#endif + int fd; + + if (strlen(dev) >= IFNAMSIZ) + return -EINVAL; + + fd = socket(AF_INET, SOCK_DGRAM, 0); + if (fd < 0) + return -EIO; + + memset(&ifr, 0, sizeof(ifr)); + + strcpy(ifr.ifr_name, dev); + +#ifdef __linux__ + if (ioctl(fd, SIOCGIFHWADDR, &ifr) < 0) { + log_err("Failed to get hardware address of %s.", dev); + goto fail_ioctl; + } + + memcpy(mac, ifr.ifr_hwaddr.sa_data, POA_MAC_SIZE); +#else + if (getifaddrs(&ifas) < 0) + goto fail_ioctl; + + for (ifa = ifas; ifa != NULL; ifa = ifa->ifa_next) { + struct sockaddr_dl * dl; + + if (ifa->ifa_addr == NULL) + continue; + + if (ifa->ifa_addr->sa_family != AF_LINK) + continue; + + if (strcmp(ifa->ifa_name, dev) != 0) + continue; + + dl = (struct sockaddr_dl *) ifa->ifa_addr; + if (dl->sdl_alen != POA_MAC_SIZE) + continue; + + memcpy(mac, LLADDR(dl), POA_MAC_SIZE); + + found = 1; + break; + } + + freeifaddrs(ifas); + + if (!found) { + log_err("No hardware address for %s.", dev); + goto fail_ioctl; + } +#endif + if (ioctl(fd, SIOCGIFMTU, &ifr) < 0) { + log_err("Failed to get MTU of %s.", dev); + goto fail_ioctl; + } + + close(fd); + + if (eth_dev_mac(mac) < 0) + return -1; + + *mtu = MIN(MIN(ETH_MAX_MTU, POA_ETH_RD_BUF), (uint32_t) ifr.ifr_mtu); + if (memcmp(dev, "lo", 2) == 0 && *mtu > POA_ETH_LO_MTU) + *mtu = POA_ETH_LO_MTU; + + return 0; + + fail_ioctl: + close(fd); + + return -EIO; +} + +static int eth_nm_open(struct eth_priv * priv, + const char * dev) +{ + char ifn[IFNAMSIZ + sizeof("netmap:")]; + + strcpy(ifn, "netmap:"); + strcat(ifn, dev); + + priv->nmd = nm_open(ifn, NULL, 0, NULL); + if (priv->nmd == NULL) { + log_err("Failed to open netmap device for %s.", dev); + return -1; + } + + memset(&priv->poll_in, 0, sizeof(priv->poll_in)); + memset(&priv->poll_out, 0, sizeof(priv->poll_out)); + + priv->poll_in.fd = NETMAP_FD(priv->nmd); + priv->poll_in.events = POLLIN; + priv->poll_out.fd = NETMAP_FD(priv->nmd); + priv->poll_out.events = POLLOUT; + + return 0; +} + +static int eth_attach(struct poa * poa, + const struct poa_spec * spec) +{ + const struct eth_poa * c = &spec->eth; + struct eth_priv * priv; + uint8_t mac[POA_MAC_SIZE]; + uint32_t mtu; + + if (strnlen(c->dev, sizeof(c->dev)) > DEV_NAME_SIZE) + return -EINVAL; + + if (c->ethertype < 0x0600 || c->ethertype == 0xFFFF) { + log_err("Invalid Ethertype 0x%x.", c->ethertype); + return -EINVAL; + } + + priv = malloc(sizeof(*priv)); + if (priv == NULL) + return -ENOMEM; + + memset(priv, 0, sizeof(*priv)); + + priv->poa = poa; + + priv->ethertype = htons(c->ethertype); + + if (eth_dev_info(c->dev, mac, &mtu) < 0) + goto fail_conf; + + priv->mtu = mtu; + + memcpy(priv->hw_addr, mac, POA_MAC_SIZE); + + if (eth_nm_open(priv, c->dev) < 0) + goto fail_conf; + + poa->priv = priv; + poa->local.type = poa->type; + poa->local.eth.src.ethertype = c->ethertype; + + memcpy(poa->local.eth.src.mac, mac, POA_MAC_SIZE); + strcpy(poa->local.eth.src.dev, c->dev); + + log_info("Using netmap on %s.", c->dev); + + return 0; + + fail_conf: + free(priv); + + return -EIO; +} + +static void eth_detach(struct poa * poa) +{ + struct eth_priv * priv = (struct eth_priv *) poa->priv; + + if (priv == NULL) + return; + + nm_close(priv->nmd); + + free(priv); + + poa->priv = NULL; +} + +static uint32_t eth_mtu(struct poa * poa, + const struct poa_addr * dst) +{ + struct eth_priv * priv = (struct eth_priv *) poa->priv; + + (void) dst; + + return priv->mtu - POA_HDR_SIZE; +} + +/* The ring is drained by the NIC; there is no queue to report. */ +static size_t eth_qlen(struct poa * poa) +{ + (void) poa; + + return 0; +} + +#endif /* HAVE_RAW_SOCKETS */ + +#ifndef HAVE_RAW_SOCKETS + +/* Only netlink reports link events; no other backend has a monitor. */ +int poa_monitor_open(void) +{ + return -1; +} + +void poa_monitor_read(int fd) +{ + (void) fd; +} + +#endif + +/* One reader per socket, so a flow cannot be reordered on receive. */ +static int eth_start(struct poa * poa) +{ + struct eth_priv * priv = (struct eth_priv *) poa->priv; + + if (pthread_create(&priv->reader, NULL, eth_reader, poa) != 0) + return -1; + + priv->running = true; + + return 0; +} + +static void eth_stop(struct poa * poa) +{ + struct eth_priv * priv = (struct eth_priv *) poa->priv; + + if (!priv->running) + return; + + pthread_cancel(priv->reader); + pthread_join(priv->reader, NULL); + + priv->running = false; +} + +static int eth_send(struct poa * poa, + const struct poa_addr * dst, + uint32_t eid, + struct ssm_pk_buff * spb, + bool block, + const struct timespec * abstime) +{ + return eth_sendv((struct eth_priv *) poa->priv, dst->eth.dst.mac, eid, + ssm_pk_buff_head(spb), ssm_pk_buff_len(spb), + block, abstime); +} + +static int eth_send_mgmt(struct poa * poa, + const struct poa_addr * dst, + const uint8_t * buf, + size_t len) +{ + struct timespec timeo = TIMESPEC_INIT_MS(POA_MGMT_SND_TIMEO); + struct timespec abstime; + + clock_gettime(PTHREAD_COND_CLOCK, &abstime); + ts_add(&abstime, &timeo, &abstime); + + return eth_sendv((struct eth_priv *) poa->priv, dst->eth.dst.mac, + POA_MGMT_EID, buf, len, true, &abstime); +} + +const struct poa_ops eth_poa_ops = { + .poa_attach = eth_attach, + .poa_detach = eth_detach, + .poa_start = eth_start, + .poa_stop = eth_stop, + .poa_send = eth_send, + .poa_send_mgmt = eth_send_mgmt, + .poa_query = eth_query, + .poa_mtu = eth_mtu, + .poa_qlen = eth_qlen, +#ifdef HAVE_RAW_SOCKETS + .poa_qpkts = eth_qpkts, +#endif +#ifndef HAVE_NETMAP + .poa_rib = eth_rib, +#endif + .poa_spec = eth_spec, + .poa_has_id = eth_has_id, + .poa_match = eth_match, + .poa_link_match = eth_link_match, + .mpl = POA_ETH_MPL +}; diff --git a/src/lib/poa/poa.c b/src/lib/poa/poa.c new file mode 100644 index 00000000..1fd91660 --- /dev/null +++ b/src/lib/poa/poa.c @@ -0,0 +1,2498 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Points of attachment (PoA) - transport independent core + * + * Included by dev.c; uses dev.c statics (proc, flow_init, ...). + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * version 2.1 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#define POA_MAX_EIDS PROC_MAX_FLOWS +#define POA_ALLOC_TIMEO 10000 /* ms, overall FLOW_REQ deadline */ +#define POA_RETRY_TIMEO 300 /* ms, FLOW_REQ retransmit period */ +/* Must fit a certificate chain: post-quantum ones are large. */ +#define POA_MGMT_BUFSZ POA_MGMT_FRAME_SIZE +#define POA_MGMT_QMAX 64 /* queued management frames per PoA */ +#define POA_PEND_TIMEO 10 /* s, reap a request that never completes */ +#define POA_SWEEP_TIMEO 1000 /* ms, sweep interval */ +#define POA_SWEEP_MAX 16 /* requests reaped per sweep */ +#define POA_DEFER_MAX 64 /* replies waiting for their flow id */ + +/* EWMA over 8 samples. */ +#define POA_AVG_SHIFT 3 +/* Queue cost is sampled every 64th packet: qlen is a syscall. */ +#define POA_COST_MASK 63 +/* Reuse a qlen read for this long; the mark moves on doublings. */ + +#define POA_RIB "poa" +/* Fits the RIB labels below with 20-digit counters. */ +#define POA_RIB_STRLEN 2048 + +enum poa_flow_state { + POA_FLOW_NULL = 0, + POA_FLOW_PENDING, + POA_FLOW_ALLOCATED, + POA_FLOW_DEAD +}; + +enum poa_state { + POA_NULL = 0, + POA_INIT, /* poa_init(); this process may attach */ + POA_RUNNING, /* the threads are up */ + POA_OPERATIONAL /* bootstrapped or enrolled in a layer */ +}; + +struct poa_flow { + struct poa * poa; + + int flow_id; + uint32_t eid; + uint32_t r_eid; + struct poa_addr r_addr; + + enum poa_flow_state state; + + struct ssm_rbuff * rx_rb; + + struct list_head pend; /* on poa->pending while unanswered */ + struct timespec t0; + + /* Our answer, kept to re-send when a request is repeated. */ + bool answered; + bool answer_sent; + + /* Handed to a caller that will attach it; not the sweeper's. */ + bool claimed; + int answer; + buffer_t answer_data; + + /* Handshake rendezvous with the reader thread. */ + pthread_mutex_t mtx; + pthread_cond_t cond; + bool replied; + bool pending; + int response; + buffer_t resp_data; +}; + +/* + * An answer can be ready before the flow it answers has an id, and + * the caller must not be kept waiting for one: it answers to the IRMd, + * which gives up long before we would. + */ +struct poa_deferred { + struct list_head next; + struct timespec t0; + int flow_id; + int response; + buffer_t data; +}; + +struct poa_mgmt_frame { + struct list_head next; + struct poa * poa; + struct poa_addr src; + size_t len; + uint8_t buf[POA_MGMT_BUFSZ]; +}; + +/* + * A detach is performed by the management thread, so that it cannot + * run alongside the accept side. The caller waits for the result. + */ +struct poa_detach_req { + struct list_head next; + struct poa_spec spec; + int result; + bool done; +}; + +static struct { + struct list_head list; + + struct poa_flow * id_to_pf[SYS_MAX_FLOWS]; + + struct llist deferred; + + pthread_mutex_t mtx; /* guards id_to_pf */ + pthread_cond_t cond; + + /* One management thread and one link monitor for all PoAs. */ + struct llist mgmt_frames; + struct list_head detach; + pthread_mutex_t mgmt_mtx; + pthread_cond_t mgmt_cond; /* work for the handler */ + pthread_cond_t done_cond; /* a detach has completed */ + pthread_t mgmt_handler; + bool mgmt_stop; + + pthread_t monitor; + int mon_fd; + + enum poa_state state; + + /* Hashes of the names this process answers queries for. */ + uint8_t name_hash[POA_QUERY_HLEN]; + uint8_t layer_hash[POA_QUERY_HLEN]; + + pthread_rwlock_t lock; /* the PoAs and their flows */ +} poas; + +static int mgmt_send(struct poa * poa, + const struct poa_addr * dst, + uint8_t code, + uint32_t s_eid, + uint32_t d_eid, + qosspec_t qs, + int response, + const buffer_t * data) +{ + uint8_t buf[POA_MGMT_BUFSZ]; + struct poa_mgmt_msg * msg = (struct poa_mgmt_msg *) buf; + size_t len; + + len = sizeof(*msg); + if (data != NULL && data->len > 0) { + if (len + data->len > sizeof(buf)) + return -EMSGSIZE; + memcpy(buf + len, data->data, data->len); + + len += data->len; + } + + poa_mgmt_msg_ser(msg, code, s_eid, d_eid, qs, response, + data != NULL ? data->len : 0); + + POA_STAT_BUMP(poa, mgmt_tx); + + return poa->ops->poa_send_mgmt(poa, dst, buf, len); +} + +/* Caller holds poas.lock for writing. */ +static struct poa_flow * pf_create(struct poa * poa, + const struct poa_addr * r_addr) +{ + struct poa_flow * pf; + pthread_condattr_t cattr; + int eid; + + pf = malloc(sizeof(*pf)); + if (pf == NULL) + goto fail_malloc; + + memset(pf, 0, sizeof(*pf)); + + if (pthread_mutex_init(&pf->mtx, NULL) != 0) + goto fail_mtx; + + if (pthread_condattr_init(&cattr) != 0) + goto fail_cond; +#ifndef __APPLE__ + pthread_condattr_setclock(&cattr, PTHREAD_COND_CLOCK); +#endif + if (pthread_cond_init(&pf->cond, &cattr) != 0) { + pthread_condattr_destroy(&cattr); + goto fail_cond; + } + + pthread_condattr_destroy(&cattr); + + eid = bmp_allocate(poa->eids); + if (!bmp_is_id_valid(poa->eids, eid)) + goto fail_eid; + + pf->poa = poa; + pf->eid = (uint32_t) eid; + pf->flow_id = -1; + pf->state = POA_FLOW_PENDING; + pf->r_addr = *r_addr; + + clock_gettime(PTHREAD_COND_CLOCK, &pf->t0); + + rcu_wrlock(&poa->guard); + + rcu_publish(pf); + rcu_assign(poa->eid_to_pf[eid], pf); + + rcu_wrunlock(&poa->guard); + + list_add_tail(&pf->pend, &poa->flows); + + POA_STAT_BUMP(poa, n_flows); + + return pf; + + fail_eid: + pthread_cond_destroy(&pf->cond); + fail_cond: + pthread_mutex_destroy(&pf->mtx); + fail_mtx: + free(pf); + fail_malloc: + return NULL; +} + +/* Caller holds poas.mtx. */ +static void deferred_purge(int flow_id) +{ + struct list_head * p; + struct list_head * h; + + list_for_each_safe(p, h, &poas.deferred.list) { + struct poa_deferred * d; + + d = list_entry(p, struct poa_deferred, next); + if (d->flow_id != flow_id) + continue; + + llist_del(&d->next, &poas.deferred); + freebuf(d->data); + free(d); + } +} + +/* The id may have moved to a newer flow; only its owner clears it. */ +static void pf_destroy(struct poa_flow * pf) +{ + struct poa * poa = pf->poa; + + pthread_rwlock_wrlock(&poas.lock); + + rcu_wrlock(&poa->guard); + + rcu_assign(poa->eid_to_pf[pf->eid], NULL); + rcu_assign(pf->rx_rb, NULL); + + rcu_wrunlock(&poa->guard); + + bmp_release(poa->eids, pf->eid); + + if (!list_is_empty(&pf->pend)) { + list_del(&pf->pend); + POA_STAT_SUB(poa, n_flows, 1); + } + + pthread_rwlock_unlock(&poas.lock); + + pthread_mutex_lock(&poas.mtx); + + if (pf->flow_id >= 0 && poas.id_to_pf[pf->flow_id] == pf) { + poas.id_to_pf[pf->flow_id] = NULL; + + deferred_purge(pf->flow_id); + } + + pthread_mutex_unlock(&poas.mtx); + + rcu_reclaim(&poa->guard); /* a receive may still hold pf */ + + freebuf(pf->resp_data); + freebuf(pf->answer_data); + + pthread_cond_destroy(&pf->cond); + pthread_mutex_destroy(&pf->mtx); + + free(pf); +} + +/* + * Keeps the answer for a repeat, then sends it. An accept must wait + * until the flow can receive; a refusal needs no receiver. + */ +static int pf_answer(struct poa_flow * pf, + int response, + const buffer_t * data) +{ + int err; + + pthread_rwlock_wrlock(&poas.lock); + + freebuf(pf->answer_data); + + if (data != NULL && data->len > 0) { + pf->answer_data.data = malloc(data->len); + if (pf->answer_data.data != NULL) { + memcpy(pf->answer_data.data, data->data, data->len); + + pf->answer_data.len = data->len; + } + } + + pf->answer = response; + pf->answered = true; + if (response == 0 && pf->state != POA_FLOW_ALLOCATED) { + pthread_rwlock_unlock(&poas.lock); + return 0; + } + + pf->answer_sent = true; + + pthread_rwlock_unlock(&poas.lock); + + err = mgmt_send(pf->poa, &pf->r_addr, POA_FLOW_REPLY, pf->eid, + pf->r_eid, qos_raw, response, data); + + if (err == -ETIMEDOUT || err == -EAGAIN) + err = 0; /* stored; a repeat request resends it */ + + return err; +} + +/* Takes an answer left for a flow that had no id yet. */ +static struct poa_deferred * deferred_take(int flow_id) +{ + struct list_head * p; + struct list_head * h; + + list_for_each_safe(p, h, &poas.deferred.list) { + struct poa_deferred * d; + + d = list_entry(p, struct poa_deferred, next); + if (d->flow_id != flow_id) + continue; + + llist_del(&d->next, &poas.deferred); + + return d; + } + + return NULL; +} + +/* Publishes the flow_id so an answer can find this flow. */ +static void pf_set_flow_id(struct poa_flow * pf, + int flow_id) +{ + struct poa_deferred * d; + + pthread_mutex_lock(&poas.mtx); + + pf->flow_id = flow_id; + poas.id_to_pf[flow_id] = pf; + + d = deferred_take(flow_id); + + pthread_cond_broadcast(&poas.cond); + + pthread_mutex_unlock(&poas.mtx); + + if (d != NULL) { + pf_answer(pf, d->response, &d->data); + freebuf(d->data); + free(d); + } +} + +/* + * Between the request arriving and the accept returning, the flow has + * an id but no fd yet; flow_init claims the PoA here. + */ +static void pf_set_pending(struct poa_flow * pf) +{ + pthread_mutex_lock(&poas.mtx); + + pf->pending = true; + + pthread_mutex_unlock(&poas.mtx); +} + +static void pf_clr_pending(struct poa_flow * pf) +{ + pthread_mutex_lock(&poas.mtx); + + pf->pending = false; + + pthread_mutex_unlock(&poas.mtx); +} + +/* A process that attaches no PoA has nothing pending. */ +struct poa_flow * poa_flow_take_pending(int flow_id) +{ + struct poa_flow * pf; + + if (poas.state == POA_NULL) + return NULL; + + if (flow_id < 0 || flow_id >= SYS_MAX_FLOWS) + return NULL; + + pthread_mutex_lock(&poas.mtx); + + pf = poas.id_to_pf[flow_id]; + if (pf != NULL && pf->pending) + pf->pending = false; + else + pf = NULL; + + pthread_mutex_unlock(&poas.mtx); + + return pf; +} + +static struct poa_flow * pf_get(int flow_id) +{ + struct poa_flow * pf; + + if (flow_id < 0 || flow_id >= SYS_MAX_FLOWS) + return NULL; + + pthread_mutex_lock(&poas.mtx); + + pf = poas.id_to_pf[flow_id]; + + pthread_mutex_unlock(&poas.mtx); + + return pf; +} + +#ifdef PROC_FLOW_STATS + +/* Caller holds poas.lock. */ +static struct poa * poa_by_rib_name(const char * name) +{ + struct list_head * p; + + list_for_each(p, &poas.list) { + struct poa * poa = list_entry(p, struct poa, next); + + if (strcmp(poa->name, name) == 0) + return poa; + } + + return NULL; +} + +static int poa_rib_read(const char * path, + char * buf, + size_t len) +{ + struct poa * poa; + const char * entry; + size_t qlen; + size_t avg; + size_t cost; + int size; + int ret; + + entry = strstr(path, RIB_SEPARATOR) + 1; + + if (len < POA_RIB_STRLEN) + return -1; + + pthread_rwlock_rdlock(&poas.lock); + + poa = poa_by_rib_name(entry); + if (poa == NULL) + goto fail; + + qlen = poa->ops->poa_qlen(poa); + avg = poa->avg_len; + cost = poa->avg_len + poa->avg_ovh; + + size = snprintf(buf, len, + "Active flows: %zu\n" + "Packets received: %zu\n" + "Bytes received: %zu\n" + "Packets sent: %zu\n" + "Bytes sent: %zu\n" + "Management frames rcvd: %zu\n" + "Management frames sent: %zu\n" + "Bad EID packets: %zu\n" + "Delivery (N+1) failures: %zu\n" + "Buffer alloc failures: %zu\n" + "Packet read failures: %zu\n" + "Packet send failures: %zu\n" + "Name queries sent: %zu\n" + "Name queries received: %zu\n" + "Name replies sent: %zu\n" + "Name replies received: %zu\n" + "Queued (transport): %zu\n" + "Queued (packets): %zu\n" + "Mean packet size (bytes): %zu\n" + "Mean packet cost: %zu\n", + POA_STAT_LOAD(poa, n_flows), + POA_STAT_LOAD(poa, rx_pkts), + POA_STAT_LOAD(poa, rx_bytes), + POA_STAT_LOAD(poa, tx_pkts), + POA_STAT_LOAD(poa, tx_bytes), + POA_STAT_LOAD(poa, mgmt_rx), + POA_STAT_LOAD(poa, mgmt_tx), + POA_STAT_LOAD(poa, bad_eid), + POA_STAT_LOAD(poa, dlv_fail), + POA_STAT_LOAD(poa, buf_fail), + POA_STAT_LOAD(poa, rcv_fail), + POA_STAT_LOAD(poa, snd_fail), + POA_STAT_LOAD(poa, qry_tx), + POA_STAT_LOAD(poa, qry_rx), + POA_STAT_LOAD(poa, rep_tx), + POA_STAT_LOAD(poa, rep_rx), + qlen, cost > 0 ? qlen / cost : 0, avg, cost); + if (size < 0 || (size_t) size >= len) + goto fail; + + if (poa->ops->poa_rib != NULL) { + ret = poa->ops->poa_rib(poa, buf + size, len - size); + if (ret < 0) + goto fail; + + size += ret; + } + + pthread_rwlock_unlock(&poas.lock); + + return size; + + fail: + pthread_rwlock_unlock(&poas.lock); + + return -1; +} + +static int poa_rib_readdir(char *** buf) +{ + struct list_head * p; + size_t n = 0; + int idx = 0; + + pthread_rwlock_rdlock(&poas.lock); + + list_for_each(p, &poas.list) + ++n; + + if (n == 0) { + *buf = NULL; + goto no_poas; + } + + *buf = malloc(sizeof(**buf) * n); + if (*buf == NULL) + goto fail_entries; + + list_for_each(p, &poas.list) { + struct poa * poa = list_entry(p, struct poa, next); + + (*buf)[idx] = malloc(strlen(poa->name) + 1); + if ((*buf)[idx] == NULL) + goto fail_entry; + + strcpy((*buf)[idx++], poa->name); + } + no_poas: + pthread_rwlock_unlock(&poas.lock); + + return idx; + + fail_entry: + while (idx-- > 0) + free((*buf)[idx]); + + free(*buf); + fail_entries: + pthread_rwlock_unlock(&poas.lock); + + return -ENOMEM; +} + +static int poa_rib_getattr(const char * path, + struct rib_attr * attr) +{ + (void) path; + + attr->size = POA_RIB_STRLEN; + attr->mtime = 0; + + return 0; +} + +static struct rib_ops poa_r_ops = { + .read = poa_rib_read, + .readdir = poa_rib_readdir, + .getattr = poa_rib_getattr +}; + +#endif /* PROC_FLOW_STATS */ + +int poa_init(const char * name) +{ + pthread_condattr_t cattr; + + assert(name != NULL); + + memset(&poas, 0, sizeof(poas)); + + str_hash(HASH_SHA3_256, poas.name_hash, name); + + poas.mon_fd = -1; + if (pthread_mutex_init(&poas.mtx, NULL) != 0) + goto fail_mtx; + + if (pthread_cond_init(&poas.cond, NULL) != 0) + goto fail_cond; + + if (pthread_mutex_init(&poas.mgmt_mtx, NULL) != 0) + goto fail_mgmt_mtx; + + if (pthread_condattr_init(&cattr) != 0) + goto fail_cattr; +#ifndef __APPLE__ + pthread_condattr_setclock(&cattr, PTHREAD_COND_CLOCK); +#endif + if (pthread_cond_init(&poas.mgmt_cond, &cattr) != 0) { + pthread_condattr_destroy(&cattr); + goto fail_cattr; + } + + pthread_condattr_destroy(&cattr); + + if (pthread_cond_init(&poas.done_cond, NULL) != 0) + goto fail_done_cond; + + if (pthread_rwlock_init(&poas.lock, NULL) != 0) + goto fail_lock; + + list_head_init(&poas.list); + llist_init(&poas.deferred); + llist_init(&poas.mgmt_frames); + list_head_init(&poas.detach); + + poas.state = POA_INIT; + +#ifdef PROC_FLOW_STATS + if (rib_reg(POA_RIB, &poa_r_ops) < 0) + goto fail_rib; +#endif + return 0; + +#ifdef PROC_FLOW_STATS + fail_rib: + pthread_rwlock_destroy(&poas.lock); +#endif + + fail_lock: + pthread_cond_destroy(&poas.done_cond); + fail_done_cond: + pthread_cond_destroy(&poas.mgmt_cond); + fail_cattr: + pthread_mutex_destroy(&poas.mgmt_mtx); + fail_mgmt_mtx: + pthread_cond_destroy(&poas.cond); + fail_cond: + pthread_mutex_destroy(&poas.mtx); + fail_mtx: + return -1; +} + +int poa_set_layer(const char * layer) +{ + if (layer == NULL) + return -EINVAL; + + pthread_rwlock_wrlock(&poas.lock); + + str_hash(HASH_SHA3_256, poas.layer_hash, layer); + + poas.state = POA_OPERATIONAL; + + pthread_rwlock_unlock(&poas.lock); + + return 0; +} + +bool poa_has_name(const uint8_t * hash) +{ + bool match = false; + + pthread_rwlock_rdlock(&poas.lock); + + if (poas.state >= POA_INIT) + match = memcmp(hash, poas.name_hash, POA_QUERY_HLEN) == 0; + + if (!match && poas.state >= POA_OPERATIONAL) + match = memcmp(hash, poas.layer_hash, POA_QUERY_HLEN) == 0; + + pthread_rwlock_unlock(&poas.lock); + + return match; +} + +int poa_spb_reserve(struct ssm_pk_buff ** spb, + size_t len) +{ + return ssm_pool_alloc_b(proc.pool, len, NULL, spb, NULL) < 0 ? -1 : 0; +} + +void poa_spb_release(struct ssm_pk_buff * spb) +{ + ssm_pool_remove(proc.pool, ssm_pk_buff_get_off(spb)); +} + +void poa_rx_pkt(struct poa * poa, + uint32_t eid, + struct ssm_pk_buff * spb) +{ + struct poa_flow * pf; + struct ssm_rbuff * rx_rb; + size_t len; + + len = ssm_pk_buff_len(spb); /* the ring write takes it over */ + + if (eid >= poa->n_eids) { + POA_STAT_BUMP(poa, bad_eid); + poa_spb_release(spb); + return; + } + + rcu_rdlock(&poa->guard); + + pf = rcu_deref(poa->eid_to_pf[eid]); + + rcu_consume(pf); + + if (pf == NULL) + goto fail_eid; + + rx_rb = rcu_deref(pf->rx_rb); + + rcu_consume(rx_rb); + + if (rx_rb == NULL) + goto fail_eid; + + if (ssm_rbuff_write(rx_rb, ssm_pk_buff_get_off(spb)) < 0) { + POA_STAT_BUMP(poa, dlv_fail); + rcu_rdunlock(&poa->guard); + poa_spb_release(spb); + return; + } + + POA_STAT_BUMP(poa, rx_pkts); + POA_STAT_ADD(poa, rx_bytes, len); + + ssm_flow_set_notify(proc.fqset, pf->flow_id, FLOW_PKT); + + rcu_rdunlock(&poa->guard); + + return; + + fail_eid: + POA_STAT_BUMP(poa, bad_eid); + + rcu_rdunlock(&poa->guard); + + poa_spb_release(spb); +} + +static int poa_flow_req_arr(struct poa_flow * pf, + qosspec_t qs, + uint32_t mtu, + const buffer_t * data) +{ + struct flow_info flow; + uint8_t buf[SOCK_BUF_SIZE]; + buffer_t msg = {SOCK_BUF_SIZE, buf}; + buffer_t out = BUF_INIT; + int err; + + memset(&flow, 0, sizeof(flow)); + + flow.n_pid = getpid(); + flow.n_1_pid = getpid(); + flow.qs = qs; + flow.mpl = pf->poa->mpl; + flow.mtu = mtu; + if (ipcp_poa_flow_req_arr__irm_req_ser(&msg, &flow, data) < 0) + return -ENOMEM; + + pf_set_pending(pf); + + err = send_recv_msg(&msg); + if (err < 0) + goto fail; + + err = poa_flow__irm_result_des(&msg, &flow, &out); + if (err < 0) + goto fail; + + freebuf(out); + + if (flow.id < 0 || flow.id >= SYS_MAX_FLOWS) { + err = -EBADF; + goto fail; + } + + pf_set_flow_id(pf, flow.id); + + return 0; + fail: + pf_clr_pending(pf); + return err; +} + +static void handle_flow_req(struct poa * poa, + const struct poa_addr * src, + const struct poa_mgmt_msg * msg, + const uint8_t * data, + size_t data_len) +{ + struct list_head * p; + struct poa_flow * pf = NULL; + qosspec_t qs; + buffer_t buf; + buffer_t answer_data = BUF_INIT; + uint32_t r_eid; + uint32_t mtu; + uint32_t eid = 0; + int answer = 0; + bool found = false; + bool served = false; + + r_eid = ntoh32(msg->s_eid); + + poa_mgmt_msg_qos(msg, &qs); + + pthread_rwlock_wrlock(&poas.lock); + + list_for_each(p, &poa->flows) { + pf = list_entry(p, struct poa_flow, pend); + if (pf->r_eid != r_eid || poa_addr_cmp(&pf->r_addr, src) != 0) + continue; + + answer = pf->answer; + eid = pf->eid; + served = true; + + found = pf->answer_sent; + if (found && pf->answer_data.len > 0) { + answer_data.data = malloc(pf->answer_data.len); + if (answer_data.data != NULL) { + memcpy(answer_data.data, pf->answer_data.data, + pf->answer_data.len); + answer_data.len = pf->answer_data.len; + } + } + break; + } + + if (!served) { + pf = pf_create(poa, src); + if (pf != NULL) + pf->r_eid = r_eid; + } + + pthread_rwlock_unlock(&poas.lock); + + if (found) { + mgmt_send(poa, src, POA_FLOW_REPLY, eid, r_eid, qos_raw, answer, + answer_data.len > 0 ? &answer_data : NULL); + freebuf(answer_data); + } + + if (served || pf == NULL) + return; + + buf.len = data_len; + buf.data = (uint8_t *) data; + mtu = poa->ops->poa_mtu(poa, src); + if (poa_flow_req_arr(pf, qs, mtu, &buf) < 0) { + mgmt_send(poa, src, POA_FLOW_REPLY, pf->eid, r_eid, qos_raw, + -1, NULL); + pf_destroy(pf); + } +} + +static void handle_flow_reply(struct poa * poa, + const struct poa_addr * src, + const struct poa_mgmt_msg * msg, + const uint8_t * data, + size_t data_len) +{ + struct poa_flow * pf; + uint32_t eid; + + eid = ntoh32(msg->d_eid); + + pthread_rwlock_rdlock(&poas.lock); + + pf = eid < poa->n_eids ? poa->eid_to_pf[eid] : NULL; + if (pf == NULL || pf->state != POA_FLOW_PENDING) { + pthread_rwlock_unlock(&poas.lock); + return; + } + + if (poa_addr_cmp(&pf->r_addr, src) != 0) { + pthread_rwlock_unlock(&poas.lock); + return; + } + + pthread_mutex_lock(&pf->mtx); + + if (pf->replied) { + pthread_mutex_unlock(&pf->mtx); + pthread_rwlock_unlock(&poas.lock); + return; + } + + if (data_len > 0) { + pf->resp_data.data = malloc(data_len); + if (pf->resp_data.data != NULL) { + memcpy(pf->resp_data.data, data, data_len); + + pf->resp_data.len = data_len; + } + } + + pf->r_eid = ntoh32(msg->s_eid); + pf->response = ntoh32(msg->response); + pf->replied = true; + + pthread_cond_broadcast(&pf->cond); + + pthread_mutex_unlock(&pf->mtx); + + pthread_rwlock_unlock(&poas.lock); +} + +static void handle_flow_update(struct poa * poa, + const struct poa_addr * src, + const struct poa_mgmt_msg * msg, + const uint8_t * data, + size_t data_len) +{ + struct poa_flow * pf; + buffer_t buf; + uint32_t eid; + + eid = ntoh32(msg->d_eid); + + pthread_rwlock_rdlock(&poas.lock); + + pf = eid < poa->n_eids ? poa->eid_to_pf[eid] : NULL; + if (pf == NULL || pf->state != POA_FLOW_ALLOCATED) { + pthread_rwlock_unlock(&poas.lock); + return; + } + + if (poa_addr_cmp(&pf->r_addr, src) != 0) { + pthread_rwlock_unlock(&poas.lock); + return; + } + + eid = (uint32_t) pf->flow_id; + + pthread_rwlock_unlock(&poas.lock); + + buf.len = data_len; + buf.data = (uint8_t *) data; + + ipcp_flow_update_arr((int) eid, &buf); +} + +static void mgmt_frame_handle(struct poa_mgmt_frame * frame) +{ + const struct poa_mgmt_msg * msg; + const uint8_t * data; + size_t data_len; + + msg = (const struct poa_mgmt_msg *) frame->buf; + if (frame->len < sizeof(*msg)) + return; + + data_len = ntoh16(msg->data_len); + if (data_len > frame->len - sizeof(*msg)) + return; + + data = frame->buf + sizeof(*msg); + + switch (msg->code) { + case POA_FLOW_REQ: + handle_flow_req(frame->poa, &frame->src, msg, data, data_len); + break; + case POA_FLOW_REPLY: + handle_flow_reply(frame->poa, &frame->src, msg, data, data_len); + break; + case POA_FLOW_UPDATE: + handle_flow_update(frame->poa, &frame->src, msg, data, + data_len); + break; + default: + break; + } +} + +static bool pf_steal(struct poa_flow * pf) +{ + bool stolen = false; + + pthread_mutex_lock(&poas.mtx); + + if (pf->pending) { + pf->pending = false; + poas.id_to_pf[pf->flow_id] = NULL; + + deferred_purge(pf->flow_id); + + stolen = true; + } + + pthread_mutex_unlock(&poas.mtx); + + return stolen; +} + +static void sweep_pending(void) +{ + struct poa_flow * dead[POA_SWEEP_MAX]; + struct list_head * p; + struct list_head * q; + struct timespec now; + size_t n = 0; + size_t i; + + clock_gettime(PTHREAD_COND_CLOCK, &now); + + pthread_rwlock_rdlock(&poas.lock); + + list_for_each(p, &poas.list) { + struct poa * poa = list_entry(p, struct poa, next); + + list_for_each(q, &poa->flows) { + struct poa_flow * pf; + + if (n == POA_SWEEP_MAX) + break; + + pf = list_entry(q, struct poa_flow, pend); + if (pf->state != POA_FLOW_PENDING || pf->claimed) + continue; + + if (now.tv_sec - pf->t0.tv_sec < POA_PEND_TIMEO) + continue; + + if (pf->flow_id >= 0 && !pf_steal(pf)) + continue; + + dead[n++] = pf; + } + + if (n == POA_SWEEP_MAX) + break; + } + + pthread_rwlock_unlock(&poas.lock); + + for (i = 0; i < n; ++i) + pf_destroy(dead[i]); + + pthread_mutex_lock(&poas.mtx); + + list_for_each_safe(p, q, &poas.deferred.list) { + struct poa_deferred * d; + + d = list_entry(p, struct poa_deferred, next); + if (now.tv_sec - d->t0.tv_sec < POA_PEND_TIMEO) + continue; + + llist_del(&d->next, &poas.deferred); + freebuf(d->data); + free(d); + } + + pthread_mutex_unlock(&poas.mtx); +} + +void poa_rx_mgmt(struct poa * poa, + const struct poa_addr * src, + const uint8_t * buf, + size_t len) +{ + struct poa_mgmt_frame * frame; + + if (len < sizeof(struct poa_mgmt_msg) || len > POA_MGMT_BUFSZ) + return; + + POA_STAT_BUMP(poa, mgmt_rx); + + frame = malloc(offsetof(struct poa_mgmt_frame, buf) + len); + if (frame == NULL) + return; + + frame->poa = poa; + frame->src = *src; + frame->len = len; + + memcpy(frame->buf, buf, len); + + pthread_mutex_lock(&poas.mgmt_mtx); + + if (poa->n_mgmt >= POA_MGMT_QMAX) { + pthread_mutex_unlock(&poas.mgmt_mtx); + free(frame); + return; + } + + ++poa->n_mgmt; + + llist_add_tail(&frame->next, &poas.mgmt_frames); + + pthread_cond_signal(&poas.mgmt_cond); + + pthread_mutex_unlock(&poas.mgmt_mtx); +} + +static void mgmt_frames_purge(const struct poa * poa) +{ + struct list_head * p; + struct list_head * h; + + pthread_mutex_lock(&poas.mgmt_mtx); + + list_for_each_safe(p, h, &poas.mgmt_frames.list) { + struct poa_mgmt_frame * frame; + + frame = list_entry(p, struct poa_mgmt_frame, next); + if (frame->poa != poa) + continue; + + --frame->poa->n_mgmt; + + llist_del(&frame->next, &poas.mgmt_frames); + + free(frame); + } + + pthread_mutex_unlock(&poas.mgmt_mtx); +} + +static bool poa_has_id(const struct poa * poa, + const struct poa_spec * spec) +{ + if (poa->type != spec->type) + return false; + + return poa->ops->poa_has_id(poa, spec); +} + +/* + * The PoA carrying dst, as the transport judges it. -EPERM if none + * matches, -EINVAL if several do. Caller holds poas.lock. + */ +static int poa_lookup(const struct poa_addr * dst, + struct poa ** out) +{ + struct list_head * p; + struct poa * found = NULL; + + list_for_each(p, &poas.list) { + struct poa * poa = list_entry(p, struct poa, next); + + if (poa->type != dst->type) + continue; + + if (!poa->ops->poa_match(poa, dst)) + continue; + + if (found != NULL) /* nothing given, two candidates */ + return -EINVAL; + + found = poa; + } + + if (found == NULL) + return -EPERM; + + *out = found; + + return 0; +} + +/* Sends are deadlined, bounding the lock hold on a full queue. */ +int poa_bcast_mgmt(const struct poa_addr * dst, + const uint8_t * buf, + size_t len) +{ + struct list_head * p; + int n = 0; + + pthread_rwlock_rdlock(&poas.lock); + + list_for_each(p, &poas.list) { + struct poa * poa = list_entry(p, struct poa, next); + + if (poa->type != dst->type) + continue; + + if (!poa->ops->poa_match(poa, dst)) + continue; + + if (poa->ops->poa_send_mgmt(poa, dst, buf, len) < 0) + continue; + + /* All management broadcasts are name queries. */ + POA_STAT_BUMP(poa, qry_tx); + ++n; + } + + pthread_rwlock_unlock(&poas.lock); + + return n; +} + +static bool deadline_is_malformed(const struct timespec * timeo) +{ + if (timeo == NULL) + return false; + + if (timeo->tv_sec < 0 || timeo->tv_nsec < 0) + return true; + + return timeo->tv_nsec >= BILLION; +} + +/* + * Complete addr for dst on any backend that can query. The ops are + * collected under poas.lock but called outside it: a query blocks up + * to its deadline and takes the lock again to broadcast. The tables + * are static, so nothing dangles; a struct poa cannot be carried + * across the unlock. The deadline applies per backend. + */ +int poa_query(const char * dst, + const struct timespec * timeo, + struct poa_addr * addr) +{ + const struct poa_ops * cand[POA_MAX_POAS]; + enum poa_type type[POA_MAX_POAS]; + struct list_head * p; + size_t n = 0; + size_t i; + int err = -ENOTSUP; + + if (dst == NULL || addr == NULL) + return -EINVAL; + + if (deadline_is_malformed(timeo)) + return -EINVAL; + + pthread_rwlock_rdlock(&poas.lock); + + list_for_each(p, &poas.list) { + struct poa * poa = list_entry(p, struct poa, next); + + if (poa->ops->poa_query == NULL) + continue; + + for (i = 0; i < n && cand[i] != poa->ops; i++) + ; + if (i < n) + continue; + + /* One type per backend: eth. Revisit if that changes. */ + cand[n] = poa->ops; + type[n++] = poa->type; + } + + pthread_rwlock_unlock(&poas.lock); + + for (i = 0; i < n; i++) { + memset(addr, 0, sizeof(*addr)); + + addr->type = type[i]; + + err = cand[i]->poa_query(dst, timeo, addr); + if (err == 0) + return 0; + } + + return err; +} + +static int poa_check(const struct poa_addr * dst) +{ + struct poa * poa; + int err; + + pthread_rwlock_rdlock(&poas.lock); + + err = poa_lookup(dst, &poa); + + pthread_rwlock_unlock(&poas.lock); + + return err; +} + +static int poa_alloc(const struct poa_addr * dst, + qosspec_t qs, + const buffer_t * req, + buffer_t * resp, + struct poa_flow ** pf_out, + uint32_t * mtu, + const struct timespec * timeo) +{ + struct timespec dflt = TIMESPEC_INIT_MS(POA_ALLOC_TIMEO); + struct timespec rintv = TIMESPEC_INIT_MS(POA_RETRY_TIMEO); + struct poa_flow * pf; + struct poa * poa; + struct timespec abstime; + struct timespec now; + struct timespec retry; + int err; + + pthread_rwlock_wrlock(&poas.lock); + + err = poa_lookup(dst, &poa); + if (err < 0) { + pthread_rwlock_unlock(&poas.lock); + return err; + } + + pf = pf_create(poa, dst); + if (pf == NULL) { + pthread_rwlock_unlock(&poas.lock); + return -ENOMEM; + } + + pf->claimed = true; + + pthread_rwlock_unlock(&poas.lock); + + clock_gettime(PTHREAD_COND_CLOCK, &abstime); + ts_add(&abstime, timeo != NULL ? timeo : &dflt, &abstime); + + pthread_mutex_lock(&pf->mtx); + + while (!pf->replied) { + pthread_mutex_unlock(&pf->mtx); + + err = mgmt_send(poa, dst, POA_FLOW_REQ, pf->eid, 0, qs, 0, req); + + pthread_mutex_lock(&pf->mtx); + + if (err < 0 && err != -ETIMEDOUT && err != -EAGAIN) { + err = -EIO; + goto fail; + } + + if (pf->replied) + break; + + clock_gettime(PTHREAD_COND_CLOCK, &now); + ts_add(&now, &rintv, &retry); + + if (ts_diff_ns(&retry, &abstime) > 0) + retry = abstime; + + pthread_cond_timedwait(&pf->cond, &pf->mtx, &retry); + + if (pf->replied) + break; + + clock_gettime(PTHREAD_COND_CLOCK, &now); + + if (ts_diff_ns(&now, &abstime) >= 0) { + err = -ETIMEDOUT; + goto fail; + } + } + + if (pf->response != 0) { + err = -ECONNREFUSED; + goto fail; + } + + *resp = pf->resp_data; + pf->resp_data.len = 0; + pf->resp_data.data = NULL; + + pthread_mutex_unlock(&pf->mtx); + + *mtu = poa->ops->poa_mtu(poa, dst); + *pf_out = pf; + + return 0; + + fail: + pthread_mutex_unlock(&pf->mtx); + pf_destroy(pf); + return err; +} + +static void poa_alloc_fail(struct poa_flow * pf) +{ + pf_destroy(pf); +} + +void poa_flow_attach(struct poa_flow * pf, + int flow_id, + struct ssm_rbuff * rx_rb) +{ + struct poa * poa = pf->poa; + + if (pf->flow_id != flow_id) + pf_set_flow_id(pf, flow_id); + + pthread_rwlock_wrlock(&poas.lock); + + pf->state = POA_FLOW_ALLOCATED; + + rcu_wrlock(&poa->guard); + + rcu_publish(rx_rb); + rcu_assign(pf->rx_rb, rx_rb); + + rcu_wrunlock(&poa->guard); + + pthread_rwlock_unlock(&poas.lock); +} + +void poa_flow_detach(struct poa_flow * pf) +{ + struct poa * poa = pf->poa; + + pthread_rwlock_wrlock(&poas.lock); + + pf->state = POA_FLOW_DEAD; + + rcu_wrlock(&poa->guard); + + rcu_assign(pf->rx_rb, NULL); + + rcu_wrunlock(&poa->guard); + + pthread_rwlock_unlock(&poas.lock); + + pf_destroy(pf); +} + +static size_t flows_updown(struct poa * poa, + bool up) +{ + struct list_head * p; + size_t n = 0; + + list_for_each(p, &poa->flows) { + struct poa_flow * pf; + struct flow * flow; + + pf = list_entry(p, struct poa_flow, pend); + if (pf->state != POA_FLOW_ALLOCATED || pf->flow_id < 0) + continue; + + flow = &proc.flows[proc.id_to_fd[pf->flow_id].fd]; + if (flow->info.id != pf->flow_id) + continue; + + if (((flow->oflags & FLOWFDOWN) != 0) != !up) + ++n; + + if (up) { + flow->oflags &= ~FLOWFDOWN; + + ssm_rbuff_clr_flags(flow->rx_rb, RB_FLOWDOWN); + } else { + flow->oflags |= FLOWFDOWN; + + ssm_rbuff_set_flags(flow->rx_rb, RB_FLOWDOWN); + } + + ssm_flow_set_notify(proc.fqset, pf->flow_id, + up ? FLOW_UP : FLOW_DOWN); + } + + return n; +} + +size_t poa_link_updown(int id, + bool up) +{ + struct list_head * p; + size_t n = 0; + + pthread_rwlock_wrlock(&proc.lock); + pthread_rwlock_rdlock(&poas.lock); + + list_for_each(p, &poas.list) { + struct poa * poa = list_entry(p, struct poa, next); + + if (poa->ops->poa_link_match == NULL) + continue; + + if (!poa->ops->poa_link_match(poa, id)) + continue; + + n += flows_updown(poa, up); + } + + pthread_rwlock_unlock(&poas.lock); + pthread_rwlock_unlock(&proc.lock); + + return n; +} + +static size_t poa_ewma(size_t avg, + size_t sz) +{ + if (avg == 0) + return sz; + + avg = avg + (sz >> POA_AVG_SHIFT) - (avg >> POA_AVG_SHIFT); + + return avg == 0 ? 1 : avg; +} + +static void poa_avg_len_update(struct poa * poa, + size_t sz) +{ + STORE_RELAXED(&poa->avg_len, poa_ewma(LOAD_RELAXED(&poa->avg_len), sz)); +} + +static size_t poa_qlen(struct poa * poa) +{ + struct timespec now; + uint64_t ns; + size_t qlen; + + clock_gettime(PTHREAD_COND_CLOCK, &now); + + ns = TS_TO_UINT64(now); + if (ns - LOAD_RELAXED(&poa->q_time) < POA_QLEN_GATE) + return LOAD_RELAXED(&poa->q_cache); + + qlen = poa->ops->poa_qlen(poa); + + STORE_RELAXED(&poa->q_cache, qlen); + STORE_RELAXED(&poa->q_time, ns); + + return qlen; +} + +static void poa_cost_sample(struct poa * poa, + size_t before, + size_t len) +{ + size_t after; + + after = poa->ops->poa_qlen(poa); + if (after <= before) + return; /* drained; nothing to learn */ + + after -= before; + if (after < len || after > (len << 2) + 1024) + return; + + STORE_RELAXED(&poa->avg_ovh, + poa_ewma(LOAD_RELAXED(&poa->avg_ovh), after - len)); +} + +int poa_flow_tx(struct poa_flow * pf, + struct ssm_pk_buff * spb, + bool block, + const struct timespec * abstime) +{ + struct poa * poa = pf->poa; + size_t len = ssm_pk_buff_len(spb); + size_t before = 0; + bool sample; + int ret; + + sample = (LOAD_RELAXED(&poa->n_tx) & POA_COST_MASK) == 0; + if (sample) + before = poa->ops->poa_qlen(poa); + + ret = poa->ops->poa_send(poa, &pf->r_addr, pf->r_eid, spb, block, + abstime); + if (ret < 0) { /* the caller releases the buffer */ + POA_STAT_BUMP(poa, snd_fail); + return ret; + } + + POA_STAT_BUMP(poa, tx_pkts); + POA_STAT_ADD(poa, tx_bytes, len); + + FETCH_ADD_RELAXED(&poa->n_tx, 1); + + poa_avg_len_update(poa, len); + + if (sample) + poa_cost_sample(poa, before, len); + + poa_spb_release(spb); + + return 0; +} + +size_t poa_flow_qlen(const struct poa_flow * pf) +{ + struct poa * poa = pf->poa; + uint64_t bytes; + size_t cost; + size_t pkts; + size_t byts; + + if (poa->ops->poa_qpkts != NULL && + poa->ops->poa_qpkts(poa, &pkts, &byts) == 0) + return byts; + + cost = LOAD_RELAXED(&poa->avg_len) + LOAD_RELAXED(&poa->avg_ovh); + if (LOAD_RELAXED(&poa->avg_ovh) == 0 || cost == 0) + return poa_qlen(poa); /* overstated beats false empty */ + + bytes = (uint64_t) poa_qlen(poa) * LOAD_RELAXED(&poa->avg_len); + + return (size_t) (bytes / cost); +} + +size_t poa_flow_qpkts(const struct poa_flow * pf) +{ + struct poa * poa = pf->poa; + size_t cost; + size_t pkts; + size_t byts; + + if (poa->ops->poa_qpkts != NULL && + poa->ops->poa_qpkts(poa, &pkts, &byts) == 0) + return pkts; + + cost = LOAD_RELAXED(&poa->avg_len) + LOAD_RELAXED(&poa->avg_ovh); + if (LOAD_RELAXED(&poa->avg_ovh) == 0 || cost == 0) + return 0; + + return poa_qlen(poa) / cost; +} + +size_t poa_flow_mean_len(const struct poa_flow * pf) +{ + return LOAD_RELAXED(&pf->poa->avg_len); +} + +struct cap_est * poa_flow_cap_est(struct poa_flow * pf) +{ + return &pf->poa->cap; +} + +void poa_flow_ready(struct poa_flow * pf) +{ + buffer_t data; + int answer; + + if (pf == NULL) + return; + + clrbuf(data); + + pthread_rwlock_wrlock(&poas.lock); + + if (!pf->answered || pf->answer_sent) { + pthread_rwlock_unlock(&poas.lock); + return; + } + + answer = pf->answer; + + if (pf->answer_data.len > 0) { + data.data = malloc(pf->answer_data.len); + if (data.data != NULL) { + memcpy(data.data, pf->answer_data.data, + pf->answer_data.len); + data.len = pf->answer_data.len; + } + } + + pf->answer_sent = true; + + pthread_rwlock_unlock(&poas.lock); + + mgmt_send(pf->poa, &pf->r_addr, POA_FLOW_REPLY, pf->eid, + pf->r_eid, qos_raw, answer, &data); + + freebuf(data); +} + +int poa_flow_alloc_resp(int flow_id, + int response, + const buffer_t * data) +{ + struct poa_deferred * d; + struct poa_flow * pf; + + if (flow_id < 0 || flow_id >= SYS_MAX_FLOWS) + return -EPERM; + + pthread_mutex_lock(&poas.mtx); + + pf = poas.id_to_pf[flow_id]; + if (pf != NULL) { + pthread_mutex_unlock(&poas.mtx); + return pf_answer(pf, response, data); + } + + if (poas.deferred.len >= POA_DEFER_MAX) { + pthread_mutex_unlock(&poas.mtx); + return -ENOMEM; + } + + d = malloc(sizeof(*d)); + if (d == NULL) { + pthread_mutex_unlock(&poas.mtx); + return -ENOMEM; + } + + memset(d, 0, sizeof(*d)); + + clock_gettime(PTHREAD_COND_CLOCK, &d->t0); + + d->flow_id = flow_id; + d->response = response; + + if (data != NULL && data->len > 0) { + d->data.data = malloc(data->len); + if (d->data.data == NULL) { + free(d); + pthread_mutex_unlock(&poas.mtx); + return -ENOMEM; + } + memcpy(d->data.data, data->data, data->len); + + d->data.len = data->len; + } + + llist_add_tail(&d->next, &poas.deferred); + + pthread_mutex_unlock(&poas.mtx); + + return 0; +} + +int poa_flow_update(int flow_id, + const buffer_t * data) +{ + struct poa_flow * pf; + + pf = pf_get(flow_id); + if (pf == NULL) + return -EPERM; + + return mgmt_send(pf->poa, &pf->r_addr, POA_FLOW_UPDATE, pf->eid, + pf->r_eid, qos_raw, 0, data); +} + +/* The PoA state is released when the flow itself is torn down. */ +int poa_flow_dealloc(int flow_id) +{ + (void) flow_id; + + return 0; +} + +/* PoA id 0 = management channel. */ +static struct poa * poa_create(enum poa_type type, + const struct poa_ops * ops, + size_t n_eids) +{ + struct poa * poa; + + poa = malloc(sizeof(*poa)); + if (poa == NULL) + goto fail_malloc; + + memset(poa, 0, sizeof(*poa)); + + poa->eid_to_pf = malloc(sizeof(*poa->eid_to_pf) * n_eids); + if (poa->eid_to_pf == NULL) + goto fail_map; + + memset(poa->eid_to_pf, 0, sizeof(*poa->eid_to_pf) * n_eids); + + poa->eids = bmp_create(n_eids - 1, 1); + if (poa->eids == NULL) + goto fail_bmp; + + if (rcu_guard_init(&poa->guard) != 0) + goto fail_guard; + + list_head_init(&poa->next); + list_head_init(&poa->flows); + + poa->type = type; + poa->ops = ops; + poa->mpl = ops->mpl; + poa->n_eids = n_eids; + + return poa; + + fail_guard: + bmp_destroy(poa->eids); + fail_bmp: + free(poa->eid_to_pf); + fail_map: + free(poa); + fail_malloc: + return NULL; +} + +static void poa_destroy(struct poa * poa) +{ + rcu_guard_fini(&poa->guard); + + bmp_destroy(poa->eids); + + free(poa->eid_to_pf); + free(poa); +} + +static void poa_teardown(struct poa * poa) +{ + if (poas.state >= POA_RUNNING) + poa->ops->poa_stop(poa); + + mgmt_frames_purge(poa); + + poa->ops->poa_detach(poa); + + poa_destroy(poa); +} + +static void poa_detach_all(void) +{ + pthread_rwlock_wrlock(&poas.lock); + + while (!list_is_empty(&poas.list)) { + struct poa * poa; + + poa = list_first_entry(&poas.list, struct poa, next); + + list_del(&poa->next); + + pthread_rwlock_unlock(&poas.lock); + + poa_teardown(poa); + + pthread_rwlock_wrlock(&poas.lock); + } + + pthread_rwlock_unlock(&poas.lock); +} + +static int poa_do_detach(const struct poa_detach_req * req) +{ + struct list_head * p; + struct poa * found = NULL; + + pthread_rwlock_wrlock(&poas.lock); + + list_for_each(p, &poas.list) { + struct poa * poa = list_entry(p, struct poa, next); + + if (!poa_has_id(poa, &req->spec)) + continue; + + found = poa; + break; + } + + if (found == NULL) { + pthread_rwlock_unlock(&poas.lock); + return -ENOENT; + } + + if (!list_is_empty(&found->flows)) { + pthread_rwlock_unlock(&poas.lock); + return -EBUSY; + } + + list_del(&found->next); + + pthread_rwlock_unlock(&poas.lock); + + poa_teardown(found); + + return 0; +} + +static int poa_del(const struct poa_spec * spec) +{ + struct poa_detach_req req; + int cs; + int ret; + + memset(&req, 0, sizeof(req)); + + req.spec = *spec; + + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cs); + + if (poas.state < POA_RUNNING) { /* set before workers run */ + ret = poa_do_detach(&req); + goto out; + } + + pthread_mutex_lock(&poas.mgmt_mtx); + + if (poas.mgmt_stop) { /* stopped: poa_fini reaps these */ + pthread_mutex_unlock(&poas.mgmt_mtx); + + ret = -EBUSY; + goto out; + } + + list_add_tail(&req.next, &poas.detach); + + pthread_cond_signal(&poas.mgmt_cond); + + while (!req.done) + pthread_cond_wait(&poas.done_cond, &poas.mgmt_mtx); + + pthread_mutex_unlock(&poas.mgmt_mtx); + + ret = req.result; + out: + pthread_setcancelstate(cs, NULL); + + return ret; +} + +static __inline__ bool mgmt_idle(void) +{ + if (poas.mgmt_stop) + return false; + + if (!llist_is_empty(&poas.mgmt_frames)) + return false; + + return list_is_empty(&poas.detach); +} + +static void detach_run(void) +{ + while (!list_is_empty(&poas.detach)) { + struct poa_detach_req * req; + + req = list_first_entry(&poas.detach, struct poa_detach_req, + next); + list_del(&req->next); + + pthread_mutex_unlock(&poas.mgmt_mtx); + + req->result = poa_do_detach(req); + + pthread_mutex_lock(&poas.mgmt_mtx); + + req->done = true; + + pthread_cond_broadcast(&poas.done_cond); + } +} + +static void * mgmt_handler(void * o) +{ + struct timespec intv = TIMESPEC_INIT_MS(POA_SWEEP_TIMEO); + + (void) o; + + while (true) { + struct poa_mgmt_frame * frame; + struct timespec abstime; + + pthread_mutex_lock(&poas.mgmt_mtx); + + detach_run(); + + while (mgmt_idle()) { + clock_gettime(PTHREAD_COND_CLOCK, &abstime); + ts_add(&abstime, &intv, &abstime); + + if (pthread_cond_timedwait(&poas.mgmt_cond, + &poas.mgmt_mtx, + &abstime) == ETIMEDOUT) { + pthread_mutex_unlock(&poas.mgmt_mtx); + sweep_pending(); + pthread_mutex_lock(&poas.mgmt_mtx); + } + } + + if (poas.mgmt_stop) { + detach_run(); /* nobody else serves these */ + + pthread_mutex_unlock(&poas.mgmt_mtx); + break; + } + + if (llist_is_empty(&poas.mgmt_frames)) { + pthread_mutex_unlock(&poas.mgmt_mtx); + continue; + } + + frame = llist_first_entry(&poas.mgmt_frames, + struct poa_mgmt_frame, next); + llist_del(&frame->next, &poas.mgmt_frames); + + --frame->poa->n_mgmt; + + pthread_mutex_unlock(&poas.mgmt_mtx); + + mgmt_frame_handle(frame); + + free(frame); + } + + return (void *) 0; +} + +#ifndef HAVE_ETH +/* Only the Ethernet transport reports link events. */ +int poa_monitor_open(void) +{ + return -1; +} + +void poa_monitor_read(int fd) +{ + (void) fd; +} +#endif + +static void * poa_monitor(void * o) +{ + (void) o; + + while (true) + poa_monitor_read(poas.mon_fd); + + return (void *) 0; +} + +static int monitor_start(void) +{ + poas.mon_fd = poa_monitor_open(); + if (poas.mon_fd < 0) + return 0; + + if (pthread_create(&poas.monitor, NULL, poa_monitor, NULL) == 0) + return 0; + + close(poas.mon_fd); + + poas.mon_fd = -1; + + return -1; +} + +static void monitor_stop(void) +{ + if (poas.mon_fd < 0) + return; + + pthread_cancel(poas.monitor); + pthread_join(poas.monitor, NULL); + + close(poas.mon_fd); + + poas.mon_fd = -1; +} + +int poa_start(void) +{ + struct list_head * p; + struct list_head * h; + + if (poas.state == POA_NULL) + return 0; + + if (pthread_create(&poas.mgmt_handler, NULL, mgmt_handler, NULL) != 0) + goto fail_mgmt; + + if (monitor_start() < 0) + goto fail_monitor; + + pthread_rwlock_rdlock(&poas.lock); + + list_for_each(p, &poas.list) { + struct poa * poa = list_entry(p, struct poa, next); + + if (poa->ops->poa_start(poa) < 0) + goto fail_reader; + } + + poas.state = POA_RUNNING; + + pthread_rwlock_unlock(&poas.lock); + + return 0; + + fail_reader: + list_for_each(h, &poas.list) { + struct poa * poa = list_entry(h, struct poa, next); + + if (h == p) + break; + + poa->ops->poa_stop(poa); + } + + pthread_rwlock_unlock(&poas.lock); + + monitor_stop(); + fail_monitor: + pthread_mutex_lock(&poas.mgmt_mtx); + + poas.mgmt_stop = true; + + pthread_cond_broadcast(&poas.mgmt_cond); + pthread_mutex_unlock(&poas.mgmt_mtx); + + pthread_join(poas.mgmt_handler, NULL); + fail_mgmt: + return -1; +} + +void poa_stop(void) +{ + struct list_head * p; + + if (poas.state < POA_RUNNING) + return; + + pthread_mutex_lock(&poas.mgmt_mtx); + + poas.mgmt_stop = true; + + pthread_cond_broadcast(&poas.mgmt_cond); + pthread_mutex_unlock(&poas.mgmt_mtx); + + pthread_join(poas.mgmt_handler, NULL); + + monitor_stop(); + + pthread_rwlock_rdlock(&poas.lock); + + list_for_each(p, &poas.list) { + struct poa * poa = list_entry(p, struct poa, next); + + poa->ops->poa_stop(poa); + } + + pthread_rwlock_unlock(&poas.lock); + + poas.state = POA_INIT; +} + +void poa_fini(void) +{ + if (poas.state == POA_NULL) + return; + + poa_stop(); + + poas.state = POA_NULL; + + poa_detach_all(); + +#ifdef PROC_FLOW_STATS + rib_unreg(POA_RIB); +#endif + + pthread_rwlock_destroy(&poas.lock); + pthread_cond_destroy(&poas.done_cond); + pthread_cond_destroy(&poas.mgmt_cond); + pthread_mutex_destroy(&poas.mgmt_mtx); + pthread_cond_destroy(&poas.cond); + pthread_mutex_destroy(&poas.mtx); +} + +static size_t poa_count(void) +{ + struct list_head * p; + size_t n = 0; + + list_for_each(p, &poas.list) + n++; + + return n; +} + +static int poa_add(const struct poa_spec * spec, + const struct poa_ops * ops, + size_t n_eids) +{ + struct list_head * p; + struct poa * poa; + int err; + + pthread_rwlock_rdlock(&poas.lock); + + list_for_each(p, &poas.list) { + poa = list_entry(p, struct poa, next); + if (poa_has_id(poa, spec)) { + pthread_rwlock_unlock(&poas.lock); + return -EPERM; + } + } + + pthread_rwlock_unlock(&poas.lock); + + poa = poa_create(spec->type, ops, n_eids); + if (poa == NULL) + return -ENOMEM; + + err = poa->ops->poa_attach(poa, spec); + if (err < 0) + goto fail_bind; + + err = poa_addr_name(&poa->local, poa->name, sizeof(poa->name)); + if (err < 0) + goto fail_start; + + err = -1; + + pthread_rwlock_wrlock(&poas.lock); + + if (poa_count() >= POA_MAX_POAS) { + pthread_rwlock_unlock(&poas.lock); + goto fail_start; + } + + if (poas.state >= POA_RUNNING && poa->ops->poa_start(poa) < 0) { + pthread_rwlock_unlock(&poas.lock); + goto fail_start; + } + + list_add_tail(&poa->next, &poas.list); + + pthread_rwlock_unlock(&poas.lock); + + return 0; + + fail_start: + poa->ops->poa_detach(poa); + fail_bind: + poa_destroy(poa); + return err; +} + +/* The single place a type is bound to its transport. */ +int poa_attach(const struct poa_spec * poa) +{ + if (poa == NULL) + return -EINVAL; + + switch (poa->type) { + case POA_UDP4: + /* FALLTHRU */ + case POA_UDP6: + return poa_add(poa, &udp_poa_ops, POA_MAX_EIDS); + case POA_ETH: +#ifdef HAVE_ETH + return poa_add(poa, ð_poa_ops, POA_MAX_EIDS); +#else + return -ENOTSUP; +#endif + default: + return -ENOTSUP; + } +} + +int poa_detach(const struct poa_spec * poa) +{ + if (poa == NULL) + return -EINVAL; + + return poa_del(poa); +} + +ssize_t poa_list(struct poa_spec * specs, + size_t max) +{ + struct list_head * p; + size_t n = 0; + + if (specs == NULL) + return -EINVAL; + + pthread_rwlock_rdlock(&poas.lock); + + list_for_each(p, &poas.list) { + struct poa * poa = list_entry(p, struct poa, next); + + if (n++ >= max) + continue; + + memset(specs, 0, sizeof(*specs)); + + poa->ops->poa_spec(poa, specs); + + specs++; + } + + pthread_rwlock_unlock(&poas.lock); + + return (ssize_t) n; +} + +/* + * Complete peer for dst on the backend serving its type. The ops are + * borrowed under poas.lock and called outside it (see poa_query); if + * every PoA of the type detaches in between, the query's broadcast + * reaches nothing and reports -EPERM, as the lookup would. + */ +static int poa_peer_resolve(const char * dst, + struct poa_addr * peer) +{ + const struct poa_ops * ops = NULL; + struct list_head * p; + + pthread_rwlock_rdlock(&poas.lock); + + list_for_each(p, &poas.list) { + struct poa * poa = list_entry(p, struct poa, next); + + if (poa->type == peer->type) { + ops = poa->ops; + break; + } + } + + pthread_rwlock_unlock(&poas.lock); + + if (ops == NULL) /* nothing could carry the flow */ + return -EPERM; + + if (ops->poa_query == NULL) /* these addresses arrive complete */ + return 0; + + return ops->poa_query(dst, NULL, peer); +} + +/* + * Three steps: the IRMd creates the flow and prepares the key exchange, + * the PoA handshakes with the peer, the IRMd completes the + * exchange and hands us the key. + */ +int poa_flow_alloc(const char * dst, + const struct poa_addr * addr, + qosspec_t * qs, + const struct timespec * timeo) +{ + struct flow_info flow; + struct poa_flow * pf; + struct poa_addr peer; + struct crypt_sk crypt; + struct timespec t0; + struct timespec t1; + uint8_t key[SYMMKEYSZ]; + uint8_t buf[SOCK_BUF_SIZE]; + buffer_t msg = {SOCK_BUF_SIZE, buf}; + buffer_t req; + buffer_t resp; + uint32_t mtu = 0; + int err; + + if (addr == NULL) + return -EINVAL; + + if (qs != NULL && qs->service == SVC_STREAM && qs->loss != 0) + return -EINVAL; + + peer = *addr; + + err = poa_peer_resolve(dst, &peer); + if (err < 0) + return err; + + addr = &peer; + + err = poa_check(addr); + if (err < 0) + return err; + + memset(&flow, 0, sizeof(flow)); + + flow.n_pid = getpid(); + flow.n_1_pid = getpid(); + flow.qs = qs == NULL ? qos_raw : *qs; + if (poa_flow_alloc__irm_req_ser(&msg, &flow, dst) < 0) + return -ENOMEM; + + err = send_recv_msg(&msg); + if (err < 0) + return err; + + clrbuf(req); + clrbuf(resp); + + err = poa_flow__irm_result_des(&msg, &flow, &req); + if (err < 0) + return err; + + clock_gettime(PTHREAD_COND_CLOCK, &t0); + + err = poa_alloc(addr, flow.qs, &req, &resp, &pf, &mtu, timeo); + + freebuf(req); + + if (err < 0) + goto fail_alloc; + + clock_gettime(PTHREAD_COND_CLOCK, &t1); + + flow.mtu = mtu; + flow.mpl = pf->poa->mpl; + msg.len = SOCK_BUF_SIZE; + msg.data = buf; + if (poa_flow_alloc_r__irm_req_ser(&msg, &flow, &resp, 0) < 0) { + err = -ENOMEM; + goto fail_resp; + } + + freebuf(resp); + + err = send_recv_msg(&msg); + if (err < 0) + goto fail_msg; + + crypt.key = key; + crypt.epoch = 0; + crypt.role = CRYPT_ROLE_INIT; + + err = flow__irm_result_des(&msg, &flow, &crypt); + if (err < 0) + goto fail_msg; + + err = flow_init(&flow, &crypt, ts_diff_ns(&t1, &t0), pf); + + crypt_secure_clear(key, SYMMKEYSZ); + + if (err < 0) + goto fail_msg; + + if (qs != NULL) + *qs = flow.qs; + + return err; + + fail_resp: + freebuf(resp); + fail_msg: + poa_alloc_fail(pf); + return err; + fail_alloc: + msg.len = SOCK_BUF_SIZE; + msg.data = buf; + if (poa_flow_alloc_r__irm_req_ser(&msg, &flow, NULL, err) == 0) + send_recv_msg(&msg); + + return err; +} + diff --git a/src/lib/poa/poa.h b/src/lib/poa/poa.h new file mode 100644 index 00000000..9edb0335 --- /dev/null +++ b/src/lib/poa/poa.h @@ -0,0 +1,366 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Points of attachment (PoA) - internal API + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * version 2.1 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#ifndef OUROBOROS_LIB_POA_POA_H +#define OUROBOROS_LIB_POA_POA_H + +#include <ouroboros/atomics.h> +#include <ouroboros/ipcp-dev.h> +#include <ouroboros/list.h> +#include <ouroboros/qos.h> +#include <ouroboros/rcu.h> +#include <ouroboros/ssm_flow_set.h> +#include <ouroboros/ssm_pool.h> +#include <ouroboros/ssm_rbuff.h> +#include <ouroboros/time.h> +#include <ouroboros/utils.h> + +#include "../cap.h" + +#include <errno.h> +#include <limits.h> +#include <poll.h> +#include <pthread.h> +#include <stdbool.h> +#include <stdint.h> + +#define POA_MGMT_EID 0 /* reserved for the mgmt channel */ + +#define POA_FLOW_REQ 1 +#define POA_FLOW_REPLY 2 +#define POA_FLOW_UPDATE 3 +#define POA_NAME_QUERY 4 +#define POA_NAME_REPLY 5 + +#define POA_QUERY_HLEN 32 /* SHA3-256, the query hash algorithm */ + +/* Fits "udp6.<ip6>.<port>", the longest display/RIB entry name. */ +#define POA_NAME_STRLEN 63 + +/* Management message; every transport frames it the same way. */ +struct poa_mgmt_msg { + uint8_t code; + uint8_t resv[3]; + uint32_t s_eid; + uint32_t d_eid; + int32_t response; + uint64_t bandwidth; + uint32_t delay; + uint32_t loss; + uint32_t ber; + uint32_t max_gap; + uint32_t timeout; + uint8_t availability; + uint8_t service; + uint16_t data_len; +} __attribute__((packed)); + +struct poa; +struct poa_flow; + +#ifdef PROC_FLOW_STATS +#define POA_STAT_BUMP(poa, field) FETCH_ADD_RELAXED(&(poa)->stat.field, 1) +#define POA_STAT_ADD(poa, field, v) FETCH_ADD_RELAXED(&(poa)->stat.field, (v)) +#define POA_STAT_SUB(poa, field, v) FETCH_SUB_RELAXED(&(poa)->stat.field, (v)) +#define POA_STAT_LOAD(poa, field) LOAD_RELAXED(&(poa)->stat.field) +#else +#define POA_STAT_BUMP(poa, field) ((void) (poa)) +#define POA_STAT_ADD(poa, field, v) ((void) (poa), (void) (v)) +#define POA_STAT_SUB(poa, field, v) ((void) (poa), (void) (v)) +#define POA_STAT_LOAD(poa, field) ((void) (poa), (size_t) 0) +#endif + +struct poa_stat { + size_t n_flows; /* gauge; the RIB reads it without poas.lock */ + size_t rx_pkts; /* packets delivered to a flow */ + size_t rx_bytes; /* payload bytes delivered */ + size_t tx_pkts; /* packets handed to the transport */ + size_t tx_bytes; /* payload bytes handed to the transport */ + size_t mgmt_rx; /* frames queued for the mgmt handler */ + size_t mgmt_tx; /* management frames sent */ + size_t bad_eid; /* no flow on the EID a peer sent */ + size_t dlv_fail; /* the rx ring above would not take it */ + size_t buf_fail; /* no buffer to receive into */ + size_t rcv_fail; /* transport read failed; the reader exits */ + size_t snd_fail; /* transport send failed */ + size_t qry_tx; /* name queries broadcast */ + size_t qry_rx; /* name queries received */ + size_t rep_tx; /* name replies sent, query matched */ + size_t rep_rx; /* name replies received */ +}; + +/* Spacing between transmit-depth samples; a depth costs a syscall. */ +#define POA_QLEN_GATE (100 * 1000) /* ns */ + + +/* Transport operations; public poa_X() dispatches to ops->poa_X. */ +struct poa_ops { + /* Parse own arm of the spec; validate; fill local and priv. */ + int (* poa_attach)(struct poa * poa, + const struct poa_spec * spec); + + void (* poa_detach)(struct poa * poa); + + int (* poa_start)(struct poa * poa); + + void (* poa_stop)(struct poa * poa); + + /* Full queue: -EAGAIN unless block; then wait, to abstime if set. */ + int (* poa_send)(struct poa * poa, + const struct poa_addr * dst, + uint32_t eid, + struct ssm_pk_buff * spb, + bool block, + const struct timespec * abstime); + + int (* poa_send_mgmt)(struct poa * poa, + const struct poa_addr * dst, + const uint8_t * buf, + size_t len); + + int (* poa_query)(const char * dst, + const struct timespec * timeo, + struct poa_addr * addr); + + uint32_t (* poa_mtu)(struct poa * poa, + const struct poa_addr * dst); + + /* Bytes queued in the transmit path of the PoA. */ + size_t (* poa_qlen)(struct poa * poa); + + /* Depth from the queue itself; NULL infers it from qlen. */ + int (* poa_qpkts)(struct poa * poa, + size_t * pkts, + size_t * byts); + + int (* poa_rib)(struct poa * poa, + char * buf, + size_t len); + + /* Identity as a spec, e.g. for poa_list. */ + void (* poa_spec)(const struct poa * poa, + struct poa_spec * spec); + + /* Same identity as spec? Caller matched poa->type already. */ + bool (* poa_has_id)(const struct poa * poa, + const struct poa_spec * spec); + + /* Carries dst? Caller matched poa->type already. */ + bool (* poa_match)(const struct poa * poa, + const struct poa_addr * dst); + + /* + * Flows ride the link this id names; NULL: no link events. + * Ids are meaningful only to the backend whose monitor + * produced them; a single backend owns the monitor. + */ + bool (* poa_link_match)(const struct poa * poa, + int id); + + /* Maximum packet lifetime in the transport, seconds. */ + time_t mpl; +}; +struct poa { + struct list_head next; + + enum poa_type type; + const struct poa_ops * ops; + void * priv; + + struct poa_addr local; /* what peers dial us on */ + + /* Display/RIB entry name for local, e.g. "udp4.<ip>.<port>". */ + char name[POA_NAME_STRLEN + 1]; + + time_t mpl; + + /* Mean sent packet size (bytes), EWMA over the send path. */ + size_t avg_len; + /* Cost of one packet in the queue, in the transport's terms. */ + size_t avg_ovh; + size_t n_tx; + + /* Last queue depth read, and when, in the transport's terms. */ + size_t q_cache; + uint64_t q_time; + + /* Capacity estimator of the queue the flows on this PoA share. */ + struct cap_est cap; + + /* Queued management frames, capped; poas.mgmt_mtx guards. */ + size_t n_mgmt; + +#ifdef PROC_FLOW_STATS + struct poa_stat stat; +#endif + + struct bmp * eids; + struct poa_flow ** eid_to_pf; + size_t n_eids; + + struct list_head flows; /* live flows, for repeats */ + + /* Keeps a flow and its ring alive under the receive path. */ + struct rcu_guard guard; +}; + +/* poa/poa.c is part of the dev.c translation unit. */ +int poa_init(const char * name); + +int poa_start(void); + +void poa_stop(void); + +void poa_fini(void); + +/* Also answer name queries for the layer once enrolled. */ +int poa_set_layer(const char * layer); + +/* Hash of a name this process answers queries for? */ +bool poa_has_name(const uint8_t * hash); + +int poa_flow_tx(struct poa_flow * pf, + struct ssm_pk_buff * spb, + bool block, + const struct timespec * abstime); + +size_t poa_flow_qlen(const struct poa_flow * pf); + +size_t poa_flow_qpkts(const struct poa_flow * pf); + +struct cap_est * poa_flow_cap_est(struct poa_flow * pf); + +size_t poa_flow_mean_len(const struct poa_flow * pf); + +void poa_flow_attach(struct poa_flow * pf, + int flow_id, + struct ssm_rbuff * rx_rb); + +void poa_flow_ready(struct poa_flow * pf); + +void poa_flow_detach(struct poa_flow * pf); + +struct poa_flow * poa_flow_take_pending(int flow_id); + +/* Addresses and management messages (poa/addr.c). */ +int poa_addr_cmp(const struct poa_addr * a, + const struct poa_addr * b); + +/* Display/RIB entry name, e.g. "udp4.<ip>.<port>". */ +int poa_addr_name(const struct poa_addr * a, + char * buf, + size_t len); + +void poa_mgmt_msg_ser(struct poa_mgmt_msg * msg, + uint8_t code, + uint32_t s_eid, + uint32_t d_eid, + qosspec_t qs, + int response, + size_t data_len); + +void poa_mgmt_msg_qos(const struct poa_mgmt_msg * msg, + qosspec_t * qs); + +/* Called by the transports. */ +void poa_rx_pkt(struct poa * poa, + uint32_t eid, + struct ssm_pk_buff * spb); + +void poa_rx_mgmt(struct poa * poa, + const struct poa_addr * src, + const uint8_t * buf, + size_t len); + +/* Reserve a buffer for a received packet, with transport headroom. */ +int poa_spb_reserve(struct ssm_pk_buff ** spb, + size_t len); + +void poa_spb_release(struct ssm_pk_buff * spb); + +/* + * All flows on PoAs whose poa_link_match reports this link id go up + * or down with it. Returns the number of flows whose state changed. + */ +size_t poa_link_updown(int id, + bool up); + +/* + * Link monitor: one socket for the whole subsystem, opened by + * poa_start(). Returns -1 where the transport has no monitor. + */ +int poa_monitor_open(void); + +/* Reads one batch of link events; cancellation point. */ +void poa_monitor_read(int fd); + +/* Broadcast a mgmt frame on every PoA matching dst; # sent. */ +int poa_bcast_mgmt(const struct poa_addr * dst, + const uint8_t * buf, + size_t len); + +/* Transport op tables. */ +extern const struct poa_ops udp_poa_ops; +extern const struct poa_ops eth_poa_ops; + +/* + * Waits for a descriptor to take another packet, up to abstime. + * A NULL deadline waits indefinitely. Transports call this when + * their send reports the transmit queue full. + */ +static __inline__ int poa_wait_out(int fd, + const struct timespec * abstime) +{ + struct pollfd pfd; + struct timespec now; + long ms = -1; + bool clamped = false; + int ret; + + if (abstime != NULL) { + clock_gettime(PTHREAD_COND_CLOCK, &now); + + if (ts_diff_ns(abstime, &now) <= 0) + return -ETIMEDOUT; + + ms = ts_diff_ms(abstime, &now) + 1; /* sub-ms must wait */ + if (ms > INT_MAX) { /* poll takes an int */ + ms = INT_MAX; + clamped = true; + } + } + + pfd.fd = fd; + pfd.events = POLLOUT; + pfd.revents = 0; + + ret = poll(&pfd, 1, (int) ms); + if (ret < 0) + return errno == EINTR ? 0 : -EIO; + + if (ret == 0) + return clamped ? 0 : -ETIMEDOUT; /* clamped: retry */ + + return 0; +} + +#endif /* OUROBOROS_LIB_POA_POA_H */ diff --git a/src/lib/poa/udp.c b/src/lib/poa/udp.c new file mode 100644 index 00000000..6753347a --- /dev/null +++ b/src/lib/poa/udp.c @@ -0,0 +1,633 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Points of attachment (PoA) - UDP transport + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * version 2.1 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#if defined(__APPLE__) +#define _BSD_SOURCE +#define _DARWIN_C_SOURCE +#elif defined(__FreeBSD__) +#define __BSD_VISIBLE 1 +#elif defined(__linux__) || defined(__CYGWIN__) +#ifndef _DEFAULT_SOURCE +#define _DEFAULT_SOURCE +#endif +#else +#ifndef _POSIX_C_SOURCE +#define _POSIX_C_SOURCE 200809L +#endif +#endif + +#include "config.h" + +#define OUROBOROS_PREFIX "poa-udp" + +#include <ouroboros/endian.h> +#include <ouroboros/errno.h> +#include <ouroboros/logs.h> +#include <ouroboros/time.h> +#include <ouroboros/utils.h> + +#include "poa.h" + +#ifdef __linux__ +#include <linux/sockios.h> +#endif +#include <arpa/inet.h> +#include <netinet/in.h> +#include <sys/ioctl.h> +#include <sys/socket.h> +#include <sys/uio.h> + +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +#define UDP_HDR_LEN sizeof(uint32_t) /* PoA id */ +#define UDP_MAX_PAYLOAD (POA_UDP_RD_BUF - UDP_HDR_LEN) +/* The reader buffer must fit a full mgmt frame at any tuning. */ +#define UDP_MAX_PACKET MAX(POA_UDP_RD_BUF, POA_MGMT_FRAME_SIZE + UDP_HDR_LEN) +#define UDP_IP4_OVERH 28U /* IPv4 + UDP */ +#define UDP_IP6_OVERH 48U /* IPv6 + UDP */ +/* Wait for the link to come back before reading it again. */ +#define UDP_DOWN_TIMEO 100 /* ms */ + +union udp_saddr { + struct sockaddr sa; + struct sockaddr_in in; + struct sockaddr_in6 in6; +}; + +struct udp_priv { + int s_fd; + int af; + union udp_saddr s_saddr; + pthread_t reader; + bool running; +}; + +static socklen_t saddr_len(int af) +{ + if (af == AF_INET) + return sizeof(struct sockaddr_in); + + return sizeof(struct sockaddr_in6); +} + +static void addr_to_saddr(const struct poa_addr * addr, + union udp_saddr * saddr) +{ + memset(saddr, 0, sizeof(*saddr)); + + if (addr->type == POA_UDP4) { + saddr->in.sin_family = AF_INET; + saddr->in.sin_addr = addr->udp4.ip_addr; + saddr->in.sin_port = htons(addr->udp4.port); + } else { + saddr->in6.sin6_family = AF_INET6; + saddr->in6.sin6_addr = addr->udp6.ip_addr; + saddr->in6.sin6_port = htons(addr->udp6.port); + } +} + +static void saddr_to_addr(const union udp_saddr * saddr, + struct poa_addr * addr) +{ + memset(addr, 0, sizeof(*addr)); + + if (saddr->sa.sa_family == AF_INET) { + addr->type = POA_UDP4; + addr->udp4.ip_addr = saddr->in.sin_addr; + addr->udp4.port = ntohs(saddr->in.sin_port); + } else { + addr->type = POA_UDP6; + addr->udp6.ip_addr = saddr->in6.sin6_addr; + addr->udp6.port = ntohs(saddr->in6.sin6_port); + } +} + +/* A datagram longer than the buffer arrives truncated: drop it. */ +static void * udp_reader(void * o) +{ + struct poa * poa = (struct poa *) o; + struct udp_priv * priv = (struct udp_priv *) poa->priv; + struct timespec down = TIMESPEC_INIT_MS(UDP_DOWN_TIMEO); + uint8_t * buf; + + buf = malloc(UDP_MAX_PACKET); + if (buf == NULL) + return (void *) -1; + + pthread_cleanup_push(free, buf); + + while (true) { + struct ssm_pk_buff * spb; + union udp_saddr r_saddr; + struct poa_addr src; + struct msghdr mh; + struct iovec iov; + ssize_t n; + uint32_t eid; + size_t plen; + + iov.iov_base = buf; + iov.iov_len = UDP_MAX_PACKET; + + memset(&mh, 0, sizeof(mh)); + + mh.msg_name = &r_saddr; + mh.msg_namelen = sizeof(r_saddr); + mh.msg_iov = &iov; + mh.msg_iovlen = 1; + + n = recvmsg(priv->s_fd, &mh, 0); + if (n < 0) { + if (errno == EINTR) + continue; + + POA_STAT_BUMP(poa, rcv_fail); + + if (errno == ENETDOWN) { + nanosleep(&down, NULL); + continue; + } + + log_err("Reader stopped: %s.", strerror(errno)); + break; + } + + if ((mh.msg_flags & MSG_TRUNC) != 0) { + POA_STAT_BUMP(poa, buf_fail); + continue; + } + + if ((size_t) n < UDP_HDR_LEN) + continue; + + eid = ntoh32(*(uint32_t *) buf); + plen = (size_t) n - UDP_HDR_LEN; + + saddr_to_addr(&r_saddr, &src); + + if (eid == POA_MGMT_EID) { + poa_rx_mgmt(poa, &src, buf + UDP_HDR_LEN, plen); + continue; + } + + if (poa_spb_reserve(&spb, plen) < 0) { + POA_STAT_BUMP(poa, buf_fail); + continue; + } + + memcpy(ssm_pk_buff_head(spb), buf + UDP_HDR_LEN, plen); + + poa_rx_pkt(poa, eid, spb); + } + + pthread_cleanup_pop(true); + + return (void *) 0; +} + +/* Reads the bound address back: an ephemeral port is only known after. */ +static int udp_attach(struct poa * poa, + const struct poa_spec * spec) +{ + struct udp_priv * priv; + socklen_t len; + int af; + + af = spec->type == POA_UDP4 ? AF_INET : AF_INET6; + + priv = malloc(sizeof(*priv)); + if (priv == NULL) + return -ENOMEM; + + memset(priv, 0, sizeof(*priv)); + + priv->af = af; + priv->s_fd = socket(af, SOCK_DGRAM, IPPROTO_UDP); + if (priv->s_fd < 0) { + log_err("Failed to create socket: %s.", strerror(errno)); + goto fail_socket; + } + + if (af == AF_INET) { + priv->s_saddr.in.sin_family = AF_INET; + priv->s_saddr.in.sin_addr = spec->udp4.ip_addr; + priv->s_saddr.in.sin_port = htons(spec->udp4.port); + } else { + int on = 1; + + if (setsockopt(priv->s_fd, IPPROTO_IPV6, IPV6_V6ONLY, + &on, sizeof(on)) < 0) { + log_err("Failed to set IPV6_V6ONLY: %s.", + strerror(errno)); + goto fail_bind; + } + + priv->s_saddr.in6.sin6_family = AF_INET6; + priv->s_saddr.in6.sin6_addr = spec->udp6.ip_addr; + priv->s_saddr.in6.sin6_port = htons(spec->udp6.port); + } + + if (bind(priv->s_fd, &priv->s_saddr.sa, saddr_len(af)) < 0) { + log_err("Failed to bind: %s.", strerror(errno)); + goto fail_bind; + } + + poa->priv = priv; + len = saddr_len(af); + if (getsockname(priv->s_fd, &priv->s_saddr.sa, &len) < 0) + log_warn("Failed to read the bound address: %s.", + strerror(errno)); + + saddr_to_addr(&priv->s_saddr, &poa->local); + + return 0; + + fail_bind: + close(priv->s_fd); + fail_socket: + poa->priv = NULL; + + free(priv); + + return -EIO; +} + +static void udp_detach(struct poa * poa) +{ + struct udp_priv * priv = (struct udp_priv *) poa->priv; + + if (priv == NULL) + return; + + close(priv->s_fd); + + free(priv); + + poa->priv = NULL; +} + +/* One reader per socket, so a flow cannot be reordered on receive. */ +static int udp_start(struct poa * poa) +{ + struct udp_priv * priv = (struct udp_priv *) poa->priv; + + if (pthread_create(&priv->reader, NULL, udp_reader, poa) != 0) + return -1; + + priv->running = true; + + return 0; +} + +static void udp_stop(struct poa * poa) +{ + struct udp_priv * priv = (struct udp_priv *) poa->priv; + + if (!priv->running) + return; + + pthread_cancel(priv->reader); + pthread_join(priv->reader, NULL); + + priv->running = false; +} + +/* + * The PoA id is a separate iovec, so the buffer needs no headroom. + * MSG_DONTWAIT: the reader blocks on this socket. + */ +static int udp_sendv(struct udp_priv * priv, + const struct poa_addr * dst, + uint32_t eid, + const uint8_t * body, + size_t len, + bool block, + const struct timespec * abstime) +{ + union udp_saddr saddr; + struct msghdr msg; + struct iovec iov[2]; + uint32_t hdr; + int ret; + + if (len > UDP_MAX_PAYLOAD) + return -EMSGSIZE; + + addr_to_saddr(dst, &saddr); + + hdr = hton32(eid); + + iov[0].iov_base = &hdr; + iov[0].iov_len = sizeof(hdr); + iov[1].iov_base = (void *) body; + iov[1].iov_len = len; + + memset(&msg, 0, sizeof(msg)); + + msg.msg_name = &saddr; + msg.msg_namelen = saddr_len(priv->af); + msg.msg_iov = iov; + msg.msg_iovlen = len > 0 ? 2 : 1; + while (sendmsg(priv->s_fd, &msg, MSG_DONTWAIT) < 0) { + if (errno != EAGAIN && errno != EWOULDBLOCK) + return -EIO; + + if (!block) + return -EAGAIN; + + ret = poa_wait_out(priv->s_fd, abstime); + if (ret < 0) + return ret; + } + + return 0; +} + +static int udp_send(struct poa * poa, + const struct poa_addr * dst, + uint32_t eid, + struct ssm_pk_buff * spb, + bool block, + const struct timespec * abstime) +{ + return udp_sendv((struct udp_priv *) poa->priv, dst, eid, + ssm_pk_buff_head(spb), ssm_pk_buff_len(spb), + block, abstime); +} + +static int udp_send_mgmt(struct poa * poa, + const struct poa_addr * dst, + const uint8_t * buf, + size_t len) +{ + struct timespec timeo = TIMESPEC_INIT_MS(POA_MGMT_SND_TIMEO); + struct timespec abstime; + + clock_gettime(PTHREAD_COND_CLOCK, &abstime); + ts_add(&abstime, &timeo, &abstime); + + return udp_sendv((struct udp_priv *) poa->priv, dst, POA_MGMT_EID, + buf, len, true, &abstime); +} + +/* The PoA id header eats into the usable MTU. */ +static uint32_t udp_mtu(struct poa * poa, + const struct poa_addr * dst) +{ + struct udp_priv * priv = (struct udp_priv *) poa->priv; + uint32_t fallback; + uint32_t overh; +#if defined(__linux__) && (defined(IP_MTU) || defined(IPV6_MTU)) + union udp_saddr saddr; + socklen_t len; + int sock; + int mtu = 0; +#endif + if (priv->af == AF_INET) { + fallback = POA_UDP4_MTU; + overh = UDP_IP4_OVERH; + } else { + fallback = POA_UDP6_MTU; + overh = UDP_IP6_OVERH; + } + + fallback -= UDP_HDR_LEN; + if (fallback > UDP_MAX_PAYLOAD) + fallback = UDP_MAX_PAYLOAD; + +#if defined(__linux__) && (defined(IP_MTU) || defined(IPV6_MTU)) + + addr_to_saddr(dst, &saddr); + + sock = socket(priv->af, SOCK_DGRAM, IPPROTO_UDP); + if (sock < 0) + return fallback; + + if (connect(sock, &saddr.sa, saddr_len(priv->af)) < 0) + goto fallback; + + len = sizeof(mtu); + +#if defined(IP_MTU) + if (priv->af == AF_INET) { + if (getsockopt(sock, IPPROTO_IP, IP_MTU, &mtu, &len) < 0) + goto fallback; + } +#endif +#if defined(IPV6_MTU) + if (priv->af == AF_INET6) { + if (getsockopt(sock, IPPROTO_IPV6, IPV6_MTU, &mtu, &len) < 0) + goto fallback; + } +#endif + close(sock); + + if (mtu <= (int) (overh + UDP_HDR_LEN)) + return fallback; + + return MIN((uint32_t) mtu - overh - UDP_HDR_LEN, UDP_MAX_PAYLOAD); + + fallback: + close(sock); +#else + (void) dst; + (void) overh; +#endif + return fallback; +} + +/* All flows on the PoA share the socket, so this is aggregate. */ +static size_t udp_qlen(struct poa * poa) +{ +#if defined(__linux__) && defined(SIOCOUTQ) + struct udp_priv * priv = (struct udp_priv *) poa->priv; + int qlen; + + qlen = 0; + if (ioctl(priv->s_fd, SIOCOUTQ, &qlen) < 0) + return 0; + + return (size_t) qlen; +#else + (void) poa; + + return 0; +#endif +} + +/* The kernel keeps no per-socket drop count for UDP. */ +static int udp_rib(struct poa * poa, + char * buf, + size_t len) +{ + struct udp_priv * priv = (struct udp_priv *) poa->priv; + socklen_t optlen; + size_t sndbuf = 0; + size_t rcvbuf = 0; + int val; + int size; + + optlen = sizeof(val); + if (getsockopt(priv->s_fd, SOL_SOCKET, SO_SNDBUF, &val, &optlen) == 0) + sndbuf = (size_t) val; + + optlen = sizeof(val); + if (getsockopt(priv->s_fd, SOL_SOCKET, SO_RCVBUF, &val, &optlen) == 0) + rcvbuf = (size_t) val; + + size = snprintf(buf, len, + "Socket sndbuf (bytes): %zu\n" + "Socket rcvbuf (bytes): %zu\n", + sndbuf, rcvbuf); + if (size < 0 || (size_t) size >= len) + return -1; + + return size; +} + +/* + * Asks the kernel which address it would send from: connect() does the + * real route lookup and sends nothing, so this honours the default + * route, metrics and policy rules alike. + */ +static int udp_src_addr(const struct poa_addr * dst, + struct poa_addr * src) +{ + union udp_saddr saddr; + socklen_t len; + int af; + int fd; + + if (dst->type != POA_UDP4 && dst->type != POA_UDP6) + return -EINVAL; + + af = dst->type == POA_UDP4 ? AF_INET : AF_INET6; + + addr_to_saddr(dst, &saddr); + + fd = socket(af, SOCK_DGRAM, IPPROTO_UDP); + if (fd < 0) + return -EIO; + + if (connect(fd, &saddr.sa, saddr_len(af)) < 0) + goto fail; + + len = saddr_len(af); + if (getsockname(fd, &saddr.sa, &len) < 0) + goto fail; + + close(fd); + + saddr_to_addr(&saddr, src); + + return 0; + + fail: + close(fd); + + return -EIO; +} + +static void udp_spec(const struct poa * poa, + struct poa_spec * spec) +{ + spec->type = poa->type; + + if (poa->type == POA_UDP4) + spec->udp4 = poa->local.udp4; + else + spec->udp6 = poa->local.udp6; +} + +static bool udp_has_id(const struct poa * poa, + const struct poa_spec * spec) +{ + if (poa->type == POA_UDP4) { + if (poa->local.udp4.port != spec->udp4.port) + return false; + + return memcmp(&poa->local.udp4.ip_addr, + &spec->udp4.ip_addr, + sizeof(spec->udp4.ip_addr)) == 0; + } + + if (poa->local.udp6.port != spec->udp6.port) + return false; + + return memcmp(&poa->local.udp6.ip_addr, &spec->udp6.ip_addr, + sizeof(spec->udp6.ip_addr)) == 0; +} + +static bool udp_addr_is_any(const struct poa_addr * addr) +{ + static const struct in6_addr any6 = IN6ADDR_ANY_INIT; + + if (addr->type == POA_UDP4) + return addr->udp4.ip_addr.s_addr == htonl(INADDR_ANY); + + return memcmp(&addr->udp6.ip_addr, &any6, sizeof(any6)) == 0; +} + +/* + * Our end of the flow: the IP the kernel would send to dst from. A + * PoA bound to the wildcard is the catch-all and a failed probe + * matches any. Ports are not compared: the probe's is ephemeral. + */ +static bool udp_match(const struct poa * poa, + const struct poa_addr * dst) +{ + struct poa_addr src; + + if (udp_addr_is_any(&poa->local)) + return true; + + if (udp_src_addr(dst, &src) < 0) + return true; + + if (poa->type == POA_UDP4) + return memcmp(&poa->local.udp4.ip_addr, + &src.udp4.ip_addr, + sizeof(src.udp4.ip_addr)) == 0; + + return memcmp(&poa->local.udp6.ip_addr, &src.udp6.ip_addr, + sizeof(src.udp6.ip_addr)) == 0; +} + +const struct poa_ops udp_poa_ops = { + .poa_attach = udp_attach, + .poa_detach = udp_detach, + .poa_start = udp_start, + .poa_stop = udp_stop, + .poa_send = udp_send, + .poa_send_mgmt = udp_send_mgmt, + .poa_mtu = udp_mtu, + .poa_qlen = udp_qlen, + .poa_rib = udp_rib, + .poa_spec = udp_spec, + .poa_has_id = udp_has_id, + .poa_match = udp_match, + .mpl = POA_UDP_MPL +}; diff --git a/src/lib/protobuf.c b/src/lib/protobuf.c index a824d357..6eec1159 100644 --- a/src/lib/protobuf.c +++ b/src/lib/protobuf.c @@ -22,6 +22,7 @@ #define _DEFAULT_SOURCE +#include <ouroboros/errno.h> #include <ouroboros/protobuf.h> #include <ouroboros/crypt.h> #include <ouroboros/proc.h> @@ -82,6 +83,7 @@ flow_info_msg_t * flow_info_s_to_msg(const struct flow_info * s) msg->state = s->state; msg->uid = s->uid; msg->mtu = s->mtu; + msg->max_rtt = s->max_rtt; msg->qos = qos_spec_s_to_msg(&s->qs); if (msg->qos == NULL) goto fail_msg; @@ -109,6 +111,7 @@ struct flow_info flow_info_msg_to_s(const flow_info_msg_t * msg) s.state = msg->state; s.uid = msg->uid; s.mtu = msg->mtu; + s.max_rtt = msg->max_rtt; s.qs = qos_spec_msg_to_s(msg->qos); return s; @@ -215,6 +218,166 @@ struct layer_info layer_info_msg_to_s(const layer_info_msg_t * msg) return s; } + +static bool mac_is_zero(const uint8_t * mac) +{ + static const uint8_t zero[POA_MAC_SIZE] = { 0 }; + + return memcmp(mac, zero, POA_MAC_SIZE) == 0; +} + +poa_addr_msg_t * poa_addr_s_to_msg(const struct poa_addr * s) +{ + poa_addr_msg_t * msg; + + assert(s != NULL); + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + goto fail_malloc; + + poa_addr_msg__init(msg); + + msg->type = s->type; + + if (s->hostname[0] != '\0') { + msg->hostname = strdup(s->hostname); + if (msg->hostname == NULL) + goto fail_msg; + } + + switch (s->type) { + case POA_UDP4: + msg->has_ip4 = true; + msg->ip4 = s->udp4.ip_addr.s_addr; + msg->has_port = true; + msg->port = s->udp4.port; + break; + case POA_UDP6: + msg->has_ip6 = true; + msg->ip6.len = sizeof(s->udp6.ip_addr); + msg->ip6.data = malloc(msg->ip6.len); + if (msg->ip6.data == NULL) + goto fail_msg; + memcpy(msg->ip6.data, &s->udp6.ip_addr, msg->ip6.len); + + msg->has_port = true; + msg->port = s->udp6.port; + break; + case POA_ETH: + msg->has_ethertype = true; + msg->ethertype = s->eth.dst.ethertype; + msg->has_mac = true; + msg->mac.len = POA_MAC_SIZE; + msg->mac.data = malloc(POA_MAC_SIZE); + if (msg->mac.data == NULL) + goto fail_msg; + memcpy(msg->mac.data, s->eth.dst.mac, POA_MAC_SIZE); + + if (!mac_is_zero(s->eth.src.mac)) { + msg->has_src_mac = true; + msg->src_mac.len = POA_MAC_SIZE; + msg->src_mac.data = malloc(POA_MAC_SIZE); + if (msg->src_mac.data == NULL) + goto fail_msg; + memcpy(msg->src_mac.data, s->eth.src.mac, POA_MAC_SIZE); + } + + if (s->eth.src.dev[0] != '\0') { + msg->dev = strdup(s->eth.src.dev); + if (msg->dev == NULL) + goto fail_msg; + } + break; + case POA_UDP: + msg->has_port = true; + msg->port = s->udp4.port; + break; + default: + goto fail_msg; + } + + return msg; + + fail_msg: + poa_addr_msg__free_unpacked(msg, NULL); + fail_malloc: + return NULL; +} + +struct poa_addr poa_addr_msg_to_s(const poa_addr_msg_t * msg) +{ + struct poa_addr s; + + assert(msg != NULL); + + memset(&s, 0, sizeof(s)); + + s.type = POA_INVALID; + + if (msg->hostname != NULL) { + if (strlen(msg->hostname) > POA_HOST_STRLEN) + return s; + strcpy(s.hostname, msg->hostname); + } + + switch (msg->type) { + case POA_UDP4: + if (!msg->has_ip4 || !msg->has_port) + return s; + + s.udp4.ip_addr.s_addr = msg->ip4; + s.udp4.port = msg->port; + break; + case POA_UDP6: + if (!msg->has_ip6 || !msg->has_port) + return s; + + if (msg->ip6.len != sizeof(s.udp6.ip_addr) + || msg->ip6.data == NULL) + return s; + + memcpy(&s.udp6.ip_addr, msg->ip6.data, msg->ip6.len); + + s.udp6.port = msg->port; + break; + case POA_ETH: + if (!msg->has_ethertype) + return s; + + s.eth.src.ethertype = msg->ethertype; + s.eth.dst.ethertype = msg->ethertype; + + if (msg->mac.len != POA_MAC_SIZE || msg->mac.data == NULL) + return s; + + memcpy(s.eth.dst.mac, msg->mac.data, POA_MAC_SIZE); + + if (msg->src_mac.len == POA_MAC_SIZE) + memcpy(s.eth.src.mac, msg->src_mac.data, POA_MAC_SIZE); + + if (msg->dev != NULL) { + if (strlen(msg->dev) > DEV_NAME_SIZE) + return s; + strcpy(s.eth.src.dev, msg->dev); + } + break; + case POA_UDP: + if (msg->hostname == NULL || msg->hostname[0] == '\0' + || !msg->has_port) + return s; + + s.udp4.port = msg->port; + break; + default: + return s; + } + + s.type = msg->type; + + return s; +} + ipcp_info_msg_t * ipcp_info_s_to_msg(const struct ipcp_info * s) { ipcp_info_msg_t * msg; @@ -362,6 +525,8 @@ dt_config_msg_t * dt_config_s_to_msg(const struct dt_config * s) msg->addr_size = s->addr_size; msg->eid_size = s->eid_size; msg->max_ttl = s->max_ttl; + msg->has_max_rtt = true; + msg->max_rtt = s->max_rtt; msg->routing = routing_config_s_to_msg(&s->routing); if (msg->routing == NULL) goto fail_routing; @@ -381,27 +546,12 @@ struct dt_config dt_config_msg_to_s(const dt_config_msg_t * msg) s.addr_size = msg->addr_size; s.eid_size = msg->eid_size; s.max_ttl = msg->max_ttl; + s.max_rtt = msg->has_max_rtt ? msg->max_rtt : 200; /* ms */ s.routing = routing_config_msg_to_s(msg->routing); return s; } -struct dir_dht_config dir_dht_config_msg_to_s(const dir_dht_config_msg_t * msg) -{ - struct dir_dht_config s; - - assert(msg != NULL); - - s.params.alpha = msg->alpha; - s.params.k = msg->k; - s.params.t_expire = msg->t_expire; - s.params.t_refresh = msg->t_refresh; - s.params.t_replicate = msg->t_replicate; - s.peer = msg->peer; - - return s; -} - dir_dht_config_msg_t * dir_dht_config_s_to_msg(const struct dir_dht_config * s) { dir_dht_config_msg_t * msg; @@ -424,22 +574,18 @@ dir_dht_config_msg_t * dir_dht_config_s_to_msg(const struct dir_dht_config * s) return msg; } -struct dir_config dir_config_msg_to_s(const dir_config_msg_t * msg) +struct dir_dht_config dir_dht_config_msg_to_s(const dir_dht_config_msg_t * msg) { - struct dir_config s; + struct dir_dht_config s; assert(msg != NULL); - switch (msg->pol) { - case DIR_DHT: - s.dht = dir_dht_config_msg_to_s(msg->dht); - break; - default: - /* No checks here */ - break; - } - - s.pol = msg->pol; + s.params.alpha = msg->alpha; + s.params.k = msg->k; + s.params.t_expire = msg->t_expire; + s.params.t_refresh = msg->t_refresh; + s.params.t_replicate = msg->t_replicate; + s.peer = msg->peer; return s; } @@ -476,6 +622,25 @@ dir_config_msg_t * dir_config_s_to_msg(const struct dir_config * s) return NULL; } +struct dir_config dir_config_msg_to_s(const dir_config_msg_t * msg) +{ + struct dir_config s; + + assert(msg != NULL); + + switch (msg->pol) { + case DIR_DHT: + s.dht = dir_dht_config_msg_to_s(msg->dht); + break; + default: + /* No checks here */ + break; + } + + s.pol = msg->pol; + + return s; +} uni_config_msg_t * uni_config_s_to_msg(const struct uni_config * s) { @@ -522,9 +687,9 @@ struct uni_config uni_config_msg_to_s(const uni_config_msg_t * msg) return s; } -udp4_config_msg_t * udp4_config_s_to_msg(const struct udp4_config * s) +udp4_poa_msg_t * udp4_poa_s_to_msg(const struct udp4_poa * s) { - udp4_config_msg_t * msg; + udp4_poa_msg_t * msg; assert(s != NULL); @@ -532,32 +697,30 @@ udp4_config_msg_t * udp4_config_s_to_msg(const struct udp4_config * s) if (msg == NULL) return NULL; - udp4_config_msg__init(msg); + udp4_poa_msg__init(msg); - msg->ip_addr = s->ip_addr.s_addr; - msg->dns_addr = s->dns_addr.s_addr; - msg->port = s->port; + msg->ip_addr = s->ip_addr.s_addr; + msg->port = s->port; return msg; } -struct udp4_config udp4_config_msg_to_s(const udp4_config_msg_t * msg) +struct udp4_poa udp4_poa_msg_to_s(const udp4_poa_msg_t * msg) { - struct udp4_config s; + struct udp4_poa s; assert(msg != NULL); - s.ip_addr.s_addr = msg->ip_addr; - s.dns_addr.s_addr = msg->dns_addr; - s.port = msg->port; + s.ip_addr.s_addr = msg->ip_addr; + s.port = msg->port; return s; } #define IN6_LEN (size_t) sizeof(struct in6_addr) -udp6_config_msg_t * udp6_config_s_to_msg(const struct udp6_config * s) +udp6_poa_msg_t * udp6_poa_s_to_msg(const struct udp6_poa * s) { - udp6_config_msg_t * msg; + udp6_poa_msg_t * msg; assert(s != NULL); @@ -565,7 +728,7 @@ udp6_config_msg_t * udp6_config_s_to_msg(const struct udp6_config * s) if (msg == NULL) goto fail_malloc; - udp6_config_msg__init(msg); + udp6_poa_msg__init(msg); msg->ip_addr.data = malloc(IN6_LEN); if (msg->ip_addr.data == NULL) @@ -574,42 +737,33 @@ udp6_config_msg_t * udp6_config_s_to_msg(const struct udp6_config * s) msg->ip_addr.len = IN6_LEN; memcpy(msg->ip_addr.data, &s->ip_addr.s6_addr, IN6_LEN); - msg->dns_addr.data = malloc(IN6_LEN); - if (msg->dns_addr.data == NULL) - goto fail_msg; - - msg->dns_addr.len = IN6_LEN; - memcpy(msg->dns_addr.data, &s->dns_addr.s6_addr, IN6_LEN); - msg->port = s->port; return msg; fail_msg: - udp6_config_msg__free_unpacked(msg, NULL); + udp6_poa_msg__free_unpacked(msg, NULL); fail_malloc: return NULL; } -struct udp6_config udp6_config_msg_to_s(const udp6_config_msg_t * msg) +struct udp6_poa udp6_poa_msg_to_s(const udp6_poa_msg_t * msg) { - struct udp6_config s; + struct udp6_poa s; assert(msg != NULL); assert(msg->ip_addr.len == IN6_LEN); - assert(msg->dns_addr.len == IN6_LEN); memcpy(&s.ip_addr.s6_addr, msg->ip_addr.data, IN6_LEN); - memcpy(&s.dns_addr.s6_addr, msg->dns_addr.data, IN6_LEN); s.port = msg->port; return s; } -eth_config_msg_t * eth_config_s_to_msg(const struct eth_config * s) +eth_poa_msg_t * eth_poa_s_to_msg(const struct eth_poa * s) { - eth_config_msg_t * msg; + eth_poa_msg_t * msg; assert(s != NULL); @@ -617,7 +771,7 @@ eth_config_msg_t * eth_config_s_to_msg(const struct eth_config * s) if (msg == NULL) goto fail_malloc; - eth_config_msg__init(msg); + eth_poa_msg__init(msg); msg->dev = strdup(s->dev); if (msg->dev == NULL) @@ -625,28 +779,115 @@ eth_config_msg_t * eth_config_s_to_msg(const struct eth_config * s) msg->ethertype = s->ethertype; + msg->has_mac = true; + msg->mac.len = POA_MAC_SIZE; + msg->mac.data = malloc(POA_MAC_SIZE); + if (msg->mac.data == NULL) + goto fail_msg; + + memcpy(msg->mac.data, s->mac, POA_MAC_SIZE); + return msg; fail_msg: - eth_config_msg__free_unpacked(msg, NULL); + eth_poa_msg__free_unpacked(msg, NULL); fail_malloc: return NULL; } -struct eth_config eth_config_msg_to_s(const eth_config_msg_t * msg) +struct eth_poa eth_poa_msg_to_s(const eth_poa_msg_t * msg) { - struct eth_config s; + struct eth_poa s; assert(msg != NULL); assert(strlen(msg->dev) <= DEV_NAME_SIZE); + memset(&s, 0, sizeof(s)); + strcpy(s.dev, msg->dev); + s.ethertype = msg->ethertype; + if (msg->mac.len == POA_MAC_SIZE) + memcpy(s.mac, msg->mac.data, POA_MAC_SIZE); + return s; } +poa_spec_msg_t * poa_spec_s_to_msg(const struct poa_spec * s) +{ + poa_spec_msg_t * msg; + + assert(s != NULL); + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + goto fail_malloc; + + poa_spec_msg__init(msg); + + switch (s->type) { + case POA_UDP4: + msg->udp4 = udp4_poa_s_to_msg(&s->udp4); + if (msg->udp4 == NULL) + goto fail_msg; + break; + case POA_UDP6: + msg->udp6 = udp6_poa_s_to_msg(&s->udp6); + if (msg->udp6 == NULL) + goto fail_msg; + break; + case POA_ETH: + msg->eth = eth_poa_s_to_msg(&s->eth); + if (msg->eth == NULL) + goto fail_msg; + break; + default: + goto fail_msg; + } + + return msg; + + fail_msg: + poa_spec_msg__free_unpacked(msg, NULL); + fail_malloc: + return NULL; +} + +struct poa_spec poa_spec_msg_to_s(const poa_spec_msg_t * msg) +{ + struct poa_spec s; + + memset(&s, 0, sizeof(s)); + + s.type = POA_INVALID; + + if (msg == NULL) + return s; + + if (msg->udp4 != NULL) { + s.type = POA_UDP4; + s.udp4 = udp4_poa_msg_to_s(msg->udp4); + } else if (msg->udp6 != NULL) { + if (msg->udp6->ip_addr.len != IN6_LEN + || msg->udp6->ip_addr.data == NULL) + return s; + + s.type = POA_UDP6; + s.udp6 = udp6_poa_msg_to_s(msg->udp6); + } else if (msg->eth != NULL) { + if (msg->eth->dev == NULL + || strlen(msg->eth->dev) > DEV_NAME_SIZE) + return s; + + s.type = POA_ETH; + s.eth = eth_poa_msg_to_s(msg->eth); + } + + return s; +} + ipcp_config_msg_t * ipcp_config_s_to_msg(const struct ipcp_config * s) { ipcp_config_msg_t * msg; @@ -669,23 +910,6 @@ ipcp_config_msg_t * ipcp_config_s_to_msg(const struct ipcp_config * s) break; case IPCP_BROADCAST: break; - case IPCP_ETH_LLC: - /* FALLTHRU */ - case IPCP_ETH_DIX: - msg->eth = eth_config_s_to_msg(&s->eth); - if (msg->eth == NULL) - goto fail_msg; - break; - case IPCP_UDP4: - msg->udp4 = udp4_config_s_to_msg(&s->udp4); - if (msg->udp4 == NULL) - goto fail_msg; - break; - case IPCP_UDP6: - msg->udp6 = udp6_config_s_to_msg(&s->udp6); - if (msg->udp6 == NULL) - goto fail_msg; - break; default: /* No checks here */ break; @@ -711,6 +935,8 @@ struct ipcp_config ipcp_config_msg_to_s(const ipcp_config_msg_t * msg) assert(msg != NULL); + memset(&s, 0, sizeof(s)); + s.type = msg->ipcp_type; s.layer_info = layer_info_msg_to_s(msg->layer_info); @@ -721,17 +947,6 @@ struct ipcp_config ipcp_config_msg_to_s(const ipcp_config_msg_t * msg) case IPCP_UNICAST: s.unicast = uni_config_msg_to_s(msg->unicast); break; - case IPCP_ETH_LLC: - /* FALLTHRU */ - case IPCP_ETH_DIX: - s.eth = eth_config_msg_to_s(msg->eth); - break; - case IPCP_UDP4: - s.udp4 = udp4_config_msg_to_s(msg->udp4); - break; - case IPCP_UDP6: - s.udp6 = udp6_config_msg_to_s(msg->udp6); - break; case IPCP_BROADCAST: break; default: diff --git a/src/lib/random.c b/src/lib/random.c index 2c9a6c0d..a132f470 100644 --- a/src/lib/random.c +++ b/src/lib/random.c @@ -28,6 +28,8 @@ #include <stdlib.h> #elif defined(HAVE_SYS_RANDOM) #include <sys/random.h> +#include <errno.h> +#include <stdint.h> #elif defined(HAVE_LIBGCRYPT) #include <gcrypt.h> #elif defined(HAVE_OPENSSL_RNG) @@ -42,7 +44,21 @@ int random_buffer(void * buf, arc4random_buf(buf, len); return 0; #elif defined(HAVE_SYS_RANDOM) - return getrandom(buf, len, GRND_NONBLOCK); + size_t off = 0; + ssize_t ret; + + while (off < len) { + ret = getrandom((uint8_t *) buf + off, len - off, + GRND_NONBLOCK); + if (ret < 0) { + if (errno == EINTR) + continue; + return -1; + } + off += (size_t) ret; + } + + return 0; #elif defined(HAVE_LIBGCRYPT) gcry_randomize(buf, len, GCRY_STRONG_RANDOM); return 0; diff --git a/src/lib/serdes-irm.c b/src/lib/serdes-irm.c index 65f2c02d..20b25a1b 100644 --- a/src/lib/serdes-irm.c +++ b/src/lib/serdes-irm.c @@ -174,6 +174,54 @@ int flow__irm_result_des(buffer_t * buf, else memset(sk->key, 0, SYMMKEYSZ); + sk->epoch = msg->has_generation ? (uint8_t) msg->generation : 0; + + if (msg->sym_key.len == SYMMKEYSZ) + crypt_secure_clear(msg->sym_key.data, msg->sym_key.len); + + irm_msg__free_unpacked(msg, NULL); + + return 0; + fail: + irm_msg__free_unpacked(msg, NULL); + fail_msg: + return err; +} + +int flow_rekey__irm_result_des(buffer_t * buf, + struct crypt_sk * sk, + bool * has_key, + bool * initiator) +{ + irm_msg_t * msg; + int err; + + msg = irm_msg__unpack(NULL, buf->len, buf->data); + if (msg == NULL) { + err = -EIRMD; + goto fail_msg; + } + + if (!msg->has_result) { + err = -EIRMD; + goto fail; + } + + if (msg->result < 0) { + err = msg->result; + goto fail; + } + + *has_key = msg->has_sym_key && msg->sym_key.len == SYMMKEYSZ; + if (*has_key) { + memcpy(sk->key, msg->sym_key.data, SYMMKEYSZ); + sk->nid = NID_undef; + sk->epoch = msg->has_generation ? + (uint8_t) msg->generation : 0; + *initiator = msg->has_rk_initiator && msg->rk_initiator; + crypt_secure_clear(msg->sym_key.data, msg->sym_key.len); + } + irm_msg__free_unpacked(msg, NULL); return 0; @@ -222,6 +270,44 @@ int flow_dealloc__irm_req_ser(buffer_t * buf, return -ENOMEM; } +int flow_update__irm_req_ser(buffer_t * buf, + const struct flow_info * flow, + bool rekey) +{ + irm_msg_t * msg; + size_t len; + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + goto fail_malloc; + + irm_msg__init(msg); + + msg->code = IRM_MSG_CODE__IRM_FLOW_UPDATE; + msg->flow_info = flow_info_s_to_msg(flow); + if (msg->flow_info == NULL) + goto fail_msg; + + msg->has_rekey = true; + msg->rekey = rekey; + + len = irm_msg__get_packed_size(msg); + if (len == 0 || len > buf->len) + goto fail_msg; + + buf->len = len; + + irm_msg__pack(msg, buf->data); + irm_msg__free_unpacked(msg, NULL); + + return 0; + + fail_msg: + irm_msg__free_unpacked(msg, NULL); + fail_malloc: + return -ENOMEM; +} + int ipcp_flow_dealloc__irm_req_ser(buffer_t * buf, const struct flow_info * flow) { @@ -355,6 +441,7 @@ int proc_exit__irm_req_ser(buffer_t * buf) return -ENOMEM; } +/* data is borrowed from the caller; detach before free. */ int ipcp_flow_req_arr__irm_req_ser(buffer_t * buf, const buffer_t * dst, const struct flow_info * flow, @@ -398,6 +485,188 @@ int ipcp_flow_req_arr__irm_req_ser(buffer_t * buf, return 0; fail_msg: + /* hash/pk are borrowed from the caller; detach before free. */ + msg->hash.len = 0; + msg->hash.data = NULL; + msg->pk.len = 0; + msg->pk.data = NULL; + irm_msg__free_unpacked(msg, NULL); + fail_malloc: + return -ENOMEM; +} + +static int __ep_flow_ser(buffer_t * buf, + const struct flow_info * flow, + const buffer_t * data, + int response, + const char * dst, + int code) +{ + irm_msg_t * msg; + size_t len; + int err = -ENOMEM; + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + goto fail_malloc; + + irm_msg__init(msg); + + msg->code = code; + msg->flow_info = flow_info_s_to_msg(flow); + if (msg->flow_info == NULL) + goto fail_msg; + + if (dst != NULL) { + msg->dst = strdup(dst); + if (msg->dst == NULL) + goto fail_msg; + } + + if (data != NULL) { + msg->has_pk = true; + msg->pk.len = data->len; + msg->pk.data = data->data; + } + + msg->has_response = true; + msg->response = response; + + len = irm_msg__get_packed_size(msg); + if (len == 0 || len > buf->len) + goto fail_msg; + + buf->len = len; + + irm_msg__pack(msg, buf->data); + + err = 0; + fail_msg: + msg->pk.len = 0; + msg->pk.data = NULL; + + irm_msg__free_unpacked(msg, NULL); + fail_malloc: + return err; +} + +int poa_flow_alloc__irm_req_ser(buffer_t * buf, + const struct flow_info * flow, + const char * dst) +{ + return __ep_flow_ser(buf, flow, NULL, 0, dst, + IRM_MSG_CODE__IRM_POA_FLOW_ALLOC); +} + +int poa_flow_alloc_r__irm_req_ser(buffer_t * buf, + const struct flow_info * flow, + const buffer_t * data, + int response) +{ + return __ep_flow_ser(buf, flow, data, response, NULL, + IRM_MSG_CODE__IRM_POA_FLOW_ALLOC_R); +} + +int ipcp_poa_flow_req_arr__irm_req_ser(buffer_t * buf, + const struct flow_info * flow, + const buffer_t * data) +{ + return __ep_flow_ser(buf, flow, data, 0, NULL, + IRM_MSG_CODE__IPCP_POA_FLOW_REQ_ARR); +} + +int poa_flow__irm_result_des(buffer_t * buf, + struct flow_info * flow, + buffer_t * data) +{ + irm_msg_t * msg; + int err; + + msg = irm_msg__unpack(NULL, buf->len, buf->data); + if (msg == NULL) { + err = -EIRMD; + goto fail_msg; + } + + if (!msg->has_result) { + err = -EIRMD; + goto fail; + } + + if (msg->result < 0) { + err = msg->result; + goto fail; + } + + if (msg->flow_info == NULL) { + err = -EBADF; + goto fail; + } + + *flow = flow_info_msg_to_s(msg->flow_info); + + clrbuf(*data); + + if (msg->has_pk && msg->pk.len > 0) { + data->data = malloc(msg->pk.len); + if (data->data == NULL) { + err = -ENOMEM; + goto fail; + } + memcpy(data->data, msg->pk.data, msg->pk.len); + + data->len = msg->pk.len; + } + + irm_msg__free_unpacked(msg, NULL); + + return 0; + fail: + irm_msg__free_unpacked(msg, NULL); + fail_msg: + return err; +} + +int ipcp_flow_update_arr__irm_req_ser(buffer_t * buf, + const struct flow_info * flow, + const buffer_t * data) +{ + irm_msg_t * msg; + size_t len; + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + goto fail_malloc; + + irm_msg__init(msg); + + msg->code = IRM_MSG_CODE__IPCP_FLOW_UPDATE_ARR; + msg->flow_info = flow_info_s_to_msg(flow); + if (msg->flow_info == NULL) + goto fail_msg; + + msg->has_pk = true; + msg->pk.len = data->len; + msg->pk.data = data->data; + + len = irm_msg__get_packed_size(msg); + if (len == 0 || len > buf->len) + goto fail_msg; + + buf->len = len; + + irm_msg__pack(msg, buf->data); + + /* Don't free data! */ + msg->pk.len = 0; + msg->pk.data = NULL; + irm_msg__free_unpacked(msg, NULL); + + return 0; + fail_msg: + /* pk.data is borrowed from the caller; detach before free. */ + msg->pk.len = 0; + msg->pk.data = NULL; irm_msg__free_unpacked(msg, NULL); fail_malloc: return -ENOMEM; diff --git a/src/lib/ssm/flow_set.c b/src/lib/ssm/flow_set.c index cb38e6fd..2e33b408 100644 --- a/src/lib/ssm/flow_set.c +++ b/src/lib/ssm/flow_set.c @@ -299,26 +299,34 @@ void ssm_flow_set_notify(struct ssm_flow_set * set, int event) { struct flowevent * e; + ssize_t idx; assert(set); assert(!(flow_id < 0) && flow_id < SYS_MAX_FLOWS); pthread_mutex_lock(set->lock); - if (set->mtable[flow_id] == -1) { + idx = set->mtable[flow_id]; + if (idx == -1) { pthread_mutex_unlock(set->lock); return; } - e = fqueue_ptr(set, set->mtable[flow_id]) + - set->heads[set->mtable[flow_id]]; + /* Ring full: drop redundant FLOW_PKT, reserve a slot for ctrl. */ + if (set->heads[idx] >= SSM_RBUFF_SIZE + || (event == FLOW_PKT && set->heads[idx] >= SSM_RBUFF_SIZE - 1)) { + pthread_mutex_unlock(set->lock); + return; + } + + e = fqueue_ptr(set, idx) + set->heads[idx]; e->flow_id = flow_id; e->event = event; - ++set->heads[set->mtable[flow_id]]; + ++set->heads[idx]; - pthread_cond_signal(&set->conds[set->mtable[flow_id]]); + pthread_cond_signal(&set->conds[idx]); pthread_mutex_unlock(set->lock); } diff --git a/src/lib/ssm/pool.c b/src/lib/ssm/pool.c index 5607a360..705de147 100644 --- a/src/lib/ssm/pool.c +++ b/src/lib/ssm/pool.c @@ -38,10 +38,20 @@ #include <stdio.h> #include <stdlib.h> #include <string.h> +#include <time.h> #include <unistd.h> #include <sys/mman.h> #include <sys/stat.h> +static __inline__ uint64_t pool_now_ns(void) +{ + struct timespec ts; + + clock_gettime(CLOCK_MONOTONIC, &ts); + + return (uint64_t) ts.tv_sec * 1000000000ULL + (uint64_t) ts.tv_nsec; +} + /* Global Shared Packet Pool (GSPP) configuration */ static const struct ssm_size_class_cfg ssm_gspp_cfg[SSM_POOL_MAX_CLASSES] = { { (1 << 8), SSM_GSPP_256_BLOCKS }, @@ -236,6 +246,7 @@ static void init_size_classes(struct ssm_pool * pool) STORE(&blk->refcount, 0); blk->allocator_pid = 0; + blk->alloc_ts = 0; STORE(&blk->next_offset, 0); list_add_head(&sc->shards[0].free_list, blk, @@ -266,19 +277,31 @@ static size_t reclaim_pid_from_sc(struct _ssm_size_class * sc, size_t i; size_t recovered = 0; struct ssm_pk_buff * blk; + uint64_t now; + uint64_t min_age_ns; - region = (uint8_t *) pool_base + sc->pool_start; + region = (uint8_t *) pool_base + sc->pool_start; + now = pool_now_ns(); + min_age_ns = (uint64_t) SSM_POOL_RECLAIM_AGE_S * 1000000000ULL; for (i = 0; i < sc->object_count; ++i) { blk = (struct ssm_pk_buff *)(region + i * sc->object_size); - if (blk->allocator_pid == pid && LOAD(&blk->refcount) > 0) { - STORE(&blk->refcount, 0); - blk->allocator_pid = 0; - list_add_head(&shard->free_list, blk, pool_base); - FETCH_ADD(&shard->free_count, 1); - recovered++; - } + if (blk->allocator_pid != pid) + continue; + + if (LOAD(&blk->refcount) == 0) + continue; + + /* Recent: a live consumer may still hold the handoff. */ + if (now - blk->alloc_ts < min_age_ns) + continue; + + STORE(&blk->refcount, 0); + blk->allocator_pid = 0; + list_add_head(&shard->free_list, blk, pool_base); + FETCH_ADD(&shard->free_count, 1); + recovered++; } return recovered; @@ -339,6 +362,7 @@ static __inline__ ssize_t init_block(struct ssm_pool * pool, { STORE(&blk->refcount, 1); blk->allocator_pid = getpid(); + blk->alloc_ts = pool_now_ns(); blk->size = (uint32_t) (sc->object_size - sizeof(struct ssm_pk_buff)); blk->pk_head = SSM_PK_BUFF_HEADSPACE; diff --git a/src/lib/ssm/rbuff.c b/src/lib/ssm/rbuff.c index c149c306..0480bce1 100644 --- a/src/lib/ssm/rbuff.c +++ b/src/lib/ssm/rbuff.c @@ -27,6 +27,7 @@ #include <ouroboros/ssm_rbuff.h> #include <ouroboros/lockfile.h> +#include <ouroboros/atomics.h> #include <ouroboros/errno.h> #include <ouroboros/fccntl.h> #include <ouroboros/pthread.h> @@ -53,11 +54,6 @@ #define MODB(x) ((x) & (SSM_RBUFF_SIZE - 1)) -#define LOAD_RELAXED(ptr) (__atomic_load_n(ptr, __ATOMIC_RELAXED)) -#define LOAD_ACQUIRE(ptr) (__atomic_load_n(ptr, __ATOMIC_ACQUIRE)) -#define STORE_RELEASE(ptr, val) \ - (__atomic_store_n(ptr, val, __ATOMIC_RELEASE)) - #define HEAD(rb) (rb->shm_base[LOAD_RELAXED(rb->head)]) #define TAIL(rb) (rb->shm_base[LOAD_RELAXED(rb->tail)]) #define HEAD_IDX(rb) (LOAD_ACQUIRE(rb->head)) @@ -67,20 +63,45 @@ #define ADVANCE_TAIL(rb) \ (STORE_RELEASE(rb->tail, MODB(LOAD_RELAXED(rb->tail) + 1))) #define QUEUED(rb) (MODB(HEAD_IDX(rb) - TAIL_IDX(rb))) -#define IS_FULL(rb) (QUEUED(rb) == (SSM_RBUFF_SIZE - 1)) #define IS_EMPTY(rb) (HEAD_IDX(rb) == TAIL_IDX(rb)) + +/* Delay-bound the TX queue delay at rate * target. */ +#define TXQ_MIN_SLOTS 4 /* floor: jitter margin */ +#define TXQ_INIT_SLOTS 64 /* ceiling until measured */ +#define TXQ_EWMA_N 4 /* EWMA weight 1/4 */ +#define TXQ_SPW_SHIFT 3 /* aim: 8 samples per window */ +#define TXQ_PERIOD_INIT 16 /* writes between samples */ +#define TXQ_PERIOD_MIN 4 +#define TXQ_PERIOD_MAX 64 +#define TXQ_MIN_DT_NS 1000LL /* shorter windows are noise */ +#define TXQ_MAX_RATE BILLION /* keeps rate * target in s64 */ +#define TXQ_UNLIMITED (SSM_RBUFF_SIZE - 1) +#define TXQ_DATA_MAX (TXQ_UNLIMITED - SSM_RBUFF_TXQ_RESERVE) + struct ssm_rbuff { ssize_t * shm_base; /* start of shared memory */ size_t * head; /* start of ringbuffer */ size_t * tail; - size_t * acl; /* access control */ + size_t * flags; /* out-of-band flags (RB_*) */ pthread_mutex_t * mtx; /* lock for cond vars only */ pthread_cond_t * add; /* signal when new data */ pthread_cond_t * del; /* signal when data removed */ pid_t pid; /* pid of the owner */ int flow_id; /* flow_id of the flow */ size_t n_users; /* in-flight users */ + struct { + uint64_t target; /* target queue delay, ns */ + uint64_t rate; /* EWMA drain rate, slots/s */ + uint64_t ns; /* window start, 0 = unset */ + size_t limit; /* current occupancy limit */ + size_t wr; /* writes this window */ + size_t due; /* sample when wr hits this */ + size_t period; /* writes between samples */ + size_t q0; /* queued at window start */ + bool idle; /* ring ran empty this one */ + bool measured; /* rate holds a measurement */ + } txq; /* tx delay limiter state */ }; #define MM_FLAGS (PROT_READ | PROT_WRITE) @@ -114,18 +135,29 @@ static struct ssm_rbuff * rbuff_create(pid_t pid, rb->shm_base = shm_base; rb->head = (size_t *) (rb->shm_base + (SSM_RBUFF_SIZE)); rb->tail = (size_t *) (rb->head + 1); - rb->acl = (size_t *) (rb->tail + 1); - rb->mtx = (pthread_mutex_t *) (rb->acl + 1); + rb->flags = (size_t *) (rb->tail + 1); + rb->mtx = (pthread_mutex_t *) (rb->flags + 1); rb->add = (pthread_cond_t *) (rb->mtx + 1); rb->del = rb->add + 1; rb->pid = pid; rb->flow_id = flow_id; rb->n_users = 0; + rb->txq.target = 0; + rb->txq.rate = 0; + rb->txq.ns = 0; + rb->txq.limit = TXQ_INIT_SLOTS; + rb->txq.wr = 0; + rb->txq.due = TXQ_PERIOD_INIT; + rb->txq.period = TXQ_PERIOD_INIT; + rb->txq.q0 = 0; + rb->txq.idle = false; + rb->txq.measured = false; return rb; fail_truncate: close(fd); + if (flags & O_CREAT) shm_unlink(fn); fail_open: @@ -158,30 +190,30 @@ struct ssm_rbuff * ssm_rbuff_create(pid_t pid, if (rb == NULL) goto fail_rb; - if (pthread_mutexattr_init(&mattr)) + if (pthread_mutexattr_init(&mattr) != 0) goto fail_mattr; pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED); #ifdef HAVE_ROBUST_MUTEX pthread_mutexattr_setrobust(&mattr, PTHREAD_MUTEX_ROBUST); #endif - if (pthread_mutex_init(rb->mtx, &mattr)) + if (pthread_mutex_init(rb->mtx, &mattr) != 0) goto fail_mutex; - if (pthread_condattr_init(&cattr)) + if (pthread_condattr_init(&cattr) != 0) goto fail_cattr; pthread_condattr_setpshared(&cattr, PTHREAD_PROCESS_SHARED); #ifndef __APPLE__ pthread_condattr_setclock(&cattr, PTHREAD_COND_CLOCK); #endif - if (pthread_cond_init(rb->add, &cattr)) + if (pthread_cond_init(rb->add, &cattr) != 0) goto fail_add; - if (pthread_cond_init(rb->del, &cattr)) + if (pthread_cond_init(rb->del, &cattr) != 0) goto fail_del; - *rb->acl = ACL_RDWR; + *rb->flags = RB_RDWR; *rb->head = 0; *rb->tail = 0; @@ -230,44 +262,231 @@ void ssm_rbuff_close(struct ssm_rbuff * rb) { assert(rb); - /* - * Caller must set ACL_FLOWDOWN first; if a user becomes - * cancellable, push a cleanup that decrements n_users. - */ - while (__atomic_load_n(&rb->n_users, __ATOMIC_SEQ_CST) > 0) { - struct timespec tic = { 0, 100000 }; + while (LOAD(&rb->n_users) > 0) { + struct timespec tic = TIMESPEC_INIT_US(100); + nanosleep(&tic, NULL); } rbuff_destroy(rb); } -int ssm_rbuff_write(struct ssm_rbuff * rb, - size_t off) +/* Cancel cleanup for a blocked reader: unlock mtx AND drop the n_users ref. */ +static void __cleanup_rbuff_reader(void * o) +{ + struct ssm_rbuff * rb = (struct ssm_rbuff *) o; + + pthread_mutex_unlock(rb->mtx); + FETCH_SUB(&rb->n_users, 1); +} + +static bool txq_is_on(struct ssm_rbuff * rb) +{ + return LOAD_RELAXED(&rb->txq.target) != 0; +} + +/* Occupancy that holds the delay at the target; rate 0 gets the floor. */ +static size_t rbuff_txq_slots(uint64_t rate, + uint64_t target) +{ + uint64_t slots; + + slots = rate * target / BILLION; + if (slots < TXQ_MIN_SLOTS) + return TXQ_MIN_SLOTS; + + return slots > TXQ_UNLIMITED ? TXQ_UNLIMITED : (size_t) slots; +} + +/* Ceiling for one write (taking into account priority). */ +static size_t rbuff_txq_ceiling(struct ssm_rbuff * rb, + bool prio) +{ + size_t lim; + size_t max; + + if (!txq_is_on(rb)) + return TXQ_UNLIMITED; + + if (!rb->txq.measured) + lim = TXQ_INIT_SLOTS; + else + lim = LOAD_RELAXED(&rb->txq.limit); + + max = TXQ_DATA_MAX; + + if (prio) { + lim *= SSM_RBUFF_TXQ_PRIO_MUL; + max = TXQ_UNLIMITED; + } + + return lim > max ? max : lim; +} + +/* Opens a measurement window at now_ns. Caller holds rb->mtx. */ +static void rbuff_txq_anchor(struct ssm_rbuff * rb, + uint64_t now_ns, + size_t queued) +{ + rb->txq.ns = now_ns; + rb->txq.q0 = queued; + rb->txq.wr = 0; + rb->txq.due = rb->txq.period; + rb->txq.idle = false; +} + +/* Enough dequeues to resolve a rate? */ +static bool txq_is_blind(struct ssm_rbuff * rb, + int64_t drained, + int64_t dt_ns) +{ + int64_t target = (int64_t) LOAD_RELAXED(&rb->txq.target); + + if (drained * 2 >= (int64_t) rb->txq.period) + return false; + + return dt_ns < (target >> TXQ_SPW_SHIFT); +} + +/* Leaves the window open and retries a period later. */ +static void rbuff_txq_defer(struct ssm_rbuff * rb) +{ + rb->txq.due = rb->txq.wr + rb->txq.period; +} + +/* Aims the sample period at 1 << TXQ_SPW_SHIFT per target window. */ +static size_t rbuff_txq_retune(size_t period, + int64_t dt_ns, + int64_t target) +{ + if (dt_ns > (target >> TXQ_SPW_SHIFT)) { + period /= 2; + return period < TXQ_PERIOD_MIN ? TXQ_PERIOD_MIN : period; + } + + if (dt_ns < (target >> (TXQ_SPW_SHIFT + 1))) { + period *= 2; + return period > TXQ_PERIOD_MAX ? TXQ_PERIOD_MAX : period; + } + + return period; +} + +/* Only raise the estimate if the window ran empty. Call holding rb->mtx. */ +static void rbuff_txq_sample(struct ssm_rbuff * rb, + size_t queued) +{ + struct timespec now; + uint64_t now_ns; + int64_t dt_ns; + int64_t target; + int64_t drained; + int64_t sample; + int64_t rate; + size_t limit; + size_t was; + + clock_gettime(PTHREAD_COND_CLOCK, &now); + + now_ns = TS_TO_UINT64(now); + + dt_ns = (int64_t) (now_ns - rb->txq.ns); + if (rb->txq.ns == 0 || dt_ns < 0) { + rbuff_txq_anchor(rb, now_ns, queued); + return; + } + + if (dt_ns < TXQ_MIN_DT_NS) { + rbuff_txq_defer(rb); + return; + } + + target = (int64_t) LOAD_RELAXED(&rb->txq.target); + drained = (int64_t) rb->txq.wr + (int64_t) rb->txq.q0 + - (int64_t) queued; + assert(drained >= 0); + + sample = drained * BILLION / dt_ns; + rate = (int64_t) rb->txq.rate; + if (sample > rate && txq_is_blind(rb, drained, dt_ns)) { + rbuff_txq_defer(rb); + return; + } + + if (!rb->txq.measured) + rate = sample; + else if (rb->txq.idle && queued <= TXQ_MIN_SLOTS) + rate = sample > rate ? sample : rate; + else + rate = (rate * (TXQ_EWMA_N - 1) + sample) / TXQ_EWMA_N; + + if (rate > TXQ_MAX_RATE) + rate = TXQ_MAX_RATE; + + limit = rbuff_txq_slots((uint64_t) rate, (uint64_t) target); + was = rbuff_txq_ceiling(rb, false); + + rb->txq.rate = (uint64_t) rate; + rb->txq.period = rbuff_txq_retune(rb->txq.period, dt_ns, target); + + STORE_RELAXED(&rb->txq.limit, limit); + STORE_RELAXED(&rb->txq.measured, true); + + if (rbuff_txq_ceiling(rb, false) > was) + pthread_cond_broadcast(rb->del); + + rbuff_txq_anchor(rb, now_ns, queued); +} + +/* + * Counts one enqueue. A prio write triggers no sample: it is the only + * traffic left in a stall, and would shrink the ceiling it needs. + */ +static void rbuff_txq_touch(struct ssm_rbuff * rb, + bool was_empty, + bool prio) +{ + ++rb->txq.wr; + + if (was_empty) + rb->txq.idle = true; + + if (prio) + return; + + if (rb->txq.wr >= rb->txq.due) + rbuff_txq_sample(rb, QUEUED(rb)); +} + +/* prio outranks new data up to its own, higher, ceiling. */ +static int rbuff_write_nb(struct ssm_rbuff * rb, + size_t off, + bool prio) { - size_t acl; + size_t flags; bool was_empty; int ret = 0; assert(rb != NULL); - __atomic_fetch_add(&rb->n_users, 1, __ATOMIC_SEQ_CST); + FETCH_ADD(&rb->n_users, 1); - acl = __atomic_load_n(rb->acl, __ATOMIC_SEQ_CST); - if (acl != ACL_RDWR) { - if (acl & ACL_FLOWDOWN) { + flags = LOAD(rb->flags); + if (flags != RB_RDWR) { + if (flags & RB_FLOWDOWN) { ret = -EFLOWDOWN; - goto fail_acl; + goto fail_flags; } - if (acl & ACL_RDONLY) { + + if (!(flags & RB_WR)) { ret = -ENOTALLOC; - goto fail_acl; + goto fail_flags; } } robust_mutex_lock(rb->mtx); - if (IS_FULL(rb)) { + if (QUEUED(rb) >= rbuff_txq_ceiling(rb, prio)) { ret = -EAGAIN; goto fail_mutex; } @@ -275,91 +494,126 @@ int ssm_rbuff_write(struct ssm_rbuff * rb, was_empty = IS_EMPTY(rb); HEAD(rb) = (ssize_t) off; + ADVANCE_HEAD(rb); if (was_empty) pthread_cond_broadcast(rb->add); + if (txq_is_on(rb)) + rbuff_txq_touch(rb, was_empty, prio); + pthread_mutex_unlock(rb->mtx); - __atomic_fetch_sub(&rb->n_users, 1, __ATOMIC_SEQ_CST); + FETCH_SUB(&rb->n_users, 1); + return 0; fail_mutex: pthread_mutex_unlock(rb->mtx); - fail_acl: - __atomic_fetch_sub(&rb->n_users, 1, __ATOMIC_SEQ_CST); + fail_flags: + FETCH_SUB(&rb->n_users, 1); return ret; } +int ssm_rbuff_write(struct ssm_rbuff * rb, + size_t off) +{ + return rbuff_write_nb(rb, off, false); +} + +/* For a packet the peer is already waiting on; skips the limit. */ +int ssm_rbuff_write_prio(struct ssm_rbuff * rb, + size_t off) +{ + return rbuff_write_nb(rb, off, true); +} + int ssm_rbuff_write_b(struct ssm_rbuff * rb, size_t off, const struct timespec * abstime) { - size_t acl; + size_t flags; int ret = 0; + int err; bool was_empty; assert(rb != NULL); - __atomic_fetch_add(&rb->n_users, 1, __ATOMIC_SEQ_CST); + FETCH_ADD(&rb->n_users, 1); - acl = __atomic_load_n(rb->acl, __ATOMIC_SEQ_CST); - if (acl != ACL_RDWR) { - if (acl & ACL_FLOWDOWN) { + flags = LOAD(rb->flags); + if (flags != RB_RDWR) { + if (flags & RB_FLOWDOWN) { ret = -EFLOWDOWN; - goto fail_acl; + goto fail_flags; } - if (acl & ACL_RDONLY) { + + if (!(flags & RB_WR)) { ret = -ENOTALLOC; - goto fail_acl; + goto fail_flags; } } robust_mutex_lock(rb->mtx); - pthread_cleanup_push(__cleanup_mutex_unlock, rb->mtx); + pthread_cleanup_push(__cleanup_rbuff_reader, rb); - while (IS_FULL(rb) && ret != -ETIMEDOUT) { - acl = __atomic_load_n(rb->acl, __ATOMIC_SEQ_CST); - if (acl & ACL_FLOWDOWN) { + while (QUEUED(rb) >= rbuff_txq_ceiling(rb, false)) { + flags = LOAD(rb->flags); + if (flags & RB_FLOWDOWN) { ret = -EFLOWDOWN; break; } - ret = -robust_wait(rb->del, rb->mtx, abstime); + + err = robust_wait(rb->del, rb->mtx, abstime); + if (err == EOWNERDEAD) + continue; + + if (err != 0) { + ret = -err; + break; + } } pthread_cleanup_pop(false); - if (ret != -ETIMEDOUT && ret != -EFLOWDOWN) { + if (ret == 0) { was_empty = IS_EMPTY(rb); HEAD(rb) = (ssize_t) off; + ADVANCE_HEAD(rb); + if (was_empty) pthread_cond_broadcast(rb->add); + + if (txq_is_on(rb)) + rbuff_txq_touch(rb, was_empty, false); } pthread_mutex_unlock(rb->mtx); - fail_acl: - __atomic_fetch_sub(&rb->n_users, 1, __ATOMIC_SEQ_CST); + fail_flags: + FETCH_SUB(&rb->n_users, 1); return ret; } -static int check_rb_acl(struct ssm_rbuff * rb) +static int check_rb_flags(struct ssm_rbuff * rb) { - size_t acl; + size_t flags; assert(rb != NULL); - acl = __atomic_load_n(rb->acl, __ATOMIC_SEQ_CST); - - if (acl & ACL_FLOWDOWN) + flags = LOAD(rb->flags); + if (flags & RB_FLOWDOWN) return -EFLOWDOWN; - if (acl & ACL_FLOWPEER) + if (flags & RB_FLOWPEER) return -EFLOWPEER; + if (!(flags & RB_RD)) + return -ENOTALLOC; + return -EAGAIN; } @@ -369,10 +623,10 @@ ssize_t ssm_rbuff_read(struct ssm_rbuff * rb) assert(rb != NULL); - __atomic_fetch_add(&rb->n_users, 1, __ATOMIC_SEQ_CST); + FETCH_ADD(&rb->n_users, 1); if (IS_EMPTY(rb)) { - ret = check_rb_acl(rb); + ret = check_rb_flags(rb); goto out; } @@ -380,11 +634,13 @@ ssize_t ssm_rbuff_read(struct ssm_rbuff * rb) if (IS_EMPTY(rb)) { pthread_mutex_unlock(rb->mtx); - ret = check_rb_acl(rb); + + ret = check_rb_flags(rb); goto out; } ret = TAIL(rb); + ADVANCE_TAIL(rb); pthread_cond_broadcast(rb->del); @@ -392,7 +648,8 @@ ssize_t ssm_rbuff_read(struct ssm_rbuff * rb) pthread_mutex_unlock(rb->mtx); out: - __atomic_fetch_sub(&rb->n_users, 1, __ATOMIC_SEQ_CST); + FETCH_SUB(&rb->n_users, 1); + return ret; } @@ -400,25 +657,29 @@ ssize_t ssm_rbuff_read_b(struct ssm_rbuff * rb, const struct timespec * abstime) { ssize_t idx = -1; - size_t acl; + size_t flags; assert(rb != NULL); - __atomic_fetch_add(&rb->n_users, 1, __ATOMIC_SEQ_CST); + FETCH_ADD(&rb->n_users, 1); - acl = __atomic_load_n(rb->acl, __ATOMIC_SEQ_CST); - if (IS_EMPTY(rb) && (acl & ACL_FLOWDOWN)) { + flags = LOAD(rb->flags); + if (IS_EMPTY(rb) && (flags & RB_FLOWDOWN)) { idx = -EFLOWDOWN; goto out; } robust_mutex_lock(rb->mtx); - pthread_cleanup_push(__cleanup_mutex_unlock, rb->mtx); + pthread_cleanup_push(__cleanup_rbuff_reader, rb); + + while (IS_EMPTY(rb)) { + if (idx == -ETIMEDOUT) + break; + + if (check_rb_flags(rb) != -EAGAIN) + break; - while (IS_EMPTY(rb) && - idx != -ETIMEDOUT && - check_rb_acl(rb) == -EAGAIN) { idx = -robust_wait(rb->add, rb->mtx, abstime); } @@ -426,10 +687,11 @@ ssize_t ssm_rbuff_read_b(struct ssm_rbuff * rb, if (!IS_EMPTY(rb)) { idx = TAIL(rb); + ADVANCE_TAIL(rb); pthread_cond_broadcast(rb->del); } else if (idx != -ETIMEDOUT) { - idx = check_rb_acl(rb); + idx = check_rb_flags(rb); } pthread_mutex_unlock(rb->mtx); @@ -437,45 +699,114 @@ ssize_t ssm_rbuff_read_b(struct ssm_rbuff * rb, assert(idx != -EAGAIN); out: - __atomic_fetch_sub(&rb->n_users, 1, __ATOMIC_SEQ_CST); + FETCH_SUB(&rb->n_users, 1); return idx; } -void ssm_rbuff_set_acl(struct ssm_rbuff * rb, - uint32_t flags) +void ssm_rbuff_set_flags(struct ssm_rbuff * rb, + uint32_t flags) { assert(rb != NULL); robust_mutex_lock(rb->mtx); - __atomic_store_n(rb->acl, (size_t) flags, __ATOMIC_SEQ_CST); + + FETCH_OR(rb->flags, (size_t) flags); pthread_cond_broadcast(rb->add); pthread_cond_broadcast(rb->del); + pthread_mutex_unlock(rb->mtx); } -uint32_t ssm_rbuff_get_acl(struct ssm_rbuff * rb) +void ssm_rbuff_clr_flags(struct ssm_rbuff * rb, + uint32_t flags) { assert(rb != NULL); - return (uint32_t) __atomic_load_n(rb->acl, __ATOMIC_SEQ_CST); + robust_mutex_lock(rb->mtx); + + FETCH_AND(rb->flags, ~(size_t) flags); + pthread_cond_broadcast(rb->add); + pthread_cond_broadcast(rb->del); + + pthread_mutex_unlock(rb->mtx); +} + +uint32_t ssm_rbuff_get_flags(struct ssm_rbuff * rb) +{ + assert(rb != NULL); + + return (uint32_t) LOAD(rb->flags); +} + +/* Current occupancy limit; SSM_RBUFF_SIZE - 1 when unlimited. */ +size_t ssm_rbuff_get_limit(struct ssm_rbuff * rb) +{ + assert(rb != NULL); + + return rbuff_txq_ceiling(rb, false); +} + +/* Wakes up writers because target may have changed. */ +void ssm_rbuff_set_txq_target(struct ssm_rbuff * rb, + const struct timespec * ts) +{ + uint64_t target; + size_t limit; + + assert(rb != NULL); + assert(ts != NULL); + assert(ts->tv_sec >= 0); + assert(ts->tv_nsec >= 0); + assert(ts->tv_nsec < BILLION); + + target = TS_TO_UINT64(*ts); + + assert(target <= SSM_RBUFF_TXQ_MAX_DELAY); + + robust_mutex_lock(rb->mtx); + + limit = rbuff_txq_slots(rb->txq.rate, target); + + rb->txq.period = TXQ_PERIOD_INIT; + + rbuff_txq_anchor(rb, 0, QUEUED(rb)); + + STORE_RELAXED(&rb->txq.limit, limit); + STORE_RELAXED(&rb->txq.target, target); + + pthread_cond_broadcast(rb->del); + + pthread_mutex_unlock(rb->mtx); +} + +/* Current target queueing delay for the tx occupancy limiter. */ +void ssm_rbuff_get_txq_target(struct ssm_rbuff * rb, + struct timespec * ts) +{ + assert(rb != NULL); + assert(ts != NULL); + + UINT64_TO_TS(LOAD_RELAXED(&rb->txq.target), ts); } void ssm_rbuff_fini(struct ssm_rbuff * rb) { assert(rb != NULL); - __atomic_fetch_add(&rb->n_users, 1, __ATOMIC_SEQ_CST); + FETCH_ADD(&rb->n_users, 1); robust_mutex_lock(rb->mtx); - pthread_cleanup_push(__cleanup_mutex_unlock, rb->mtx); + pthread_cleanup_push(__cleanup_rbuff_reader, rb); while (!IS_EMPTY(rb)) robust_wait(rb->del, rb->mtx, NULL); - pthread_cleanup_pop(true); + pthread_cleanup_pop(false); + + pthread_mutex_unlock(rb->mtx); - __atomic_fetch_sub(&rb->n_users, 1, __ATOMIC_SEQ_CST); + FETCH_SUB(&rb->n_users, 1); } size_t ssm_rbuff_queued(struct ssm_rbuff * rb) diff --git a/src/lib/ssm/ssm.h.in b/src/lib/ssm/ssm.h.in index b86327a1..a17c8edd 100644 --- a/src/lib/ssm/ssm.h.in +++ b/src/lib/ssm/ssm.h.in @@ -39,6 +39,8 @@ #define SSM_FLOW_SET_PREFIX "@SSM_FLOW_SET_PREFIX@" #define SSM_POOL_NAME "@SSM_POOL_NAME@" #define SSM_RBUFF_SIZE @SSM_RBUFF_SIZE@ +#define SSM_RBUFF_TXQ_PRIO_MUL @SSM_RBUFF_TXQ_PRIO_MUL@ +#define SSM_RBUFF_TXQ_RESERVE @SSM_RBUFF_TXQ_RESERVE@ /* Packet buffer space reservation */ #define SSM_PK_BUFF_HEADSPACE @SSM_PK_BUFF_HEADSPACE@ @@ -83,6 +85,7 @@ /* Size class configuration */ #define SSM_POOL_MAX_CLASSES 9 #define SSM_POOL_SHARDS @SSM_POOL_SHARDS@ +#define SSM_POOL_RECLAIM_AGE_S @SSM_POOL_RECLAIM_AGE_S@ /* Internal structures - exposed for testing */ #ifdef __cplusplus @@ -125,6 +128,7 @@ struct ssm_pk_buff { uint32_t pk_head; /* Head offset into data */ uint32_t pk_tail; /* Tail offset into data */ uint32_t off; /* Block offset in pool */ + uint64_t alloc_ts; /* CLOCK_MONOTONIC ns at alloc */ uint8_t data[]; /* Packet data */ }; diff --git a/src/lib/ssm/tests/pool_test.c b/src/lib/ssm/tests/pool_test.c index 0f9db24d..f86fbd9e 100644 --- a/src/lib/ssm/tests/pool_test.c +++ b/src/lib/ssm/tests/pool_test.c @@ -956,6 +956,8 @@ static int test_ssm_pool_reclaim_orphans(void) ssize_t ret3; pid_t my_pid; pid_t fake_pid = 99999; + struct timespec now; + uint64_t old_ts; TEST_START(); @@ -976,9 +978,15 @@ static int test_ssm_pool_reclaim_orphans(void) goto fail_alloc; } - /* Simulate blocks from another process by changing allocator_pid */ + /* Simulate blocks leaked by a dead process: foreign pid, aged out. */ + clock_gettime(CLOCK_MONOTONIC, &now); + old_ts = ((uint64_t) now.tv_sec - (SSM_POOL_RECLAIM_AGE_S + 1)) + * 1000000000ULL + (uint64_t) now.tv_nsec; + spb1->allocator_pid = fake_pid; spb2->allocator_pid = fake_pid; + spb1->alloc_ts = old_ts; + spb2->alloc_ts = old_ts; /* Keep spb3 with our pid */ /* Reclaim orphans from fake_pid */ diff --git a/src/lib/ssm/tests/rbuff_test.c b/src/lib/ssm/tests/rbuff_test.c index 58cb39c3..b7ef3dfb 100644 --- a/src/lib/ssm/tests/rbuff_test.c +++ b/src/lib/ssm/tests/rbuff_test.c @@ -34,6 +34,10 @@ #include <ouroboros/errno.h> #include <ouroboros/time.h> +/* Mirrors TXQ_MIN_SLOTS in ssm/rbuff.c; keep in sync. */ +#define FLOOR_SLOTS 4 +#define CEIL_SLOTS (SSM_RBUFF_SIZE - 1 - SSM_RBUFF_TXQ_RESERVE) + #include <errno.h> #include <stdio.h> #include <unistd.h> @@ -54,6 +58,7 @@ static int test_ssm_rbuff_create_destroy(void) ssm_rbuff_destroy(rb); TEST_SUCCESS(); + return TEST_RC_SUCCESS; fail: @@ -100,6 +105,7 @@ static int test_ssm_rbuff_write_read(void) ssm_rbuff_destroy(rb); TEST_SUCCESS(); + return TEST_RC_SUCCESS; fail_rb: @@ -131,6 +137,7 @@ static int test_ssm_rbuff_read_empty(void) ssm_rbuff_destroy(rb); TEST_SUCCESS(); + return TEST_RC_SUCCESS; fail_rb: @@ -160,6 +167,7 @@ static int test_ssm_rbuff_fill_drain(void) i, ssm_rbuff_queued(rb)); goto fail_rb; } + if (ssm_rbuff_write(rb, i) < 0) { printf("Failed to write at index %zu.\n", i); goto fail_rb; @@ -195,21 +203,23 @@ static int test_ssm_rbuff_fill_drain(void) ssm_rbuff_destroy(rb); TEST_SUCCESS(); + return TEST_RC_SUCCESS; fail_rb: while (ssm_rbuff_read(rb) >= 0) ; + ssm_rbuff_destroy(rb); fail: TEST_FAIL(); return TEST_RC_FAIL; } -static int test_ssm_rbuff_acl(void) +static int test_ssm_rbuff_flags(void) { struct ssm_rbuff * rb; - uint32_t acl; + uint32_t flags; TEST_START(); @@ -219,16 +229,17 @@ static int test_ssm_rbuff_acl(void) goto fail; } - acl = ssm_rbuff_get_acl(rb); - if (acl != ACL_RDWR) { - printf("Expected ACL_RDWR, got %u.\n", acl); + flags = ssm_rbuff_get_flags(rb); + if (flags != RB_RDWR) { + printf("Expected RB_RDWR, got %u.\n", flags); goto fail_rb; } - ssm_rbuff_set_acl(rb, ACL_RDONLY); - acl = ssm_rbuff_get_acl(rb); - if (acl != ACL_RDONLY) { - printf("Expected ACL_RDONLY, got %u.\n", acl); + ssm_rbuff_clr_flags(rb, RB_WR); + + flags = ssm_rbuff_get_flags(rb); + if (flags != RB_RD) { + printf("Expected RB_RD, got %u.\n", flags); goto fail_rb; } @@ -237,7 +248,8 @@ static int test_ssm_rbuff_acl(void) goto fail_rb; } - ssm_rbuff_set_acl(rb, ACL_FLOWDOWN); + ssm_rbuff_set_flags(rb, RB_FLOWDOWN); + if (ssm_rbuff_write(rb, 1) != -EFLOWDOWN) { printf("Expected -EFLOWDOWN on FLOWDOWN.\n"); goto fail_rb; @@ -251,6 +263,7 @@ static int test_ssm_rbuff_acl(void) ssm_rbuff_destroy(rb); TEST_SUCCESS(); + return TEST_RC_SUCCESS; fail_rb: @@ -302,6 +315,7 @@ static int test_ssm_rbuff_open_close(void) ssm_rbuff_destroy(rb1); TEST_SUCCESS(); + return TEST_RC_SUCCESS; fail_rb2: @@ -348,8 +362,10 @@ static void * reader_thread(void * arg) val = ssm_rbuff_read(args->rb); while (val < 0) { nanosleep(&delay, NULL); + val = ssm_rbuff_read(args->rb); } + if (val != i) { printf("Expected %d, got %zd.\n", i, val); return (void *) -1; @@ -359,7 +375,7 @@ static void * reader_thread(void * arg) return NULL; } -static void * blocking_writer_thread(void * arg) +static void * blocking_wr_thread(void * arg) { struct thread_args * args = (struct thread_args *) arg; int i; @@ -372,7 +388,7 @@ static void * blocking_writer_thread(void * arg) return NULL; } -static void * blocking_reader_thread(void * arg) +static void * blocking_rd_thread(void * arg) { struct thread_args * args = (struct thread_args *) arg; int i; @@ -391,13 +407,13 @@ static void * blocking_reader_thread(void * arg) static int test_ssm_rbuff_blocking(void) { - struct ssm_rbuff * rb; - pthread_t wthread; - pthread_t rthread; - struct thread_args args; - struct timespec delay = {0, 10 * MILLION}; - void * ret_w; - void * ret_r; + struct ssm_rbuff * rb; + pthread_t wthread; + pthread_t rthread; + struct thread_args args; + struct timespec delay = {0, 10 * MILLION}; + void * ret_w; + void * ret_r; TEST_START(); @@ -410,15 +426,14 @@ static int test_ssm_rbuff_blocking(void) args.rb = rb; args.iterations = 50; args.delay_us = 0; - - if (pthread_create(&rthread, NULL, blocking_reader_thread, &args)) { + if (pthread_create(&rthread, NULL, blocking_rd_thread, &args) != 0) { printf("Failed to create reader thread.\n"); goto fail_rthread; } nanosleep(&delay, NULL); - if (pthread_create(&wthread, NULL, blocking_writer_thread, &args)) { + if (pthread_create(&wthread, NULL, blocking_wr_thread, &args) != 0) { printf("Failed to create writer thread.\n"); pthread_cancel(rthread); goto fail_wthread; @@ -435,6 +450,7 @@ static int test_ssm_rbuff_blocking(void) ssm_rbuff_destroy(rb); TEST_SUCCESS(); + return TEST_RC_SUCCESS; fail_ret: @@ -482,8 +498,7 @@ static int test_ssm_rbuff_blocking_timeout(void) (end.tv_nsec - start.tv_nsec) / 1000000L; if (elapsed_ms < 90 || elapsed_ms > 200) { - printf("Timeout took %ld ms, expected ~100 ms.\n", - elapsed_ms); + printf("Timeout took %ld ms, expected ~100 ms.\n", elapsed_ms); goto fail_rb; } @@ -502,8 +517,7 @@ static int test_ssm_rbuff_blocking_timeout(void) clock_gettime(PTHREAD_COND_CLOCK, &end); if (ret != -ETIMEDOUT) { - printf("Expected -ETIMEDOUT on full buffer, got %zd.\n", - ret); + printf("Expected -ETIMEDOUT on full buffer, got %zd.\n", ret); goto fail_rb; } @@ -522,11 +536,13 @@ static int test_ssm_rbuff_blocking_timeout(void) ssm_rbuff_destroy(rb); TEST_SUCCESS(); + return TEST_RC_SUCCESS; fail_rb: while (ssm_rbuff_read(rb) >= 0) ; + ssm_rbuff_destroy(rb); fail: TEST_FAIL(); @@ -553,7 +569,7 @@ static int test_ssm_rbuff_blocking_flowdown(void) clock_gettime(PTHREAD_COND_CLOCK, &now); ts_add(&now, &interval, &abs_timeout); - ssm_rbuff_set_acl(rb, ACL_FLOWDOWN); + ssm_rbuff_set_flags(rb, RB_FLOWDOWN); ret = ssm_rbuff_read_b(rb, &abs_timeout); if (ret != -EFLOWDOWN) { @@ -561,7 +577,7 @@ static int test_ssm_rbuff_blocking_flowdown(void) goto fail_rb; } - ssm_rbuff_set_acl(rb, ACL_RDWR); + ssm_rbuff_clr_flags(rb, RB_FLOWDOWN); for (i = 0; i < SSM_RBUFF_SIZE - 1; ++i) { if (ssm_rbuff_write(rb, i) < 0) { @@ -573,7 +589,7 @@ static int test_ssm_rbuff_blocking_flowdown(void) clock_gettime(PTHREAD_COND_CLOCK, &now); ts_add(&now, &interval, &abs_timeout); - ssm_rbuff_set_acl(rb, ACL_FLOWDOWN); + ssm_rbuff_set_flags(rb, RB_FLOWDOWN); ret = ssm_rbuff_write_b(rb, 999, &abs_timeout); if (ret != -EFLOWDOWN) { @@ -581,18 +597,21 @@ static int test_ssm_rbuff_blocking_flowdown(void) goto fail_rb; } - ssm_rbuff_set_acl(rb, ACL_RDWR); + ssm_rbuff_clr_flags(rb, RB_FLOWDOWN); + while (ssm_rbuff_read(rb) >= 0) ; ssm_rbuff_destroy(rb); TEST_SUCCESS(); + return TEST_RC_SUCCESS; fail_rb: while (ssm_rbuff_read(rb) >= 0) ; + ssm_rbuff_destroy(rb); fail: TEST_FAIL(); @@ -601,12 +620,12 @@ static int test_ssm_rbuff_blocking_flowdown(void) static int test_ssm_rbuff_threaded(void) { - struct ssm_rbuff * rb; - pthread_t wthread; - pthread_t rthread; - struct thread_args args; - void * ret_w; - void * ret_r; + struct ssm_rbuff * rb; + pthread_t wthread; + pthread_t rthread; + struct thread_args args; + void * ret_w; + void * ret_r; TEST_START(); @@ -619,13 +638,12 @@ static int test_ssm_rbuff_threaded(void) args.rb = rb; args.iterations = 100; args.delay_us = 100; - - if (pthread_create(&wthread, NULL, writer_thread, &args)) { + if (pthread_create(&wthread, NULL, writer_thread, &args) != 0) { printf("Failed to create writer thread.\n"); goto fail_rb; } - if (pthread_create(&rthread, NULL, reader_thread, &args)) { + if (pthread_create(&rthread, NULL, reader_thread, &args) != 0) { printf("Failed to create reader thread.\n"); pthread_cancel(wthread); pthread_join(wthread, NULL); @@ -643,9 +661,393 @@ static int test_ssm_rbuff_threaded(void) ssm_rbuff_destroy(rb); TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + + fail_rb: + ssm_rbuff_destroy(rb); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_ssm_rbuff_limit_off(void) +{ + struct ssm_rbuff * rb; + size_t i; + + TEST_START(); + + rb = ssm_rbuff_create(getpid(), 11); + if (rb == NULL) { + printf("Failed to create rbuff.\n"); + goto fail; + } + + if (ssm_rbuff_get_limit(rb) != SSM_RBUFF_SIZE - 1) { + printf("Expected default limit %d, got %zu.\n", + SSM_RBUFF_SIZE - 1, ssm_rbuff_get_limit(rb)); + goto fail_rb; + } + + for (i = 0; i < SSM_RBUFF_SIZE - 1; ++i) { + if (ssm_rbuff_write(rb, i) < 0) { + printf("Failed to write at index %zu.\n", i); + goto fail_rb; + } + } + + if (ssm_rbuff_write(rb, 999) != -EAGAIN) { + printf("Expected -EAGAIN on physically full buffer.\n"); + goto fail_rb; + } + + while (ssm_rbuff_read(rb) >= 0) + ; + + ssm_rbuff_destroy(rb); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + + fail_rb: + while (ssm_rbuff_read(rb) >= 0) + ; + + ssm_rbuff_destroy(rb); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_ssm_rbuff_limit_slow(void) +{ + struct ssm_rbuff * rb; + struct timespec dfl = TIMESPEC_INIT_MS(SSM_RBUFF_TXQ_DELAY); + struct timespec delay = {0, 10 * MILLION}; + size_t limit; + size_t i; + + TEST_START(); + + rb = ssm_rbuff_create(getpid(), 12); + if (rb == NULL) { + printf("Failed to create rbuff.\n"); + goto fail; + } + + ssm_rbuff_set_txq_target(rb, &dfl); + + for (i = 0; i < 32; ++i) { + if (ssm_rbuff_write_b(rb, i, NULL) < 0) { + printf("Failed to write at index %zu.\n", i); + goto fail_rb; + } + nanosleep(&delay, NULL); + + if (ssm_rbuff_read(rb) < 0) { + printf("Failed to read at index %zu.\n", i); + goto fail_rb; + } + } + + limit = ssm_rbuff_get_limit(rb); + if (limit > FLOOR_SLOTS) { + printf("Expected limit near the floor, got %zu.\n", limit); + goto fail_rb; + } + + ssm_rbuff_destroy(rb); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + + fail_rb: + while (ssm_rbuff_read(rb) >= 0) + ; + + ssm_rbuff_destroy(rb); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_ssm_rbuff_limit_fast(void) +{ + struct ssm_rbuff * rb; + struct timespec dfl = TIMESPEC_INIT_MS(SSM_RBUFF_TXQ_DELAY); + size_t limit; + size_t i; + + TEST_START(); + + rb = ssm_rbuff_create(getpid(), 13); + if (rb == NULL) { + printf("Failed to create rbuff.\n"); + goto fail; + } + + ssm_rbuff_set_txq_target(rb, &dfl); + + for (i = 0; i < 200; ++i) { + if (ssm_rbuff_write_b(rb, i, NULL) < 0) { + printf("Failed to write at index %zu.\n", i); + goto fail_rb; + } + + if (ssm_rbuff_read(rb) < 0) { + printf("Failed to read at index %zu.\n", i); + goto fail_rb; + } + } + + limit = ssm_rbuff_get_limit(rb); + if (limit != CEIL_SLOTS) { + printf("Expected limit %d, got %zu.\n", CEIL_SLOTS, limit); + goto fail_rb; + } + + ssm_rbuff_destroy(rb); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + + fail_rb: + while (ssm_rbuff_read(rb) >= 0) + ; + + ssm_rbuff_destroy(rb); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_ssm_rbuff_limit_floor(void) +{ + struct ssm_rbuff * rb; + struct timespec dfl = TIMESPEC_INIT_MS(SSM_RBUFF_TXQ_DELAY); + struct timespec interval = {0, 50 * MILLION}; + struct timespec now; + struct timespec abs_timeout; + size_t limit; + int ret = 0; + size_t i; + + TEST_START(); + + rb = ssm_rbuff_create(getpid(), 14); + if (rb == NULL) { + printf("Failed to create rbuff.\n"); + goto fail; + } + + ssm_rbuff_set_txq_target(rb, &dfl); + + clock_gettime(PTHREAD_COND_CLOCK, &now); + ts_add(&now, &interval, &abs_timeout); + + for (i = 0; i < SSM_RBUFF_SIZE; ++i) { + ret = ssm_rbuff_write_b(rb, i, &abs_timeout); + if (ret == -ETIMEDOUT) + break; + + if (ret < 0) { + printf("Write failed at index %zu: %d.\n", i, ret); + goto fail_rb; + } + } + + if (ret != -ETIMEDOUT) { + printf("Expected the limiter to block the ring.\n"); + goto fail_rb; + } + + limit = ssm_rbuff_get_limit(rb); + if (limit > FLOOR_SLOTS) { + printf("Expected floor limit, got %zu.\n", limit); + goto fail_rb; + } + + while (ssm_rbuff_read(rb) >= 0) + ; + + ssm_rbuff_destroy(rb); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; fail_rb: + while (ssm_rbuff_read(rb) >= 0) + ; + + ssm_rbuff_destroy(rb); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* A fresh ring is unlimited; rx rings must not inherit a bound. */ +static int test_ssm_rbuff_txq_target(void) +{ + struct ssm_rbuff * rb; + struct timespec dfl = TIMESPEC_INIT_MS(SSM_RBUFF_TXQ_DELAY); + struct timespec delay = {0, 5 * MILLION}; + struct timespec small = {0, 2 * MILLION}; + struct timespec big = {0, 200 * MILLION}; + struct timespec def; + struct timespec got; + size_t limit_small; + size_t limit_big; + size_t i; + + TEST_START(); + + rb = ssm_rbuff_create(getpid(), 15); + if (rb == NULL) { + printf("Failed to create rbuff.\n"); + goto fail; + } + + ssm_rbuff_get_txq_target(rb, &got); + + if (got.tv_sec != 0 || got.tv_nsec != 0) { + printf("A new ring is not unlimited.\n"); + goto fail_rb; + } + + ssm_rbuff_set_txq_target(rb, &dfl); + ssm_rbuff_get_txq_target(rb, &def); + + ssm_rbuff_set_txq_target(rb, &small); + + for (i = 0; i < 64; ++i) { + if (ssm_rbuff_write_b(rb, i, NULL) < 0) { + printf("Failed to write at index %zu.\n", i); + goto fail_rb; + } + nanosleep(&delay, NULL); + + if (ssm_rbuff_read(rb) < 0) { + printf("Failed to read at index %zu.\n", i); + goto fail_rb; + } + } + + limit_small = ssm_rbuff_get_limit(rb); + + ssm_rbuff_set_txq_target(rb, &big); + + for (i = 0; i < 64; ++i) { + if (ssm_rbuff_write_b(rb, i, NULL) < 0) { + printf("Failed to write at index %zu.\n", i); + goto fail_rb; + } + nanosleep(&delay, NULL); + + if (ssm_rbuff_read(rb) < 0) { + printf("Failed to read at index %zu.\n", i); + goto fail_rb; + } + } + + limit_big = ssm_rbuff_get_limit(rb); + if (limit_big <= limit_small) { + printf("Expected a larger target to grow the limit: " + "%zu -> %zu.\n", limit_small, limit_big); + goto fail_rb; + } + + ssm_rbuff_set_txq_target(rb, &dfl); + ssm_rbuff_get_txq_target(rb, &got); + + if (got.tv_sec != def.tv_sec || got.tv_nsec != def.tv_nsec) { + printf("NULL did not restore the default target.\n"); + goto fail_rb; + } + + ssm_rbuff_destroy(rb); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + + fail_rb: + while (ssm_rbuff_read(rb) >= 0) + ; + + ssm_rbuff_destroy(rb); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* Ages the seed sample past the estimator's dt floor at write 16. */ +static int test_ssm_rbuff_write_over_limit(void) +{ + struct ssm_rbuff * rb; + struct timespec dfl = TIMESPEC_INIT_MS(SSM_RBUFF_TXQ_DELAY); + struct timespec age = {0, 20 * 1000}; + size_t count; + int ret = 0; + + TEST_START(); + + rb = ssm_rbuff_create(getpid(), 16); + if (rb == NULL) { + printf("Failed to create rbuff.\n"); + goto fail; + } + + ssm_rbuff_set_txq_target(rb, &dfl); + + for (count = 0; count < SSM_RBUFF_SIZE; ++count) { + ret = ssm_rbuff_write(rb, count); + if (ret == -EAGAIN) + break; + + if (ret < 0) { + printf("Write failed at index %zu: %d.\n", count, ret); + goto fail_rb; + } + + if (count == 16) + nanosleep(&age, NULL); + } + + if (ret != -EAGAIN) { + printf("Expected the limiter to reject a write.\n"); + goto fail_rb; + } + + if (count >= SSM_RBUFF_SIZE / 2) { + printf("Expected -EAGAIN well before a full ring, " + "got %zu writes.\n", count); + goto fail_rb; + } + + if (ssm_rbuff_queued(rb) != count) { + printf("Queued %zu does not match write count %zu.\n", + ssm_rbuff_queued(rb), count); + goto fail_rb; + } + + while (ssm_rbuff_read(rb) >= 0) + ; + + ssm_rbuff_destroy(rb); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + + fail_rb: + while (ssm_rbuff_read(rb) >= 0) + ; + ssm_rbuff_destroy(rb); fail: TEST_FAIL(); @@ -664,12 +1066,18 @@ int rbuff_test(int argc, ret |= test_ssm_rbuff_write_read(); ret |= test_ssm_rbuff_read_empty(); ret |= test_ssm_rbuff_fill_drain(); - ret |= test_ssm_rbuff_acl(); + ret |= test_ssm_rbuff_flags(); ret |= test_ssm_rbuff_open_close(); ret |= test_ssm_rbuff_threaded(); ret |= test_ssm_rbuff_blocking(); ret |= test_ssm_rbuff_blocking_timeout(); ret |= test_ssm_rbuff_blocking_flowdown(); + ret |= test_ssm_rbuff_limit_off(); + ret |= test_ssm_rbuff_limit_slow(); + ret |= test_ssm_rbuff_limit_fast(); + ret |= test_ssm_rbuff_limit_floor(); + ret |= test_ssm_rbuff_txq_target(); + ret |= test_ssm_rbuff_write_over_limit(); return ret; } diff --git a/src/lib/tests/CMakeLists.txt b/src/lib/tests/CMakeLists.txt index 32836589..d470d539 100644 --- a/src/lib/tests/CMakeLists.txt +++ b/src/lib/tests/CMakeLists.txt @@ -10,10 +10,13 @@ create_test_sourcelist(${PARENT_DIR}_tests test_suite.c auth_test_slh_dsa.c bitmap_test.c btree_test.c + cap_test.c crypt_test.c + poa_test.c hash_test.c kex_test.c kex_test_ml_kem.c + keyrot_test.c md5_test.c sha3_test.c sockets_test.c @@ -24,6 +27,15 @@ create_test_sourcelist(${PARENT_DIR}_tests test_suite.c add_executable(${PARENT_DIR}_test ${${PARENT_DIR}_tests}) +if(HAVE_LIBURCU) + # poa_test.c pulls in poa.h, whose urcu guard needs C99. + set_source_files_properties(poa_test.c PROPERTIES + COMPILE_OPTIONS "-std=gnu99") +endif() + +target_include_directories(${PARENT_DIR}_test PRIVATE + ${CMAKE_SOURCE_DIR}/src/lib) + disable_test_logging_for_target(${PARENT_DIR}_test) target_link_libraries(${PARENT_DIR}_test ouroboros-common) diff --git a/src/lib/tests/auth_test.c b/src/lib/tests/auth_test.c index 0f3ef715..61f97683 100644 --- a/src/lib/tests/auth_test.c +++ b/src/lib/tests/auth_test.c @@ -24,11 +24,14 @@ #include <test/test.h> #include <ouroboros/crypt.h> +#include <ouroboros/name.h> #include <ouroboros/random.h> #include <ouroboros/utils.h> #include <test/certs/ecdsa.h> +#include <string.h> + #define TEST_MSG_SIZE 1500 static int test_auth_create_destroy_ctx(void) @@ -138,6 +141,47 @@ static int test_check_crt_name(void) return TEST_RC_FAIL; } +static int test_crt_name_confusion(void) +{ + char name[NAME_SIZE + 1]; + void * crt; + + TEST_START(); + + if (crypt_load_crt_str(confused_crt_ec, &crt) < 0) { + printf("Failed to load name-confusion certificate.\n"); + goto fail_load; + } + + /* Must extract the real CN, not the "CN=" decoy in the O field. */ + if (crypt_get_crt_name(crt, name) < 0) { + printf("Failed to extract name from certificate.\n"); + goto fail_check; + } + + if (strcmp(name, "attacker.unittest.o7s") != 0) { + printf("Extracted '%s', expected real CN.\n", name); + goto fail_check; + } + + /* The decoy name in the O field must never authenticate. */ + if (crypt_check_crt_name(crt, "victim.unittest.o7s") == 0) { + printf("Accepted spoofed name from O field.\n"); + goto fail_check; + } + + crypt_free_crt(crt); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_check: + crypt_free_crt(crt); + fail_load: + TEST_FAIL(); + return TEST_RC_FAIL; +} + static int test_load_free_privkey(void) { void * key; @@ -260,7 +304,7 @@ static int test_store_add(void) fail_add: crypt_free_crt(_root_ca_crt); fail_load: - crypt_free_crt(_root_ca_crt); + auth_destroy_ctx(ctx); fail_create: TEST_FAIL(); return TEST_RC_FAIL; @@ -400,6 +444,98 @@ static int test_verify_crt_missing_root_ca(void) return TEST_RC_FAIL; } +/* auth_verify_crt_pin: pin must lie in the verified chain (NULL: any) */ +static int test_verify_crt_pin(void) +{ + struct auth_ctx * auth; + void * _root_ca_crt; + void * _im_ca_crt; + void * _signed_server_crt; + void * _other_ca_crt; + + TEST_START(); + + auth = auth_create_ctx(); + if (auth == NULL) { + printf("Failed to create auth context.\n"); + goto fail_create_ctx; + } + + if (crypt_load_crt_str(root_ca_crt_ec, &_root_ca_crt) < 0) { + printf("Failed to load root crt from string.\n"); + goto fail_load_root_ca; + } + + if (crypt_load_crt_str(im_ca_crt_ec, &_im_ca_crt) < 0) { + printf("Failed to load intermediate crt from string.\n"); + goto fail_load_im_ca; + } + + if (crypt_load_crt_str(signed_server_crt_ec, &_signed_server_crt) < 0) { + printf("Failed to load signed crt from string.\n"); + goto fail_load_signed; + } + + if (crypt_load_crt_str(other_ca_crt_ec, &_other_ca_crt) < 0) { + printf("Failed to load out-of-chain crt from string.\n"); + goto fail_load_other; + } + + if (auth_add_crt_to_store(auth, _root_ca_crt) < 0) { + printf("Failed to add root ca crt to auth store.\n"); + goto fail_verify; + } + + if (auth_add_crt_to_store(auth, _im_ca_crt) < 0) { + printf("Failed to add intermediate ca crt to auth store.\n"); + goto fail_verify; + } + + if (auth_verify_crt_pin(auth, _signed_server_crt, _im_ca_crt) < 0) { + printf("Failed to accept pin on intermediate CA.\n"); + goto fail_verify; + } + + if (auth_verify_crt_pin(auth, _signed_server_crt, _root_ca_crt) < 0) { + printf("Failed to accept pin on root CA.\n"); + goto fail_verify; + } + + if (auth_verify_crt_pin(auth, _signed_server_crt, _other_ca_crt) == 0) { + printf("Failed to reject out-of-chain pin.\n"); + goto fail_verify; + } + + if (auth_verify_crt_pin(auth, _signed_server_crt, NULL) < 0) { + printf("Failed to accept NULL (any) pin.\n"); + goto fail_verify; + } + + crypt_free_crt(_other_ca_crt); + crypt_free_crt(_signed_server_crt); + crypt_free_crt(_im_ca_crt); + crypt_free_crt(_root_ca_crt); + + auth_destroy_ctx(auth); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_verify: + crypt_free_crt(_other_ca_crt); + fail_load_other: + crypt_free_crt(_signed_server_crt); + fail_load_signed: + crypt_free_crt(_im_ca_crt); + fail_load_im_ca: + crypt_free_crt(_root_ca_crt); + fail_load_root_ca: + auth_destroy_ctx(auth); + fail_create_ctx: + TEST_FAIL(); + return TEST_RC_FAIL; +} + int test_auth_sign(void) { uint8_t buf[TEST_MSG_SIZE]; @@ -573,6 +709,7 @@ int auth_test(int argc, #ifdef HAVE_OPENSSL ret |= test_load_free_crt(); ret |= test_check_crt_name(); + ret |= test_crt_name_confusion(); ret |= test_crypt_get_pubkey_crt(); ret |= test_load_free_privkey(); ret |= test_load_free_pubkey(); @@ -580,12 +717,14 @@ int auth_test(int argc, ret |= test_store_add(); ret |= test_verify_crt(); ret |= test_verify_crt_missing_root_ca(); + ret |= test_verify_crt_pin(); ret |= test_auth_sign(); ret |= test_auth_bad_signature(); ret |= test_crt_str(); #else (void) test_load_free_crt; (void) test_check_crt_name; + (void) test_crt_name_confusion; (void) test_crypt_get_pubkey_crt; (void) test_load_free_privkey; (void) test_load_free_pubkey; @@ -593,11 +732,13 @@ int auth_test(int argc, (void) test_store_add; (void) test_verify_crt; (void) test_verify_crt_missing_root_ca; + (void) test_verify_crt_pin; (void) test_auth_sign; (void) test_auth_bad_signature; (void) test_crt_str; - ret = TEST_RC_SKIP; + if (ret == 0) + ret = TEST_RC_SKIP; #endif return ret; } diff --git a/src/lib/tests/auth_test_ml_dsa.c b/src/lib/tests/auth_test_ml_dsa.c index cc72e61b..e324c32d 100644 --- a/src/lib/tests/auth_test_ml_dsa.c +++ b/src/lib/tests/auth_test_ml_dsa.c @@ -333,7 +333,7 @@ int auth_test_ml_dsa(int argc, (void) argc; (void) argv; -#ifdef HAVE_OPENSSL_ML_DSA +#ifdef HAVE_ML ret |= test_auth_create_destroy_ctx(); ret |= test_load_free_crt(); ret |= test_load_free_privkey(); diff --git a/src/lib/tests/auth_test_slh_dsa.c b/src/lib/tests/auth_test_slh_dsa.c index 511d20fe..e9af8da8 100644 --- a/src/lib/tests/auth_test_slh_dsa.c +++ b/src/lib/tests/auth_test_slh_dsa.c @@ -344,7 +344,7 @@ int auth_test_slh_dsa(int argc, (void) argc; (void) argv; -#ifdef HAVE_OPENSSL_SLH_DSA +#ifdef HAVE_SLH ret |= test_auth_create_destroy_ctx(); ret |= test_load_free_crt(); ret |= test_load_free_privkey(); diff --git a/src/lib/tests/cap_test.c b/src/lib/tests/cap_test.c new file mode 100644 index 00000000..ea0e1fef --- /dev/null +++ b/src/lib/tests/cap_test.c @@ -0,0 +1,427 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Unit tests for link capacity estimation + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * version 2.1 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#include "../cap.c" + +#include <test/test.h> + +#include <inttypes.h> +#include <stdbool.h> + +#define TICK (50 * 1000ULL) /* 50 us between packets */ +#define LEN 1000ULL /* default packet size (B) */ +#define QLEN (8 * LEN) /* steady backlog (bytes) */ +#define RATE (LEN * BILLION / TICK) /* LEN per TICK = 20 MB/s */ + +#define SHP_LEN 1250ULL /* shaped-link packet (B) */ +#define SHP_STEP 20 /* packets per shaped window */ +#define SHP_RATE (SHP_LEN * BILLION / (SHP_STEP * TICK)) + +/* Draining CAP_N_MIN of these outlasts CAP_T_MAX without a gap. */ +#define LOW_STEP (250 * TICK) /* 12.5 ms between packets */ +#define LOW_RATE (LEN * BILLION / LOW_STEP) + +/* Within the quarter-log2 band the wire code publishes. */ +static bool rate_is_near(uint64_t got, + uint64_t exp) +{ + return got >= exp - exp / 8 && got <= exp + exp / 8; +} + +static int test_cap_est_clear(void) +{ + struct cap_est e; + size_t i; + + TEST_START(); + + cap_clear(&e); + + if (cap_rate(&e) != 0) { + printf("Fresh estimator not unknown.\n"); + goto fail; + } + + for (i = 1; i <= 40; i++) + cap_update_at(&e, QLEN, LEN, i * TICK); + + if (cap_rate(&e) == 0) { + printf("No estimate to clear.\n"); + goto fail; + } + + cap_clear(&e); + + if (cap_rate(&e) != 0) { + printf("Clear did not drop the estimate.\n"); + goto fail; + } + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* 1000 B every 50 us, ring steady at 8: drain = 20 MB/s. */ +static int test_cap_est_busy_window(void) +{ + struct cap_est e; + size_t i; + + TEST_START(); + + cap_clear(&e); + + for (i = 1; i <= 40; i++) + cap_update_at(&e, QLEN, LEN, i * TICK); + + if (!rate_is_near(cap_rate(&e), RATE)) { + printf("Estimated rate: exp %" PRIu64 ", got %" PRIu64 ".\n", + (uint64_t) RATE, cap_rate(&e)); + goto fail; + } + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_cap_est_idle_tolerated(void) +{ + struct cap_est e; + size_t i; + + TEST_START(); + + cap_clear(&e); + + for (i = 1; i <= 40; i++) + cap_update_at(&e, i == 21 ? 0 : QLEN, LEN, i * TICK); + + if (!rate_is_near(cap_rate(&e), RATE)) { + printf("Grazed window: exp %" PRIu64 ", got %" PRIu64 ".\n", + (uint64_t) RATE, cap_rate(&e)); + goto fail; + } + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_cap_est_mostly_idle_rejects(void) +{ + struct cap_est e; + size_t i; + + TEST_START(); + + cap_clear(&e); + + for (i = 1; i <= 100; i++) + cap_update_at(&e, 0, LEN, i * TICK); + + if (cap_rate(&e) != 0) { + printf("Idle ring estimated %" PRIu64 ".\n", cap_rate(&e)); + goto fail; + } + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* 1000 B every 100 us: 10 slots/ms closes on a 2 ms window. */ +static int test_cap_est_slow_link_extends(void) +{ + struct cap_est e; + size_t i; + + TEST_START(); + + cap_clear(&e); + + for (i = 1; i <= 30; i++) + cap_update_at(&e, QLEN, LEN, i * 2 * TICK); + + if (!rate_is_near(cap_rate(&e), RATE / 2)) { + printf("Slow link: exp %" PRIu64 ", got %" PRIu64 ".\n", + (uint64_t) (RATE / 2), cap_rate(&e)); + goto fail; + } + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* 1250 B every ms; one empty observation per 20 packets. */ +static int test_cap_est_shaped_link(void) +{ + struct cap_est e; + size_t i; + + TEST_START(); + + cap_clear(&e); + + for (i = 1; i <= 100; i++) + cap_update_at(&e, i % SHP_STEP == 0 ? 0 : 6 * SHP_LEN, + SHP_LEN, i * SHP_STEP * TICK); + + if (!rate_is_near(cap_rate(&e), SHP_RATE)) { + printf("Shaped link: exp %" PRIu64 ", got %" PRIu64 ".\n", + (uint64_t) SHP_RATE, cap_rate(&e)); + goto fail; + } + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* Open a window, trickle 4 slots, then ~200 ms of silence. */ +static int test_cap_est_stale_discard(void) +{ + struct cap_est e; + uint64_t t; + size_t i; + + TEST_START(); + + cap_clear(&e); + + for (i = 1; i <= 5; i++) + cap_update_at(&e, QLEN, LEN, i * CAP_T_MIN); + + t = 205 * CAP_T_MIN; + + cap_update_at(&e, QLEN, LEN, t); + + if (cap_rate(&e) != 0) { + printf("Gap window estimated %" PRIu64 ".\n", cap_rate(&e)); + goto fail; + } + + for (i = 1; i <= 40; i++) + cap_update_at(&e, QLEN, LEN, t + i * TICK); + + if (!rate_is_near(cap_rate(&e), RATE)) { + printf("Post-gap: exp %" PRIu64 ", got %" PRIu64 ".\n", + (uint64_t) RATE, cap_rate(&e)); + goto fail; + } + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_cap_est_empty_start_no_raise(void) +{ + struct cap_est e; + size_t i; + + TEST_START(); + + cap_clear(&e); + + cap_update_at(&e, 0, LEN, CAP_T_MIN); + + for (i = 1; i <= 40; i++) + cap_update_at(&e, QLEN, LEN, CAP_T_MIN + i * TICK); + + if (cap_rate(&e) != 0) { + printf("Empty-start window raised to %" PRIu64 ".\n", + cap_rate(&e)); + goto fail; + } + + for (i = 41; i <= 60; i++) + cap_update_at(&e, QLEN, LEN, CAP_T_MIN + i * TICK); + + if (!rate_is_near(cap_rate(&e), RATE)) { + printf("Backlogged window: exp %" PRIu64 ", got %" PRIu64 + ".\n", (uint64_t) RATE, cap_rate(&e)); + goto fail; + } + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* + * Max filter: fast attack on a high sample, slow release on the + * lower samples from a halved packet size (10 MB/s). + */ +static int test_cap_est_max_filter(void) +{ + struct cap_est e; + uint64_t high; + size_t i; + + TEST_START(); + + cap_clear(&e); + + for (i = 1; i <= 40; i++) + cap_update_at(&e, QLEN, LEN, i * TICK); + + high = cap_rate(&e); + if (!rate_is_near(high, RATE)) { + printf("Attack missed: exp %" PRIu64 ", got %" PRIu64 ".\n", + (uint64_t) RATE, high); + goto fail; + } + + for (i = 41; i <= 80; i++) + cap_update_at(&e, QLEN, LEN / 2, i * TICK); + + if (cap_rate(&e) >= high) { + printf("Release did not decay: %" PRIu64 ".\n", cap_rate(&e)); + goto fail; + } + + if (cap_rate(&e) <= RATE / 2) { + printf("Release collapsed to %" PRIu64 ".\n", cap_rate(&e)); + goto fail; + } + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* No window close within CAP_T_MIN of the last one. */ +static int test_cap_est_gate(void) +{ + struct cap_est e; + size_t i; + + TEST_START(); + + cap_clear(&e); + + cap_update_at(&e, QLEN, LEN, CAP_T_MIN); + + for (i = 0; i < 5; i++) + cap_update_at(&e, QLEN, LEN, CAP_T_MIN + CAP_T_MIN / 2); + + if (e.t_gate != CAP_T_MIN) { + printf("Window closed inside the gate.\n"); + goto fail; + } + + if (LOAD_RELAXED(&e.c_pkt) != 6) { + printf("Gated packets not counted.\n"); + goto fail; + } + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* + * A link slow enough that CAP_N_MIN packets take longer than + * CAP_T_MAX to drain still publishes, as long as the sender keeps + * offering: only silence voids a window. + */ +static int test_cap_est_low_rate_publishes(void) +{ + struct cap_est e; + size_t i; + + TEST_START(); + + cap_clear(&e); + + for (i = 1; i <= 20; i++) + cap_update_at(&e, QLEN, LEN, i * LOW_STEP); + + if (!rate_is_near(cap_rate(&e), LOW_RATE)) { + printf("Low rate: exp %" PRIu64 ", got %" PRIu64 ".\n", + (uint64_t) LOW_RATE, cap_rate(&e)); + goto fail; + } + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +int cap_test(int argc, + char ** argv) +{ + int ret = 0; + + (void) argc; + (void) argv; + + ret |= test_cap_est_clear(); + ret |= test_cap_est_busy_window(); + ret |= test_cap_est_idle_tolerated(); + ret |= test_cap_est_mostly_idle_rejects(); + ret |= test_cap_est_slow_link_extends(); + ret |= test_cap_est_shaped_link(); + ret |= test_cap_est_stale_discard(); + ret |= test_cap_est_empty_start_no_raise(); + ret |= test_cap_est_max_filter(); + ret |= test_cap_est_gate(); + ret |= test_cap_est_low_rate_publishes(); + + return ret; +} diff --git a/src/lib/tests/crypt_test.c b/src/lib/tests/crypt_test.c index 028c4eb5..88c9634a 100644 --- a/src/lib/tests/crypt_test.c +++ b/src/lib/tests/crypt_test.c @@ -30,6 +30,7 @@ #include <stdio.h> #define TEST_PACKET_SIZE 1500 +#define TEST_N_PACKETS 1000 extern const uint16_t crypt_supported_nids[]; extern const uint16_t md_supported_nids[]; @@ -39,9 +40,10 @@ static int test_crypt_create_destroy(void) struct crypt_ctx * ctx; uint8_t key[SYMMKEYSZ]; struct crypt_sk sk = { - .nid = NID_aes_256_gcm, - .key = key, - .rot_bit = KEY_ROTATION_BIT + .nid = NID_aes_256_gcm, + .key = key, + .epoch = 0, + .role = CRYPT_ROLE_INIT }; TEST_START(); @@ -49,12 +51,20 @@ static int test_crypt_create_destroy(void) memset(key, 0, sizeof(key)); ctx = crypt_create_ctx(&sk); +#ifdef HAVE_OPENSSL if (ctx == NULL) { printf("Failed to initialize cryptography.\n"); goto fail; } crypt_destroy_ctx(ctx); +#else + if (ctx != NULL) { + printf("Created cipher context without a backend.\n"); + crypt_destroy_ctx(ctx); + goto fail; + } +#endif TEST_SUCCESS(); @@ -67,18 +77,27 @@ static int test_crypt_create_destroy(void) static int test_crypt_encrypt_decrypt(int nid) { uint8_t pkt[TEST_PACKET_SIZE]; - struct crypt_ctx * ctx; + struct crypt_ctx * tx; + struct crypt_ctx * rx; uint8_t key[SYMMKEYSZ]; - struct crypt_sk sk = { - .nid = NID_aes_256_gcm, - .key = key, - .rot_bit = KEY_ROTATION_BIT + struct crypt_sk sk_tx = { + .key = key, + .epoch = 0, + .role = CRYPT_ROLE_INIT + }; + struct crypt_sk sk_rx = { + .key = key, + .epoch = 0, + .role = CRYPT_ROLE_RESP }; buffer_t in; buffer_t out; buffer_t out2; const char * cipher; + sk_tx.nid = nid; + sk_rx.nid = nid; + cipher = crypt_nid_to_str(nid); TEST_START("(%s)", cipher); @@ -92,53 +111,63 @@ static int test_crypt_encrypt_decrypt(int nid) goto fail_init; } - ctx = crypt_create_ctx(&sk); - if (ctx == NULL) { - printf("Failed to initialize cryptography.\n"); + tx = crypt_create_ctx(&sk_tx); + if (tx == NULL) { + printf("Failed to initialize TX cryptography.\n"); goto fail_init; } + rx = crypt_create_ctx(&sk_rx); + if (rx == NULL) { + printf("Failed to initialize RX cryptography.\n"); + goto fail_tx; + } + in.len = sizeof(pkt); in.data = pkt; - if (crypt_encrypt(ctx, in, &out) < 0) { + if (crypt_encrypt(tx, in, &out) < 0) { printf("Encryption failed.\n"); goto fail_encrypt; } if (out.len < in.len) { printf("Encryption returned too little data.\n"); - goto fail_encrypt; + goto fail_chk; } - if (crypt_decrypt(ctx, out, &out2) < 0) { + if (crypt_decrypt(rx, out, &out2) < 0) { printf("Decryption failed.\n"); goto fail_decrypt; } if (out2.len != in.len) { printf("Decrypted data length does not match original.\n"); - goto fail_chk; + goto fail_chk2; } if (memcmp(in.data, out2.data, in.len) != 0) { printf("Decrypted data does not match original.\n"); - goto fail_chk; + goto fail_chk2; } - crypt_destroy_ctx(ctx); freebuf(out2); freebuf(out); + crypt_destroy_ctx(rx); + crypt_destroy_ctx(tx); TEST_SUCCESS("(%s)", cipher); return TEST_RC_SUCCESS; - fail_chk: + fail_chk2: freebuf(out2); fail_decrypt: + fail_chk: freebuf(out); fail_encrypt: - crypt_destroy_ctx(ctx); + crypt_destroy_ctx(rx); + fail_tx: + crypt_destroy_ctx(tx); fail_init: TEST_FAIL("(%s)", cipher); return TEST_RC_FAIL; @@ -155,6 +184,230 @@ static int test_encrypt_decrypt_all(void) return ret; } +static int test_crypt_multi_packet(int nid) +{ + uint8_t pkt[TEST_PACKET_SIZE]; + struct crypt_ctx * tx; + struct crypt_ctx * rx; + uint8_t key[SYMMKEYSZ]; + struct crypt_sk sk_tx = { + .key = key, + .epoch = 0, + .role = CRYPT_ROLE_INIT + }; + struct crypt_sk sk_rx = { + .key = key, + .epoch = 0, + .role = CRYPT_ROLE_RESP + }; + buffer_t in; + buffer_t enc; + buffer_t dec; + const char * cipher; + int i; + + sk_tx.nid = nid; + sk_rx.nid = nid; + + cipher = crypt_nid_to_str(nid); + TEST_START("(%s)", cipher); + + if (random_buffer(key, sizeof(key)) < 0) { + printf("Failed to generate random key.\n"); + goto fail_init; + } + + if (random_buffer(pkt, sizeof(pkt)) < 0) { + printf("Failed to generate random data.\n"); + goto fail_init; + } + + tx = crypt_create_ctx(&sk_tx); + if (tx == NULL) { + printf("Failed to create TX context.\n"); + goto fail_init; + } + + rx = crypt_create_ctx(&sk_rx); + if (rx == NULL) { + printf("Failed to create RX context.\n"); + goto fail_tx; + } + + in.len = sizeof(pkt); + in.data = pkt; + + for (i = 0; i < TEST_N_PACKETS; i++) { + if (crypt_encrypt(tx, in, &enc) < 0) { + printf("Encryption failed at packet %d.\n", i); + goto fail_rx; + } + + if (crypt_decrypt(rx, enc, &dec) < 0) { + printf("Decryption failed at packet %d.\n", i); + freebuf(enc); + goto fail_rx; + } + + if (dec.len != in.len || + memcmp(in.data, dec.data, in.len) != 0) { + printf("Data mismatch at packet %d.\n", i); + freebuf(dec); + freebuf(enc); + goto fail_rx; + } + + freebuf(dec); + freebuf(enc); + } + + crypt_destroy_ctx(rx); + crypt_destroy_ctx(tx); + + TEST_SUCCESS("(%s)", cipher); + + return TEST_RC_SUCCESS; + fail_rx: + crypt_destroy_ctx(rx); + fail_tx: + crypt_destroy_ctx(tx); + fail_init: + TEST_FAIL("(%s)", cipher); + return TEST_RC_FAIL; +} + +static int test_multi_packet_all(void) +{ + int ret = 0; + int i; + + for (i = 0; crypt_supported_nids[i] != NID_undef; i++) + ret |= test_crypt_multi_packet(crypt_supported_nids[i]); + + return ret; +} + +static int test_crypt_aad_tamper(int nid) +{ + uint8_t pkt[TEST_PACKET_SIZE]; + struct crypt_ctx * tx; + struct crypt_ctx * rx; + uint8_t key[SYMMKEYSZ]; + struct crypt_sk sk_tx = { + .key = key, + .epoch = 0, + .role = CRYPT_ROLE_INIT + }; + struct crypt_sk sk_rx = { + .key = key, + .epoch = 0, + .role = CRYPT_ROLE_RESP + }; + buffer_t in; + buffer_t enc; + buffer_t dec; + const char * cipher; + + sk_tx.nid = nid; + sk_rx.nid = nid; + + cipher = crypt_nid_to_str(nid); + TEST_START("(%s)", cipher); + + if (random_buffer(key, sizeof(key)) < 0) { + printf("Failed to generate random key.\n"); + goto fail_init; + } + + if (random_buffer(pkt, sizeof(pkt)) < 0) { + printf("Failed to generate random data.\n"); + goto fail_init; + } + + tx = crypt_create_ctx(&sk_tx); + if (tx == NULL) { + printf("Failed to create TX context.\n"); + goto fail_init; + } + + rx = crypt_create_ctx(&sk_rx); + if (rx == NULL) { + printf("Failed to create RX context.\n"); + goto fail_tx; + } + + /* Only AEAD ciphers bind the selector as AAD. */ + if (crypt_get_tagsz(tx) == 0) { + crypt_destroy_ctx(rx); + crypt_destroy_ctx(tx); + + TEST_SUCCESS("(%s)", cipher); + + return TEST_RC_SUCCESS; + } + + in.len = sizeof(pkt); + in.data = pkt; + + if (crypt_encrypt(tx, in, &enc) < 0) { + printf("Encryption failed.\n"); + goto fail_rx; + } + + /* Flip a seq byte: epoch/node stay valid so the AEAD tag rejects. */ + enc.data[5] ^= 0x01; + + if (crypt_decrypt(rx, enc, &dec) == 0) { + printf("Decryption accepted a tampered selector.\n"); + freebuf(dec); + freebuf(enc); + goto fail_rx; + } + + enc.data[5] ^= 0x01; + + if (crypt_decrypt(rx, enc, &dec) < 0) { + printf("Decryption failed after a rejected packet.\n"); + freebuf(enc); + goto fail_rx; + } + + if (dec.len != in.len || memcmp(dec.data, in.data, in.len) != 0) { + printf("Decrypted data mismatch after rejection.\n"); + freebuf(dec); + freebuf(enc); + goto fail_rx; + } + + freebuf(dec); + freebuf(enc); + + crypt_destroy_ctx(rx); + crypt_destroy_ctx(tx); + + TEST_SUCCESS("(%s)", cipher); + + return TEST_RC_SUCCESS; + fail_rx: + crypt_destroy_ctx(rx); + fail_tx: + crypt_destroy_ctx(tx); + fail_init: + TEST_FAIL("(%s)", cipher); + return TEST_RC_FAIL; +} + +static int test_aad_tamper_all(void) +{ + int ret = 0; + int i; + + for (i = 0; crypt_supported_nids[i] != NID_undef; i++) + ret |= test_crypt_aad_tamper(crypt_supported_nids[i]); + + return ret; +} + #ifdef HAVE_OPENSSL #include <openssl/evp.h> #include <openssl/obj_mac.h> @@ -256,22 +509,17 @@ static int test_md_nid_values(void) } #endif -static int test_key_rotation(void) +static int test_crypt_headsz(void) { - uint8_t pkt[TEST_PACKET_SIZE]; - struct crypt_ctx * tx_ctx; - struct crypt_ctx * rx_ctx; - uint8_t key[SYMMKEYSZ]; - struct crypt_sk sk = { - .nid = NID_aes_256_gcm, - .key = key, - .rot_bit = 7 + struct crypt_ctx * ctx; + uint8_t key[SYMMKEYSZ]; + struct crypt_sk sk = { + .nid = NID_aes_256_gcm, + .key = key, + .epoch = 0, + .role = CRYPT_ROLE_INIT }; - buffer_t in; - buffer_t enc; - buffer_t dec; - uint32_t i; - uint32_t threshold; + int headsz; TEST_START(); @@ -280,155 +528,66 @@ static int test_key_rotation(void) goto fail; } - if (random_buffer(pkt, sizeof(pkt)) < 0) { - printf("Failed to generate random data.\n"); - goto fail; - } - - tx_ctx = crypt_create_ctx(&sk); - if (tx_ctx == NULL) { - printf("Failed to create TX context.\n"); + ctx = crypt_create_ctx(&sk); + if (ctx == NULL) { + printf("Failed to initialize cryptography.\n"); goto fail; } - rx_ctx = crypt_create_ctx(&sk); - if (rx_ctx == NULL) { - printf("Failed to create RX context.\n"); - goto fail_tx; - } - - in.len = sizeof(pkt); - in.data = pkt; - - threshold = (1U << sk.rot_bit); - - /* Encrypt and decrypt across multiple rotations */ - for (i = 0; i < threshold * 3; i++) { - if (crypt_encrypt(tx_ctx, in, &enc) < 0) { - printf("Encryption failed at packet %u.\n", i); - goto fail_rx; - } - - if (crypt_decrypt(rx_ctx, enc, &dec) < 0) { - printf("Decryption failed at packet %u.\n", i); - freebuf(enc); - goto fail_rx; - } - - if (dec.len != in.len || - memcmp(in.data, dec.data, in.len) != 0) { - printf("Data mismatch at packet %u.\n", i); - freebuf(dec); - freebuf(enc); - goto fail_rx; - } - - freebuf(dec); - freebuf(enc); + headsz = crypt_get_headsz(ctx); + if (headsz != 6) { + printf("Unexpected header size: %d (expected 6).\n", headsz); + goto fail_ctx; } - crypt_destroy_ctx(rx_ctx); - crypt_destroy_ctx(tx_ctx); + crypt_destroy_ctx(ctx); TEST_SUCCESS(); return TEST_RC_SUCCESS; - fail_rx: - crypt_destroy_ctx(rx_ctx); - fail_tx: - crypt_destroy_ctx(tx_ctx); + fail_ctx: + crypt_destroy_ctx(ctx); fail: TEST_FAIL(); return TEST_RC_FAIL; } -static int test_key_phase_bit(void) +static int test_crypt_ct_cmp(void) { - uint8_t pkt[TEST_PACKET_SIZE]; - struct crypt_ctx * ctx; - uint8_t key[SYMMKEYSZ]; - struct crypt_sk sk = { - .nid = NID_aes_256_gcm, - .key = key, - .rot_bit = 7 - }; - buffer_t in; - buffer_t out; - uint32_t count; - uint32_t threshold; - uint8_t phase_before; - uint8_t phase_after; - int ivsz; + uint8_t a[64]; + uint8_t b[64]; + size_t i; TEST_START(); - if (random_buffer(key, sizeof(key)) < 0) { - printf("Failed to generate random key.\n"); - goto fail; - } + for (i = 0; i < sizeof(a); i++) + a[i] = (uint8_t) i; - if (random_buffer(pkt, sizeof(pkt)) < 0) { - printf("Failed to generate random data.\n"); - goto fail; - } + memcpy(b, a, sizeof(a)); - ctx = crypt_create_ctx(&sk); - if (ctx == NULL) { - printf("Failed to initialize cryptography.\n"); + if (crypt_ct_cmp(a, b, sizeof(a)) != 0) { + printf("Equal buffers should compare equal.\n"); goto fail; } - ivsz = crypt_get_ivsz(ctx); - if (ivsz <= 0) { - printf("Invalid IV size.\n"); - goto fail_ctx; + if (crypt_ct_cmp(a, b, 0) != 0) { + printf("Zero length should compare equal.\n"); + goto fail; } - in.len = sizeof(pkt); - in.data = pkt; - - /* Encrypt packets up to just before rotation threshold */ - threshold = (1U << sk.rot_bit); - - /* Encrypt threshold - 1 packets (indices 0 to threshold-2) */ - for (count = 0; count < threshold - 1; count++) { - if (crypt_encrypt(ctx, in, &out) < 0) { - printf("Encryption failed at count %u.\n", count); - goto fail_ctx; + for (i = 0; i < sizeof(a); i++) { + b[i] ^= 0x01; + if (crypt_ct_cmp(a, b, sizeof(a)) == 0) { + printf("Difference at byte %zu not detected.\n", i); + goto fail; } - freebuf(out); - } - - /* Packet at index threshold-1: phase should still be initial */ - if (crypt_encrypt(ctx, in, &out) < 0) { - printf("Encryption failed before rotation.\n"); - goto fail_ctx; - } - phase_before = (out.data[0] & 0x80) ? 1 : 0; - freebuf(out); - /* Packet at index threshold: phase should have toggled */ - if (crypt_encrypt(ctx, in, &out) < 0) { - printf("Encryption failed at rotation threshold.\n"); - goto fail_ctx; + b[i] ^= 0x01; } - phase_after = (out.data[0] & 0x80) ? 1 : 0; - freebuf(out); - - /* Phase bit should have toggled */ - if (phase_before == phase_after) { - printf("Phase bit did not toggle: before=%u, after=%u.\n", - phase_before, phase_after); - goto fail_ctx; - } - - crypt_destroy_ctx(ctx); TEST_SUCCESS(); return TEST_RC_SUCCESS; - fail_ctx: - crypt_destroy_ctx(ctx); fail: TEST_FAIL(); return TEST_RC_FAIL; @@ -444,16 +603,20 @@ int crypt_test(int argc, ret |= test_crypt_create_destroy(); ret |= test_encrypt_decrypt_all(); + ret |= test_crypt_ct_cmp(); #ifdef HAVE_OPENSSL ret |= test_cipher_nid_values(); ret |= test_md_nid_values(); - ret |= test_key_rotation(); - ret |= test_key_phase_bit(); + ret |= test_multi_packet_all(); + ret |= test_aad_tamper_all(); + ret |= test_crypt_headsz(); #else - (void) test_key_rotation; - (void) test_key_phase_bit; + (void) test_multi_packet_all; + (void) test_aad_tamper_all; + (void) test_crypt_headsz; - return TEST_RC_SKIP; + if (ret == 0) + ret = TEST_RC_SKIP; #endif return ret; } diff --git a/src/lib/tests/hash_test.c b/src/lib/tests/hash_test.c index 451d3c25..a2ba62cc 100644 --- a/src/lib/tests/hash_test.c +++ b/src/lib/tests/hash_test.c @@ -39,6 +39,11 @@ struct vec_entry { char * out; }; +struct mix_entry { + uint64_t in; + uint64_t out; +}; + static int test_crc8(void) { int ret = 0; @@ -288,6 +293,36 @@ static int test_sha3(void) return ret; } +static int test_mix64(void) +{ + int ret = 0; + + struct mix_entry vec [] = { + { 0x0000000000000000ULL, 0x0000000000000000ULL }, + { 0x123456789abcdefeULL, 0xb1943cfea4f78f08ULL } + }; + + size_t n = sizeof(vec) / sizeof(vec[0]); + size_t i; + + TEST_START(); + + for (i = 0; i < n; i++) { + uint64_t res = hash_mix64(vec[i].in); + + if (res != vec[i].out) { + printf("Mix failed %016llx != %016llx.\n", + (unsigned long long) res, + (unsigned long long) vec[i].out); + ret |= -1; + } + } + + TEST_END(ret); + + return ret; +} + int hash_test(int argc, char ** argv) { @@ -308,5 +343,7 @@ int hash_test(int argc, ret |= test_sha3(); + ret |= test_mix64(); + return ret; } diff --git a/src/lib/tests/kex_test.c b/src/lib/tests/kex_test.c index 6a4f802e..d7629f95 100644 --- a/src/lib/tests/kex_test.c +++ b/src/lib/tests/kex_test.c @@ -44,6 +44,9 @@ #define KEX_CONFIG_NONE \ "none\n" +#define KEX_CONFIG_NO_ENC \ + "encryption=none\n" + #define KEX_CONFIG_WHITESPACE \ "# Comment line\n" \ "kex = X448" \ @@ -58,6 +61,31 @@ "kex=X25519\n" \ "digest=sha384\n" +#define KEX_CONFIG_AUTH \ + "auth=required\n" + +#define KEX_CONFIG_AUTH_INVALID \ + "auth=mandatory\n" + +#define KEX_CONFIG_AUTH_OPTIONAL \ + "auth=optional\n" + +#define KEX_CONFIG_AUTH_THEN_NO_ENC \ + "auth=required\n" \ + "digest=sha512\n" \ + "encryption=none\n" + +#define KEX_CONFIG_NO_ENC_THEN_AUTH \ + "encryption=none\n" \ + "auth=required\n" \ + "digest=sha512\n" + +#define KEX_CONFIG_CACERT \ + "cacert=/etc/ouroboros/security/cacert/ca.crt\n" + +#define KEX_CONFIG_UNKNOWN_KEY \ + "autth=required\n" + /* Test key material for key loading tests */ #define X25519_PRIVKEY_PEM \ "-----BEGIN PRIVATE KEY-----\n" \ @@ -77,6 +105,9 @@ extern const uint16_t kex_supported_nids[]; int parse_sec_config(struct sec_config * cfg, FILE * fp); +int crypt_load_sec_config(struct sec_config * cfg, + FILE * fp); + static int test_kex_create_destroy(void) { struct sec_config cfg; @@ -151,17 +182,9 @@ static int test_kex_get_algo_from_pk(const char * algo) pk.len = (size_t) len; pk.data = buf; - /* Use raw decode for hybrid KEMs, DER for others */ - if (IS_HYBRID_KEM(algo)) { - if (kex_get_algo_from_pk_raw(pk, extracted_algo) < 0) { - printf("Failed to extract algo from pk.\n"); - goto fail_pkp; - } - } else { - if (kex_get_algo_from_pk_der(pk, extracted_algo) < 0) { - printf("Failed to extract algo from pk.\n"); - goto fail_pkp; - } + if (kex_get_algo_from_pk_der(pk, extracted_algo) < 0) { + printf("Failed to extract algo from pk.\n"); + goto fail_pkp; } /* All algorithms should now return the specific group name */ @@ -190,6 +213,11 @@ static int test_kex_get_algo_from_pk_all(void) for (i = 0; kex_supported_nids[i] != NID_undef; i++) { const char * algo = kex_nid_to_str(kex_supported_nids[i]); + + /* Raw hybrid PKs are opaque, OAP carries the NID */ + if (IS_HYBRID_KEM(algo)) + continue; + ret |= test_kex_get_algo_from_pk(algo); } @@ -213,6 +241,7 @@ static int test_kex_dhe_derive(const char * algo) memset(&kex, 0, sizeof(kex)); SET_KEX_ALGO(&kex, algo); + SET_KEX_KDF_NID(&kex, NID_sha256); len = kex_pkp_create(&kex, &pkp1, buf1); if (len < 0) { @@ -276,7 +305,7 @@ static int test_kex_validate_algo(void) goto fail; } -#ifdef HAVE_OPENSSL_ML_KEM +#ifdef HAVE_ML if (kex_validate_algo("ML-KEM-768") != 0) { printf("ML-KEM-768 should be valid.\n"); goto fail; @@ -324,6 +353,7 @@ static int test_kex_dhe_corrupted_pubkey(const char * algo) memset(&kex, 0, sizeof(kex)); SET_KEX_ALGO(&kex, algo); + SET_KEX_KDF_NID(&kex, NID_sha256); len = kex_pkp_create(&kex, &pkp, buf); if (len < 0) { @@ -375,6 +405,8 @@ static int test_kex_dhe_wrong_algo(void) memset(&kex2, 0, sizeof(kex2)); SET_KEX_ALGO(&kex1, algo1); SET_KEX_ALGO(&kex2, algo2); + SET_KEX_KDF_NID(&kex1, NID_sha256); + SET_KEX_KDF_NID(&kex2, NID_sha256); if (kex_pkp_create(&kex1, &pkp1, buf1) < 0) { printf("Failed to create first key pair.\n"); @@ -411,6 +443,57 @@ static int test_kex_dhe_wrong_algo(void) return TEST_RC_FAIL; } +static int test_kex_dhe_no_kdf(void) +{ + struct sec_config kex; + void * pkp1; + void * pkp2; + buffer_t pk2; + ssize_t len; + uint8_t buf1[CRYPT_KEY_BUFSZ]; + uint8_t buf2[CRYPT_KEY_BUFSZ]; + uint8_t s[SYMMKEYSZ]; + + TEST_START(); + + memset(&kex, 0, sizeof(kex)); + SET_KEX_ALGO(&kex, "X25519"); + + if (kex_pkp_create(&kex, &pkp1, buf1) < 0) { + printf("Failed to create first key pair.\n"); + goto fail; + } + + len = kex_pkp_create(&kex, &pkp2, buf2); + if (len < 0) { + printf("Failed to create second key pair.\n"); + goto fail_pkp1; + } + + pk2.len = (size_t) len; + pk2.data = buf2; + + /* No KDF configured: derive must fail, not fall back. */ + if (kex_dhe_derive(&kex, pkp1, pk2, s) == 0) { + printf("Derive succeeded without a KDF.\n"); + goto fail_pkp2; + } + + kex_pkp_destroy(pkp2); + kex_pkp_destroy(pkp1); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_pkp2: + kex_pkp_destroy(pkp2); + fail_pkp1: + kex_pkp_destroy(pkp1); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + static int test_kex_load_dhe_privkey(void) { void * key; @@ -639,7 +722,8 @@ static int test_kex_parse_config_custom(void) return TEST_RC_FAIL; } -static int test_kex_parse_config_none(void) +/* The old bare 'none' keyword must be rejected loudly */ +static int test_kex_parse_config_none_rejected(void) { struct sec_config kex; FILE * fp; @@ -654,14 +738,51 @@ static int test_kex_parse_config_none(void) goto fail; } + if (parse_sec_config(&kex, fp) == 0) { + printf("Bare 'none' keyword should be rejected.\n"); + fclose(fp); + goto fail; + } + + fclose(fp); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_kex_parse_config_no_enc(void) +{ + struct sec_config kex; + FILE * fp; + + TEST_START(); + + memset(&kex, 0, sizeof(kex)); + + fp = FMEMOPEN_STR(KEX_CONFIG_NO_ENC); + if (fp == NULL) { + printf("Failed to open memory stream.\n"); + goto fail; + } + if (parse_sec_config(&kex, fp) < 0) { - printf("Failed to parse 'none' config.\n"); + printf("Failed to parse encryption=none config.\n"); fclose(fp); goto fail; } - if (kex.x.nid != NID_undef) { - printf("'none' keyword should disable encryption.\n"); + if (kex.x.nid != NID_undef || kex.c.nid != NID_undef) { + printf("encryption=none should disable encryption.\n"); + fclose(fp); + goto fail; + } + + if (kex.d.nid != NID_sha256) { + printf("encryption=none should keep the digest.\n"); fclose(fp); goto fail; } @@ -799,6 +920,309 @@ static int test_kex_parse_config_digest(void) return TEST_RC_FAIL; } +static int test_kex_parse_config_auth(void) +{ + struct sec_config kex; + FILE * fp; + + TEST_START(); + + memset(&kex, 0, sizeof(kex)); + + fp = FMEMOPEN_STR(KEX_CONFIG_AUTH); + if (fp == NULL) { + printf("Failed to open memory stream.\n"); + goto fail; + } + + if (parse_sec_config(&kex, fp) < 0) { + printf("Failed to parse auth config.\n"); + fclose(fp); + goto fail; + } + + if (!kex.a.req) { + printf("auth=required not parsed correctly.\n"); + fclose(fp); + goto fail; + } + + fclose(fp); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_kex_parse_config_auth_invalid(void) +{ + struct sec_config kex; + FILE * fp; + + TEST_START(); + + memset(&kex, 0, sizeof(kex)); + + fp = FMEMOPEN_STR(KEX_CONFIG_AUTH_INVALID); + if (fp == NULL) { + printf("Failed to open memory stream.\n"); + goto fail; + } + + if (parse_sec_config(&kex, fp) == 0) { + printf("Invalid auth value should be rejected.\n"); + fclose(fp); + goto fail; + } + + fclose(fp); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* A caller-seeded req_auth survives parsing when no auth= line is set */ +static int test_kex_parse_config_auth_seed(void) +{ + struct sec_config kex; + FILE * fp; + + TEST_START(); + + memset(&kex, 0, sizeof(kex)); + kex.a.req = true; + + fp = FMEMOPEN_STR(KEX_CONFIG_NO_ENC); + if (fp == NULL) { + printf("Failed to open memory stream.\n"); + goto fail; + } + + if (parse_sec_config(&kex, fp) < 0) { + printf("Failed to parse config.\n"); + fclose(fp); + goto fail; + } + + if (!kex.a.req) { + printf("Seeded req_auth should survive parsing.\n"); + fclose(fp); + goto fail; + } + + fclose(fp); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* An explicit auth=optional clears a caller-seeded req_auth */ +static int test_kex_parse_config_auth_optional(void) +{ + struct sec_config kex; + FILE * fp; + + TEST_START(); + + memset(&kex, 0, sizeof(kex)); + kex.a.req = true; + + fp = FMEMOPEN_STR(KEX_CONFIG_AUTH_OPTIONAL); + if (fp == NULL) { + printf("Failed to open memory stream.\n"); + goto fail; + } + + if (parse_sec_config(&kex, fp) < 0) { + printf("Failed to parse auth=optional config.\n"); + fclose(fp); + goto fail; + } + + if (kex.a.req) { + printf("auth=optional should clear req_auth.\n"); + fclose(fp); + goto fail; + } + + fclose(fp); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* encryption=none must not drop auth=required or the digest */ +static int test_kex_parse_config_auth_no_enc(const char * config) +{ + struct sec_config kex; + FILE * fp; + + TEST_START(); + + memset(&kex, 0, sizeof(kex)); + + fp = FMEMOPEN_STR(config); + if (fp == NULL) { + printf("Failed to open memory stream.\n"); + goto fail; + } + + if (parse_sec_config(&kex, fp) < 0) { + printf("Failed to parse auth + encryption=none.\n"); + fclose(fp); + goto fail; + } + + if (!kex.a.req) { + printf("encryption=none should not drop required auth.\n"); + fclose(fp); + goto fail; + } + + if (kex.x.nid != NID_undef) { + printf("encryption=none should disable encryption.\n"); + fclose(fp); + goto fail; + } + + if (kex.d.nid != NID_sha512) { + printf("encryption=none should keep the digest.\n"); + fclose(fp); + goto fail; + } + + fclose(fp); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_kex_parse_config_cacert(void) +{ + struct sec_config kex; + FILE * fp; + + TEST_START(); + + memset(&kex, 0, sizeof(kex)); + + fp = FMEMOPEN_STR(KEX_CONFIG_CACERT); + if (fp == NULL) { + printf("Failed to open memory stream.\n"); + goto fail; + } + + if (parse_sec_config(&kex, fp) < 0) { + printf("Failed to parse cacert config.\n"); + fclose(fp); + goto fail; + } + + if (strcmp(kex.a.cacert, + "/etc/ouroboros/security/cacert/ca.crt") != 0) { + printf("cacert not parsed correctly.\n"); + fclose(fp); + goto fail; + } + + if (kex.a.req) { + printf("cacert must not imply req_auth.\n"); + fclose(fp); + goto fail; + } + + fclose(fp); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_kex_parse_config_unknown_key(void) +{ + struct sec_config kex; + FILE * fp; + + TEST_START(); + + memset(&kex, 0, sizeof(kex)); + + fp = FMEMOPEN_STR(KEX_CONFIG_UNKNOWN_KEY); + if (fp == NULL) { + printf("Failed to open memory stream.\n"); + goto fail; + } + + if (parse_sec_config(&kex, fp) == 0) { + printf("Unknown key should be rejected.\n"); + fclose(fp); + goto fail; + } + + fclose(fp); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +#ifndef HAVE_OPENSSL +/* A present security config must be refused without a backend. */ +static int test_kex_load_config_enotsup(void) +{ + struct sec_config kex; + FILE * fp; + + TEST_START(); + + fp = FMEMOPEN_STR(KEX_CONFIG_CUSTOM); + if (fp == NULL) { + printf("Failed to open config stream.\n"); + goto fail; + } + + if (crypt_load_sec_config(&kex, fp) != -ENOTSUP) { + printf("Loaded a config without a crypto backend.\n"); + fclose(fp); + goto fail; + } + + fclose(fp); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} +#endif + int kex_test(int argc, char ** argv) { @@ -808,9 +1232,18 @@ int kex_test(int argc, (void) argv; ret |= test_kex_create_destroy(); - ret |= test_kex_parse_config_empty(); - ret |= test_kex_parse_config_none(); #ifdef HAVE_OPENSSL + ret |= test_kex_parse_config_empty(); + ret |= test_kex_parse_config_none_rejected(); + ret |= test_kex_parse_config_no_enc(); + ret |= test_kex_parse_config_auth(); + ret |= test_kex_parse_config_auth_invalid(); + ret |= test_kex_parse_config_auth_seed(); + ret |= test_kex_parse_config_auth_optional(); + ret |= test_kex_parse_config_auth_no_enc(KEX_CONFIG_AUTH_THEN_NO_ENC); + ret |= test_kex_parse_config_auth_no_enc(KEX_CONFIG_NO_ENC_THEN_AUTH); + ret |= test_kex_parse_config_cacert(); + ret |= test_kex_parse_config_unknown_key(); ret |= test_kex_parse_config_custom(); ret |= test_kex_parse_config_whitespace(); ret |= test_kex_parse_config_cipher(); @@ -821,10 +1254,23 @@ int kex_test(int argc, ret |= test_kex_validate_algo(); ret |= test_kex_get_algo_from_pk_all(); ret |= test_kex_dhe_wrong_algo(); + ret |= test_kex_dhe_no_kdf(); ret |= test_kex_dhe_corrupted_pubkey_all(); ret |= test_kex_load_dhe_privkey(); ret |= test_kex_load_dhe_pubkey(); #else + ret |= test_kex_load_config_enotsup(); + + (void) test_kex_parse_config_empty; + (void) test_kex_parse_config_none_rejected; + (void) test_kex_parse_config_no_enc; + (void) test_kex_parse_config_auth; + (void) test_kex_parse_config_auth_invalid; + (void) test_kex_parse_config_auth_seed; + (void) test_kex_parse_config_auth_optional; + (void) test_kex_parse_config_auth_no_enc; + (void) test_kex_parse_config_cacert; + (void) test_kex_parse_config_unknown_key; (void) test_kex_parse_config_custom; (void) test_kex_parse_config_whitespace; (void) test_kex_parse_config_cipher; @@ -833,12 +1279,11 @@ int kex_test(int argc, (void) test_kex_all; (void) test_kex_validate_algo; (void) test_kex_get_algo_from_pk_all; - (void) test_kex_dhe_wrong_algo(); + (void) test_kex_dhe_wrong_algo; + (void) test_kex_dhe_no_kdf; (void) test_kex_dhe_corrupted_pubkey_all; (void) test_kex_load_dhe_privkey; (void) test_kex_load_dhe_pubkey; - - ret = TEST_RC_SKIP; #endif return ret; } diff --git a/src/lib/tests/kex_test_ml_kem.c b/src/lib/tests/kex_test_ml_kem.c index 7761c3dc..3059946c 100644 --- a/src/lib/tests/kex_test_ml_kem.c +++ b/src/lib/tests/kex_test_ml_kem.c @@ -220,7 +220,7 @@ static int test_kex_kem(const char * algo) pk.data = buf1; if (IS_HYBRID_KEM(algo)) - ct_len = kex_kem_encap_raw(pk, buf2, kdf, s1); + ct_len = kex_kem_encap_raw(algo, pk, buf2, kdf, s1); else ct_len = kex_kem_encap(pk, buf2, kdf, s1); @@ -267,6 +267,7 @@ static int test_kex_kem_corrupted_ciphertext(const char * algo) uint8_t s1[SYMMKEYSZ]; uint8_t s2[SYMMKEYSZ]; int kdf; + int ret; TEST_START("(%s)", algo); @@ -285,7 +286,7 @@ static int test_kex_kem_corrupted_ciphertext(const char * algo) pk.data = buf1; if (IS_HYBRID_KEM(algo)) - ct_len = kex_kem_encap_raw(pk, buf2, kdf, s1); + ct_len = kex_kem_encap_raw(algo, pk, buf2, kdf, s1); else ct_len = kex_kem_encap(pk, buf2, kdf, s1); @@ -301,14 +302,15 @@ static int test_kex_kem_corrupted_ciphertext(const char * algo) buf2[0] ^= 0xFF; buf2[ct_len - 1] ^= 0xFF; - /* ML-KEM uses implicit rejection */ - if (kex_kem_decap(pkp, ct, kdf, s2) < 0) { + /* EC hybrids may reject the corrupted point explicitly */ + ret = kex_kem_decap(pkp, ct, kdf, s2); + if (strstr(algo, "SecP") == NULL && ret < 0) { printf("Decapsulation failed unexpectedly.\n"); goto fail_pkp; } - /* The shared secrets should NOT match with corrupted CT */ - if (memcmp(s1, s2, SYMMKEYSZ) == 0) { + /* Corrupted CT must never yield the original secret */ + if (ret == 0 && memcmp(s1, s2, SYMMKEYSZ) == 0) { printf("Corrupted ciphertext produced same secret.\n"); goto fail_pkp; } @@ -360,7 +362,7 @@ static int test_kex_kem_wrong_keypair(const char * algo) } if (IS_HYBRID_KEM(algo)) - ct_len = kex_kem_encap_raw(pk1, buf3, NID_sha256, s1); + ct_len = kex_kem_encap_raw(algo, pk1, buf3, NID_sha256, s1); else ct_len = kex_kem_encap(pk1, buf3, NID_sha256, s1); @@ -422,7 +424,7 @@ static int test_kex_kem_truncated_ciphertext(const char * algo) pk.data = buf1; if (IS_HYBRID_KEM(algo)) - ct_len = kex_kem_encap_raw(pk, buf2, NID_sha256, s1); + ct_len = kex_kem_encap_raw(algo, pk, buf2, NID_sha256, s1); else ct_len = kex_kem_encap(pk, buf2, NID_sha256, s1); @@ -528,7 +530,7 @@ int kex_test_ml_kem(int argc, (void) argc; (void) argv; -#ifdef HAVE_OPENSSL_ML_KEM +#ifdef HAVE_ML ret |= test_kex_load_kem_privkey(); ret |= test_kex_load_kem_pubkey(); ret |= test_kex_kem_all(); diff --git a/src/lib/tests/keyrot_test.c b/src/lib/tests/keyrot_test.c new file mode 100644 index 00000000..efdc718e --- /dev/null +++ b/src/lib/tests/keyrot_test.c @@ -0,0 +1,1234 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Test of the key-rotation schedule + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#define _POSIX_C_SOURCE 200809L + +#include "config.h" + +#include <test/test.h> + +#ifdef HAVE_OPENSSL +#include <ouroboros/crypt.h> +#include <ouroboros/errno.h> +#include <ouroboros/pthread.h> + +#include "crypt/keyrot.h" + +#include <stdbool.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> + +static const uint8_t SEED_A[SYMMKEYSZ] = { + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, + 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, + 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20 +}; + +static const uint8_t SEED_B[SYMMKEYSZ] = { + 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, + 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, + 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, + 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0 +}; + +static int test_create_destroy(void) +{ + struct keyrot * kr; + + TEST_START(); + + kr = keyrot_create(SEED_A, 0, 0); + if (kr == NULL) + goto fail; + + keyrot_destroy(kr); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_epoch_range(void) +{ + struct keyrot * a; + + TEST_START(); + + /* epoch is a 4-bit wire field; 16 and up must be refused. */ + if (keyrot_create(SEED_A, 16, 0) != NULL) + goto fail; + + a = keyrot_create(SEED_A, 0, 0); + if (a == NULL) + goto fail; + + if (keyrot_rekey(a, SEED_A, 16) == 0) + goto fail_a; + + keyrot_destroy(a); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_a: + keyrot_destroy(a); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* Epochs of the live batches (cur, prev) must stay unique. */ +static int test_rekey_dup_epoch(void) +{ + struct keyrot * a; + + TEST_START(); + + a = keyrot_create(SEED_A, 0, 0); + if (a == NULL) + goto fail; + + if (keyrot_rekey(a, SEED_B, 0) != -1) { + printf("New key to the current epoch not a conflict.\n"); + goto fail_a; + } + + if (keyrot_rekey(a, SEED_B, 1) != 0) { + printf("Re-key to a fresh epoch refused.\n"); + goto fail_a; + } + + if (keyrot_rekey(a, SEED_B, 1) != -EREPLAY) { + printf("Same key to the current epoch not a replay.\n"); + goto fail_a; + } + + if (keyrot_rekey(a, SEED_A, 1) != -1) { + printf("New key to the current epoch not a conflict.\n"); + goto fail_a; + } + + if (keyrot_rekey(a, SEED_A, 0) != -EREPLAY) { + printf("Same key to the previous epoch not a replay.\n"); + goto fail_a; + } + + if (keyrot_rekey(a, SEED_B, 0) != -1) { + printf("New key to the previous epoch not a conflict.\n"); + goto fail_a; + } + + keyrot_destroy(a); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_a: + keyrot_destroy(a); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* The 4-bit wire epoch legitimately wraps 15 -> 0. */ +static int test_rekey_epoch_wrap(void) +{ + struct keyrot * a; + uint8_t sel[KR_SELECTOR_LEN]; + uint8_t n[KR_NONCE_LEN]; + const uint8_t * k; + + TEST_START(); + + a = keyrot_create(SEED_A, 14, 0); + if (a == NULL) + goto fail; + + if (keyrot_rekey(a, SEED_B, 15) != 0) + goto fail_a; + + if (keyrot_rekey(a, SEED_A, 0) != 0) { + printf("Epoch wrap 15 -> 0 refused.\n"); + goto fail_a; + } + + keyrot_tx_promote(a); + + if (keyrot_tx_next(a, sel, &k, n) != 0) { + printf("TX failed after epoch wrap.\n"); + goto fail_a; + } + + keyrot_destroy(a); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_a: + keyrot_destroy(a); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_tx_deterministic(void) +{ + struct keyrot * a; + struct keyrot * b; + uint8_t sela[KR_SELECTOR_LEN]; + uint8_t selb[KR_SELECTOR_LEN]; + uint8_t na[KR_NONCE_LEN]; + uint8_t nb[KR_NONCE_LEN]; + uint8_t ka[SYMMKEYSZ]; + const uint8_t * pa; + const uint8_t * pb; + + TEST_START(); + + a = keyrot_create(SEED_A, 0, 0); + if (a == NULL) + goto fail; + + b = keyrot_create(SEED_A, 0, 0); + if (b == NULL) + goto fail_a; + + if (keyrot_tx_next(a, sela, &pa, na) != 0) + goto fail_b; + + /* Copy out: pa points into the tcache, pb may reuse the slot. */ + memcpy(ka, pa, SYMMKEYSZ); + if (keyrot_tx_next(b, selb, &pb, nb) != 0) + goto fail_b; + + if (memcmp(sela, selb, KR_SELECTOR_LEN) != 0) + goto fail_b; + + if (memcmp(ka, pb, SYMMKEYSZ) != 0) + goto fail_b; + + if (memcmp(na, nb, KR_NONCE_LEN) != 0) + goto fail_b; + + keyrot_destroy(b); + keyrot_destroy(a); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_b: + keyrot_destroy(b); + fail_a: + keyrot_destroy(a); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_selector_layout(void) +{ + struct keyrot * a; + uint8_t sel[KR_SELECTOR_LEN]; + uint8_t nonce[KR_NONCE_LEN]; + const uint8_t * k; + + TEST_START(); + + a = keyrot_create(SEED_A, 3, 0); + if (a == NULL) + goto fail; + + /* First packet: epoch 3, node 0, seq 0 */ + if (keyrot_tx_next(a, sel, &k, nonce) != 0) + goto fail_a; + + if ((sel[0] >> 4) != 3) /* epoch */ + goto fail_a; + + if ((((sel[0] & 0x0F) << 8) | sel[1]) != 0) /* node */ + goto fail_a; + + if (sel[2] != 0 || sel[3] != 0 || sel[4] != 0 || sel[5] != 0) + goto fail_a; + + /* Second packet: seq advances to 1 */ + if (keyrot_tx_next(a, sel, &k, nonce) != 0) + goto fail_a; + + if (sel[5] != 1) + goto fail_a; + + keyrot_destroy(a); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_a: + keyrot_destroy(a); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_nodes_left_initial(void) +{ + struct keyrot * a; + + TEST_START(); + + a = keyrot_create(SEED_A, 0, 0); + if (a == NULL) + goto fail; + + if (keyrot_tx_nodes_left(a) != KEY_NODE_COUNT) + goto fail_a; + + keyrot_destroy(a); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_a: + keyrot_destroy(a); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_roundtrip(void) +{ + struct keyrot * a; /* role 0 */ + struct keyrot * b; /* role 1 */ + uint8_t sel[KR_SELECTOR_LEN]; + uint8_t ntx[KR_NONCE_LEN]; + uint8_t nrx[KR_NONCE_LEN]; + uint8_t ktx[SYMMKEYSZ]; + const uint8_t * ptx; + const uint8_t * prx; + struct kr_rx rx; + int i; + + TEST_START(); + + a = keyrot_create(SEED_A, 0, 0); + if (a == NULL) + goto fail; + + b = keyrot_create(SEED_A, 0, 1); + if (b == NULL) + goto fail_a; + + for (i = 0; i < 256; i++) { + if (keyrot_tx_next(a, sel, &ptx, ntx) != 0) + goto fail_b; + memcpy(ktx, ptx, SYMMKEYSZ); + if (keyrot_rx_lookup(b, sel, &prx, nrx, &rx) != 0) + goto fail_b; + if (keyrot_rx_commit(b, &rx) != 0) + goto fail_b; + if (memcmp(ktx, prx, SYMMKEYSZ) != 0) + goto fail_b; + if (memcmp(ntx, nrx, KR_NONCE_LEN) != 0) + goto fail_b; + } + + keyrot_destroy(b); + keyrot_destroy(a); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_b: + keyrot_destroy(b); + fail_a: + keyrot_destroy(a); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_direction_separation(void) +{ + struct keyrot * a; /* role 0 */ + struct keyrot * b; /* role 1 */ + uint8_t sela[KR_SELECTOR_LEN]; + uint8_t selb[KR_SELECTOR_LEN]; + uint8_t n[KR_NONCE_LEN]; + uint8_t ka[SYMMKEYSZ]; + const uint8_t * pa; + const uint8_t * pb; + + TEST_START(); + + a = keyrot_create(SEED_A, 0, 0); + if (a == NULL) + goto fail; + + b = keyrot_create(SEED_A, 0, 1); + if (b == NULL) + goto fail_a; + + if (keyrot_tx_next(a, sela, &pa, n) != 0) + goto fail_b; + + memcpy(ka, pa, SYMMKEYSZ); + if (keyrot_tx_next(b, selb, &pb, n) != 0) + goto fail_b; + + /* Same position, different role -> different leaf key */ + if (memcmp(ka, pb, SYMMKEYSZ) == 0) + goto fail_b; + + keyrot_destroy(b); + keyrot_destroy(a); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_b: + keyrot_destroy(b); + fail_a: + keyrot_destroy(a); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* Build a selector by hand (test knows the wire format). */ +static void mk_sel(uint8_t epoch, + uint16_t node, + uint32_t seq, + uint8_t sel[KR_SELECTOR_LEN]) +{ + sel[0] = (uint8_t) ((epoch << 4) | ((node >> 8) & 0x0F)); + sel[1] = (uint8_t) (node & 0xFF); + sel[2] = (uint8_t) (seq >> 24); + sel[3] = (uint8_t) (seq >> 16); + sel[4] = (uint8_t) (seq >> 8); + sel[5] = (uint8_t) (seq); +} + +static int test_random_access(void) +{ + struct keyrot * b; + uint8_t s0[KR_SELECTOR_LEN]; + uint8_t s5[KR_SELECTOR_LEN]; + uint8_t n[KR_NONCE_LEN]; + uint8_t k_first[SYMMKEYSZ]; + uint8_t k_node5[SYMMKEYSZ]; + const uint8_t * p; + struct kr_rx rx; + + TEST_START(); + + b = keyrot_create(SEED_A, 0, 1); + if (b == NULL) + goto fail; + + mk_sel(0, 0, 0, s0); + mk_sel(0, 5, 12345, s5); /* a far-ahead node, mid-span */ + + /* Jump straight to node 0 */ + if (keyrot_rx_lookup(b, s0, &p, n, &rx) != 0) + goto fail_b; + + memcpy(k_first, p, SYMMKEYSZ); + + /* Jump forward to node 5 (simulates a burst skip) */ + if (keyrot_rx_lookup(b, s5, &p, n, &rx) != 0) + goto fail_b; + + memcpy(k_node5, p, SYMMKEYSZ); + + /* Different nodes must yield different keys */ + if (memcmp(k_first, k_node5, SYMMKEYSZ) == 0) + goto fail_b; + + /* Jump back to node 0: still works, identical (no wedge) */ + if (keyrot_rx_lookup(b, s0, &p, n, &rx) != 0) + goto fail_b; + + if (memcmp(k_first, p, SYMMKEYSZ) != 0) + goto fail_b; + + /* Out-of-range node must be rejected */ + mk_sel(0, KEY_NODE_COUNT, 0, s0); + if (keyrot_rx_lookup(b, s0, &p, n, &rx) == 0) + goto fail_b; + + keyrot_destroy(b); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_b: + keyrot_destroy(b); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* + * Look up and commit one within-node counter on epoch 0. Returns 0 on + * accept, 1 on a rejected commit (replay or too old), and -1 if the + * lookup itself failed - kept distinct so a reject assertion can never + * pass on an unrelated lookup miss. + */ +static int commit_ctr(struct keyrot * kr, + uint32_t ctr) +{ + uint8_t sel[KR_SELECTOR_LEN]; + uint8_t n[KR_NONCE_LEN]; + const uint8_t * k; + struct kr_rx rx; + + mk_sel(0, 0, ctr, sel); + + if (keyrot_rx_lookup(kr, sel, &k, n, &rx) != 0) + return -1; + + return keyrot_rx_commit(kr, &rx) == 0 ? 0 : 1; +} + +static int test_replay_window(void) +{ + struct keyrot * b; + struct keyrot * c; + uint32_t base; + uint32_t jump; + + TEST_START(); + + b = keyrot_create(SEED_A, 0, 1); + if (b == NULL) + goto fail; + + /* Fresh counters accepted; an immediate replay is rejected. */ + if (commit_ctr(b, 100) != 0) + goto fail_b; + + if (commit_ctr(b, 100) != 1) + goto fail_b; + + /* In-window reorder: accepted once, rejected on replay. */ + if (commit_ctr(b, 105) != 0) + goto fail_b; + + if (commit_ctr(b, 102) != 0) + goto fail_b; + + if (commit_ctr(b, 102) != 1) + goto fail_b; + + /* Too-old boundary: the window edge is rejected, just inside is not. */ + base = 4 * KEY_REPLAY_WINDOW; + if (commit_ctr(b, base) != 0) + goto fail_b; + + if (commit_ctr(b, base - (KEY_REPLAY_WINDOW - 64)) != 1) + goto fail_b; + + if (commit_ctr(b, base - (KEY_REPLAY_WINDOW - 64) + 1) != 0) + goto fail_b; + + /* + * RFC 6479 slack-word regression: two low counters, then a + * forward jump of a full bitmap that aliases their slot, then a + * replay of a low counter. Without the reserved slack word this + * replay is wrongly accepted. + */ + c = keyrot_create(SEED_A, 0, 1); + if (c == NULL) + goto fail_b; + + if (commit_ctr(c, 70) != 0) + goto fail_c; + + if (commit_ctr(c, 74) != 0) + goto fail_c; + + jump = KEY_REPLAY_WINDOW + 63; + if (commit_ctr(c, jump) != 0) + goto fail_c; + + if (commit_ctr(c, 74) != 1) + goto fail_c; + + keyrot_destroy(c); + keyrot_destroy(b); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_c: + keyrot_destroy(c); + fail_b: + keyrot_destroy(b); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_lookup_no_commit(void) +{ + struct keyrot * b; + uint8_t sel[KR_SELECTOR_LEN]; + uint8_t n[KR_NONCE_LEN]; + const uint8_t * k; + struct kr_rx rx; + int i; + + TEST_START(); + + b = keyrot_create(SEED_A, 0, 1); + if (b == NULL) + goto fail; + + mk_sel(0, 0, 100, sel); + + /* Repeated lookups are pre-AEAD and must not consume the slot. */ + for (i = 0; i < 4; i++) { + if (keyrot_rx_lookup(b, sel, &k, n, &rx) != 0) + goto fail_b; + } + + /* The slot is still fresh, so the first commit accepts ... */ + if (keyrot_rx_commit(b, &rx) != 0) + goto fail_b; + + /* ... and only the commit advanced it, so the next is a replay. */ + if (keyrot_rx_commit(b, &rx) == 0) + goto fail_b; + + keyrot_destroy(b); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_b: + keyrot_destroy(b); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_commit_prev_batch(void) +{ + struct keyrot * b; + uint8_t sel[KR_SELECTOR_LEN]; + uint8_t n[KR_NONCE_LEN]; + const uint8_t * k; + struct kr_rx rx; + + TEST_START(); + + b = keyrot_create(SEED_A, 0, 1); + if (b == NULL) + goto fail; + + /* Capture a packet under cur (epoch 0). */ + mk_sel(0, 0, 7, sel); + if (keyrot_rx_lookup(b, sel, &k, n, &rx) != 0) + goto fail_b; + + /* Re-key: the captured batch becomes prev and the flag clears. */ + if (keyrot_rekey(b, SEED_B, 1) != 0) + goto fail_b; + + /* The straggler commits under prev without claiming a switch. */ + if (keyrot_rx_commit(b, &rx) != 0) + goto fail_b; + + if (keyrot_peer_switched(b)) + goto fail_b; + + /* prev still holds a replay window: its replay is rejected. */ + if (keyrot_rx_commit(b, &rx) == 0) + goto fail_b; + + keyrot_destroy(b); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_b: + keyrot_destroy(b); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_replay_forward_clear(void) +{ + struct keyrot * d; + uint32_t low; + uint32_t alias; + uint32_t jump; + + TEST_START(); + + d = keyrot_create(SEED_A, 0, 1); + if (d == NULL) + goto fail; + + /* alias shares low's slot a window away; the jump must clear it. */ + low = 10; + alias = low + KEY_REPLAY_WINDOW; + jump = alias + KEY_REPLAY_WINDOW / 2; + + if (commit_ctr(d, low) != 0) + goto fail_d; + + if (commit_ctr(d, jump) != 0) + goto fail_d; + + if (commit_ctr(d, alias) != 0) + goto fail_d; + + if (commit_ctr(d, alias) != 1) + goto fail_d; + + keyrot_destroy(d); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_d: + keyrot_destroy(d); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_rekey_overlap(void) +{ + struct keyrot * a; /* role 0 */ + struct keyrot * b; /* role 1 */ + uint8_t old_sel[KR_SELECTOR_LEN]; + uint8_t sel[KR_SELECTOR_LEN]; + uint8_t ntx[KR_NONCE_LEN]; + uint8_t nrx[KR_NONCE_LEN]; + uint8_t ktx[SYMMKEYSZ]; + const uint8_t * ptx; + const uint8_t * prx; + struct kr_rx rx; + + TEST_START(); + + a = keyrot_create(SEED_A, 0, 0); + if (a == NULL) + goto fail; + + b = keyrot_create(SEED_A, 0, 1); + if (b == NULL) + goto fail_a; + + /* Send one gen-0 packet; keep its selector for the overlap. */ + if (keyrot_tx_next(a, old_sel, &ptx, ntx) != 0) + goto fail_b; + + memcpy(ktx, ptx, SYMMKEYSZ); + if (keyrot_rx_lookup(b, old_sel, &prx, nrx, &rx) != 0) + goto fail_b; + + if (memcmp(ktx, prx, SYMMKEYSZ) != 0) + goto fail_b; + + /* Both ends re-key to epoch 1 with a fresh seed. */ + if (keyrot_rekey(a, SEED_B, 1) != 0) + goto fail_b; + + if (keyrot_rekey(b, SEED_B, 1) != 0) + goto fail_b; + + /* TX is gated until promotion; promote a to emit the new epoch. */ + keyrot_tx_promote(a); + + /* New gen-1 traffic works. */ + if (keyrot_tx_next(a, sel, &ptx, ntx) != 0) + goto fail_b; + + memcpy(ktx, ptx, SYMMKEYSZ); + if (keyrot_rx_lookup(b, sel, &prx, nrx, &rx) != 0) + goto fail_b; + + if (memcmp(ktx, prx, SYMMKEYSZ) != 0) + goto fail_b; + + /* A straggling gen-0 packet still decrypts (overlap window). */ + if (keyrot_rx_lookup(b, old_sel, &prx, nrx, &rx) != 0) + goto fail_b; + + /* An unknown epoch is rejected. */ + mk_sel(7, 0, 0, sel); + if (keyrot_rx_lookup(b, sel, &prx, nrx, &rx) == 0) + goto fail_b; + + keyrot_destroy(b); + keyrot_destroy(a); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_b: + keyrot_destroy(b); + fail_a: + keyrot_destroy(a); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_tx_gate(void) +{ + struct keyrot * a; /* role 0 */ + struct keyrot * b; /* role 1 */ + uint8_t sel[KR_SELECTOR_LEN]; + uint8_t n[KR_NONCE_LEN]; + const uint8_t * p; + struct kr_rx rx; + + TEST_START(); + + a = keyrot_create(SEED_A, 0, 0); + if (a == NULL) + goto fail; + + b = keyrot_create(SEED_A, 0, 1); + if (b == NULL) + goto fail_a; + + /* Both re-key to epoch 1; TX must stay on epoch 0 until promoted. */ + if (keyrot_rekey(a, SEED_B, 1) != 0) + goto fail_b; + + if (keyrot_rekey(b, SEED_B, 1) != 0) + goto fail_b; + + /* a's TX still stamps the old epoch (0). */ + if (keyrot_tx_next(a, sel, &p, n) != 0) + goto fail_b; + + if ((sel[0] >> 4) != 0) + goto fail_b; + + /* b decrypts the old-epoch packet via its prev batch. */ + if (keyrot_rx_lookup(b, sel, &p, n, &rx) != 0) + goto fail_b; + + if (keyrot_rx_commit(b, &rx) != 0) + goto fail_b; + + /* b has not yet seen the new epoch from a. */ + if (keyrot_peer_switched(b)) + goto fail_b; + + /* a promotes; its TX now stamps the new epoch (1). */ + keyrot_tx_promote(a); + if (keyrot_tx_next(a, sel, &p, n) != 0) + goto fail_b; + + if ((sel[0] >> 4) != 1) + goto fail_b; + + /* b sees the new epoch and reports the peer switched. */ + if (keyrot_rx_lookup(b, sel, &p, n, &rx) != 0) + goto fail_b; + + if (keyrot_rx_commit(b, &rx) != 0) + goto fail_b; + + if (!keyrot_peer_switched(b)) + goto fail_b; + + keyrot_destroy(b); + keyrot_destroy(a); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_b: + keyrot_destroy(b); + fail_a: + keyrot_destroy(a); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_peer_switched_commit_only(void) +{ + struct keyrot * b; + uint8_t sel[KR_SELECTOR_LEN]; + uint8_t n[KR_NONCE_LEN]; + const uint8_t * k; + struct kr_rx rx; + + TEST_START(); + + b = keyrot_create(SEED_A, 0, 1); + if (b == NULL) + goto fail; + + /* A re-key clears the flag until a packet is seen on cur. */ + if (keyrot_rekey(b, SEED_B, 1) != 0) + goto fail_b; + + if (keyrot_peer_switched(b)) + goto fail_b; + + mk_sel(1, 0, 0, sel); + + /* Lookup is pre-AEAD: selecting a key must not flip the flag. */ + if (keyrot_rx_lookup(b, sel, &k, n, &rx) != 0) + goto fail_b; + + if (keyrot_peer_switched(b)) + goto fail_b; + + /* Commit runs post-AEAD and is what records the peer switched. */ + if (keyrot_rx_commit(b, &rx) != 0) + goto fail_b; + + if (!keyrot_peer_switched(b)) + goto fail_b; + + keyrot_destroy(b); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_b: + keyrot_destroy(b); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_commit_evicted(void) +{ + struct keyrot * b; + uint8_t sel[KR_SELECTOR_LEN]; + uint8_t n[KR_NONCE_LEN]; + const uint8_t * k; + struct kr_rx rx; + + TEST_START(); + + b = keyrot_create(SEED_A, 0, 1); + if (b == NULL) + goto fail; + + mk_sel(0, 0, 3, sel); + if (keyrot_rx_lookup(b, sel, &k, n, &rx) != 0) + goto fail_b; + + /* Two re-keys drop the captured batch from both cur and prev. */ + if (keyrot_rekey(b, SEED_B, 1) != 0) + goto fail_b; + + if (keyrot_rekey(b, SEED_A, 2) != 0) + goto fail_b; + + /* Commit on an evicted batch is a silent no-op, not a fault. */ + if (keyrot_rx_commit(b, &rx) != 0) + goto fail_b; + + keyrot_destroy(b); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_b: + keyrot_destroy(b); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* TX fails closed when the tx_epoch batch is evicted, until promote. */ +static int test_tx_fail_closed(void) +{ + struct keyrot * b; + uint8_t sel[KR_SELECTOR_LEN]; + uint8_t n[KR_NONCE_LEN]; + const uint8_t * k; + + TEST_START(); + + b = keyrot_create(SEED_A, 0, 0); + if (b == NULL) + goto fail; + + if (keyrot_rekey(b, SEED_B, 1) != 0) + goto fail_b; + + if (keyrot_tx_next(b, sel, &k, n) != 0) { + printf("TX should keep the old epoch after one re-key.\n"); + goto fail_b; + } + + /* Second re-key without promote evicts the TX epoch-0 batch. */ + if (keyrot_rekey(b, SEED_A, 2) != 0) + goto fail_b; + + if (keyrot_tx_next(b, sel, &k, n) == 0) { + printf("TX should fail closed with tx_epoch evicted.\n"); + goto fail_b; + } + + keyrot_tx_promote(b); + + if (keyrot_tx_next(b, sel, &k, n) != 0) { + printf("TX should resync after promote.\n"); + goto fail_b; + } + + keyrot_destroy(b); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_b: + keyrot_destroy(b); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* + * Concurrency: many TX threads + RX + re-key share one keyrot. The + * (epoch, counter) the TX side stamps must be globally unique (no AEAD + * nonce reuse). Capped below 16 re-keys so epoch maps 1:1 to a batch and + * the wire epoch never wraps (a wrapped epoch under a fresh key is not + * reuse but would false-trip the uniqueness check). Run under TSan to + * catch data races the static reviews can't. + */ +#define CT_THREADS 4 +#define CT_PKTS 2000 +#define CT_REKEYS 8 + +struct ct_rec { + uint8_t epoch; + uint64_t ctr; +}; + +struct ct_arg { + struct keyrot * kr; + struct ct_rec * recs; + size_t n; +}; + +static void * ct_tx_thread(void * a) +{ + struct ct_arg * arg = a; + uint8_t sel[KR_SELECTOR_LEN]; + uint8_t nonce[KR_NONCE_LEN]; + const uint8_t * k; + uint64_t ctr; + size_t i; + size_t j; + + for (i = 0; i < CT_PKTS; i++) { + if (keyrot_tx_next(arg->kr, sel, &k, nonce) != 0) + continue; + + ctr = 0; + for (j = 0; j < 8; j++) + ctr = (ctr << 8) | nonce[j]; + + arg->recs[arg->n].epoch = (uint8_t) (sel[0] >> 4); + arg->recs[arg->n].ctr = ctr; + arg->n++; + } + + return NULL; +} + +static void * ct_rx_thread(void * a) +{ + struct keyrot * kr = a; + uint8_t sel[KR_SELECTOR_LEN]; + uint8_t nonce[KR_NONCE_LEN]; + const uint8_t * k; + struct kr_rx rx; + size_t i; + + /* Exercise rx_lookup against re-key reclaim; results ignored. */ + for (i = 0; i < CT_PKTS; i++) { + mk_sel((uint8_t) (i % 16), 0, (uint32_t) i, sel); + if (keyrot_rx_lookup(kr, sel, &k, nonce, &rx) == 0) + (void) keyrot_rx_commit(kr, &rx); + } + + return NULL; +} + +static void * ct_rekey_thread(void * a) +{ + struct keyrot * kr = a; + struct timespec t; + int e; + + t.tv_sec = 0; + t.tv_nsec = 2 * 1000 * 1000; /* 2 ms */ + + for (e = 1; e <= CT_REKEYS; e++) { + nanosleep(&t, NULL); + if (keyrot_rekey(kr, (e & 1) ? SEED_B : SEED_A, + (uint8_t) e) != 0) + break; + keyrot_tx_promote(kr); + } + + return NULL; +} + +static int ct_cmp(const void * x, + const void * y) +{ + const struct ct_rec * a = x; + const struct ct_rec * b = y; + + if (a->epoch != b->epoch) + return a->epoch < b->epoch ? -1 : 1; + + if (a->ctr != b->ctr) + return a->ctr < b->ctr ? -1 : 1; + + return 0; +} + +static int test_concurrent_nonce_unique(void) +{ + struct keyrot * kr; + struct ct_arg arg[CT_THREADS]; + pthread_t tx[CT_THREADS]; + pthread_t rx; + pthread_t rk; + struct ct_rec * all; + size_t total; + size_t i; + bool reuse = false; + + TEST_START(); + + kr = keyrot_create(SEED_A, 0, 0); + if (kr == NULL) + goto fail; + + all = malloc(sizeof(*all) * CT_THREADS * CT_PKTS); + if (all == NULL) + goto fail_kr; + + for (i = 0; i < CT_THREADS; i++) { + arg[i].kr = kr; + arg[i].n = 0; + arg[i].recs = all + i * CT_PKTS; + } + + for (i = 0; i < CT_THREADS; i++) + pthread_create(&tx[i], NULL, ct_tx_thread, &arg[i]); + + pthread_create(&rx, NULL, ct_rx_thread, kr); + pthread_create(&rk, NULL, ct_rekey_thread, kr); + + for (i = 0; i < CT_THREADS; i++) + pthread_join(tx[i], NULL); + + pthread_join(rx, NULL); + pthread_join(rk, NULL); + + total = 0; + for (i = 0; i < CT_THREADS; i++) { + memmove(all + total, all + i * CT_PKTS, + arg[i].n * sizeof(*all)); + total += arg[i].n; + } + + qsort(all, total, sizeof(*all), ct_cmp); + + for (i = 1; i < total; i++) + if (ct_cmp(&all[i - 1], &all[i]) == 0) { + printf("(epoch %u, ctr %llu) reused\n", + all[i].epoch, + (unsigned long long) all[i].ctr); + reuse = true; + break; + } + + free(all); + + if (reuse) + goto fail_kr; + + keyrot_destroy(kr); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_kr: + keyrot_destroy(kr); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} +#endif /* HAVE_OPENSSL */ + +int keyrot_test(int argc, + char ** argv) +{ + int ret = 0; + + (void) argc; + (void) argv; + +#ifdef HAVE_OPENSSL + ret |= test_create_destroy(); + ret |= test_epoch_range(); + ret |= test_rekey_dup_epoch(); + ret |= test_rekey_epoch_wrap(); + ret |= test_tx_deterministic(); + ret |= test_selector_layout(); + ret |= test_nodes_left_initial(); + ret |= test_roundtrip(); + ret |= test_direction_separation(); + ret |= test_random_access(); + ret |= test_peer_switched_commit_only(); + ret |= test_commit_evicted(); + ret |= test_tx_fail_closed(); + ret |= test_replay_window(); + ret |= test_lookup_no_commit(); + ret |= test_commit_prev_batch(); + ret |= test_replay_forward_clear(); + ret |= test_rekey_overlap(); + ret |= test_tx_gate(); + ret |= test_concurrent_nonce_unique(); +#endif + return ret; +} diff --git a/src/lib/tests/poa_test.c b/src/lib/tests/poa_test.c new file mode 100644 index 00000000..99886769 --- /dev/null +++ b/src/lib/tests/poa_test.c @@ -0,0 +1,307 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Flow PoA tests + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#if defined(__linux__) || defined(__CYGWIN__) +#define _DEFAULT_SOURCE +#else +#define _POSIX_C_SOURCE 200809L +#endif + +#include <test/test.h> + +#include "poa/addr.c" +#ifdef HAVE_RAW_SOCKETS +#include "poa/eth.c" +#endif + +#include <arpa/inet.h> +#include <stdio.h> + +#ifdef HAVE_RAW_SOCKETS +static const uint8_t eth_our_mac[POA_MAC_SIZE] = + { 0x02, 0x00, 0x00, 0x00, 0x00, 0x01 }; +static const uint8_t eth_far_mac[POA_MAC_SIZE] = + { 0x02, 0x00, 0x00, 0x00, 0x00, 0x02 }; +static const uint8_t eth_bc_mac[POA_MAC_SIZE] = + { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; +#endif + +/* PoA core stubs: the reader threads never run in this test. */ +void poa_rx_pkt(struct poa * poa, + uint32_t eid, + struct ssm_pk_buff * spb) +{ + (void) poa; + (void) eid; + (void) spb; +} + +void poa_rx_mgmt(struct poa * poa, + const struct poa_addr * src, + const uint8_t * buf, + size_t len) +{ + (void) poa; + (void) src; + (void) buf; + (void) len; +} + +int poa_spb_reserve(struct ssm_pk_buff ** spb, + size_t len) +{ + (void) spb; + (void) len; + + return -1; +} + +size_t poa_link_updown(int ifindex, + bool up) +{ + (void) ifindex; + (void) up; + + return 0; +} + +bool poa_has_name(const uint8_t * hash) +{ + (void) hash; + + return false; +} + +int poa_bcast_mgmt(const struct poa_addr * dst, + const uint8_t * buf, + size_t len) +{ + (void) dst; + (void) buf; + (void) len; + + return 0; +} + +static void udp4_addr(struct poa_addr * addr, + const char * ip, + uint16_t port) +{ + memset(addr, 0, sizeof(*addr)); + + addr->type = POA_UDP4; + addr->udp4.port = port; + + inet_pton(AF_INET, ip, &addr->udp4.ip_addr); +} + +static int test_poa_addr_cmp(void) +{ + struct poa_addr a; + struct poa_addr b; + + TEST_START(); + + udp4_addr(&a, "10.0.0.10", 3435); + udp4_addr(&b, "10.0.0.10", 3435); + + if (poa_addr_cmp(&a, &b) != 0) { + printf("Identical addresses did not match.\n"); + goto fail; + } + + udp4_addr(&b, "10.0.0.10", 3436); + + if (poa_addr_cmp(&a, &b) == 0) { + printf("Addresses with a different port matched.\n"); + goto fail; + } + + udp4_addr(&b, "10.0.0.11", 3435); + + if (poa_addr_cmp(&a, &b) == 0) { + printf("Addresses with a different ip matched.\n"); + goto fail; + } + + b.type = POA_UDP6; + if (poa_addr_cmp(&a, &b) == 0) { + printf("Addresses of a different type matched.\n"); + goto fail; + } + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_poa_mgmt_msg_qos(void) +{ + struct poa_mgmt_msg msg; + qosspec_t qs; + + TEST_START(); + + poa_mgmt_msg_ser(&msg, POA_FLOW_REQ, 1, 2, qos_stream, 0, 0); + + if (msg.code != POA_FLOW_REQ) { + printf("Wrong code in management message.\n"); + goto fail; + } + + if (ntoh32(msg.s_eid) != 1 || ntoh32(msg.d_eid) != 2) { + printf("Wrong PoA ids in management message.\n"); + goto fail; + } + + memset(&qs, 0, sizeof(qs)); + + poa_mgmt_msg_qos(&msg, &qs); + + if (memcmp(&qs, &qos_stream, sizeof(qs)) != 0) { + printf("QoS did not survive the management message.\n"); + goto fail; + } + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +#ifdef HAVE_RAW_SOCKETS + +static void eth_fake_priv(struct eth_priv * priv, + struct poa * e) +{ + memset(priv, 0, sizeof(*priv)); + memset(e, 0, sizeof(*e)); + + e->type = POA_ETH; + + strcpy(e->local.eth.src.dev, "test0"); + + priv->poa = e; + priv->ethertype = htons(0xA000); + priv->mtu = 1500; + + memcpy(priv->hw_addr, eth_our_mac, POA_MAC_SIZE); +} + +static int test_eth_frame(void) +{ + struct eth_priv priv; + struct poa e; + uint8_t buf[64]; + struct eth_hdr * hdr = (struct eth_hdr *) buf; + const char * body = "hello"; + size_t n; + size_t plen; + uint32_t eid; + + TEST_START(); + + eth_fake_priv(&priv, &e); + + eth_hdr_ser(&priv, hdr, eth_our_mac, 7, strlen(body)); + + memcpy(buf + ETH_HDR_TOT_SIZE, body, strlen(body)); + + n = ETH_HDR_TOT_SIZE + strlen(body); + if (frame_parse(&priv, buf, n, &eid, &plen) < 0) { + printf("Failed to parse a valid frame.\n"); + goto fail; + } + + if (eid != 7 || plen != strlen(body)) { + printf("Expected eid 7 len %zu, got %u len %zu.\n", + strlen(body), eid, plen); + goto fail; + } + + hdr->poa.hcs ^= 0xFF; + + if (frame_parse(&priv, buf, n, &eid, &plen) == 0) { + printf("Accepted a corrupt header checksum.\n"); + goto fail; + } + + hdr->poa.hcs ^= 0xFF; + hdr->ethertype ^= 0xFF; + + if (frame_parse(&priv, buf, n, &eid, &plen) == 0) { + printf("Accepted a foreign Ethertype.\n"); + goto fail; + } + + hdr->ethertype ^= 0xFF; + + if (frame_parse(&priv, buf, ETH_HDR_SIZE + 2, &eid, &plen) == 0) { + printf("Accepted a truncated frame.\n"); + goto fail; + } + + eth_hdr_ser(&priv, hdr, eth_far_mac, 7, strlen(body)); + + if (frame_parse(&priv, buf, n, &eid, &plen) == 0) { + printf("Accepted a frame for another host.\n"); + goto fail; + } + + eth_hdr_ser(&priv, hdr, eth_bc_mac, 7, strlen(body)); + + if (frame_parse(&priv, buf, n, &eid, &plen) < 0) { + printf("Rejected a broadcast frame.\n"); + goto fail; + } + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +#endif /* HAVE_RAW_SOCKETS */ + +int poa_test(int argc, + char ** argv) +{ + int ret = 0; + + (void) argc; + (void) argv; + + ret |= test_poa_addr_cmp(); + ret |= test_poa_mgmt_msg_qos(); +#ifdef HAVE_RAW_SOCKETS + ret |= test_eth_frame(); +#endif + + return ret; +} |