From 4d030871c004c917458b5e1c296edcfc689a2e66 Mon Sep 17 00:00:00 2001 From: Dirk Engling Date: Sat, 31 Oct 2026 15:59:41 +0100 Subject: Make tcp worker multithreaded, too This shamelessly steals the model established in the UDP workers, i.e. if the kernel is able to bind multiple listening sockets to the same endpoint, we will share them across threads. If it can't, all threads will accept on the same socket. The fast path handles most of the connections, only the rare case of no data yet on the socket is offloaded to the slow path (which used to be the mainloop) for completion. This scales nicely, as the higher the load is, the higher chances are that an accept happens only after the network roundtrip to the peer completes, leading to more connections that already have data waiting for us. We also consolidate the workstruct init and deinit code, preparation for later unifying all thread state structs for all protocols (mca, udc and tcp). The config file gains a per tcp connection worker count and some docs explaining what this knob buys and costs, nudging the user to either keep it 0 or dial it up to at least 2. --- ot_fdstate.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'ot_fdstate.c') diff --git a/ot_fdstate.c b/ot_fdstate.c index 7d742da..09daf50 100644 --- a/ot_fdstate.c +++ b/ot_fdstate.c @@ -91,7 +91,7 @@ uint32_t fdstate_open(int fd, void *cookie, ot_time expiry) { } word = atomic_load_explicit(&slot->state, memory_order_relaxed); - atomic_store_explicit(&slot->cookie, cookie, memory_order_relaxed); + atomic_store_explicit(&slot->cookie, cookie, memory_order_release); atomic_store_explicit(&slot->expiry, expiry, memory_order_relaxed); /* Publishes the two above to whoever claims the slot next */ atomic_store_explicit(&slot->state, FD_WORD(FD_GEN(word), FD_IDLE), memory_order_release); @@ -183,14 +183,17 @@ int fdstate_next_expired(ot_time now, int *cursor) { void *fdstate_cookie(int fd) { ot_fdslot *slot = fdstate_slot(fd); - return slot ? atomic_load_explicit(&slot->cookie, memory_order_relaxed) : NULL; + /* Pairs with the release in fdstate_set_cookie and fdstate_open */ + return slot ? atomic_load_explicit(&slot->cookie, memory_order_acquire) : NULL; } void fdstate_set_cookie(int fd, void *cookie) { ot_fdslot *slot = fdstate_slot(fd); + /* Publishes what the cookie points at to whoever reads it next, which may + be another thread entirely */ if (slot) - atomic_store_explicit(&slot->cookie, cookie, memory_order_relaxed); + atomic_store_explicit(&slot->cookie, cookie, memory_order_release); } void fdstate_set_expiry(int fd, ot_time expiry) { -- cgit v1.3