14 files changed, 421 insertions, 135 deletions

diff --git a/DESIGN b/DESIGN
new file mode 100644
index 0000000..ff37706
--- /dev/null
+++ b/DESIGN
@@ -0,0 +1,99 @@
+opentracker design notes
+========================
+
+Decisions that cost measurement to reach, and the alternatives that were
+tried and rejected.
+
+What the tracker is optimised for
+---------------------------------
+
+UDP carries most of the traffic. An HTTP request costs about ten times a UDP
+announce, 26 microseconds of tracker CPU against 2.5. Nearly all of that is
+the kernel setting up and tearing down a connection rather than tracker
+code.
+
+A connection carries one request and closes. Keep-alive works and is rarely
+asked for.
+
+Statistics are approximate on purpose. Counters are per thread and summed
+when read, so a /stats read can under-report once. Nothing drifts.
+
+
+Decisions
+---------
+
+One UDP socket per worker, where the kernel spreads datagrams over a group
+bound to one address. Cost per datagram then stays flat as workers are added,
+0.71 to 0.72 microseconds from one worker to eight. Sharing one socket grows
+it eightfold over the same range, 0.71 to 6.07.
+
+listen.udp.workers defaults to 1. One worker beats none by 19 to 37 per cent
+everywhere, because a blocking read costs one system call where the event
+loop costs two. Beyond one the gain depends on the kernel balancing, so
+raising it is the operator's decision.
+
+A thread parked in a blocking read is woken by a signal, not by closing or
+shutting down its socket:
+
+    mechanism           Linux     FreeBSD   macOS
+    close()             blocked   blocked   0 ms
+    shutdown(SHUT_RD)   1 ms      1 ms      blocked
+    SO_RCVTIMEO         works     works     works
+    pthread_kill        1 ms      1 ms      1 ms
+
+Only the signal works everywhere. A receive timeout also works, but wakes
+every thread on every tick for the life of the process to catch something
+that happens once: at eight workers it doubled idle context switches, 255 to
+495 over thirty seconds.
+
+Each binary is built in one compiler invocation. Without object files a build
+cannot link objects compiled with different FEATURES, which was a real
+failure mode. It costs incremental rebuilds, 0.075 seconds becoming 1.56.
+
+A socket is read or written, never both. http_interest_set replaces an
+add-and-drop pair so that the invariant cannot be broken by accident, and the
+event layer hears about a change once instead of twice.
+
+Accept waits until the client's request has arrived: TCP_DEFER_ACCEPT on
+Linux, the accf_data filter on FreeBSD. Measured against a flood of silent
+connections, the filter serves more requests than no filter, not fewer, so it
+is not the denial of service vector that FreeBSD-SA-02:26 once described.
+
+
+Paths not taken
+---------------
+
+accf_http, the HTTP-aware accept filter. It duplicates a parser the tracker
+has anyway, puts more kernel code in the path, and holds a connection that
+sends "GET /" and stops, which accf_data hands straight over.
+
+recvmmsg and sendmmsg. No measurable gain on either FreeBSD machine, between
+-4 and +10 per cent, and they do not exist on Darwin.
+
+Link time optimisation. It cuts .text by 13 per cent, 93,497 bytes to 81,630,
+and changes throughput not at all. A tracker waits on memory and system
+calls, not on instruction fetch: the announce path is 6.6 KB of code against
+a 32 KB instruction cache at worst.
+
+A directory for object files. bmake has no pattern rules and the Makefile
+builds under FreeBSD's make today. Building each binary in one invocation
+clears the objects out of the tree without giving that up.
+
+A src/ directory. 52 files sharing an ot_ prefix are a module list rather
+than clutter, and build output was most of what made the tree look crowded.
+
+
+Platform differences that cost us code
+--------------------------------------
+
+SO_REUSEPORT means three different things. Linux spreads datagrams over the
+group; FreeBSD permits the bind and needs SO_REUSEPORT_LB to spread. Darwin
+hands every datagram to whichever socket bound last, so a group there starves
+all but one reader.
+
+Waking a blocked reader differs per kernel. see the table above.
+
+Deferring accept is TCP_DEFER_ACCEPT, or accf_data, or nothing.
+
+make is bmake on FreeBSD and GNU make elsewhere, and the Makefile has to
+build under both.
diff --git a/Makefile b/Makefile
index 2e52367..8f4dcf6 100644
--- a/Makefile
+++ b/Makefile
@@ -69,8 +69,8 @@ CFLAGS_libowfat=-I$(LIBOWFAT_HEADERS)
 LDFLAGS_libowfat=-L$(LIBOWFAT_LIBRARY) -lowfat
 
 BINARY =opentracker
-HEADERS=trackerlogic.h ot_urlencoded.h ot_batch.h ot_buf.h ot_event.h ot_fdstate.h ot_fmt.h ot_hex.h ot_ip.h ot_config.h ot_scan.h ot_socket.h ot_mutex.h ot_random.h ot_stats.h ot_vector.h ot_clean.h ot_udp.h ot_iovec.h ot_fullscrape.h ot_accesslist.h ot_http.h ot_livesync.h ot_rijndael.h
-SOURCES=opentracker.c trackerlogic.c ot_urlencoded.c ot_batch.c ot_buf.c ot_event.c ot_fdstate.c ot_fmt.c ot_hex.c ot_ip.c ot_config.c ot_scan.c ot_socket.c ot_mutex.c ot_random.c ot_stats.c ot_vector.c ot_clean.c ot_udp.c ot_iovec.c ot_fullscrape.c ot_accesslist.c ot_http.c ot_livesync.c ot_rijndael.c
+HEADERS=trackerlogic.h ot_urlencoded.h ot_batch.h ot_buf.h ot_event.h ot_fdstate.h ot_fmt.h ot_hex.h ot_ip.h ot_config.h ot_scan.h ot_socket.h ot_tcp.h ot_mutex.h ot_random.h ot_stats.h ot_vector.h ot_clean.h ot_udp.h ot_iovec.h ot_fullscrape.h ot_accesslist.h ot_http.h ot_livesync.h ot_rijndael.h
+SOURCES=opentracker.c trackerlogic.c ot_urlencoded.c ot_batch.c ot_buf.c ot_event.c ot_fdstate.c ot_fmt.c ot_hex.c ot_ip.c ot_config.c ot_scan.c ot_socket.c ot_tcp.c ot_mutex.c ot_random.c ot_stats.c ot_vector.c ot_clean.c ot_udp.c ot_iovec.c ot_fullscrape.c ot_accesslist.c ot_http.c ot_livesync.c ot_rijndael.c
 SOURCES_proxy=proxy.c ot_batch.c ot_fdstate.c ot_vector.c ot_mutex.c ot_random.c ot_scan.c ot_ip.c ot_socket.c
 
 all: $(BINARY) $(BINARY).debug
diff --git a/opentracker.c b/opentracker.c
index ec92fe0..2eaf988 100644
--- a/opentracker.c
+++ b/opentracker.c
@@ -35,6 +35,7 @@
 #include "ot_fdstate.h"
 #include "ot_socket.h"
 #include "ot_stats.h"
+#include "ot_tcp.h"
 #include "ot_udp.h"
 #include "trackerlogic.h"
 
@@ -96,126 +97,6 @@ static void install_signal_handlers(void) {
 }
 
 
-static ssize_t header_complete(char *request, ssize_t byte_count) {
-  ssize_t i = 0, state = 0;
-
-  for (i = 1; i < byte_count; i += 2)
-    if (request[i] <= 13) {
-      i--;
-      for (state = 0; i < byte_count; ++i) {
-        char c = request[i];
-        if (c == '\r' || c == '\n')
-          state = (state >> 2) | ((c << 6) & 0xc0);
-        else
-          break;
-        if (state >= 0xa0 || state == 0x99)
-          return i + 1;
-      }
-    }
-  return 0;
-}
-
-static void handle_read(const int64_t sock, struct ot_workstruct *ws) {
-  struct http_data *cookie     = fdstate_cookie(sock);
-  ssize_t           byte_count = read((int)sock, ws->inbuf, G_INBUF_SIZE);
-
-  if (byte_count == 0 || (byte_count == -1 && errno != EAGAIN && errno != EWOULDBLOCK && errno != EINTR)) {
-    http_close(sock);
-    return;
-  }
-
-  if (byte_count == -1)
-    return;
-
-  ws->request      = NULL;
-  ws->request_size = 0;
-  ws->header_size  = 0;
-  ws->keep_alive   = 0;
-
-  /* If we get the whole request in one packet, handle it without copying */
-  if (!cookie->request.data) {
-    if ((ws->header_size = header_complete(ws->inbuf, byte_count))) {
-      ws->request      = ws->inbuf;
-      ws->request_size = byte_count;
-      http_handle_request(sock, ws);
-    } else if (buf_cat(&cookie->request, ws->inbuf, (size_t)byte_count))
-      http_close(sock);
-    return;
-  }
-
-  if (buf_cat(&cookie->request, ws->inbuf, (size_t)byte_count) || cookie->request.size > G_INBUF_SIZE) {
-    http_issue_error(sock, ws, CODE_HTTPERROR_500);
-    return;
-  }
-
-  while ((ws->header_size = header_complete(cookie->request.data, (ssize_t)cookie->request.size))) {
-    ws->request      = cookie->request.data;
-    ws->request_size = (ssize_t)cookie->request.size;
-    http_handle_request(sock, ws);
-#ifdef WANT_KEEPALIVE
-    if (!ws->keep_alive)
-#endif
-      return;
-  }
-}
-
-static void handle_write(const int64_t sock) {
-  struct http_data *cookie     = fdstate_cookie(sock);
-  int               chunked    = 0;
-  int               keep_alive = 0;
-
-  if (cookie) {
-    chunked    = (cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER) != 0;
-    keep_alive = (cookie->flag & STRUCT_HTTP_FLAG_KEEPALIVE) != 0;
-
-    if (cookie->batch) {
-      if (batch_send(sock, cookie->batch) == BATCH_FAILED) {
-        http_close(sock);
-        return;
-      }
-      if (batch_pending(cookie->batch))
-        return;
-    }
-  }
-
-  /* In a chunked transfer, wait for the next piece once everything queued has gone out */
-  if (chunked)
-    http_interest_set(sock, cookie, 0);
-  else if (keep_alive) {
-    /* The reply is out and the client asked to keep the connection, so go
-       back to waiting for the next request */
-    http_interest_set(sock, cookie, EVENT_IN);
-    fdstate_set_expiry(sock, time_future_seconds(OT_CLIENT_TIMEOUT));
-  } else
-    http_close(sock);
-}
-
-static void handle_accept(const int64_t serversocket, struct ot_workstruct *ws) {
-  struct http_data *cookie;
-  int64_t           sock;
-  ot_ip6            ip;
-  uint16_t          port;
-
-  while ((sock = socket_accept(serversocket, ip, &port)) != -1) {
-
-    if (!(cookie = (struct http_data *)calloc(1, sizeof(struct http_data)))) {
-      close((int)sock);
-      continue;
-    }
-    memcpy(cookie->ip, ip, sizeof(ot_ip6));
-    cookie->interest = EVENT_IN;
-
-    if (event_add((int)sock, EVENT_IN, fdstate_open(sock, cookie, time_future_seconds(OT_CLIENT_TIMEOUT))) == -1) {
-      fdstate_close(sock);
-      free(cookie);
-      close((int)sock);
-      continue;
-    }
-
-    stats_issue_event(ws, EVENT_ACCEPT, FLAG_TCP, (uintptr_t)ip);
-  }
-}
-
 /* How many ready descriptors one wait hands back at a time */
 #define OT_EVENT_BATCH 512
 
@@ -261,7 +142,7 @@ static void *server_mainloop(void *args) {
       cookie = fdstate_cookie((int)sock);
 
       if ((intptr_t)cookie == FLAG_TCP)
-        handle_accept(sock, &ws);
+        tcp_accept(sock, &ws);
       else if ((intptr_t)cookie == FLAG_UDP)
         handle_udp6(sock, &ws);
       else if ((intptr_t)cookie == FLAG_SELFPIPE)
@@ -269,10 +150,10 @@ static void *server_mainloop(void *args) {
           ;
       else {
         if (ready[i].what & EVENT_IN)
-          handle_read(sock, &ws);
+          tcp_read(sock, &ws);
         /* The read may have closed it, so look again before writing */
         if ((ready[i].what & EVENT_OUT) && fdstate_generation((int)sock) == ready[i].generation)
-          handle_write(sock);
+          tcp_write(sock);
       }
     }
 
diff --git a/ot_config.c b/ot_config.c
index a0dbef1..272acf1 100644
--- a/ot_config.c
+++ b/ot_config.c
@@ -14,6 +14,7 @@
 
 /* Opentracker */
 #include "ot_accesslist.h"
+#include "ot_event.h"
 #include "ot_config.h"
 #include "ot_http.h"
 #include "ot_ip.h"
@@ -32,7 +33,8 @@ const char   *g_statefile;
 static ot_listen *g_listen;
 static size_t     g_listen_count;
 static int        g_listen_shared;
-static int        g_listen_workers = 1;
+static int        g_listen_workers_udp = 1;
+static int        g_listen_workers_tcp = 0;
 
 static void panic(const char *routine) __attribute__((noreturn));
 static void panic(const char *routine) {
@@ -196,6 +198,7 @@ typedef enum {
     CONFIG_LISTEN_TCP,
     CONFIG_LISTEN_UDP,
     CONFIG_LISTEN_UDP_WORKERS,
+    CONFIG_LISTEN_TCP_WORKERS,
     CONFIG_LISTEN_SHARED,
     CONFIG_ACCESS_WHITELIST,
     CONFIG_ACCESS_BLACKLIST,
@@ -224,6 +227,7 @@ static const config_key config_keys[] = {
     CONFIG_KEY("listen.tcp",               CONFIG_LISTEN_TCP),
     CONFIG_KEY("listen.udp",               CONFIG_LISTEN_UDP),
     CONFIG_KEY("listen.udp.workers",       CONFIG_LISTEN_UDP_WORKERS),
+    CONFIG_KEY("listen.tcp.workers",       CONFIG_LISTEN_TCP_WORKERS),
     CONFIG_KEY("listen.shared",            CONFIG_LISTEN_SHARED),
 #ifdef WANT_ACCESSLIST_WHITE
     CONFIG_KEY("access.whitelist",         CONFIG_ACCESS_WHITELIST),
@@ -318,7 +322,7 @@ static int parse_configfile(char *config_filename) {
         if (!off || !listen_scan_options(p + off, &shared, &workers))
           goto parse_error;
         if (kind != CONFIG_LISTEN_UDP)
-          listen_append(tmpip, tmpport, FLAG_TCP, shared, OT_LISTEN_INHERIT), ++bound;
+          listen_append(tmpip, tmpport, FLAG_TCP, shared, workers), ++bound;
         if (kind != CONFIG_LISTEN_TCP)
           listen_append(tmpip, tmpport, FLAG_UDP, shared, workers), ++bound;
         break;
@@ -326,7 +330,12 @@ static int parse_configfile(char *config_filename) {
       case CONFIG_LISTEN_UDP_WORKERS:
         if (!scan_uint(p, &value, UINT_MAX))
           goto parse_error;
-        g_listen_workers = (int)value;
+        g_listen_workers_udp = (int)value;
+        break;
+      case CONFIG_LISTEN_TCP_WORKERS:
+        if (!scan_uint(p, &value, UINT_MAX))
+          goto parse_error;
+        g_listen_workers_tcp = (int)value;
         break;
       case CONFIG_LISTEN_SHARED:
         if (!scan_uint(p, &value, 1))
@@ -446,7 +455,16 @@ static void listen_resolve_defaults(void) {
     if (entry->shared == OT_LISTEN_INHERIT)
       entry->shared = g_listen_shared;
     if (entry->workers == OT_LISTEN_INHERIT)
-      entry->workers = g_listen_workers;
+      entry->workers = entry->proto == FLAG_TCP ? g_listen_workers_tcp : g_listen_workers_udp;
+
+    /* A tcp worker hands a connection it cannot finish to the event loop,
+       which only one thread may touch where the set is ours rather than the
+       kernel's */
+    if (entry->proto == FLAG_TCP && entry->workers > 1 && !event_can_share()) {
+      fprintf(stderr, "Warning: Multiple workers impossible on this platform with event mechanism %s chosen.\n",
+              event_mechanism());
+      entry->workers = 1;
+    }
   }
 }
 
diff --git a/ot_event.c b/ot_event.c
index 8f55817..61cc07b 100644
--- a/ot_event.c
+++ b/ot_event.c
@@ -40,6 +40,9 @@ static struct epoll_event *g_ready;
 static int                 g_ready_space;
 
 const char *event_mechanism(void) { return "epoll"; }
+/* The set lives in the kernel, and epoll_ctl during another thread's
+   epoll_wait is what it is for */
+int         event_can_share(void) { return 1; }
 
 int event_init(void) {
   g_ready       = NULL;
@@ -118,6 +121,8 @@ static struct kevent *g_ready;
 static int            g_ready_space;
 
 const char *event_mechanism(void) { return "kqueue"; }
+/* The set lives in the kernel, and kevent may be called from any thread */
+int         event_can_share(void) { return 1; }
 
 int event_init(void) {
   g_ready       = NULL;
@@ -171,7 +176,7 @@ static int event_change(int fd, int what, uint32_t generation, int deleting) {
   return 0;
 }
 
-int event_add(int fd, int what, uint32_t generation) { return event_change(fd, what, generation, 0); }
+int event_add(int fd, int what, uint32_t generation) { return what ? event_change(fd, what, generation, 0) : 0; }
 int event_modify(int fd, int what, uint32_t generation) { return event_change(fd, what, generation, 0); }
 int event_del(int fd) { return event_change(fd, 0, 0, 1); }
 
@@ -229,6 +234,8 @@ static int           *g_where;
 static int            g_count, g_space, g_where_space;
 
 const char *event_mechanism(void) { return "poll"; }
+/* The arrays below are ours and nothing guards them */
+int         event_can_share(void) { return 0; }
 
 int event_init(void) {
   g_pfds = NULL, g_generation = NULL, g_where = NULL;
diff --git a/ot_event.h b/ot_event.h
index aa007d0..efdefff 100644
--- a/ot_event.h
+++ b/ot_event.h
@@ -38,6 +38,11 @@ int  event_close(int fd);
    out with up to max ready descriptors. Returns how many, or -1. */
 int  event_wait(ot_event *out, int max, int timeout_ms);
 
+/* Whether the registered set may be changed from another thread while this
+   one is waiting on it. True where the kernel keeps the set, false where
+   this module keeps it in memory of its own. */
+int  event_can_share(void);
+
 /* Which mechanism was compiled in, for the version string */
 const char *event_mechanism(void);
 
diff --git a/ot_fdstate.c b/ot_fdstate.c
index 537bcb0..7d742da 100644
--- a/ot_fdstate.c
+++ b/ot_fdstate.c
@@ -186,6 +186,13 @@ void *fdstate_cookie(int fd) {
   return slot ? atomic_load_explicit(&slot->cookie, memory_order_relaxed) : NULL;
 }
 
+void fdstate_set_cookie(int fd, void *cookie) {
+  ot_fdslot *slot = fdstate_slot(fd);
+
+  if (slot)
+    atomic_store_explicit(&slot->cookie, cookie, memory_order_relaxed);
+}
+
 void fdstate_set_expiry(int fd, ot_time expiry) {
   ot_fdslot *slot = fdstate_slot(fd);
 
diff --git a/ot_fdstate.h b/ot_fdstate.h
index c09f018..93890d1 100644
--- a/ot_fdstate.h
+++ b/ot_fdstate.h
@@ -56,6 +56,9 @@ int      fdstate_next_expired(ot_time now, int *cursor);
 
 /* Both only valid while the caller holds the descriptor. */
 void    *fdstate_cookie(int fd);
+
+/* Hand the slot a different cookie, for a connection whose state has moved */
+void     fdstate_set_cookie(int fd, void *cookie);
 void     fdstate_set_expiry(int fd, ot_time expiry);
 
 #endif
diff --git a/ot_http.c b/ot_http.c
index 66f0102..bb6ff24 100644
--- a/ot_http.c
+++ b/ot_http.c
@@ -39,9 +39,25 @@ ssize_t      g_stats_path_len;
 
 enum { SUCCESS_HTTP_HEADER_LENGTH = 80, SUCCESS_HTTP_SIZE_OFF = 17 };
 
-/* Give up a connection: the one place that knows everything hanging off it.
-   The slot is released before anything is freed, so that a late event or the
-   expiry sweep finds a closed descriptor rather than a freed cookie. */
+int http_adopt(const int64_t sock, struct http_data *cookie) {
+  struct http_data *heap = malloc(sizeof(*heap));
+
+  if (!heap)
+    return -1;
+
+  *heap = *cookie;
+  heap->flag &= ~STRUCT_HTTP_FLAG_ONSTACK;
+  fdstate_set_cookie((int)sock, heap);
+
+  if (event_add((int)sock, heap->interest, fdstate_generation((int)sock)) == -1) {
+    fdstate_set_cookie((int)sock, cookie);
+    free(heap);
+    return -1;
+  }
+  return 0;
+}
+
+/* Give up a connection: the one place that knows everything hanging off it. */
 void http_close(const int64_t sock) {
   struct http_data *cookie = fdstate_cookie(sock);
 
@@ -54,7 +70,8 @@ void http_close(const int64_t sock) {
     buf_reset(&cookie->request);
     if (cookie->flag & (STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER))
       mutex_workqueue_canceltask(sock);
-    free(cookie);
+    if (!(cookie->flag & STRUCT_HTTP_FLAG_ONSTACK))
+      free(cookie);
   }
 
   event_close((int)sock);
@@ -140,6 +157,10 @@ void http_interest_set(const int64_t sock, struct http_data *cookie, int what) {
   if (cookie->interest == what)
     return;
   cookie->interest = what;
+  /* Still on a stack, so not in the event loop yet: whoever adopts this
+     connection registers whatever it asked for last */
+  if (cookie->flag & STRUCT_HTTP_FLAG_ONSTACK)
+    return;
   event_modify((int)sock, what, fdstate_generation((int)sock));
 }
 
diff --git a/ot_http.h b/ot_http.h
index ba16de4..82d645b 100644
--- a/ot_http.h
+++ b/ot_http.h
@@ -22,7 +22,8 @@ typedef enum {
   STRUCT_HTTP_FLAG_ZSTD                = 8,
   STRUCT_HTTP_FLAG_CHUNKED             = 16,
   STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER = 32,
-  STRUCT_HTTP_FLAG_KEEPALIVE           = 64
+  STRUCT_HTTP_FLAG_KEEPALIVE           = 64,
+  STRUCT_HTTP_FLAG_ONSTACK             = 128
 } STRUCT_HTTP_FLAG;
 
 struct http_data {
@@ -39,6 +40,10 @@ void           http_interest_set(const int64_t s, struct http_data *cookie, int
 /* Close a connection and release everything that belongs to it */
 void           http_close(const int64_t s);
 
+/* Promote a stack cookie to a heap one for persistence and handing it over
+   to the event loop. */
+int            http_adopt(const int64_t s, struct http_data *cookie);
+
 ssize_t        http_handle_request(const int64_t s, struct ot_workstruct *ws);
 ssize_t        http_sendiovecdata(const int64_t s, struct ot_workstruct *ws, size_t iovec_entries, struct iovec *iovector, int is_partial);
 ssize_t        http_issue_error(const int64_t s, struct ot_workstruct *ws, int code);
diff --git a/ot_tcp.c b/ot_tcp.c
new file mode 100644
index 0000000..8647a47
--- /dev/null
+++ b/ot_tcp.c
@@ -0,0 +1,157 @@
+/* This software was written by Dirk Engling <erdgeist@erdgeist.org>
+   It is considered beerware. Prost. Skol. Cheers or whatever.
+
+   $id$ */
+
+/* System */
+#include <errno.h>
+#include <string.h>
+#include <unistd.h>
+
+/* Opentracker */
+#include "ot_batch.h"
+#include "ot_buf.h"
+#include "ot_event.h"
+#include "ot_fdstate.h"
+#include "ot_http.h"
+#include "ot_mutex.h"
+#include "ot_socket.h"
+#include "ot_stats.h"
+#include "ot_tcp.h"
+#include "trackerlogic.h"
+
+static ssize_t header_complete(char *request, ssize_t byte_count) {
+  ssize_t i = 0, state = 0;
+
+  for (i = 1; i < byte_count; i += 2)
+    if (request[i] <= 13) {
+      i--;
+      for (state = 0; i < byte_count; ++i) {
+        char c = request[i];
+        if (c == '\r' || c == '\n')
+          state = (state >> 2) | ((c << 6) & 0xc0);
+        else
+          break;
+        if (state >= 0xa0 || state == 0x99)
+          return i + 1;
+      }
+    }
+  return 0;
+}
+
+void tcp_read(const int64_t sock, struct ot_workstruct *ws) {
+  struct http_data *cookie     = fdstate_cookie(sock);
+  ssize_t           byte_count = read((int)sock, ws->inbuf, G_INBUF_SIZE);
+
+  if (byte_count == 0 || (byte_count == -1 && errno != EAGAIN && errno != EWOULDBLOCK && errno != EINTR)) {
+    http_close(sock);
+    return;
+  }
+
+  if (byte_count == -1)
+    return;
+
+  ws->request      = NULL;
+  ws->request_size = 0;
+  ws->header_size  = 0;
+  ws->keep_alive   = 0;
+
+  /* If we get the whole request in one packet, handle it without copying */
+  if (!cookie->request.data) {
+    if ((ws->header_size = header_complete(ws->inbuf, byte_count))) {
+      ws->request      = ws->inbuf;
+      ws->request_size = byte_count;
+      http_handle_request(sock, ws);
+    } else if (buf_cat(&cookie->request, ws->inbuf, (size_t)byte_count))
+      http_close(sock);
+    return;
+  }
+
+  if (buf_cat(&cookie->request, ws->inbuf, (size_t)byte_count) || cookie->request.size > G_INBUF_SIZE) {
+    http_issue_error(sock, ws, CODE_HTTPERROR_500);
+    return;
+  }
+
+  while ((ws->header_size = header_complete(cookie->request.data, (ssize_t)cookie->request.size))) {
+    ws->request      = cookie->request.data;
+    ws->request_size = (ssize_t)cookie->request.size;
+    http_handle_request(sock, ws);
+#ifdef WANT_KEEPALIVE
+    if (!ws->keep_alive)
+#endif
+      return;
+  }
+}
+
+void tcp_write(const int64_t sock) {
+  struct http_data *cookie     = fdstate_cookie(sock);
+  int               chunked    = 0;
+  int               keep_alive = 0;
+
+  if (cookie) {
+    chunked    = (cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER) != 0;
+    keep_alive = (cookie->flag & STRUCT_HTTP_FLAG_KEEPALIVE) != 0;
+
+    if (cookie->batch) {
+      if (batch_send(sock, cookie->batch) == BATCH_FAILED) {
+        http_close(sock);
+        return;
+      }
+      if (batch_pending(cookie->batch))
+        return;
+    }
+  }
+
+  /* In a chunked transfer, wait for the next piece once everything queued has gone out */
+  if (chunked)
+    http_interest_set(sock, cookie, 0);
+  else if (keep_alive) {
+    /* The reply is out and the client asked to keep the connection, so go
+       back to waiting for the next request */
+    http_interest_set(sock, cookie, EVENT_IN);
+    fdstate_set_expiry(sock, time_future_seconds(OT_CLIENT_TIMEOUT));
+  } else
+    http_close(sock);
+}
+
+/* The fast path, assuming that the complete request has usually arrived by the time a
+   connection is accepted, so it is answered here outside the event loop. The more load
+   a tracker has, the more likely it is that by the time it accepts, the roundtrip to
+   the peer has completed.
+   If the request could not be completely handled, the stack cookie is promoted to heap
+   and the connection handed over to the event loop to finish */
+void tcp_accept(const int64_t serversocket, struct ot_workstruct *ws) {
+  int64_t  sock;
+  ot_ip6   ip;
+  uint16_t port;
+
+  while ((sock = socket_accept(serversocket, ip, &port)) != -1) {
+    struct http_data cookie;
+    uint32_t         generation;
+
+    memset(&cookie, 0, sizeof(cookie));
+    memcpy(cookie.ip, ip, sizeof(ot_ip6));
+    cookie.interest = EVENT_IN;
+    cookie.flag     = STRUCT_HTTP_FLAG_ONSTACK;
+
+    generation = fdstate_open(sock, &cookie, time_future_seconds(OT_CLIENT_TIMEOUT));
+    if (!fdstate_claim((int)sock, generation)) {
+      fdstate_close(sock);
+      event_close((int)sock);
+      continue;
+    }
+
+    stats_issue_event(ws, EVENT_ACCEPT, FLAG_TCP, (uintptr_t)ip);
+
+    tcp_read(sock, ws);
+
+    /* Gone already if it was answered and closed inline */
+    if (fdstate_generation((int)sock) != generation)
+      continue;
+
+    if (http_adopt(sock, &cookie) == -1)
+      http_close(sock);
+    else
+      fdstate_release((int)sock);
+  }
+}
diff --git a/ot_tcp.h b/ot_tcp.h
new file mode 100644
index 0000000..b0a93f4
--- /dev/null
+++ b/ot_tcp.h
@@ -0,0 +1,22 @@
+/* This software was written by Dirk Engling <erdgeist@erdgeist.org>
+   It is considered beerware. Prost. Skol. Cheers or whatever.
+
+   $id$ */
+
+#ifndef OT_TCP_H__
+#define OT_TCP_H__
+
+#include <stdint.h>
+
+#include "trackerlogic.h"
+
+/* Take whatever connections are waiting on a listening socket and answer
+   them, each on this thread's stack, handing a connection that cannot be
+   finished to the event loop */
+void tcp_accept(const int64_t serversocket, struct ot_workstruct *ws);
+
+/* Handler connections in the event loop has become readable or writable */
+void tcp_read(const int64_t sock, struct ot_workstruct *ws);
+void tcp_write(const int64_t sock);
+
+#endif
diff --git a/tests/integration/test_config.py b/tests/integration/test_config.py
new file mode 100644
index 0000000..1a84fa7
--- /dev/null
+++ b/tests/integration/test_config.py
@@ -0,0 +1,29 @@
+"""Settings the tracker has to accept, and still work with.
+
+An unrecognised key makes opentracker refuse to start, so a test that it
+comes up at all is a test that the key was understood.
+"""
+
+import unittest
+
+from harness import Tracker, TrackerTestCase, free_port
+
+
+class TcpWorkers(TrackerTestCase):
+    config = "listen.tcp.workers 2\n"
+
+    def test_the_tracker_serves_with_tcp_workers_asked_for(self):
+        self.assertEqual(self.tracker.announce(self.hash_for("tcpworkers"))[b"complete"], 1)
+
+
+class WorkersOnAListenStatement(unittest.TestCase):
+    def test_a_listen_statement_may_carry_a_worker_count(self):
+        """The option is read off the statement rather than inherited."""
+        second = free_port()
+        tracker = Tracker("listen.tcp 127.0.0.1:%d workers=2\n" % second)
+        try:
+            self.assertEqual(tracker.announce(b"listenopts\0\0\0\0\0\0\0\0\0\0")[b"complete"], 1)
+            complaints = tracker.stop()
+            self.assertEqual(complaints, [], "the tracker reported:\n" + "\n".join(complaints))
+        finally:
+            tracker._cleanup()
diff --git a/tests/t_event.c b/tests/t_event.c
index ad3948f..1908dc9 100644
--- a/tests/t_event.c
+++ b/tests/t_event.c
@@ -241,9 +241,40 @@ static void case_close_takes_it_out(void) {
   pair_close();
 }
 
+/* A descriptor can be taken on before anybody waits for anything on it: the
+   full scrape parks a connection while a worker thread builds the reply. The
+   mechanisms disagree about what registering nothing means, so what is
+   checked is that asking for a filter afterwards still works. */
+static void case_add_nothing_then_ask(void) {
+  test_case("a descriptor added with no interest can be given some later");
+  pair_open();
+  CHECK(sv[0] != -1);
+  CHECK_INT(event_add(sv[0], 0, 7), 0);
+  CHECK_INT((int)write(sv[1], "x", 1), 1);
+  CHECK_INT(wait_for(sv[0], EVENT_IN, 200, NULL), 0);
+
+  CHECK_INT(event_modify(sv[0], EVENT_IN, 7), 0);
+  {
+    uint32_t generation = 0;
+    CHECK_INT(wait_for(sv[0], EVENT_IN, 500, &generation), 1);
+    CHECK_INT((int)generation, 7);
+  }
+  event_del(sv[0]);
+  pair_close();
+}
+
+/* Whether a worker thread may register a descriptor while this one waits
+   depends on who keeps the set. Tying the answer to the mechanism's name
+   here means a backend added later cannot quietly forget to say. */
+static void case_sharing_matches_the_mechanism(void) {
+  test_case("only a set the kernel keeps may be changed from another thread");
+  CHECK_INT(event_can_share(), strcmp(event_mechanism(), "poll") ? 1 : 0);
+}
+
 void test_event(void) {
   CHECK_INT(event_init(), 0);
   CHECK(event_mechanism() != NULL);
+  case_sharing_matches_the_mechanism();
 
   case_readable();
   case_writable();
@@ -254,6 +285,7 @@ void test_event(void) {
   case_timeout();
   case_unknown_descriptor();
   case_close_takes_it_out();
+  case_add_nothing_then_ask();
 
   event_deinit();
 }