1 files changed, 54 insertions, 22 deletions

diff --git a/ot_http.c b/ot_http.c
index e8ba3da..602ff94 100644
--- a/ot_http.c
+++ b/ot_http.c
@@ -39,9 +39,33 @@ ssize_t      g_stats_path_len;
 
 enum { SUCCESS_HTTP_HEADER_LENGTH = 80, SUCCESS_HTTP_SIZE_OFF = 17 };
 
-/* Give up a connection: the one place that knows everything hanging off it.
-   The slot is released before anything is freed, so that a late event or the
-   expiry sweep finds a closed descriptor rather than a freed cookie. */
+struct http_data *http_adopt(const int64_t sock) {
+  struct http_data *cookie = fdstate_cookie(sock), *heap;
+  int               interest;
+
+  if (!cookie || !(cookie->flag & STRUCT_HTTP_FLAG_ONSTACK))
+    return cookie;
+
+  if (!(heap = malloc(sizeof(*heap))))
+    return NULL;
+
+  *heap = *cookie;
+  heap->flag &= ~STRUCT_HTTP_FLAG_ONSTACK;
+
+  /* Publishing the cookie is what lets another thread reach this connection,
+     so nothing may touch it afterwards, not even to read what it asked for */
+  interest = heap->interest;
+  fdstate_set_cookie((int)sock, heap);
+
+  if (event_add((int)sock, interest, fdstate_generation((int)sock)) == -1) {
+    fdstate_set_cookie((int)sock, cookie);
+    free(heap);
+    return NULL;
+  }
+  return heap;
+}
+
+/* Give up a connection: the one place that knows everything hanging off it. */
 void http_close(const int64_t sock) {
   struct http_data *cookie = fdstate_cookie(sock);
 
@@ -54,7 +78,8 @@ void http_close(const int64_t sock) {
     buf_reset(&cookie->request);
     if (cookie->flag & (STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER))
       mutex_workqueue_canceltask(sock);
-    free(cookie);
+    if (!(cookie->flag & STRUCT_HTTP_FLAG_ONSTACK))
+      free(cookie);
   }
 
   event_close((int)sock);
@@ -118,11 +143,9 @@ static void http_senddata(const int64_t sock, struct ot_workstruct *ws) {
     batch_add(cookie->batch, outbuf, ws->reply_size - written_size);
 
     /* writeable short data sockets just have a tcp timeout */
-    if (!ws->keep_alive) {
+    if (!ws->keep_alive)
       fdstate_set_expiry(sock, 0);
-      http_interest_drop(sock, cookie, EVENT_IN);
-    }
-    http_interest_add(sock, cookie, EVENT_OUT);
+    http_interest_set(sock, cookie, EVENT_OUT);
   }
 }
 
@@ -135,21 +158,20 @@ static void http_senddata(const int64_t sock, struct ot_workstruct *ws) {
 #define HTTPERROR_403_IP         return http_issue_error(sock, ws, CODE_HTTPERROR_403_IP)
 #define HTTPERROR_404            return http_issue_error(sock, ws, CODE_HTTPERROR_404)
 #define HTTPERROR_500            return http_issue_error(sock, ws, CODE_HTTPERROR_500)
-static void http_interest(const int64_t sock, struct http_data *cookie, int what) {
+/* A connection is either being read or being written, never both, so what it
+   waits for is set outright rather than added to and taken away again. The
+   event layer only hears about it when the answer changes. */
+void http_interest_set(const int64_t sock, struct http_data *cookie, int what) {
   if (cookie->interest == what)
     return;
   cookie->interest = what;
+  /* Still on a stack, so not in the event loop yet: whoever adopts this
+     connection registers whatever it asked for last */
+  if (cookie->flag & STRUCT_HTTP_FLAG_ONSTACK)
+    return;
   event_modify((int)sock, what, fdstate_generation((int)sock));
 }
 
-void http_interest_add(const int64_t sock, struct http_data *cookie, int what) {
-  http_interest(sock, cookie, cookie->interest | what);
-}
-
-void http_interest_drop(const int64_t sock, struct http_data *cookie, int what) {
-  http_interest(sock, cookie, cookie->interest & ~what);
-}
-
 ssize_t http_issue_error(const int64_t sock, struct ot_workstruct *ws, int code) {
   const char *error_code[] = {"302 Found",      "400 Invalid Request", "400 Invalid Request", "400 Invalid Request",      "402 Payment Required",
                         "403 Not Modest", "403 Access Denied",   "404 Not Found",       "500 Internal Server Error"};
@@ -194,7 +216,7 @@ ssize_t http_sendiovecdata(const int64_t sock, struct ot_workstruct *ws, size_t
 
   /* If we came here, wait for the answer is over */
   if (cookie->flag & STRUCT_HTTP_FLAG_WAITINGFORTASK) {
-    http_interest_drop(sock, cookie, EVENT_IN);
+    http_interest_set(sock, cookie, 0);
     cookie->flag &= ~STRUCT_HTTP_FLAG_WAITINGFORTASK;
   }
 
@@ -245,7 +267,7 @@ ssize_t http_sendiovecdata(const int64_t sock, struct ot_workstruct *ws, size_t
   }
 
   fdstate_set_expiry(sock, time_future_seconds(OT_CLIENT_TIMEOUT_SEND));
-  http_interest_add(sock, cookie, EVENT_OUT);
+  http_interest_set(sock, cookie, EVENT_OUT);
   return 0;
 }
 
@@ -334,13 +356,17 @@ static ssize_t http_handle_stats(const int64_t sock, struct ot_workstruct *ws, c
     }
 #endif
 #endif
-    /* Pass this task to the worker thread */
     cookie->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED;
 
     /* Clients waiting for us should not easily timeout */
     fdstate_set_expiry(sock, 0);
+    http_interest_set(sock, cookie, 0);
+
+    /* Everything this connection needs has been written, so it can be given
+       a home of its own and only then be made reachable by a worker thread */
+    if (!http_adopt(sock))
+      HTTPERROR_500;
     fullscrape_deliver(sock, format);
-    http_interest_drop(sock, cookie, EVENT_IN);
     return ws->reply_size = -2;
   }
 #endif
@@ -352,6 +378,9 @@ static ssize_t http_handle_stats(const int64_t sock, struct ot_workstruct *ws, c
 
     /* Complex stats also include expensive memory debugging tools */
     fdstate_set_expiry(sock, 0);
+
+    if (!http_adopt(sock))
+      HTTPERROR_500;
     stats_deliver(sock, mode);
     return ws->reply_size = -2;
   }
@@ -434,8 +463,11 @@ static ssize_t http_handle_fullscrape(const int64_t sock, struct ot_workstruct *
   cookie->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED;
   /* Clients waiting for us should not easily timeout */
   fdstate_set_expiry(sock, 0);
+  http_interest_set(sock, cookie, 0);
+
+  if (!http_adopt(sock))
+    HTTPERROR_500;
   fullscrape_deliver(sock, format);
-  http_interest_drop(sock, cookie, EVENT_IN);
   return ws->reply_size = -2;
 }
 #endif