From 95f1780f0b6229a6f52b6dbad1a645b4e91c6b06 Mon Sep 17 00:00:00 2001
From: Dirk Engling <erdgeist@erdgeist.org>
Date: Sat, 24 Apr 2021 03:25:30 +0200
Subject: Split huge iovecs over multiple io_batches

---
 ot_http.c | 36 ++++++++++++++++++++++++++++++------
 1 file changed, 30 insertions(+), 6 deletions(-)

(limited to 'ot_http.c')

diff --git a/ot_http.c b/ot_http.c
index e05db72..390878b 100644
--- a/ot_http.c
+++ b/ot_http.c
@@ -31,6 +31,7 @@
 #include "ot_accesslist.h"
 
 #define OT_MAXMULTISCRAPE_COUNT 64
+#define OT_BATCH_LIMIT (1024*1024*16)
 extern char *g_redirecturl;
 
 char   *g_stats_path;
@@ -75,7 +76,13 @@ static void http_senddata( const int64 sock, struct ot_workstruct *ws ) {
     }
 
     memcpy( outbuf, ws->reply + written_size, ws->reply_size - written_size );
-    iob_addbuf_free( &cookie->batch, outbuf, ws->reply_size - written_size );
+    if ( !cookie->batch ) {
+        cookie->batch = malloc( sizeof(io_batch) );
+        memset( cookie->batch, 0, sizeof(io_batch) );
+        cookie->batches = 1;
+    }
+
+    iob_addbuf_free( cookie->batch, outbuf, ws->reply_size - written_size );
 
     /* writeable short data sockets just have a tcp timeout */
     if( !ws->keep_alive ) {
@@ -152,12 +159,29 @@ ssize_t http_sendiovecdata( const int64 sock, struct ot_workstruct *ws, int iove
   else
     header_size = sprintf( header, "HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\nContent-Length: %zd\r\n\r\n", size );
 
-  iob_reset( &cookie->batch );
-  iob_addbuf_free( &cookie->batch, header, header_size );
+  if (!cookie->batch ) {
+    cookie->batch = malloc( sizeof(io_batch) );
+    memset( cookie->batch, 0, sizeof(io_batch) );
+    cookie->batches = 1;
+  }
+  iob_addbuf_free( cookie->batch, header, header_size );
+
+  /* Split huge iovectors into separate io_batches */
+  for( i=0; i<iovec_entries; ++i ) {
+    io_batch *current = cookie->batch + cookie->batches;
+
+    /* If the current batch's limit is reached, try to reallocate a new batch to work on */
+    if( current->bytesleft > OT_BATCH_LIMIT ) {
+        io_batch * new_batch = realloc( current, (cookie->batches + 1) * sizeof(io_batch) );
+        if( new_batch ) {
+            cookie->batches++;
+            current = cookie->batch = new_batch;
+            memset( current, 0, sizeof(io_batch) );
+        }
+    }
 
-  /* Will move to ot_iovec.c */
-  for( i=0; i<iovec_entries; ++i )
-    iob_addbuf_munmap( &cookie->batch, iovector[i].iov_base, iovector[i].iov_len );
+    iob_addbuf_munmap( current, iovector[i].iov_base, iovector[i].iov_len );
+  }
   free( iovector );
 
   /* writeable sockets timeout after 10 minutes */
-- 
cgit v1.2.3