From 1a70d9f9ef81ac1b5e843ac71f3538f7845e03ae Mon Sep 17 00:00:00 2001 From: Dirk Engling Date: Sat, 13 Apr 2024 00:47:29 +0200 Subject: First shot on chunked transfers --- ot_http.c | 93 ++++++++++++++++++++++++++++++++++++++------------------------- 1 file changed, 56 insertions(+), 37 deletions(-) (limited to 'ot_http.c') diff --git a/ot_http.c b/ot_http.c index 61843a8..edcfadb 100644 --- a/ot_http.c +++ b/ot_http.c @@ -121,9 +121,10 @@ ssize_t http_issue_error( const int64 sock, struct ot_workstruct *ws, int code ) return ws->reply_size = -2; } -ssize_t http_sendiovecdata( const int64 sock, struct ot_workstruct *ws, int iovec_entries, struct iovec *iovector ) { +ssize_t http_sendiovecdata( const int64 sock, struct ot_workstruct *ws, int iovec_entries, struct iovec *iovector, int is_partial ) { struct http_data *cookie = io_getcookie( sock ); char *header; + const char *encoding = ""; int i; size_t header_size, size = iovec_length( &iovec_entries, (const struct iovec **)&iovector ); tai6464 t; @@ -140,54 +141,72 @@ ssize_t http_sendiovecdata( const int64 sock, struct ot_workstruct *ws, int iove /* If we came here, wait for the answer is over */ cookie->flag &= ~STRUCT_HTTP_FLAG_WAITINGFORTASK; - /* Our answers never are 0 vectors. Return an error. */ - if( !iovec_entries ) { - HTTPERROR_500; - } +fprintf(stderr, "http_sendiovecdata sending %d iovec entries found cookie->batch == %p\n", iovec_entries, cookie->batch); - /* Prepare space for http header */ - header = malloc( SUCCESS_HTTP_HEADER_LENGTH + SUCCESS_HTTP_HEADER_LENGTH_CONTENT_ENCODING ); - if( !header ) { - iovec_free( &iovec_entries, &iovector ); - HTTPERROR_500; - } + if( iovec_entries ) { - if( cookie->flag & STRUCT_HTTP_FLAG_GZIP ) - header_size = sprintf( header, "HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\nContent-Encoding: gzip\r\nContent-Length: %zd\r\n\r\n", size ); - else if( cookie->flag & STRUCT_HTTP_FLAG_BZIP2 ) - header_size = sprintf( header, "HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\nContent-Encoding: bzip2\r\nContent-Length: %zd\r\n\r\n", size ); - else - header_size = sprintf( header, "HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\nContent-Length: %zd\r\n\r\n", size ); + /* Prepare space for http header */ + header = malloc( SUCCESS_HTTP_HEADER_LENGTH + SUCCESS_HTTP_HEADER_LENGTH_CONTENT_ENCODING ); + if( !header ) { + iovec_free( &iovec_entries, &iovector ); + HTTPERROR_500; + } - if (!cookie->batch ) { - cookie->batch = malloc( sizeof(io_batch) ); - memset( cookie->batch, 0, sizeof(io_batch) ); - cookie->batches = 1; - } - iob_addbuf_free( cookie->batch, header, header_size ); + if( cookie->flag & STRUCT_HTTP_FLAG_GZIP ) + encoding = "Content-Encoding: gzip\r\n"; + else if( cookie->flag & STRUCT_HTTP_FLAG_BZIP2 ) + encoding = "Content-Encoding: bzip2\r\n"; + + if( !(cookie->flag & STRUCT_HTTP_FLAG_CHUNKED) ) + header_size = sprintf( header, "HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\n%sContent-Length: %zd\r\n\r\n", encoding, size ); + else { + if ( !(cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER )) { + header_size = sprintf( header, "HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\n%sTransfer-Encoding: chunked\r\n\r\n%zx\r\n", encoding, size ); + cookie->flag |= STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER; + } else + header_size = sprintf( header, "%zx\r\n", size ); + } - /* Split huge iovectors into separate io_batches */ - for( i=0; ibatch + cookie->batches - 1; + if (!cookie->batch ) { + cookie->batch = malloc( sizeof(io_batch) ); + memset( cookie->batch, 0, sizeof(io_batch) ); + cookie->batches = 1; + } + iob_addbuf_free( cookie->batch, header, header_size ); - /* If the current batch's limit is reached, try to reallocate a new batch to work on */ - if( current->bytesleft > OT_BATCH_LIMIT ) { - io_batch * new_batch = realloc( current, (cookie->batches + 1) * sizeof(io_batch) ); + /* Split huge iovectors into separate io_batches */ + for( i=0; ibatch + cookie->batches - 1; + + /* If the current batch's limit is reached, try to reallocate a new batch to work on */ + if( current->bytesleft > OT_BATCH_LIMIT ) { +fprintf(stderr, "http_sendiovecdata found batch above limit: %zd\n", current->bytesleft); + io_batch * new_batch = realloc( cookie->batch, (cookie->batches + 1) * sizeof(io_batch) ); if( new_batch ) { - cookie->batches++; - current = cookie->batch = new_batch; - memset( current, 0, sizeof(io_batch) ); + cookie->batch = new_batch; + current = cookie->batch + cookie->batches++; + memset( current, 0, sizeof(io_batch) ); } + } +fprintf(stderr, "http_sendiovecdata calling iob_addbuf_free with %zd\n", iovector[i].iov_len); + iob_addbuf_free( current, iovector[i].iov_base, iovector[i].iov_len ); } + free( iovector ); + if ( cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER ) + iob_addbuf(cookie->batch + cookie->batches - 1, "\r\n", 2); + } - iob_addbuf_free( current, iovector[i].iov_base, iovector[i].iov_len ); + if ((cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER) && cookie->batch && !is_partial) { +fprintf(stderr, "http_sendiovecdata adds a terminating 0 size buffer to batch\n"); + iob_addbuf(cookie->batch + cookie->batches - 1, "0\r\n\r\n", 5); + cookie->flag &= ~STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER; } - free( iovector ); /* writeable sockets timeout after 10 minutes */ taia_now( &t ); taia_addsec( &t, &t, OT_CLIENT_TIMEOUT_SEND ); io_timeout( sock, t ); io_dontwantread( sock ); +fprintf (stderr, "http_sendiovecdata marks socket %lld as wantwrite\n", sock); io_wantwrite( sock ); return 0; } @@ -254,7 +273,7 @@ static const ot_keywords keywords_format[] = #endif #endif /* Pass this task to the worker thread */ - cookie->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK; + cookie->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED; /* Clients waiting for us should not easily timeout */ taia_uint( &t, 0 ); io_timeout( sock, t ); @@ -278,7 +297,7 @@ static const ot_keywords keywords_format[] = } #ifdef WANT_MODEST_FULLSCRAPES -static pthread_mutex_t g_modest_fullscrape_mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_mutex_t g_modest_fullscrape_mutex = PTHREAD_MUTEX_INITIALIZER; static ot_vector g_modest_fullscrape_timeouts; typedef struct { ot_ip6 ip; ot_time last_fullscrape; } ot_scrape_log; #endif @@ -325,7 +344,7 @@ static ssize_t http_handle_fullscrape( const int64 sock, struct ot_workstruct *w #endif /* Pass this task to the worker thread */ - cookie->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK; + cookie->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED; /* Clients waiting for us should not easily timeout */ taia_uint( &t, 0 ); io_timeout( sock, t ); fullscrape_deliver( sock, TASK_FULLSCRAPE | format ); -- cgit v1.2.3