From bd4992435ca8343cca0b34af13cf6da331a357f8 Mon Sep 17 00:00:00 2001 From: Dirk Engling Date: Sat, 13 Apr 2024 14:04:18 +0200 Subject: Add proper parsing for the gzip content encoding --- Makefile | 4 ++-- ot_fullscrape.c | 41 ++++++++++++++++++++++------------------- ot_http.c | 10 +++++++--- tests/testsuite2.sh | 24 ++++++++++++++++-------- 4 files changed, 47 insertions(+), 32 deletions(-) diff --git a/Makefile b/Makefile index 7f43829..0150217 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ STRIP?=strip #FEATURES+=-DWANT_DYNAMIC_ACCESSLIST #FEATURES+=-DWANT_SYNC_LIVE -#FEATURES+=-DWANT_IP_FROM_QUERY_STRING +FEATURES+=-DWANT_IP_FROM_QUERY_STRING #FEATURES+=-DWANT_COMPRESSION_GZIP #FEATURES+=-DWANT_COMPRESSION_GZIP_ALWAYS #FEATURES+=-DWANT_LOG_NETWORKS @@ -48,7 +48,7 @@ FEATURES+=-DWANT_FULLSCRAPE OPTS_debug=-D_DEBUG -g -ggdb # -pg -fprofile-arcs -ftest-coverage OPTS_production=-O3 -CFLAGS+=-I$(LIBOWFAT_HEADERS) -Wall -pipe -pthread -Wextra #-ansi -pedantic +CFLAGS+=-I$(LIBOWFAT_HEADERS) -Wall -pipe -pthread -Wextra -Wincompatible-pointer-types #-ansi -pedantic LDFLAGS+=-L$(LIBOWFAT_LIBRARY) -lowfat -pthread -lz #LDFLAGS+=-lbsd diff --git a/ot_fullscrape.c b/ot_fullscrape.c index b147b6a..86b9e26 100644 --- a/ot_fullscrape.c +++ b/ot_fullscrape.c @@ -262,33 +262,36 @@ fprintf(stderr, "GZIP path\n"); if( deflate( &strm, Z_FINISH ) < Z_OK ) fprintf( stderr, "deflate() failed while in fullscrape_make()'s endgame.\n" ); - if( !strm.avail_out ) { + iovector.iov_len = (char *)strm.next_out - (char *)iovector.iov_base; + if (mutex_workqueue_pushchunked(taskid, &iovector) ) { + free(iovector.iov_base); + return mutex_bucket_unlock( bucket, 0 ); + } + + { unsigned int pending; int bits; deflatePending( &strm, &pending, &bits); pending += ( bits ? 1 : 0 ); - iovector.iov_len = (char *)strm.next_out - (char *)iovector.iov_base; - if (mutex_workqueue_pushchunked(taskid, &iovector) ) { - free(iovector.iov_base); + if (pending) { + /* Allocate a fresh output buffer */ + iovector.iov_base = malloc( pending ); + iovector.iov_len = pending; + + if( !iovector.iov_base ) { + fprintf( stderr, "Problem with iovec_fix_increase_or_free\n" ); + deflateEnd(&strm); return mutex_bucket_unlock( bucket, 0 ); - } - /* Allocate a fresh output buffer */ - iovector.iov_base = malloc( pending ); - iovector.iov_len = pending; + } + strm.next_out = iovector.iov_base; + strm.avail_out = pending; + if( deflate( &strm, Z_FINISH ) < Z_OK ) + fprintf( stderr, "deflate() failed while in fullscrape_make()'s endgame.\n" ); - if( !iovector.iov_base ) { - fprintf( stderr, "Problem with iovec_fix_increase_or_free\n" ); - deflateEnd(&strm); - return mutex_bucket_unlock( bucket, 0 ); + if( mutex_workqueue_pushchunked(taskid, &iovector) ) + free(iovector.iov_base); } - strm.next_out = iovector.iov_base; - strm.avail_out = pending; - if( deflate( &strm, Z_FINISH ) < Z_OK ) - fprintf( stderr, "deflate() failed while in fullscrape_make()'s endgame.\n" ); - - if( mutex_workqueue_pushchunked(taskid, &iovector) ) - free(iovector.iov_base); } deflateEnd(&strm); diff --git a/ot_http.c b/ot_http.c index c82bcdf..66899b1 100644 --- a/ot_http.c +++ b/ot_http.c @@ -162,7 +162,7 @@ fprintf(stderr, "http_sendiovecdata sending %d iovec entries found cookie->batch header_size = sprintf( header, "HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\n%sContent-Length: %zd\r\n\r\n", encoding, size ); else { if ( !(cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER )) { - header_size = sprintf( header, "HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\n%sTransfer-Encoding: chunked\r\n\r\n%zx\r\n", encoding, size ); + header_size = sprintf( header, "HTTP/1.0 200 OK\r\nContent-Type: application/octet-stream\r\n%sTransfer-Encoding: chunked\r\n\r\n%zx\r\n", encoding, size ); cookie->flag |= STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER; } else header_size = sprintf( header, "%zx\r\n", size ); @@ -269,12 +269,12 @@ static const ot_keywords keywords_format[] = tai6464 t; #ifdef WANT_COMPRESSION_GZIP ws->request[ws->request_size] = 0; -#ifdef WANT_COMPRESSION_GZIP_ALWAYS +#ifndef WANT_COMPRESSION_GZIP_ALWAYS if( strstr( read_ptr - 1, "gzip" ) ) { #endif cookie->flag |= STRUCT_HTTP_FLAG_GZIP; format |= TASK_FLAG_GZIP; -#ifdef WANT_COMPRESSION_GZIP_ALWAYS +#ifndef WANT_COMPRESSION_GZIP_ALWAYS } #endif #endif @@ -337,11 +337,15 @@ static ssize_t http_handle_fullscrape( const int64 sock, struct ot_workstruct *w #ifdef WANT_COMPRESSION_GZIP ws->request[ws->request_size-1] = 0; +#ifndef WANT_COMPRESSION_GZIP_ALWAYS if( strstr( ws->request, "gzip" ) ) { +#endif cookie->flag |= STRUCT_HTTP_FLAG_GZIP; format = TASK_FLAG_GZIP; stats_issue_event( EVENT_FULLSCRAPE_REQUEST_GZIP, 0, (uintptr_t)cookie->ip ); +#ifndef WANT_COMPRESSION_GZIP_ALWAYS } else +#endif #endif stats_issue_event( EVENT_FULLSCRAPE_REQUEST, 0, (uintptr_t)cookie->ip ); diff --git a/tests/testsuite2.sh b/tests/testsuite2.sh index c9a5a6a..da5181b 100644 --- a/tests/testsuite2.sh +++ b/tests/testsuite2.sh @@ -2,13 +2,21 @@ while true; do request_string="GET /announce?info_hash=012345678901234567\ -%$(printf %02X $(( $RANDOM & 0xff )) )\ -%$(printf %02X $(( $RANDOM & 0xff )) )\ -&ip=$(( $RANDOM & 0xff )).17.13.15&port=$(( $RANDOM & 0xff )) HTTP/1.0\n" - - echo $request_string - echo - echo $request_string | nc 23.23.23.237 6969 >/dev/null - echo +$(printf %02X $(( $RANDOM & 0xff )) )\ +&ip=$(( $RANDOM & 0xff )).17.13.15&port=$(( $RANDOM & 0xff )) HTTP/1.0" + +# echo $request_string +# echo + printf "%s\n\n" "$request_string" | nc 84.200.61.9 6969 | hexdump -C + + request_string="GET /announce?info_hash=012345678901234567\ +$(printf %02X $(( $RANDOM & 0xff )) )\ +&ip=2001:1608:6:27::$(( $RANDOM & 0xff ))&port=$(( $RANDOM & 0xff )) HTTP/1.0" + printf "%s\n\n" "$request_string" | nc 2001:1608:6:27::9 6969 | hexdump -C + printf "%s\n\n" "$request_string" + + request_string="GET /scrape?info_hash=012345678901234567\ +$(printf %02X $(( $RANDOM & 0xff )) ) HTTP/1.0" + printf "%s\n\n" "$request_string" | nc 2001:1608:6:27::9 6969 | hexdump -C done -- cgit v1.2.3