summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDirk Engling <erdgeist@erdgeist.org>2024-04-13 14:04:18 +0200
committerDirk Engling <erdgeist@erdgeist.org>2024-04-13 14:04:18 +0200
commitbd4992435ca8343cca0b34af13cf6da331a357f8 (patch)
tree73889845ee88f968ce14332fcfd9043d2c736a4b
parent35f55290f3832740535c7b419e494b8b7266804d (diff)
Add proper parsing for the gzip content encoding
-rw-r--r--Makefile4
-rw-r--r--ot_fullscrape.c41
-rw-r--r--ot_http.c10
-rw-r--r--tests/testsuite2.sh24
4 files changed, 47 insertions, 32 deletions
diff --git a/Makefile b/Makefile
index 7f43829..0150217 100644
--- a/Makefile
+++ b/Makefile
@@ -24,7 +24,7 @@ STRIP?=strip
24#FEATURES+=-DWANT_DYNAMIC_ACCESSLIST 24#FEATURES+=-DWANT_DYNAMIC_ACCESSLIST
25 25
26#FEATURES+=-DWANT_SYNC_LIVE 26#FEATURES+=-DWANT_SYNC_LIVE
27#FEATURES+=-DWANT_IP_FROM_QUERY_STRING 27FEATURES+=-DWANT_IP_FROM_QUERY_STRING
28#FEATURES+=-DWANT_COMPRESSION_GZIP 28#FEATURES+=-DWANT_COMPRESSION_GZIP
29#FEATURES+=-DWANT_COMPRESSION_GZIP_ALWAYS 29#FEATURES+=-DWANT_COMPRESSION_GZIP_ALWAYS
30#FEATURES+=-DWANT_LOG_NETWORKS 30#FEATURES+=-DWANT_LOG_NETWORKS
@@ -48,7 +48,7 @@ FEATURES+=-DWANT_FULLSCRAPE
48OPTS_debug=-D_DEBUG -g -ggdb # -pg -fprofile-arcs -ftest-coverage 48OPTS_debug=-D_DEBUG -g -ggdb # -pg -fprofile-arcs -ftest-coverage
49OPTS_production=-O3 49OPTS_production=-O3
50 50
51CFLAGS+=-I$(LIBOWFAT_HEADERS) -Wall -pipe -pthread -Wextra #-ansi -pedantic 51CFLAGS+=-I$(LIBOWFAT_HEADERS) -Wall -pipe -pthread -Wextra -Wincompatible-pointer-types #-ansi -pedantic
52LDFLAGS+=-L$(LIBOWFAT_LIBRARY) -lowfat -pthread -lz 52LDFLAGS+=-L$(LIBOWFAT_LIBRARY) -lowfat -pthread -lz
53#LDFLAGS+=-lbsd 53#LDFLAGS+=-lbsd
54 54
diff --git a/ot_fullscrape.c b/ot_fullscrape.c
index b147b6a..86b9e26 100644
--- a/ot_fullscrape.c
+++ b/ot_fullscrape.c
@@ -262,33 +262,36 @@ fprintf(stderr, "GZIP path\n");
262 if( deflate( &strm, Z_FINISH ) < Z_OK ) 262 if( deflate( &strm, Z_FINISH ) < Z_OK )
263 fprintf( stderr, "deflate() failed while in fullscrape_make()'s endgame.\n" ); 263 fprintf( stderr, "deflate() failed while in fullscrape_make()'s endgame.\n" );
264 264
265 if( !strm.avail_out ) { 265 iovector.iov_len = (char *)strm.next_out - (char *)iovector.iov_base;
266 if (mutex_workqueue_pushchunked(taskid, &iovector) ) {
267 free(iovector.iov_base);
268 return mutex_bucket_unlock( bucket, 0 );
269 }
270
271 {
266 unsigned int pending; 272 unsigned int pending;
267 int bits; 273 int bits;
268 deflatePending( &strm, &pending, &bits); 274 deflatePending( &strm, &pending, &bits);
269 pending += ( bits ? 1 : 0 ); 275 pending += ( bits ? 1 : 0 );
270 276
271 iovector.iov_len = (char *)strm.next_out - (char *)iovector.iov_base; 277 if (pending) {
272 if (mutex_workqueue_pushchunked(taskid, &iovector) ) { 278 /* Allocate a fresh output buffer */
273 free(iovector.iov_base); 279 iovector.iov_base = malloc( pending );
280 iovector.iov_len = pending;
281
282 if( !iovector.iov_base ) {
283 fprintf( stderr, "Problem with iovec_fix_increase_or_free\n" );
284 deflateEnd(&strm);
274 return mutex_bucket_unlock( bucket, 0 ); 285 return mutex_bucket_unlock( bucket, 0 );
275 } 286 }
276 /* Allocate a fresh output buffer */ 287 strm.next_out = iovector.iov_base;
277 iovector.iov_base = malloc( pending ); 288 strm.avail_out = pending;
278 iovector.iov_len = pending; 289 if( deflate( &strm, Z_FINISH ) < Z_OK )
290 fprintf( stderr, "deflate() failed while in fullscrape_make()'s endgame.\n" );
279 291
280 if( !iovector.iov_base ) { 292 if( mutex_workqueue_pushchunked(taskid, &iovector) )
281 fprintf( stderr, "Problem with iovec_fix_increase_or_free\n" ); 293 free(iovector.iov_base);
282 deflateEnd(&strm);
283 return mutex_bucket_unlock( bucket, 0 );
284 } 294 }
285 strm.next_out = iovector.iov_base;
286 strm.avail_out = pending;
287 if( deflate( &strm, Z_FINISH ) < Z_OK )
288 fprintf( stderr, "deflate() failed while in fullscrape_make()'s endgame.\n" );
289
290 if( mutex_workqueue_pushchunked(taskid, &iovector) )
291 free(iovector.iov_base);
292 } 295 }
293 296
294 deflateEnd(&strm); 297 deflateEnd(&strm);
diff --git a/ot_http.c b/ot_http.c
index c82bcdf..66899b1 100644
--- a/ot_http.c
+++ b/ot_http.c
@@ -162,7 +162,7 @@ fprintf(stderr, "http_sendiovecdata sending %d iovec entries found cookie->batch
162 header_size = sprintf( header, "HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\n%sContent-Length: %zd\r\n\r\n", encoding, size ); 162 header_size = sprintf( header, "HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\n%sContent-Length: %zd\r\n\r\n", encoding, size );
163 else { 163 else {
164 if ( !(cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER )) { 164 if ( !(cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER )) {
165 header_size = sprintf( header, "HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\n%sTransfer-Encoding: chunked\r\n\r\n%zx\r\n", encoding, size ); 165 header_size = sprintf( header, "HTTP/1.0 200 OK\r\nContent-Type: application/octet-stream\r\n%sTransfer-Encoding: chunked\r\n\r\n%zx\r\n", encoding, size );
166 cookie->flag |= STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER; 166 cookie->flag |= STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER;
167 } else 167 } else
168 header_size = sprintf( header, "%zx\r\n", size ); 168 header_size = sprintf( header, "%zx\r\n", size );
@@ -269,12 +269,12 @@ static const ot_keywords keywords_format[] =
269 tai6464 t; 269 tai6464 t;
270#ifdef WANT_COMPRESSION_GZIP 270#ifdef WANT_COMPRESSION_GZIP
271 ws->request[ws->request_size] = 0; 271 ws->request[ws->request_size] = 0;
272#ifdef WANT_COMPRESSION_GZIP_ALWAYS 272#ifndef WANT_COMPRESSION_GZIP_ALWAYS
273 if( strstr( read_ptr - 1, "gzip" ) ) { 273 if( strstr( read_ptr - 1, "gzip" ) ) {
274#endif 274#endif
275 cookie->flag |= STRUCT_HTTP_FLAG_GZIP; 275 cookie->flag |= STRUCT_HTTP_FLAG_GZIP;
276 format |= TASK_FLAG_GZIP; 276 format |= TASK_FLAG_GZIP;
277#ifdef WANT_COMPRESSION_GZIP_ALWAYS 277#ifndef WANT_COMPRESSION_GZIP_ALWAYS
278 } 278 }
279#endif 279#endif
280#endif 280#endif
@@ -337,12 +337,16 @@ static ssize_t http_handle_fullscrape( const int64 sock, struct ot_workstruct *w
337 337
338#ifdef WANT_COMPRESSION_GZIP 338#ifdef WANT_COMPRESSION_GZIP
339 ws->request[ws->request_size-1] = 0; 339 ws->request[ws->request_size-1] = 0;
340#ifndef WANT_COMPRESSION_GZIP_ALWAYS
340 if( strstr( ws->request, "gzip" ) ) { 341 if( strstr( ws->request, "gzip" ) ) {
342#endif
341 cookie->flag |= STRUCT_HTTP_FLAG_GZIP; 343 cookie->flag |= STRUCT_HTTP_FLAG_GZIP;
342 format = TASK_FLAG_GZIP; 344 format = TASK_FLAG_GZIP;
343 stats_issue_event( EVENT_FULLSCRAPE_REQUEST_GZIP, 0, (uintptr_t)cookie->ip ); 345 stats_issue_event( EVENT_FULLSCRAPE_REQUEST_GZIP, 0, (uintptr_t)cookie->ip );
346#ifndef WANT_COMPRESSION_GZIP_ALWAYS
344 } else 347 } else
345#endif 348#endif
349#endif
346 stats_issue_event( EVENT_FULLSCRAPE_REQUEST, 0, (uintptr_t)cookie->ip ); 350 stats_issue_event( EVENT_FULLSCRAPE_REQUEST, 0, (uintptr_t)cookie->ip );
347 351
348#ifdef _DEBUG_HTTPERROR 352#ifdef _DEBUG_HTTPERROR
diff --git a/tests/testsuite2.sh b/tests/testsuite2.sh
index c9a5a6a..da5181b 100644
--- a/tests/testsuite2.sh
+++ b/tests/testsuite2.sh
@@ -2,13 +2,21 @@
2 2
3while true; do 3while true; do
4 request_string="GET /announce?info_hash=012345678901234567\ 4 request_string="GET /announce?info_hash=012345678901234567\
5%$(printf %02X $(( $RANDOM & 0xff )) )\ 5$(printf %02X $(( $RANDOM & 0xff )) )\
6%$(printf %02X $(( $RANDOM & 0xff )) )\ 6&ip=$(( $RANDOM & 0xff )).17.13.15&port=$(( $RANDOM & 0xff )) HTTP/1.0"
7&ip=$(( $RANDOM & 0xff )).17.13.15&port=$(( $RANDOM & 0xff )) HTTP/1.0\n" 7
8 8# echo $request_string
9 echo $request_string 9# echo
10 echo 10 printf "%s\n\n" "$request_string" | nc 84.200.61.9 6969 | hexdump -C
11 echo $request_string | nc 23.23.23.237 6969 >/dev/null 11
12 echo 12 request_string="GET /announce?info_hash=012345678901234567\
13$(printf %02X $(( $RANDOM & 0xff )) )\
14&ip=2001:1608:6:27::$(( $RANDOM & 0xff ))&port=$(( $RANDOM & 0xff )) HTTP/1.0"
15 printf "%s\n\n" "$request_string" | nc 2001:1608:6:27::9 6969 | hexdump -C
16 printf "%s\n\n" "$request_string"
17
18 request_string="GET /scrape?info_hash=012345678901234567\
19$(printf %02X $(( $RANDOM & 0xff )) ) HTTP/1.0"
20 printf "%s\n\n" "$request_string" | nc 2001:1608:6:27::9 6969 | hexdump -C
13 21
14done 22done