diff options
author | Dirk Engling <erdgeist@erdgeist.org> | 2024-04-13 14:04:18 +0200 |
---|---|---|
committer | Dirk Engling <erdgeist@erdgeist.org> | 2024-04-13 14:04:18 +0200 |
commit | bd4992435ca8343cca0b34af13cf6da331a357f8 (patch) | |
tree | 73889845ee88f968ce14332fcfd9043d2c736a4b | |
parent | 35f55290f3832740535c7b419e494b8b7266804d (diff) |
Add proper parsing for the gzip content encoding
-rw-r--r-- | Makefile | 4 | ||||
-rw-r--r-- | ot_fullscrape.c | 41 | ||||
-rw-r--r-- | ot_http.c | 10 | ||||
-rw-r--r-- | tests/testsuite2.sh | 24 |
4 files changed, 47 insertions, 32 deletions
@@ -24,7 +24,7 @@ STRIP?=strip | |||
24 | #FEATURES+=-DWANT_DYNAMIC_ACCESSLIST | 24 | #FEATURES+=-DWANT_DYNAMIC_ACCESSLIST |
25 | 25 | ||
26 | #FEATURES+=-DWANT_SYNC_LIVE | 26 | #FEATURES+=-DWANT_SYNC_LIVE |
27 | #FEATURES+=-DWANT_IP_FROM_QUERY_STRING | 27 | FEATURES+=-DWANT_IP_FROM_QUERY_STRING |
28 | #FEATURES+=-DWANT_COMPRESSION_GZIP | 28 | #FEATURES+=-DWANT_COMPRESSION_GZIP |
29 | #FEATURES+=-DWANT_COMPRESSION_GZIP_ALWAYS | 29 | #FEATURES+=-DWANT_COMPRESSION_GZIP_ALWAYS |
30 | #FEATURES+=-DWANT_LOG_NETWORKS | 30 | #FEATURES+=-DWANT_LOG_NETWORKS |
@@ -48,7 +48,7 @@ FEATURES+=-DWANT_FULLSCRAPE | |||
48 | OPTS_debug=-D_DEBUG -g -ggdb # -pg -fprofile-arcs -ftest-coverage | 48 | OPTS_debug=-D_DEBUG -g -ggdb # -pg -fprofile-arcs -ftest-coverage |
49 | OPTS_production=-O3 | 49 | OPTS_production=-O3 |
50 | 50 | ||
51 | CFLAGS+=-I$(LIBOWFAT_HEADERS) -Wall -pipe -pthread -Wextra #-ansi -pedantic | 51 | CFLAGS+=-I$(LIBOWFAT_HEADERS) -Wall -pipe -pthread -Wextra -Wincompatible-pointer-types #-ansi -pedantic |
52 | LDFLAGS+=-L$(LIBOWFAT_LIBRARY) -lowfat -pthread -lz | 52 | LDFLAGS+=-L$(LIBOWFAT_LIBRARY) -lowfat -pthread -lz |
53 | #LDFLAGS+=-lbsd | 53 | #LDFLAGS+=-lbsd |
54 | 54 | ||
diff --git a/ot_fullscrape.c b/ot_fullscrape.c index b147b6a..86b9e26 100644 --- a/ot_fullscrape.c +++ b/ot_fullscrape.c | |||
@@ -262,33 +262,36 @@ fprintf(stderr, "GZIP path\n"); | |||
262 | if( deflate( &strm, Z_FINISH ) < Z_OK ) | 262 | if( deflate( &strm, Z_FINISH ) < Z_OK ) |
263 | fprintf( stderr, "deflate() failed while in fullscrape_make()'s endgame.\n" ); | 263 | fprintf( stderr, "deflate() failed while in fullscrape_make()'s endgame.\n" ); |
264 | 264 | ||
265 | if( !strm.avail_out ) { | 265 | iovector.iov_len = (char *)strm.next_out - (char *)iovector.iov_base; |
266 | if (mutex_workqueue_pushchunked(taskid, &iovector) ) { | ||
267 | free(iovector.iov_base); | ||
268 | return mutex_bucket_unlock( bucket, 0 ); | ||
269 | } | ||
270 | |||
271 | { | ||
266 | unsigned int pending; | 272 | unsigned int pending; |
267 | int bits; | 273 | int bits; |
268 | deflatePending( &strm, &pending, &bits); | 274 | deflatePending( &strm, &pending, &bits); |
269 | pending += ( bits ? 1 : 0 ); | 275 | pending += ( bits ? 1 : 0 ); |
270 | 276 | ||
271 | iovector.iov_len = (char *)strm.next_out - (char *)iovector.iov_base; | 277 | if (pending) { |
272 | if (mutex_workqueue_pushchunked(taskid, &iovector) ) { | 278 | /* Allocate a fresh output buffer */ |
273 | free(iovector.iov_base); | 279 | iovector.iov_base = malloc( pending ); |
280 | iovector.iov_len = pending; | ||
281 | |||
282 | if( !iovector.iov_base ) { | ||
283 | fprintf( stderr, "Problem with iovec_fix_increase_or_free\n" ); | ||
284 | deflateEnd(&strm); | ||
274 | return mutex_bucket_unlock( bucket, 0 ); | 285 | return mutex_bucket_unlock( bucket, 0 ); |
275 | } | 286 | } |
276 | /* Allocate a fresh output buffer */ | 287 | strm.next_out = iovector.iov_base; |
277 | iovector.iov_base = malloc( pending ); | 288 | strm.avail_out = pending; |
278 | iovector.iov_len = pending; | 289 | if( deflate( &strm, Z_FINISH ) < Z_OK ) |
290 | fprintf( stderr, "deflate() failed while in fullscrape_make()'s endgame.\n" ); | ||
279 | 291 | ||
280 | if( !iovector.iov_base ) { | 292 | if( mutex_workqueue_pushchunked(taskid, &iovector) ) |
281 | fprintf( stderr, "Problem with iovec_fix_increase_or_free\n" ); | 293 | free(iovector.iov_base); |
282 | deflateEnd(&strm); | ||
283 | return mutex_bucket_unlock( bucket, 0 ); | ||
284 | } | 294 | } |
285 | strm.next_out = iovector.iov_base; | ||
286 | strm.avail_out = pending; | ||
287 | if( deflate( &strm, Z_FINISH ) < Z_OK ) | ||
288 | fprintf( stderr, "deflate() failed while in fullscrape_make()'s endgame.\n" ); | ||
289 | |||
290 | if( mutex_workqueue_pushchunked(taskid, &iovector) ) | ||
291 | free(iovector.iov_base); | ||
292 | } | 295 | } |
293 | 296 | ||
294 | deflateEnd(&strm); | 297 | deflateEnd(&strm); |
@@ -162,7 +162,7 @@ fprintf(stderr, "http_sendiovecdata sending %d iovec entries found cookie->batch | |||
162 | header_size = sprintf( header, "HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\n%sContent-Length: %zd\r\n\r\n", encoding, size ); | 162 | header_size = sprintf( header, "HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\n%sContent-Length: %zd\r\n\r\n", encoding, size ); |
163 | else { | 163 | else { |
164 | if ( !(cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER )) { | 164 | if ( !(cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER )) { |
165 | header_size = sprintf( header, "HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\n%sTransfer-Encoding: chunked\r\n\r\n%zx\r\n", encoding, size ); | 165 | header_size = sprintf( header, "HTTP/1.0 200 OK\r\nContent-Type: application/octet-stream\r\n%sTransfer-Encoding: chunked\r\n\r\n%zx\r\n", encoding, size ); |
166 | cookie->flag |= STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER; | 166 | cookie->flag |= STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER; |
167 | } else | 167 | } else |
168 | header_size = sprintf( header, "%zx\r\n", size ); | 168 | header_size = sprintf( header, "%zx\r\n", size ); |
@@ -269,12 +269,12 @@ static const ot_keywords keywords_format[] = | |||
269 | tai6464 t; | 269 | tai6464 t; |
270 | #ifdef WANT_COMPRESSION_GZIP | 270 | #ifdef WANT_COMPRESSION_GZIP |
271 | ws->request[ws->request_size] = 0; | 271 | ws->request[ws->request_size] = 0; |
272 | #ifdef WANT_COMPRESSION_GZIP_ALWAYS | 272 | #ifndef WANT_COMPRESSION_GZIP_ALWAYS |
273 | if( strstr( read_ptr - 1, "gzip" ) ) { | 273 | if( strstr( read_ptr - 1, "gzip" ) ) { |
274 | #endif | 274 | #endif |
275 | cookie->flag |= STRUCT_HTTP_FLAG_GZIP; | 275 | cookie->flag |= STRUCT_HTTP_FLAG_GZIP; |
276 | format |= TASK_FLAG_GZIP; | 276 | format |= TASK_FLAG_GZIP; |
277 | #ifdef WANT_COMPRESSION_GZIP_ALWAYS | 277 | #ifndef WANT_COMPRESSION_GZIP_ALWAYS |
278 | } | 278 | } |
279 | #endif | 279 | #endif |
280 | #endif | 280 | #endif |
@@ -337,12 +337,16 @@ static ssize_t http_handle_fullscrape( const int64 sock, struct ot_workstruct *w | |||
337 | 337 | ||
338 | #ifdef WANT_COMPRESSION_GZIP | 338 | #ifdef WANT_COMPRESSION_GZIP |
339 | ws->request[ws->request_size-1] = 0; | 339 | ws->request[ws->request_size-1] = 0; |
340 | #ifndef WANT_COMPRESSION_GZIP_ALWAYS | ||
340 | if( strstr( ws->request, "gzip" ) ) { | 341 | if( strstr( ws->request, "gzip" ) ) { |
342 | #endif | ||
341 | cookie->flag |= STRUCT_HTTP_FLAG_GZIP; | 343 | cookie->flag |= STRUCT_HTTP_FLAG_GZIP; |
342 | format = TASK_FLAG_GZIP; | 344 | format = TASK_FLAG_GZIP; |
343 | stats_issue_event( EVENT_FULLSCRAPE_REQUEST_GZIP, 0, (uintptr_t)cookie->ip ); | 345 | stats_issue_event( EVENT_FULLSCRAPE_REQUEST_GZIP, 0, (uintptr_t)cookie->ip ); |
346 | #ifndef WANT_COMPRESSION_GZIP_ALWAYS | ||
344 | } else | 347 | } else |
345 | #endif | 348 | #endif |
349 | #endif | ||
346 | stats_issue_event( EVENT_FULLSCRAPE_REQUEST, 0, (uintptr_t)cookie->ip ); | 350 | stats_issue_event( EVENT_FULLSCRAPE_REQUEST, 0, (uintptr_t)cookie->ip ); |
347 | 351 | ||
348 | #ifdef _DEBUG_HTTPERROR | 352 | #ifdef _DEBUG_HTTPERROR |
diff --git a/tests/testsuite2.sh b/tests/testsuite2.sh index c9a5a6a..da5181b 100644 --- a/tests/testsuite2.sh +++ b/tests/testsuite2.sh | |||
@@ -2,13 +2,21 @@ | |||
2 | 2 | ||
3 | while true; do | 3 | while true; do |
4 | request_string="GET /announce?info_hash=012345678901234567\ | 4 | request_string="GET /announce?info_hash=012345678901234567\ |
5 | %$(printf %02X $(( $RANDOM & 0xff )) )\ | 5 | $(printf %02X $(( $RANDOM & 0xff )) )\ |
6 | %$(printf %02X $(( $RANDOM & 0xff )) )\ | 6 | &ip=$(( $RANDOM & 0xff )).17.13.15&port=$(( $RANDOM & 0xff )) HTTP/1.0" |
7 | &ip=$(( $RANDOM & 0xff )).17.13.15&port=$(( $RANDOM & 0xff )) HTTP/1.0\n" | 7 | |
8 | 8 | # echo $request_string | |
9 | echo $request_string | 9 | # echo |
10 | echo | 10 | printf "%s\n\n" "$request_string" | nc 84.200.61.9 6969 | hexdump -C |
11 | echo $request_string | nc 23.23.23.237 6969 >/dev/null | 11 | |
12 | echo | 12 | request_string="GET /announce?info_hash=012345678901234567\ |
13 | $(printf %02X $(( $RANDOM & 0xff )) )\ | ||
14 | &ip=2001:1608:6:27::$(( $RANDOM & 0xff ))&port=$(( $RANDOM & 0xff )) HTTP/1.0" | ||
15 | printf "%s\n\n" "$request_string" | nc 2001:1608:6:27::9 6969 | hexdump -C | ||
16 | printf "%s\n\n" "$request_string" | ||
17 | |||
18 | request_string="GET /scrape?info_hash=012345678901234567\ | ||
19 | $(printf %02X $(( $RANDOM & 0xff )) ) HTTP/1.0" | ||
20 | printf "%s\n\n" "$request_string" | nc 2001:1608:6:27::9 6969 | hexdump -C | ||
13 | 21 | ||
14 | done | 22 | done |