summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDirk Engling <erdgeist@erdgeist.org>2024-04-13 12:51:04 +0200
committerDirk Engling <erdgeist@erdgeist.org>2024-04-13 12:51:04 +0200
commit35f55290f3832740535c7b419e494b8b7266804d (patch)
treea2b5771457dfdf77b0b087fffffff5fd97e3f608
parent2eeae0a65a137ab1796ff93133201cfcf3cab750 (diff)
Make chunked transfers use gzip also
-rw-r--r--ot_fullscrape.c55
1 files changed, 33 insertions, 22 deletions
diff --git a/ot_fullscrape.c b/ot_fullscrape.c
index fafd83c..b147b6a 100644
--- a/ot_fullscrape.c
+++ b/ot_fullscrape.c
@@ -153,7 +153,7 @@ static void fullscrape_make( int taskid, ot_tasktype mode ) {
153 free(iovector.iov_base); 153 free(iovector.iov_base);
154 return mutex_bucket_unlock( bucket, 0 ); 154 return mutex_bucket_unlock( bucket, 0 );
155 } 155 }
156 /* Allocate a fresh output buffer at the end of our buffers list */ 156 /* Allocate a fresh output buffer */
157 r = iovector.iov_base = malloc( OT_SCRAPE_CHUNK_SIZE ); 157 r = iovector.iov_base = malloc( OT_SCRAPE_CHUNK_SIZE );
158 if( !r ) 158 if( !r )
159 return mutex_bucket_unlock( bucket, 0 ); 159 return mutex_bucket_unlock( bucket, 0 );
@@ -174,7 +174,7 @@ static void fullscrape_make( int taskid, ot_tasktype mode ) {
174 if( ( mode & TASK_TASK_MASK ) == TASK_FULLSCRAPE ) 174 if( ( mode & TASK_TASK_MASK ) == TASK_FULLSCRAPE )
175 r += sprintf( r, "ee" ); 175 r += sprintf( r, "ee" );
176 176
177 /* Release unused memory in current output buffer */ 177 /* Send rest of data */
178 iovector.iov_len = r - (char *)iovector.iov_base; 178 iovector.iov_len = r - (char *)iovector.iov_base;
179 if( mutex_workqueue_pushchunked(taskid, &iovector) ) 179 if( mutex_workqueue_pushchunked(taskid, &iovector) )
180 free(iovector.iov_base); 180 free(iovector.iov_base);
@@ -182,21 +182,20 @@ static void fullscrape_make( int taskid, ot_tasktype mode ) {
182 182
183#ifdef WANT_COMPRESSION_GZIP 183#ifdef WANT_COMPRESSION_GZIP
184 184
185static void fullscrape_make_gzip( int *iovec_entries, struct iovec **iovector, ot_tasktype mode ) { 185static void fullscrape_make_gzip( int taskid, ot_tasktype mode) {
186 int bucket; 186 int bucket;
187 char *r; 187 char *r;
188 struct iovec iovector = { NULL, 0 };
188 int zres; 189 int zres;
189 z_stream strm; 190 z_stream strm;
190 191fprintf(stderr, "GZIP path\n");
191 /* Setup return vector... */ 192 /* Setup return vector... */
192 *iovec_entries = 0; 193 iovector.iov_base = malloc( OT_SCRAPE_CHUNK_SIZE );
193 *iovector = NULL; 194 if( !iovector.iov_base )
194 r = iovec_increase( iovec_entries, iovector, OT_SCRAPE_CHUNK_SIZE );
195 if( !r )
196 return; 195 return;
197 196
198 byte_zero( &strm, sizeof(strm) ); 197 byte_zero( &strm, sizeof(strm) );
199 strm.next_out = (uint8_t*)r; 198 strm.next_out = (uint8_t*)iovector.iov_base;
200 strm.avail_out = OT_SCRAPE_CHUNK_SIZE; 199 strm.avail_out = OT_SCRAPE_CHUNK_SIZE;
201 if( deflateInit2(&strm,7,Z_DEFLATED,31,9,Z_DEFAULT_STRATEGY) != Z_OK ) 200 if( deflateInit2(&strm,7,Z_DEFLATED,31,9,Z_DEFAULT_STRATEGY) != Z_OK )
202 fprintf( stderr, "not ok.\n" ); 201 fprintf( stderr, "not ok.\n" );
@@ -226,15 +225,20 @@ static void fullscrape_make_gzip( int *iovec_entries, struct iovec **iovector, o
226 225
227 /* Check if there still is enough buffer left */ 226 /* Check if there still is enough buffer left */
228 while( !strm.avail_out ) { 227 while( !strm.avail_out ) {
229 /* Allocate a fresh output buffer at the end of our buffers list */ 228 iovector.iov_len = (char *)strm.next_out - (char *)iovector.iov_base;
230 r = iovec_increase( iovec_entries, iovector, OT_SCRAPE_CHUNK_SIZE ); 229
231 if( !r ) { 230 if (mutex_workqueue_pushchunked(taskid, &iovector) ) {
232 fprintf( stderr, "Problem with iovec_fix_increase_or_free\n" ); 231 free(iovector.iov_base);
233 iovec_free( iovec_entries, iovector ); 232 return mutex_bucket_unlock( bucket, 0 );
233 }
234 /* Allocate a fresh output buffer */
235 iovector.iov_base = malloc( OT_SCRAPE_CHUNK_SIZE );
236 if( !iovector.iov_base ) {
237 fprintf( stderr, "Out of memory trying to claim ouput buffer\n" );
234 deflateEnd(&strm); 238 deflateEnd(&strm);
235 return mutex_bucket_unlock( bucket, 0 ); 239 return mutex_bucket_unlock( bucket, 0 );
236 } 240 }
237 strm.next_out = (uint8_t*)r; 241 strm.next_out = (uint8_t*)iovector.iov_base;
238 strm.avail_out = OT_SCRAPE_CHUNK_SIZE; 242 strm.avail_out = OT_SCRAPE_CHUNK_SIZE;
239 zres = deflate( &strm, Z_NO_FLUSH ); 243 zres = deflate( &strm, Z_NO_FLUSH );
240 if( ( zres < Z_OK ) && ( zres != Z_BUF_ERROR ) ) 244 if( ( zres < Z_OK ) && ( zres != Z_BUF_ERROR ) )
@@ -264,21 +268,28 @@ static void fullscrape_make_gzip( int *iovec_entries, struct iovec **iovector, o
264 deflatePending( &strm, &pending, &bits); 268 deflatePending( &strm, &pending, &bits);
265 pending += ( bits ? 1 : 0 ); 269 pending += ( bits ? 1 : 0 );
266 270
267 /* Allocate a fresh output buffer at the end of our buffers list */ 271 iovector.iov_len = (char *)strm.next_out - (char *)iovector.iov_base;
268 r = iovec_fix_increase_or_free( iovec_entries, iovector, strm.next_out, pending ); 272 if (mutex_workqueue_pushchunked(taskid, &iovector) ) {
269 if( !r ) { 273 free(iovector.iov_base);
274 return mutex_bucket_unlock( bucket, 0 );
275 }
276 /* Allocate a fresh output buffer */
277 iovector.iov_base = malloc( pending );
278 iovector.iov_len = pending;
279
280 if( !iovector.iov_base ) {
270 fprintf( stderr, "Problem with iovec_fix_increase_or_free\n" ); 281 fprintf( stderr, "Problem with iovec_fix_increase_or_free\n" );
271 deflateEnd(&strm); 282 deflateEnd(&strm);
272 return mutex_bucket_unlock( bucket, 0 ); 283 return mutex_bucket_unlock( bucket, 0 );
273 } 284 }
274 strm.next_out = (uint8_t*)r; 285 strm.next_out = iovector.iov_base;
275 strm.avail_out = pending; 286 strm.avail_out = pending;
276 if( deflate( &strm, Z_FINISH ) < Z_OK ) 287 if( deflate( &strm, Z_FINISH ) < Z_OK )
277 fprintf( stderr, "deflate() failed while in fullscrape_make()'s endgame.\n" ); 288 fprintf( stderr, "deflate() failed while in fullscrape_make()'s endgame.\n" );
278 }
279 289
280 /* Release unused memory in current output buffer */ 290 if( mutex_workqueue_pushchunked(taskid, &iovector) )
281 iovec_fixlast( iovec_entries, iovector, strm.next_out ); 291 free(iovector.iov_base);
292 }
282 293
283 deflateEnd(&strm); 294 deflateEnd(&strm);
284} 295}