summaryrefslogtreecommitdiff
path: root/ot_fullscrape.c
diff options
context:
space:
mode:
authorerdgeist <>2007-11-22 04:39:08 +0000
committererdgeist <>2007-11-22 04:39:08 +0000
commitb1c8723609578b05b999f9cb58b9c90c4787f9d6 (patch)
tree83a40332304e122a7b2c6f88b3c842344126b7c7 /ot_fullscrape.c
parent4bf12406fb38d0ec0f6f9d156334d73e1b671830 (diff)
Introducing compression for fullscrapes and tpbs stats
Diffstat (limited to 'ot_fullscrape.c')
-rw-r--r--ot_fullscrape.c70
1 files changed, 64 insertions, 6 deletions
diff --git a/ot_fullscrape.c b/ot_fullscrape.c
index bb78b8a..d9c872e 100644
--- a/ot_fullscrape.c
+++ b/ot_fullscrape.c
@@ -7,8 +7,12 @@
7#include <string.h> 7#include <string.h>
8#include <pthread.h> 8#include <pthread.h>
9#include <arpa/inet.h> 9#include <arpa/inet.h>
10#ifdef WANT_COMPRESSION_GZIP
11#include <zlib.h>
12#endif
10 13
11/* Libowfat */ 14/* Libowfat */
15#include "byte.h"
12#include "textcode.h" 16#include "textcode.h"
13 17
14/* Opentracker */ 18/* Opentracker */
@@ -24,7 +28,7 @@
24#define OT_SCRAPE_CHUNK_SIZE (512*1024) 28#define OT_SCRAPE_CHUNK_SIZE (512*1024)
25 29
26/* "d8:completei%zde10:downloadedi%zde10:incompletei%zdee" */ 30/* "d8:completei%zde10:downloadedi%zde10:incompletei%zdee" */
27#define OT_FULLSCRAPE_MAXENTRYLEN 100 31#define OT_FULLSCRAPE_MAXENTRYLEN 256
28 32
29/* Forward declaration */ 33/* Forward declaration */
30static void fullscrape_make( int *iovec_entries, struct iovec **iovector, ot_tasktype mode ); 34static void fullscrape_make( int *iovec_entries, struct iovec **iovector, ot_tasktype mode );
@@ -66,8 +70,12 @@ void fullscrape_deliver( int64 socket, ot_tasktype tasktype ) {
66} 70}
67 71
68static void fullscrape_make( int *iovec_entries, struct iovec **iovector, ot_tasktype mode ) { 72static void fullscrape_make( int *iovec_entries, struct iovec **iovector, ot_tasktype mode ) {
69 int bucket; 73 int bucket;
70 char *r, *re; 74 char *r, *re;
75#ifdef WANT_COMPRESSION_GZIP
76 char compress_buffer[OT_FULLSCRAPE_MAXENTRYLEN];
77 z_stream strm;
78#endif
71 79
72 /* Setup return vector... */ 80 /* Setup return vector... */
73 *iovec_entries = 0; 81 *iovec_entries = 0;
@@ -79,8 +87,21 @@ static void fullscrape_make( int *iovec_entries, struct iovec **iovector, ot_tas
79 This works as a low watermark */ 87 This works as a low watermark */
80 re = r + OT_SCRAPE_CHUNK_SIZE; 88 re = r + OT_SCRAPE_CHUNK_SIZE;
81 89
90#ifdef WANT_COMPRESSION_GZIP
91 if( mode & TASK_FLAG_GZIP ) {
92 byte_zero( &strm, sizeof(strm) );
93 strm.next_in = (ot_byte*)r;
94 if( deflateInit2(&strm,9,Z_DEFLATED,31,8,Z_DEFAULT_STRATEGY) != Z_OK )
95 fprintf( stderr, "not ok.\n" );
96
97 strm.next_out = (unsigned char*)r;
98 strm.avail_out = OT_SCRAPE_CHUNK_SIZE;
99 r = compress_buffer;
100 }
101#endif
102
82 /* Reply dictionary only needed for bencoded fullscrape */ 103 /* Reply dictionary only needed for bencoded fullscrape */
83 if( mode == TASK_FULLSCRAPE ) { 104 if( ( mode & TASK_TASK_MASK ) == TASK_FULLSCRAPE ) {
84 memmove( r, "d5:filesd", 9 ); 105 memmove( r, "d5:filesd", 9 );
85 r += 9; 106 r += 9;
86 } 107 }
@@ -97,7 +118,7 @@ static void fullscrape_make( int *iovec_entries, struct iovec **iovector, ot_tas
97 ot_peerlist *peer_list = ( ((ot_torrent*)(torrents_list->data))[tor_offset] ).peer_list; 118 ot_peerlist *peer_list = ( ((ot_torrent*)(torrents_list->data))[tor_offset] ).peer_list;
98 ot_hash *hash =&( ((ot_torrent*)(torrents_list->data))[tor_offset] ).hash; 119 ot_hash *hash =&( ((ot_torrent*)(torrents_list->data))[tor_offset] ).hash;
99 120
100 switch( mode ) { 121 switch( mode & TASK_TASK_MASK ) {
101 case TASK_FULLSCRAPE: 122 case TASK_FULLSCRAPE:
102 default: 123 default:
103 /* push hash as bencoded string */ 124 /* push hash as bencoded string */
@@ -122,6 +143,16 @@ static void fullscrape_make( int *iovec_entries, struct iovec **iovector, ot_tas
122 break; 143 break;
123 } 144 }
124 145
146#ifdef WANT_COMPRESSION_GZIP
147 if( mode & TASK_FLAG_GZIP ) {
148 strm.next_in = (ot_byte*)compress_buffer;
149 strm.avail_in = r - compress_buffer;
150 if( deflate( &strm, Z_NO_FLUSH ) != Z_OK )
151 fprintf( stderr, "Not ok.\n" );
152 r = (char*)strm.next_out;
153 }
154#endif
155
125 /* If we reached our low watermark in buffer... */ 156 /* If we reached our low watermark in buffer... */
126 if( re - r <= OT_FULLSCRAPE_MAXENTRYLEN ) { 157 if( re - r <= OT_FULLSCRAPE_MAXENTRYLEN ) {
127 158
@@ -134,6 +165,10 @@ static void fullscrape_make( int *iovec_entries, struct iovec **iovector, ot_tas
134 /* If this fails: free buffers */ 165 /* If this fails: free buffers */
135 iovec_free( iovec_entries, iovector ); 166 iovec_free( iovec_entries, iovector );
136 167
168#ifdef WANT_COMPRESSION_GZIP
169 deflateEnd(&strm);
170#endif
171
137 /* Release lock on current bucket and return */ 172 /* Release lock on current bucket and return */
138 mutex_bucket_unlock( bucket ); 173 mutex_bucket_unlock( bucket );
139 return; 174 return;
@@ -141,7 +176,19 @@ static void fullscrape_make( int *iovec_entries, struct iovec **iovector, ot_tas
141 176
142 /* Adjust new end of output buffer */ 177 /* Adjust new end of output buffer */
143 re = r + OT_SCRAPE_CHUNK_SIZE; 178 re = r + OT_SCRAPE_CHUNK_SIZE;
179
180#ifdef WANT_COMPRESSION_GZIP
181 if( mode & TASK_FLAG_GZIP ) {
182 strm.next_out = (ot_byte*)r;
183 strm.avail_out = OT_SCRAPE_CHUNK_SIZE;
184 }
185#endif
186 }
187#ifdef WANT_COMPRESSION_GZIP
188 if( mode & TASK_FLAG_GZIP ) {
189 r = compress_buffer;
144 } 190 }
191#endif
145 } 192 }
146 193
147 /* All torrents done: release lock on currenct bucket */ 194 /* All torrents done: release lock on currenct bucket */
@@ -149,10 +196,21 @@ static void fullscrape_make( int *iovec_entries, struct iovec **iovector, ot_tas
149 } 196 }
150 197
151 /* Close bencoded scrape dictionary if necessary */ 198 /* Close bencoded scrape dictionary if necessary */
152 if( mode == TASK_FULLSCRAPE ) { 199 if( ( mode & TASK_TASK_MASK ) == TASK_FULLSCRAPE ) {
153 *r++='e'; *r++='e'; 200 *r++='e'; *r++='e';
154 } 201 }
155 202
203#ifdef WANT_COMPRESSION_GZIP
204 if( mode & TASK_FLAG_GZIP ) {
205 strm.next_in = (ot_byte*) compress_buffer;
206 strm.avail_in = r - compress_buffer;
207 if( deflate( &strm, Z_FINISH ) != Z_STREAM_END )
208 fprintf( stderr, "Not ok.\n" );
209 r = (char*)strm.next_out;
210 deflateEnd(&strm);
211 }
212#endif
213
156 /* Release unused memory in current output buffer */ 214 /* Release unused memory in current output buffer */
157 iovec_fixlast( iovec_entries, iovector, OT_SCRAPE_CHUNK_SIZE - ( re - r ) ); 215 iovec_fixlast( iovec_entries, iovector, OT_SCRAPE_CHUNK_SIZE - ( re - r ) );
158} 216}