summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--Makefile11
-rw-r--r--opentracker.c67
-rw-r--r--opentracker.conf.sample47
-rw-r--r--ot_accesslist.c287
-rw-r--r--ot_accesslist.h15
-rw-r--r--ot_clean.c4
-rw-r--r--ot_fullscrape.c286
-rw-r--r--ot_http.c43
-rw-r--r--ot_http.h3
-rw-r--r--ot_iovec.c58
-rw-r--r--ot_iovec.h2
-rw-r--r--ot_mutex.c230
-rw-r--r--ot_mutex.h6
-rw-r--r--ot_stats.c32
-rw-r--r--ot_stats.h6
-rw-r--r--ot_udp.c16
-rw-r--r--proxy.c8
-rw-r--r--trackerlogic.c30
-rw-r--r--trackerlogic.h19
19 files changed, 756 insertions, 414 deletions
diff --git a/Makefile b/Makefile
index da2c8f1..e3301a5 100644
--- a/Makefile
+++ b/Makefile
@@ -18,11 +18,13 @@ LIBOWFAT_HEADERS=$(PREFIX)/libowfat
18LIBOWFAT_LIBRARY=$(PREFIX)/libowfat 18LIBOWFAT_LIBRARY=$(PREFIX)/libowfat
19 19
20BINDIR?=$(PREFIX)/bin 20BINDIR?=$(PREFIX)/bin
21STRIP?=strip
21 22
22#FEATURES+=-DWANT_V6 23#FEATURES+=-DWANT_V6
23 24
24#FEATURES+=-DWANT_ACCESSLIST_BLACK 25#FEATURES+=-DWANT_ACCESSLIST_BLACK
25#FEATURES+=-DWANT_ACCESSLIST_WHITE 26#FEATURES+=-DWANT_ACCESSLIST_WHITE
27#FEATURES+=-DWANT_DYNAMIC_ACCESSLIST
26 28
27#FEATURES+=-DWANT_SYNC_LIVE 29#FEATURES+=-DWANT_SYNC_LIVE
28#FEATURES+=-DWANT_IP_FROM_QUERY_STRING 30#FEATURES+=-DWANT_IP_FROM_QUERY_STRING
@@ -39,6 +41,10 @@ BINDIR?=$(PREFIX)/bin
39#FEATURES+=-DWANT_DEV_RANDOM 41#FEATURES+=-DWANT_DEV_RANDOM
40FEATURES+=-DWANT_FULLSCRAPE 42FEATURES+=-DWANT_FULLSCRAPE
41 43
44# Is enabled on BSD systems by default in trackerlogic.h
45# on Linux systems you will need -lbds
46#FEATURES+=-DWANT_ARC4RANDOM
47
42#FEATURES+=-D_DEBUG_HTTPERROR 48#FEATURES+=-D_DEBUG_HTTPERROR
43 49
44OPTS_debug=-D_DEBUG -g -ggdb # -pg -fprofile-arcs -ftest-coverage 50OPTS_debug=-D_DEBUG -g -ggdb # -pg -fprofile-arcs -ftest-coverage
@@ -46,6 +52,7 @@ OPTS_production=-O3
46 52
47CFLAGS+=-I$(LIBOWFAT_HEADERS) -Wall -pipe -Wextra #-ansi -pedantic 53CFLAGS+=-I$(LIBOWFAT_HEADERS) -Wall -pipe -Wextra #-ansi -pedantic
48LDFLAGS+=-L$(LIBOWFAT_LIBRARY) -lowfat -pthread -lpthread -lz 54LDFLAGS+=-L$(LIBOWFAT_LIBRARY) -lowfat -pthread -lpthread -lz
55#LDFLAGS+=-lbsd
49 56
50BINARY =opentracker 57BINARY =opentracker
51HEADERS=trackerlogic.h scan_urlencoded_query.h ot_mutex.h ot_stats.h ot_vector.h ot_clean.h ot_udp.h ot_iovec.h ot_fullscrape.h ot_accesslist.h ot_http.h ot_livesync.h ot_rijndael.h 58HEADERS=trackerlogic.h scan_urlencoded_query.h ot_mutex.h ot_stats.h ot_vector.h ot_clean.h ot_udp.h ot_iovec.h ot_fullscrape.h ot_accesslist.h ot_http.h ot_livesync.h ot_rijndael.h
@@ -66,7 +73,7 @@ CFLAGS_debug = $(CFLAGS) $(OPTS_debug) $(FEATURES)
66 73
67$(BINARY): $(OBJECTS) $(HEADERS) 74$(BINARY): $(OBJECTS) $(HEADERS)
68 $(CC) -o $@ $(OBJECTS) $(LDFLAGS) 75 $(CC) -o $@ $(OBJECTS) $(LDFLAGS)
69 strip $@ 76 $(STRIP) $@
70$(BINARY).debug: $(OBJECTS_debug) $(HEADERS) 77$(BINARY).debug: $(OBJECTS_debug) $(HEADERS)
71 $(CC) -o $@ $(OBJECTS_debug) $(LDFLAGS) 78 $(CC) -o $@ $(OBJECTS_debug) $(LDFLAGS)
72proxy: $(OBJECTS_proxy) $(HEADERS) 79proxy: $(OBJECTS_proxy) $(HEADERS)
@@ -84,4 +91,4 @@ clean:
84 rm -rf opentracker opentracker.debug *.o *~ 91 rm -rf opentracker opentracker.debug *.o *~
85 92
86install: 93install:
87 install -m 755 opentracker $(BINDIR) 94 install -m 755 opentracker $(DESTDIR)$(BINDIR)
diff --git a/opentracker.c b/opentracker.c
index 5b9915f..2ca9e06 100644
--- a/opentracker.c
+++ b/opentracker.c
@@ -123,7 +123,7 @@ static void help( char *name ) {
123 HELPLINE("-P port","specify udp port to bind to (default: 6969, you may specify more than one)"); 123 HELPLINE("-P port","specify udp port to bind to (default: 6969, you may specify more than one)");
124 HELPLINE("-r redirecturl","specify url where / should be redirected to (default none)"); 124 HELPLINE("-r redirecturl","specify url where / should be redirected to (default none)");
125 HELPLINE("-d dir","specify directory to try to chroot to (default: \".\")"); 125 HELPLINE("-d dir","specify directory to try to chroot to (default: \".\")");
126 HELPLINE("-u user","specify user under whose priviliges opentracker should run (default: \"nobody\")"); 126 HELPLINE("-u user","specify user under whose privileges opentracker should run (default: \"nobody\")");
127 HELPLINE("-A ip","bless an ip address as admin address (e.g. to allow syncs from this address)"); 127 HELPLINE("-A ip","bless an ip address as admin address (e.g. to allow syncs from this address)");
128#ifdef WANT_ACCESSLIST_BLACK 128#ifdef WANT_ACCESSLIST_BLACK
129 HELPLINE("-b file","specify blacklist file."); 129 HELPLINE("-b file","specify blacklist file.");
@@ -135,8 +135,8 @@ static void help( char *name ) {
135} 135}
136#undef HELPLINE 136#undef HELPLINE
137 137
138static size_t header_complete( char * request, ssize_t byte_count ) { 138static ssize_t header_complete( char * request, ssize_t byte_count ) {
139 int i = 0, state = 0; 139 ssize_t i = 0, state = 0;
140 140
141 for( i=1; i < byte_count; i+=2 ) 141 for( i=1; i < byte_count; i+=2 )
142 if( request[i] <= 13 ) { 142 if( request[i] <= 13 ) {
@@ -156,7 +156,10 @@ static size_t header_complete( char * request, ssize_t byte_count ) {
156static void handle_dead( const int64 sock ) { 156static void handle_dead( const int64 sock ) {
157 struct http_data* cookie=io_getcookie( sock ); 157 struct http_data* cookie=io_getcookie( sock );
158 if( cookie ) { 158 if( cookie ) {
159 iob_reset( &cookie->batch ); 159 size_t i;
160 for ( i = 0; i < cookie->batches; ++i)
161 iob_reset( cookie->batch + i );
162 free( cookie->batch );
160 array_reset( &cookie->request ); 163 array_reset( &cookie->request );
161 if( cookie->flag & STRUCT_HTTP_FLAG_WAITINGFORTASK ) 164 if( cookie->flag & STRUCT_HTTP_FLAG_WAITINGFORTASK )
162 mutex_workqueue_canceltask( sock ); 165 mutex_workqueue_canceltask( sock );
@@ -167,13 +170,16 @@ static void handle_dead( const int64 sock ) {
167 170
168static void handle_read( const int64 sock, struct ot_workstruct *ws ) { 171static void handle_read( const int64 sock, struct ot_workstruct *ws ) {
169 struct http_data* cookie = io_getcookie( sock ); 172 struct http_data* cookie = io_getcookie( sock );
170 ssize_t byte_count; 173 ssize_t byte_count = io_tryread( sock, ws->inbuf, G_INBUF_SIZE );
171 174
172 if( ( byte_count = io_tryread( sock, ws->inbuf, G_INBUF_SIZE ) ) <= 0 ) { 175 if( byte_count == 0 || byte_count == -3 ) {
173 handle_dead( sock ); 176 handle_dead( sock );
174 return; 177 return;
175 } 178 }
176 179
180 if( byte_count == -1)
181 return;
182
177 /* If we get the whole request in one packet, handle it without copying */ 183 /* If we get the whole request in one packet, handle it without copying */
178 if( !array_start( &cookie->request ) ) { 184 if( !array_start( &cookie->request ) ) {
179 if( ( ws->header_size = header_complete( ws->inbuf, byte_count ) ) ) { 185 if( ( ws->header_size = header_complete( ws->inbuf, byte_count ) ) ) {
@@ -181,7 +187,7 @@ static void handle_read( const int64 sock, struct ot_workstruct *ws ) {
181 ws->request_size = byte_count; 187 ws->request_size = byte_count;
182 http_handle_request( sock, ws ); 188 http_handle_request( sock, ws );
183 } else 189 } else
184 array_catb( &cookie->request, ws->inbuf, byte_count ); 190 array_catb( &cookie->request, ws->inbuf, (size_t)byte_count );
185 return; 191 return;
186 } 192 }
187 193
@@ -204,8 +210,25 @@ static void handle_read( const int64 sock, struct ot_workstruct *ws ) {
204 210
205static void handle_write( const int64 sock ) { 211static void handle_write( const int64 sock ) {
206 struct http_data* cookie=io_getcookie( sock ); 212 struct http_data* cookie=io_getcookie( sock );
207 if( !cookie || ( iob_send( sock, &cookie->batch ) <= 0 ) ) 213 size_t i;
208 handle_dead( sock ); 214
215 /* Look for the first io_batch still containing bytes to write */
216 if( cookie )
217 for( i = 0; i < cookie->batches; ++i )
218 if( cookie->batch[i].bytesleft ) {
219 int64 res = iob_send( sock, cookie->batch + i );
220
221 if( res == -3 )
222 break;
223
224 if( !cookie->batch[i].bytesleft )
225 continue;
226
227 if( res == -1 || res > 0 || i < cookie->batches - 1 )
228 return;
229 }
230
231 handle_dead( sock );
209} 232}
210 233
211static void handle_accept( const int64 serversocket ) { 234static void handle_accept( const int64 serversocket ) {
@@ -239,6 +262,7 @@ static void handle_accept( const int64 serversocket ) {
239 tai_unix( &(t.sec), (g_now_seconds + OT_CLIENT_TIMEOUT) ); 262 tai_unix( &(t.sec), (g_now_seconds + OT_CLIENT_TIMEOUT) );
240 io_timeout( sock, t ); 263 io_timeout( sock, t );
241 } 264 }
265 io_eagain(serversocket);
242} 266}
243 267
244static void * server_mainloop( void * args ) { 268static void * server_mainloop( void * args ) {
@@ -255,9 +279,18 @@ static void * server_mainloop( void * args ) {
255#ifdef _DEBUG_HTTPERROR 279#ifdef _DEBUG_HTTPERROR
256 ws.debugbuf= malloc( G_DEBUGBUF_SIZE ); 280 ws.debugbuf= malloc( G_DEBUGBUF_SIZE );
257#endif 281#endif
282
258 if( !ws.inbuf || !ws.outbuf ) 283 if( !ws.inbuf || !ws.outbuf )
259 panic( "Initializing worker failed" ); 284 panic( "Initializing worker failed" );
260 285
286#ifdef WANT_ARC4RANDOM
287 arc4random_buf(&ws.rand48_state[0], 3 * sizeof(uint16_t));
288#else
289 ws.rand48_state[0] = (uint16_t)random();
290 ws.rand48_state[1] = (uint16_t)random();
291 ws.rand48_state[2] = (uint16_t)random();
292#endif
293
261 for( ; ; ) { 294 for( ; ; ) {
262 int64 sock; 295 int64 sock;
263 296
@@ -433,6 +466,12 @@ int parse_configfile( char * config_filename ) {
433 } else if(!byte_diff(p, 16, "access.blacklist" ) && isspace(p[16])) { 466 } else if(!byte_diff(p, 16, "access.blacklist" ) && isspace(p[16])) {
434 set_config_option( &g_accesslist_filename, p+17 ); 467 set_config_option( &g_accesslist_filename, p+17 );
435#endif 468#endif
469#ifdef WANT_DYNAMIC_ACCESSLIST
470 } else if(!byte_diff(p, 15, "access.fifo_add" ) && isspace(p[15])) {
471 set_config_option( &g_accesslist_pipe_add, p+16 );
472 } else if(!byte_diff(p, 18, "access.fifo_delete" ) && isspace(p[18])) {
473 set_config_option( &g_accesslist_pipe_delete, p+19 );
474#endif
436#ifdef WANT_RESTRICT_STATS 475#ifdef WANT_RESTRICT_STATS
437 } else if(!byte_diff(p, 12, "access.stats" ) && isspace(p[12])) { 476 } else if(!byte_diff(p, 12, "access.stats" ) && isspace(p[12])) {
438 if( !scan_ip6( p+13, tmpip )) goto parse_error; 477 if( !scan_ip6( p+13, tmpip )) goto parse_error;
@@ -529,12 +568,12 @@ int drop_privileges ( const char * const serveruser, const char * const serverdi
529 /* If we can't find server user, revert to nobody's default uid */ 568 /* If we can't find server user, revert to nobody's default uid */
530 if( !pws ) { 569 if( !pws ) {
531 fprintf( stderr, "Warning: Could not get password entry for %s. Reverting to uid -2.\n", serveruser ); 570 fprintf( stderr, "Warning: Could not get password entry for %s. Reverting to uid -2.\n", serveruser );
532 setegid( (gid_t)-2 ); setgid( (gid_t)-2 ); 571 if (setegid( (gid_t)-2 ) || setgid( (gid_t)-2 ) || setuid( (uid_t)-2 ) || seteuid( (uid_t)-2 ))
533 setuid( (uid_t)-2 ); seteuid( (uid_t)-2 ); 572 panic("Could not set uid to value -2");
534 } 573 }
535 else { 574 else {
536 setegid( pws->pw_gid ); setgid( pws->pw_gid ); 575 if (setegid( pws->pw_gid ) || setgid( pws->pw_gid ) || setuid( pws->pw_uid ) || seteuid( pws->pw_uid ))
537 setuid( pws->pw_uid ); seteuid( pws->pw_uid ); 576 panic("Could not set uid to specified value");
538 } 577 }
539 578
540 if( geteuid() == 0 || getegid() == 0 ) 579 if( geteuid() == 0 || getegid() == 0 )
@@ -640,6 +679,8 @@ int main( int argc, char **argv ) {
640 panic( "selfpipe failed: " ); 679 panic( "selfpipe failed: " );
641 if( !io_fd( g_self_pipe[0] ) ) 680 if( !io_fd( g_self_pipe[0] ) )
642 panic( "selfpipe io_fd failed: " ); 681 panic( "selfpipe io_fd failed: " );
682 if( !io_fd( g_self_pipe[1] ) )
683 panic( "selfpipe io_fd failed: " );
643 io_setcookie( g_self_pipe[0], (void*)FLAG_SELFPIPE ); 684 io_setcookie( g_self_pipe[0], (void*)FLAG_SELFPIPE );
644 io_wantread( g_self_pipe[0] ); 685 io_wantread( g_self_pipe[0] );
645 686
diff --git a/opentracker.conf.sample b/opentracker.conf.sample
index db45122..d44f3d4 100644
--- a/opentracker.conf.sample
+++ b/opentracker.conf.sample
@@ -44,6 +44,43 @@
44# listing, so choose one of those options at compile time. File format 44# listing, so choose one of those options at compile time. File format
45# is straight forward: "<hex info hash>\n<hex info hash>\n..." 45# is straight forward: "<hex info hash>\n<hex info hash>\n..."
46# 46#
47# IIa) You can enable dynamic changesets to accesslists by enabling
48# WANT_DYNAMIC_ACCESSLIST.
49#
50# The suggested way to work with dynamic changeset lists is to keep a
51# main accesslist file that is loaded when opentracker (re)starts and
52# reloaded infrequently (hourly or daily).
53#
54# All changes to the accesslist (e.g. from a web frontend) should be
55# both appended to or removed from that file and sent to opentracker. By
56# keeping dynamic changeset lists, you can avoid reloading huge
57# accesslists whenever just a single entry is added or removed.
58#
59# Any info_hash (format see above) written to the fifo_add file will be
60# kept on a dynamic add-changeset, removed from the dynamic
61# delete-changeset and treated as if it was in the main accesslist file.
62# The semantic of the respective dynamic changeset depends on whether
63# WANT_ACCESSLIST_WHITE or WANT_ACCESSLIST_BLACK is enabled.
64#
65# access.fifo_add /var/run/opentracker/adder.fifo
66#
67# Any info_hash (format see above) written to the fifo_delete file will
68# be kept on a dynamic delete-changeset, removed from the dynamic
69# add-changeset and treated as if it was not in the main accesslist
70# file.
71#
72# access.fifo_delete /var/run/opentracker/deleter.fifo
73#
74# If you reload the accesslist by sending SIGHUP to the tracker process,
75# the dynamic lists are flushed, as opentracker assumes thoses lists are
76# merged into the main accesslist.
77#
78# NOTE: While you can have multiple writers sending lines to the fifos,
79# any writes larger than PIPE_BUF (see your limits.h, minimally 512
80# bytes but usually 4096) may be interleaved with data sent by other
81# writers. This can lead to unparsable lines of info_hashes.
82#
83# IIb)
47# If you do not want to grant anyone access to your stats, enable the 84# If you do not want to grant anyone access to your stats, enable the
48# WANT_RESTRICT_STATS option in Makefile and bless the ip addresses 85# WANT_RESTRICT_STATS option in Makefile and bless the ip addresses
49# allowed to fetch stats here. 86# allowed to fetch stats here.
@@ -55,6 +92,16 @@
55# appear anywhere on your tracker. 92# appear anywhere on your tracker.
56# 93#
57# access.stats_path stats 94# access.stats_path stats
95#
96# IIc)
97# If opentracker lives behind one or multiple reverse proxies, you can
98# every http connection appears to come from these proxies. In order to
99# take the X-Forwarded-For address instead, compile opentracker with the
100# WANT_IP_FROM_PROXY option and set your proxy addresses here.
101#
102# access.proxy 10.0.1.23
103# access.proxy 10.0.1.24
104#
58 105
59# III) Live sync uses udp multicast packets to keep a cluster of opentrackers 106# III) Live sync uses udp multicast packets to keep a cluster of opentrackers
60# synchronized. This option tells opentracker which port to listen for 107# synchronized. This option tells opentracker which port to listen for
diff --git a/ot_accesslist.c b/ot_accesslist.c
index cdb964d..7df503f 100644
--- a/ot_accesslist.c
+++ b/ot_accesslist.c
@@ -10,6 +10,11 @@
10#include <stdio.h> 10#include <stdio.h>
11#include <signal.h> 11#include <signal.h>
12#include <unistd.h> 12#include <unistd.h>
13#ifdef WANT_DYNAMIC_ACCESSLIST
14#include <sys/types.h>
15#include <sys/stat.h>
16#include <errno.h>
17#endif
13 18
14/* Libowfat */ 19/* Libowfat */
15#include "byte.h" 20#include "byte.h"
@@ -24,20 +29,83 @@
24 29
25/* GLOBAL VARIABLES */ 30/* GLOBAL VARIABLES */
26#ifdef WANT_ACCESSLIST 31#ifdef WANT_ACCESSLIST
27 char *g_accesslist_filename; 32char *g_accesslist_filename = NULL;
28static ot_hash *g_accesslist; 33#ifdef WANT_DYNAMIC_ACCESSLIST
29static size_t g_accesslist_size; 34char *g_accesslist_pipe_add = NULL;
35char *g_accesslist_pipe_delete = NULL;
36#endif
30static pthread_mutex_t g_accesslist_mutex; 37static pthread_mutex_t g_accesslist_mutex;
31 38
39/* Accesslists are lock free linked lists. We can not make them locking, because every announce
40 would try to acquire the mutex, making it the most contested mutex in the whole of opentracker,
41 basically creating a central performance choke point.
42
43 The idea is that updating the list heads happens under the g_accesslist_mutex guard and is
44 done atomically, while consumers might potentially still hold pointers deeper inside the list.
45
46 Consumers (for now only via accesslist_hashisvalid) will always fetch the list head pointer
47 that is guaranteed to live for at least five minutes. This should be many orders of magnitudes
48 more than how long it will be needed by the bsearch done on the list. */
49struct ot_accesslist;
50typedef struct ot_accesslist ot_accesslist;
51struct ot_accesslist {
52 ot_hash *list;
53 size_t size;
54 ot_time base;
55 ot_accesslist *next;
56};
57static ot_accesslist * _Atomic g_accesslist = NULL;
58#ifdef WANT_DYNAMIC_ACCESSLIST
59static ot_accesslist * _Atomic g_accesslist_add = NULL;
60static ot_accesslist * _Atomic g_accesslist_delete = NULL;
61#endif
62
63/* Helpers to work on access lists */
32static int vector_compare_hash(const void *hash1, const void *hash2 ) { 64static int vector_compare_hash(const void *hash1, const void *hash2 ) {
33 return memcmp( hash1, hash2, OT_HASH_COMPARE_SIZE ); 65 return memcmp( hash1, hash2, OT_HASH_COMPARE_SIZE );
34} 66}
35 67
68static ot_accesslist * accesslist_free(ot_accesslist *accesslist) {
69 while (accesslist) {
70 ot_accesslist * this_accesslist = accesslist;
71 accesslist = this_accesslist->next;
72 free(this_accesslist->list);
73 free(this_accesslist);
74 }
75 return NULL;
76}
77
78static ot_accesslist * accesslist_make(ot_accesslist *next, size_t size) {
79 ot_accesslist * accesslist_new = malloc(sizeof(ot_accesslist));
80 if (accesslist_new) {
81 accesslist_new->list = size ? malloc(sizeof(ot_hash) * size) : NULL;
82 accesslist_new->size = size;
83 accesslist_new->base = g_now_minutes;
84 accesslist_new->next = next;
85 if (size && !accesslist_new->list) {
86 free(accesslist_new);
87 accesslist_new = NULL;
88 }
89 }
90 return accesslist_new;
91}
92
93/* This must be called with g_accesslist_mutex held.
94 This will never delete head, because that might still be in use. */
95static void accesslist_clean(ot_accesslist *accesslist) {
96 while (accesslist && accesslist->next) {
97 if (accesslist->next->base + 5 < g_now_minutes)
98 accesslist->next = accesslist_free(accesslist->next);
99 accesslist = accesslist->next;
100 }
101}
102
36/* Read initial access list */ 103/* Read initial access list */
37static void accesslist_readfile( void ) { 104static void accesslist_readfile( void ) {
38 ot_hash *info_hash, *accesslist_new = NULL; 105 ot_accesslist * accesslist_new;
39 char *map, *map_end, *read_offs; 106 ot_hash *info_hash;
40 size_t maplen; 107 const char *map, *map_end, *read_offs;
108 size_t maplen;
41 109
42 if( ( map = mmap_read( g_accesslist_filename, &maplen ) ) == NULL ) { 110 if( ( map = mmap_read( g_accesslist_filename, &maplen ) ) == NULL ) {
43 char *wd = getcwd( NULL, 0 ); 111 char *wd = getcwd( NULL, 0 );
@@ -48,11 +116,13 @@ static void accesslist_readfile( void ) {
48 116
49 /* You need at least 41 bytes to pass an info_hash, make enough room 117 /* You need at least 41 bytes to pass an info_hash, make enough room
50 for the maximum amount of them */ 118 for the maximum amount of them */
51 info_hash = accesslist_new = malloc( ( maplen / 41 ) * 20 ); 119 accesslist_new = accesslist_make(g_accesslist, maplen / 41);
52 if( !accesslist_new ) { 120 if( !accesslist_new ) {
53 fprintf( stderr, "Warning: Not enough memory to allocate %zd bytes for accesslist buffer. May succeed later.\n", ( maplen / 41 ) * 20 ); 121 fprintf( stderr, "Warning: Not enough memory to allocate %zd bytes for accesslist buffer. May succeed later.\n", ( maplen / 41 ) * 20 );
122 mmap_unmap( map, maplen);
54 return; 123 return;
55 } 124 }
125 info_hash = accesslist_new->list;
56 126
57 /* No use to scan if there's not enough room for another full info_hash */ 127 /* No use to scan if there's not enough room for another full info_hash */
58 map_end = map + maplen - 40; 128 map_end = map + maplen - 40;
@@ -62,18 +132,18 @@ static void accesslist_readfile( void ) {
62 while( read_offs <= map_end ) { 132 while( read_offs <= map_end ) {
63 int i; 133 int i;
64 for( i=0; i<(int)sizeof(ot_hash); ++i ) { 134 for( i=0; i<(int)sizeof(ot_hash); ++i ) {
65 int eger1 = scan_fromhex( read_offs[ 2*i ] ); 135 int eger1 = scan_fromhex( (unsigned char)read_offs[ 2*i ] );
66 int eger2 = scan_fromhex( read_offs[ 1 + 2*i ] ); 136 int eger2 = scan_fromhex( (unsigned char)read_offs[ 1 + 2*i ] );
67 if( eger1 < 0 || eger2 < 0 ) 137 if( eger1 < 0 || eger2 < 0 )
68 break; 138 break;
69 (*info_hash)[i] = eger1 * 16 + eger2; 139 (*info_hash)[i] = (uint8_t)(eger1 * 16 + eger2);
70 } 140 }
71 141
72 if( i == sizeof(ot_hash) ) { 142 if( i == sizeof(ot_hash) ) {
73 read_offs += 40; 143 read_offs += 40;
74 144
75 /* Append accesslist to accesslist vector */ 145 /* Append accesslist to accesslist vector */
76 if( read_offs == map_end || scan_fromhex( *read_offs ) < 0 ) 146 if( read_offs == map_end || scan_fromhex( (unsigned char)*read_offs ) < 0 )
77 ++info_hash; 147 ++info_hash;
78 } 148 }
79 149
@@ -81,28 +151,55 @@ static void accesslist_readfile( void ) {
81 while( read_offs <= map_end && *(read_offs++) != '\n' ); 151 while( read_offs <= map_end && *(read_offs++) != '\n' );
82 } 152 }
83#ifdef _DEBUG 153#ifdef _DEBUG
84 fprintf( stderr, "Added %zd info_hashes to accesslist\n", (size_t)(info_hash - accesslist_new) ); 154 fprintf( stderr, "Added %zd info_hashes to accesslist\n", (size_t)(info_hash - accesslist_new->list) );
85#endif 155#endif
86 156
87 mmap_unmap( map, maplen); 157 mmap_unmap( map, maplen);
88 158
89 qsort( accesslist_new, info_hash - accesslist_new, sizeof( *info_hash ), vector_compare_hash ); 159 qsort( accesslist_new->list, info_hash - accesslist_new->list, sizeof( *info_hash ), vector_compare_hash );
160 accesslist_new->size = info_hash - accesslist_new->list;
90 161
91 /* Now exchange the accesslist vector in the least race condition prone way */ 162 /* Now exchange the accesslist vector in the least race condition prone way */
92 pthread_mutex_lock(&g_accesslist_mutex); 163 pthread_mutex_lock(&g_accesslist_mutex);
93 free( g_accesslist ); 164 accesslist_new->next = g_accesslist;
94 g_accesslist = accesslist_new; 165 g_accesslist = accesslist_new; /* Only now set a new list */
95 g_accesslist_size = info_hash - accesslist_new; 166
167#ifdef WANT_DYNAMIC_ACCESSLIST
168 /* If we have dynamic accesslists, reloading a new one will always void the add/delete lists.
169 Insert empty ones at the list head */
170 if (g_accesslist_add && (accesslist_new = accesslist_make(g_accesslist_add, 0)) != NULL)
171 g_accesslist_add = accesslist_new;
172 if (g_accesslist_delete && (accesslist_new = accesslist_make(g_accesslist_delete, 0)) != NULL)
173 g_accesslist_delete = accesslist_new;
174#endif
175
176 accesslist_clean(g_accesslist);
177
96 pthread_mutex_unlock(&g_accesslist_mutex); 178 pthread_mutex_unlock(&g_accesslist_mutex);
97} 179}
98 180
99int accesslist_hashisvalid( ot_hash hash ) { 181int accesslist_hashisvalid( ot_hash hash ) {
100 void *exactmatch; 182 /* Get working copy of current access list */
183 ot_accesslist * accesslist = g_accesslist;
184#ifdef WANT_DYNAMIC_ACCESSLIST
185 ot_accesslist * accesslist_add, * accesslist_delete;
186#endif
187 void * exactmatch = NULL;
101 188
102 /* Lock should hardly ever be contended */ 189 if (accesslist)
103 pthread_mutex_lock(&g_accesslist_mutex); 190 exactmatch = bsearch( hash, accesslist->list, accesslist->size, OT_HASH_COMPARE_SIZE, vector_compare_hash );
104 exactmatch = bsearch( hash, g_accesslist, g_accesslist_size, OT_HASH_COMPARE_SIZE, vector_compare_hash ); 191
105 pthread_mutex_unlock(&g_accesslist_mutex); 192#ifdef WANT_DYNAMIC_ACCESSLIST
193 /* If we had no match on the main list, scan the list of dynamically added hashes */
194 accesslist_add = g_accesslist_add;
195 if ((exactmatch == NULL) && accesslist_add)
196 exactmatch = bsearch( hash, accesslist_add->list, accesslist_add->size, OT_HASH_COMPARE_SIZE, vector_compare_hash );
197
198 /* If we found a matching hash on the main list, scan the list of dynamically deleted hashes */
199 accesslist_delete = g_accesslist_delete;
200 if ((exactmatch != NULL) && accesslist_delete && bsearch( hash, accesslist_add->list, accesslist_add->size, OT_HASH_COMPARE_SIZE, vector_compare_hash ))
201 exactmatch = NULL;
202#endif
106 203
107#ifdef WANT_ACCESSLIST_BLACK 204#ifdef WANT_ACCESSLIST_BLACK
108 return exactmatch == NULL; 205 return exactmatch == NULL;
@@ -121,6 +218,8 @@ static void * accesslist_worker( void * args ) {
121 (void)args; 218 (void)args;
122 219
123 while( 1 ) { 220 while( 1 ) {
221 if (!g_opentracker_running)
222 return NULL;
124 223
125 /* Initial attempt to read accesslist */ 224 /* Initial attempt to read accesslist */
126 accesslist_readfile( ); 225 accesslist_readfile( );
@@ -131,18 +230,156 @@ static void * accesslist_worker( void * args ) {
131 return NULL; 230 return NULL;
132} 231}
133 232
233#ifdef WANT_DYNAMIC_ACCESSLIST
234static pthread_t thread_adder_id, thread_deleter_id;
235static void * accesslist_adddel_worker(char * fifoname, ot_accesslist * _Atomic * adding_to, ot_accesslist * _Atomic * removing_from) {
236 struct stat st;
237
238 if (!stat(fifoname, &st)) {
239 if (!S_ISFIFO(st.st_mode)) {
240 fprintf(stderr, "Error when starting dynamic accesslists: Found Non-FIFO file at %s.\nPlease remove it and restart opentracker.\n", fifoname);
241 return NULL;
242 }
243 } else {
244 int error = mkfifo(fifoname, 0755);
245 if (error && error != EEXIST) {
246 fprintf(stderr, "Error when starting dynamic accesslists: Couldn't create FIFO at %s, error: %s\n", fifoname, strerror(errno));
247 return NULL;
248 }
249 }
250
251 while (g_opentracker_running) {
252 FILE * fifo = fopen(fifoname, "r");
253 char *line = NULL;
254 size_t linecap = 0;
255 ssize_t linelen;
256
257 if (!fifo) {
258 fprintf(stderr, "Error when reading dynamic accesslists: Couldn't open FIFO at %s, error: %s\n", fifoname, strerror(errno));
259 return NULL;
260 }
261
262 while ((linelen = getline(&line, &linecap, fifo)) > 0) {
263 ot_hash info_hash;
264 int i;
265
266 printf("Got line %*s", (int)linelen, line);
267 /* We do ignore anything that is not of the form "^[:xdigit:]{40}[^:xdigit:].*"
268 If there's not enough characters for an info_hash in the line, skip it. */
269 if (linelen < 41)
270 continue;
271
272 for( i=0; i<(int)sizeof(ot_hash); ++i ) {
273 int eger1 = scan_fromhex( (unsigned char)line[ 2*i ] );
274 int eger2 = scan_fromhex( (unsigned char)line[ 1 + 2*i ] );
275 if( eger1 < 0 || eger2 < 0 )
276 break;
277 ((uint8_t*)info_hash)[i] = (uint8_t)(eger1 * 16 + eger2);
278 }
279printf("parsed info_hash %20s\n", info_hash);
280 if( i != sizeof(ot_hash) )
281 continue;
282
283 /* From now on we modify g_accesslist_add and g_accesslist_delete, so prevent the
284 other worker threads from doing the same */
285 pthread_mutex_lock(&g_accesslist_mutex);
286
287 /* If the info hash is in the removing_from list, create a new head without that entry */
288 if (*removing_from && (*removing_from)->list) {
289 ot_hash * exactmatch = bsearch( info_hash, (*removing_from)->list, (*removing_from)->size, OT_HASH_COMPARE_SIZE, vector_compare_hash );
290 if (exactmatch) {
291 ptrdiff_t off = exactmatch - (*removing_from)->list;
292 ot_accesslist * accesslist_new = accesslist_make(*removing_from, (*removing_from)->size - 1);
293 if (accesslist_new) {
294 memcpy(accesslist_new->list, (*removing_from)->list, sizeof(ot_hash) * off);
295 memcpy(accesslist_new->list + off, (*removing_from)->list + off + 1, (*removing_from)->size - off - 1);
296 *removing_from = accesslist_new;
297 }
298 }
299 }
300
301 /* Simple case: there's no adding_to list yet, create one with one member */
302 if (!*adding_to) {
303 ot_accesslist * accesslist_new = accesslist_make(NULL, 1);
304 if (accesslist_new) {
305 memcpy(accesslist_new->list, info_hash, sizeof(ot_hash));
306 *adding_to = accesslist_new;
307 }
308 } else {
309 int exactmatch = 0;
310 ot_hash * insert_point = binary_search( info_hash, (*adding_to)->list, (*adding_to)->size, OT_HASH_COMPARE_SIZE, sizeof(ot_hash), &exactmatch );
311
312 /* Only if the info hash is not in the adding_to list, create a new head with that entry */
313 if (!exactmatch) {
314 ot_accesslist * accesslist_new = accesslist_make(*adding_to, (*adding_to)->size + 1);
315 ptrdiff_t off = insert_point - (*adding_to)->list;
316 if (accesslist_new) {
317 memcpy(accesslist_new->list, (*adding_to)->list, sizeof(ot_hash) * off);
318 memcpy(accesslist_new->list + off, info_hash, sizeof(info_hash));
319 memcpy(accesslist_new->list + off + 1, (*adding_to)->list + off, (*adding_to)->size - off);
320 *adding_to = accesslist_new;
321 }
322 }
323 }
324
325 pthread_mutex_unlock(&g_accesslist_mutex);
326 }
327
328 fclose(fifo);
329 }
330 return NULL;
331}
332
333static void * accesslist_adder_worker( void * args ) {
334 (void)args;
335 return accesslist_adddel_worker(g_accesslist_pipe_add, &g_accesslist_add, &g_accesslist_delete);
336}
337static void * accesslist_deleter_worker( void * args ) {
338 (void)args;
339 return accesslist_adddel_worker(g_accesslist_pipe_delete, &g_accesslist_delete, &g_accesslist_add);
340}
341#endif
342
134static pthread_t thread_id; 343static pthread_t thread_id;
135void accesslist_init( ) { 344void accesslist_init( ) {
136 pthread_mutex_init(&g_accesslist_mutex, NULL); 345 pthread_mutex_init(&g_accesslist_mutex, NULL);
137 pthread_create( &thread_id, NULL, accesslist_worker, NULL ); 346 pthread_create( &thread_id, NULL, accesslist_worker, NULL );
347#ifdef WANT_DYNAMIC_ACCESSLIST
348 if (g_accesslist_pipe_add)
349 pthread_create( &thread_adder_id, NULL, accesslist_adder_worker, NULL );
350 if (g_accesslist_pipe_delete)
351 pthread_create( &thread_deleter_id, NULL, accesslist_deleter_worker, NULL );
352#endif
138} 353}
139 354
140void accesslist_deinit( void ) { 355void accesslist_deinit( void ) {
356 /* Wake up sleeping worker */
357 pthread_kill(thread_id, SIGHUP);
358
359 pthread_mutex_lock(&g_accesslist_mutex);
360
361 g_accesslist = accesslist_free(g_accesslist);
362
363#ifdef WANT_DYNAMIC_ACCESSLIST
364 g_accesslist_add = accesslist_free(g_accesslist_add);
365 g_accesslist_delete = accesslist_free(g_accesslist_delete);
366#endif
367
368 pthread_mutex_unlock(&g_accesslist_mutex);
141 pthread_cancel( thread_id ); 369 pthread_cancel( thread_id );
142 pthread_mutex_destroy(&g_accesslist_mutex); 370 pthread_mutex_destroy(&g_accesslist_mutex);
143 free( g_accesslist ); 371}
144 g_accesslist = 0; 372
145 g_accesslist_size = 0; 373void accesslist_cleanup( void ) {
374 pthread_mutex_lock(&g_accesslist_mutex);
375
376 accesslist_clean(g_accesslist);
377#if WANT_DYNAMIC_ACCESSLIST
378 accesslist_clean(g_accesslist_add);
379 accesslist_clean(g_accesslist_delete);
380#endif
381
382 pthread_mutex_unlock(&g_accesslist_mutex);
146} 383}
147#endif 384#endif
148 385
@@ -295,7 +532,7 @@ int accesslist_blessip( ot_ip6 ip, ot_permissions permissions ) {
295 if( permissions & OT_PERMISSION_MAY_PROXY ) off += snprintf( _debug+off, 512-off, " may_proxy" ); 532 if( permissions & OT_PERMISSION_MAY_PROXY ) off += snprintf( _debug+off, 512-off, " may_proxy" );
296 if( !permissions ) off += snprintf( _debug+off, sizeof(_debug)-off, " nothing\n" ); 533 if( !permissions ) off += snprintf( _debug+off, sizeof(_debug)-off, " nothing\n" );
297 _debug[off++] = '.'; 534 _debug[off++] = '.';
298 write( 2, _debug, off ); 535 (void)write( 2, _debug, off );
299 } 536 }
300#endif 537#endif
301 538
diff --git a/ot_accesslist.h b/ot_accesslist.h
index b38b91a..281f61b 100644
--- a/ot_accesslist.h
+++ b/ot_accesslist.h
@@ -6,19 +6,28 @@
6#ifndef OT_ACCESSLIST_H__ 6#ifndef OT_ACCESSLIST_H__
7#define OT_ACCESSLIST_H__ 7#define OT_ACCESSLIST_H__
8 8
9#if defined ( WANT_ACCESSLIST_BLACK ) && defined (WANT_ACCESSLIST_WHITE ) 9#if defined ( WANT_ACCESSLIST_BLACK ) && defined ( WANT_ACCESSLIST_WHITE )
10# error WANT_ACCESSLIST_BLACK and WANT_ACCESSLIST_WHITE are exclusive. 10# error WANT_ACCESSLIST_BLACK and WANT_ACCESSLIST_WHITE are exclusive.
11#endif 11#endif
12 12
13#if defined ( WANT_ACCESSLIST_BLACK ) || defined (WANT_ACCESSLIST_WHITE ) 13#if defined ( WANT_ACCESSLIST_BLACK ) || defined (WANT_ACCESSLIST_WHITE )
14#define WANT_ACCESSLIST 14#define WANT_ACCESSLIST
15void accesslist_init( ); 15void accesslist_init( void );
16void accesslist_deinit( ); 16void accesslist_deinit( void );
17int accesslist_hashisvalid( ot_hash hash ); 17int accesslist_hashisvalid( ot_hash hash );
18void accesslist_cleanup( void );
18 19
19extern char *g_accesslist_filename; 20extern char *g_accesslist_filename;
21#ifdef WANT_DYNAMIC_ACCESSLIST
22extern char *g_accesslist_pipe_add;
23extern char *g_accesslist_pipe_delete;
24#endif
20 25
21#else 26#else
27#ifdef WANT_DYNAMIC_ACCESSLIST
28# error WANT_DYNAMIC_ACCESSLIST needs either WANT_ACCESSLIST_BLACK or WANT_ACCESSLIST_WHITE
29#endif
30
22#define accesslist_init( accesslist_filename ) 31#define accesslist_init( accesslist_filename )
23#define accesslist_deinit( ) 32#define accesslist_deinit( )
24#define accesslist_hashisvalid( hash ) 1 33#define accesslist_hashisvalid( hash ) 1
diff --git a/ot_clean.c b/ot_clean.c
index 4c03416..139bedb 100644
--- a/ot_clean.c
+++ b/ot_clean.c
@@ -17,6 +17,7 @@
17#include "ot_vector.h" 17#include "ot_vector.h"
18#include "ot_clean.h" 18#include "ot_clean.h"
19#include "ot_stats.h" 19#include "ot_stats.h"
20#include "ot_accesslist.h"
20 21
21/* Returns amount of removed peers */ 22/* Returns amount of removed peers */
22static ssize_t clean_single_bucket( ot_peer *peers, size_t peer_count, time_t timedout, int *removed_seeders ) { 23static ssize_t clean_single_bucket( ot_peer *peers, size_t peer_count, time_t timedout, int *removed_seeders ) {
@@ -123,6 +124,9 @@ static void * clean_worker( void * args ) {
123 usleep( OT_CLEAN_SLEEP ); 124 usleep( OT_CLEAN_SLEEP );
124 } 125 }
125 stats_cleanup(); 126 stats_cleanup();
127#ifdef WANT_ACCESSLIST
128 accesslist_cleanup();
129#endif
126 } 130 }
127 return NULL; 131 return NULL;
128} 132}
diff --git a/ot_fullscrape.c b/ot_fullscrape.c
index faea4b9..5d115dc 100644
--- a/ot_fullscrape.c
+++ b/ot_fullscrape.c
@@ -30,21 +30,16 @@
30 Full scrapes usually are huge and one does not want to 30 Full scrapes usually are huge and one does not want to
31 allocate more memory. So lets get them in 512k units 31 allocate more memory. So lets get them in 512k units
32*/ 32*/
33#define OT_SCRAPE_CHUNK_SIZE (512*1024) 33#define OT_SCRAPE_CHUNK_SIZE (1024*1024)
34 34
35/* "d8:completei%zde10:downloadedi%zde10:incompletei%zdee" */ 35/* "d8:completei%zde10:downloadedi%zde10:incompletei%zdee" */
36#define OT_SCRAPE_MAXENTRYLEN 256 36#define OT_SCRAPE_MAXENTRYLEN 256
37 37
38#ifdef WANT_COMPRESSION_GZIP
39#define IF_COMPRESSION( TASK ) if( mode & TASK_FLAG_GZIP ) TASK
40#define WANT_COMPRESSION_GZIP_PARAM( param1, param2, param3 ) , param1, param2, param3
41#else
42#define IF_COMPRESSION( TASK )
43#define WANT_COMPRESSION_GZIP_PARAM( param1, param2, param3 )
44#endif
45
46/* Forward declaration */ 38/* Forward declaration */
47static void fullscrape_make( int *iovec_entries, struct iovec **iovector, ot_tasktype mode ); 39static void fullscrape_make( int *iovec_entries, struct iovec **iovector, ot_tasktype mode );
40#ifdef WANT_COMPRESSION_GZIP
41static void fullscrape_make_gzip( int *iovec_entries, struct iovec **iovector, ot_tasktype mode );
42#endif
48 43
49/* Converter function from memory to human readable hex strings 44/* Converter function from memory to human readable hex strings
50 XXX - Duplicated from ot_stats. Needs fix. */ 45 XXX - Duplicated from ot_stats. Needs fix. */
@@ -59,14 +54,17 @@ static void * fullscrape_worker( void * args ) {
59 54
60 (void) args; 55 (void) args;
61 56
62 while( 1 ) { 57 while( g_opentracker_running ) {
63 ot_tasktype tasktype = TASK_FULLSCRAPE; 58 ot_tasktype tasktype = TASK_FULLSCRAPE;
64 ot_taskid taskid = mutex_workqueue_poptask( &tasktype ); 59 ot_taskid taskid = mutex_workqueue_poptask( &tasktype );
65 fullscrape_make( &iovec_entries, &iovector, tasktype ); 60#ifdef WANT_COMPRESSION_GZIP
61 if (tasktype & TASK_FLAG_GZIP)
62 fullscrape_make_gzip( &iovec_entries, &iovector, tasktype );
63 else
64#endif
65 fullscrape_make( &iovec_entries, &iovector, tasktype );
66 if( mutex_workqueue_pushresult( taskid, iovec_entries, iovector ) ) 66 if( mutex_workqueue_pushresult( taskid, iovec_entries, iovector ) )
67 iovec_free( &iovec_entries, &iovector ); 67 iovec_free( &iovec_entries, &iovector );
68 if( !g_opentracker_running )
69 return NULL;
70 } 68 }
71 return NULL; 69 return NULL;
72} 70}
@@ -84,68 +82,57 @@ void fullscrape_deliver( int64 sock, ot_tasktype tasktype ) {
84 mutex_workqueue_pushtask( sock, tasktype ); 82 mutex_workqueue_pushtask( sock, tasktype );
85} 83}
86 84
87static int fullscrape_increase( int *iovec_entries, struct iovec **iovector, 85static char * fullscrape_write_one( ot_tasktype mode, char *r, ot_peerlist *peer_list, ot_hash *hash ) {
88 char **r, char **re WANT_COMPRESSION_GZIP_PARAM( z_stream *strm, ot_tasktype mode, int zaction ) ) { 86 switch( mode & TASK_TASK_MASK ) {
89 /* Allocate a fresh output buffer at the end of our buffers list */ 87 case TASK_FULLSCRAPE:
90 if( !( *r = iovec_fix_increase_or_free( iovec_entries, iovector, *r, OT_SCRAPE_CHUNK_SIZE ) ) ) { 88 default:
91 89 /* push hash as bencoded string */
92 /* Deallocate gzip buffers */ 90 *r++='2'; *r++='0'; *r++=':';
93 IF_COMPRESSION( deflateEnd(strm); ) 91 memcpy( r, hash, sizeof(ot_hash) ); r += sizeof(ot_hash);
94 92 /* push rest of the scrape string */
95 /* Release lock on current bucket and return */ 93 r += sprintf( r, "d8:completei%zde10:downloadedi%zde10:incompletei%zdee", peer_list->seed_count, peer_list->down_count, peer_list->peer_count-peer_list->seed_count );
96 return -1; 94
97 } 95 break;
98 96 case TASK_FULLSCRAPE_TPB_ASCII:
99 /* Adjust new end of output buffer */ 97 to_hex( r, *hash ); r+= 2 * sizeof(ot_hash);
100 *re = *r + OT_SCRAPE_CHUNK_SIZE - OT_SCRAPE_MAXENTRYLEN; 98 r += sprintf( r, ":%zd:%zd\n", peer_list->seed_count, peer_list->peer_count-peer_list->seed_count );
101 99 break;
102 /* When compressing, we have all the bytes in output buffer */ 100 case TASK_FULLSCRAPE_TPB_ASCII_PLUS:
103#ifdef WANT_COMPRESSION_GZIP 101 to_hex( r, *hash ); r+= 2 * sizeof(ot_hash);
104 if( mode & TASK_FLAG_GZIP ) { 102 r += sprintf( r, ":%zd:%zd:%zd\n", peer_list->seed_count, peer_list->peer_count-peer_list->seed_count, peer_list->down_count );
105 int zres; 103 break;
106 *re -= OT_SCRAPE_MAXENTRYLEN; 104 case TASK_FULLSCRAPE_TPB_BINARY:
107 strm->next_out = (uint8_t*)*r; 105 memcpy( r, *hash, sizeof(ot_hash) ); r += sizeof(ot_hash);
108 strm->avail_out = OT_SCRAPE_CHUNK_SIZE; 106 *(uint32_t*)(r+0) = htonl( (uint32_t) peer_list->seed_count );
109 zres = deflate( strm, zaction ); 107 *(uint32_t*)(r+4) = htonl( (uint32_t)( peer_list->peer_count-peer_list->seed_count) );
110 if( ( zres < Z_OK ) && ( zres != Z_BUF_ERROR ) ) 108 r+=8;
111 fprintf( stderr, "deflate() failed while in fullscrape_increase(%d).\n", zaction ); 109 break;
112 *r = (char*)strm->next_out; 110 case TASK_FULLSCRAPE_TPB_URLENCODED:
113 } 111 r += fmt_urlencoded( r, (char *)*hash, 20 );
114#endif 112 r += sprintf( r, ":%zd:%zd\n", peer_list->seed_count, peer_list->peer_count-peer_list->seed_count );
115 113 break;
116 return 0; 114 case TASK_FULLSCRAPE_TRACKERSTATE:
115 to_hex( r, *hash ); r+= 2 * sizeof(ot_hash);
116 r += sprintf( r, ":%zd:%zd\n", peer_list->base, peer_list->down_count );
117 break;
118 }
119 return r;
117} 120}
118 121
119static void fullscrape_make( int *iovec_entries, struct iovec **iovector, ot_tasktype mode ) { 122static void fullscrape_make( int *iovec_entries, struct iovec **iovector, ot_tasktype mode ) {
120 int bucket; 123 int bucket;
121 char *r, *re; 124 char *r, *re;
122#ifdef WANT_COMPRESSION_GZIP
123 char compress_buffer[OT_SCRAPE_MAXENTRYLEN];
124 z_stream strm;
125#endif
126 125
127 /* Setup return vector... */ 126 /* Setup return vector... */
128 *iovec_entries = 0; 127 *iovec_entries = 0;
129 *iovector = NULL; 128 *iovector = NULL;
130 if( !( r = iovec_increase( iovec_entries, iovector, OT_SCRAPE_CHUNK_SIZE ) ) ) 129 r = iovec_increase( iovec_entries, iovector, OT_SCRAPE_CHUNK_SIZE );
130 if( !r )
131 return; 131 return;
132 132
133 /* re points to low watermark */ 133 /* re points to low watermark */
134 re = r + OT_SCRAPE_CHUNK_SIZE - OT_SCRAPE_MAXENTRYLEN; 134 re = r + OT_SCRAPE_CHUNK_SIZE - OT_SCRAPE_MAXENTRYLEN;
135 135
136#ifdef WANT_COMPRESSION_GZIP
137 if( mode & TASK_FLAG_GZIP ) {
138 re += OT_SCRAPE_MAXENTRYLEN;
139 byte_zero( &strm, sizeof(strm) );
140 strm.next_in = (uint8_t*)compress_buffer;
141 strm.next_out = (uint8_t*)r;
142 strm.avail_out = OT_SCRAPE_CHUNK_SIZE;
143 if( deflateInit2(&strm,7,Z_DEFLATED,31,8,Z_DEFAULT_STRATEGY) != Z_OK )
144 fprintf( stderr, "not ok.\n" );
145 r = compress_buffer;
146 }
147#endif
148
149 if( ( mode & TASK_TASK_MASK ) == TASK_FULLSCRAPE ) 136 if( ( mode & TASK_TASK_MASK ) == TASK_FULLSCRAPE )
150 r += sprintf( r, "d5:filesd" ); 137 r += sprintf( r, "d5:filesd" );
151 138
@@ -153,66 +140,99 @@ static void fullscrape_make( int *iovec_entries, struct iovec **iovector, ot_tas
153 for( bucket=0; bucket<OT_BUCKET_COUNT; ++bucket ) { 140 for( bucket=0; bucket<OT_BUCKET_COUNT; ++bucket ) {
154 /* Get exclusive access to that bucket */ 141 /* Get exclusive access to that bucket */
155 ot_vector *torrents_list = mutex_bucket_lock( bucket ); 142 ot_vector *torrents_list = mutex_bucket_lock( bucket );
156 size_t tor_offset; 143 ot_torrent *torrents = (ot_torrent*)(torrents_list->data);
144 size_t i;
157 145
158 /* For each torrent in this bucket.. */ 146 /* For each torrent in this bucket.. */
159 for( tor_offset=0; tor_offset<torrents_list->size; ++tor_offset ) { 147 for( i=0; i<torrents_list->size; ++i ) {
160 /* Address torrents members */ 148 r = fullscrape_write_one( mode, r, torrents[i].peer_list, &torrents[i].hash );
161 ot_peerlist *peer_list = ( ((ot_torrent*)(torrents_list->data))[tor_offset] ).peer_list; 149
162 ot_hash *hash =&( ((ot_torrent*)(torrents_list->data))[tor_offset] ).hash; 150 if( r > re) {
163 151 /* Allocate a fresh output buffer at the end of our buffers list */
164 switch( mode & TASK_TASK_MASK ) { 152 r = iovec_fix_increase_or_free( iovec_entries, iovector, r, OT_SCRAPE_CHUNK_SIZE );
165 case TASK_FULLSCRAPE: 153 if( !r )
166 default: 154 return mutex_bucket_unlock( bucket, 0 );
167 /* push hash as bencoded string */ 155
168 *r++='2'; *r++='0'; *r++=':'; 156 /* re points to low watermark */
169 memcpy( r, hash, sizeof(ot_hash) ); r += sizeof(ot_hash); 157 re = r + OT_SCRAPE_CHUNK_SIZE - OT_SCRAPE_MAXENTRYLEN;
170 /* push rest of the scrape string */
171 r += sprintf( r, "d8:completei%zde10:downloadedi%zde10:incompletei%zdee", peer_list->seed_count, peer_list->down_count, peer_list->peer_count-peer_list->seed_count );
172
173 break;
174 case TASK_FULLSCRAPE_TPB_ASCII:
175 to_hex( r, *hash ); r+= 2 * sizeof(ot_hash);
176 r += sprintf( r, ":%zd:%zd\n", peer_list->seed_count, peer_list->peer_count-peer_list->seed_count );
177 break;
178 case TASK_FULLSCRAPE_TPB_ASCII_PLUS:
179 to_hex( r, *hash ); r+= 2 * sizeof(ot_hash);
180 r += sprintf( r, ":%zd:%zd:%zd\n", peer_list->seed_count, peer_list->peer_count-peer_list->seed_count, peer_list->down_count );
181 break;
182 case TASK_FULLSCRAPE_TPB_BINARY:
183 memcpy( r, *hash, sizeof(ot_hash) ); r += sizeof(ot_hash);
184 *(uint32_t*)(r+0) = htonl( (uint32_t) peer_list->seed_count );
185 *(uint32_t*)(r+4) = htonl( (uint32_t)( peer_list->peer_count-peer_list->seed_count) );
186 r+=8;
187 break;
188 case TASK_FULLSCRAPE_TPB_URLENCODED:
189 r += fmt_urlencoded( r, (char *)*hash, 20 );
190 r += sprintf( r, ":%zd:%zd\n", peer_list->seed_count, peer_list->peer_count-peer_list->seed_count );
191 break;
192 case TASK_FULLSCRAPE_TRACKERSTATE:
193 to_hex( r, *hash ); r+= 2 * sizeof(ot_hash);
194 r += sprintf( r, ":%zd:%zd\n", peer_list->base, peer_list->down_count );
195 break;
196 } 158 }
159 }
160
161 /* All torrents done: release lock on current bucket */
162 mutex_bucket_unlock( bucket, 0 );
163
164 /* Parent thread died? */
165 if( !g_opentracker_running )
166 return;
167 }
168
169 if( ( mode & TASK_TASK_MASK ) == TASK_FULLSCRAPE )
170 r += sprintf( r, "ee" );
171
172 /* Release unused memory in current output buffer */
173 iovec_fixlast( iovec_entries, iovector, r );
174}
197 175
198#ifdef WANT_COMPRESSION_GZIP 176#ifdef WANT_COMPRESSION_GZIP
199 if( mode & TASK_FLAG_GZIP ) { 177
200 int zres; 178static void fullscrape_make_gzip( int *iovec_entries, struct iovec **iovector, ot_tasktype mode ) {
201 strm.next_in = (uint8_t*)compress_buffer; 179 int bucket;
202 strm.avail_in = r - compress_buffer; 180 char *r;
181 int zres;
182 z_stream strm;
183
184 /* Setup return vector... */
185 *iovec_entries = 0;
186 *iovector = NULL;
187 r = iovec_increase( iovec_entries, iovector, OT_SCRAPE_CHUNK_SIZE );
188 if( !r )
189 return;
190
191 byte_zero( &strm, sizeof(strm) );
192 strm.next_out = (uint8_t*)r;
193 strm.avail_out = OT_SCRAPE_CHUNK_SIZE;
194 if( deflateInit2(&strm,7,Z_DEFLATED,31,9,Z_DEFAULT_STRATEGY) != Z_OK )
195 fprintf( stderr, "not ok.\n" );
196
197 if( ( mode & TASK_TASK_MASK ) == TASK_FULLSCRAPE ) {
198 strm.next_in = (uint8_t*)"d5:filesd";
199 strm.avail_in = strlen("d5:filesd");
200 zres = deflate( &strm, Z_NO_FLUSH );
201 }
202
203 /* For each bucket... */
204 for( bucket=0; bucket<OT_BUCKET_COUNT; ++bucket ) {
205 /* Get exclusive access to that bucket */
206 ot_vector *torrents_list = mutex_bucket_lock( bucket );
207 ot_torrent *torrents = (ot_torrent*)(torrents_list->data);
208 size_t i;
209
210 /* For each torrent in this bucket.. */
211 for( i=0; i<torrents_list->size; ++i ) {
212 char compress_buffer[OT_SCRAPE_MAXENTRYLEN];
213 r = fullscrape_write_one( mode, compress_buffer, torrents[i].peer_list, &torrents[i].hash );
214 strm.next_in = (uint8_t*)compress_buffer;
215 strm.avail_in = r - compress_buffer;
216 zres = deflate( &strm, Z_NO_FLUSH );
217 if( ( zres < Z_OK ) && ( zres != Z_BUF_ERROR ) )
218 fprintf( stderr, "deflate() failed while in fullscrape_make().\n" );
219
220 /* Check if there still is enough buffer left */
221 while( !strm.avail_out ) {
222 /* Allocate a fresh output buffer at the end of our buffers list */
223 r = iovec_increase( iovec_entries, iovector, OT_SCRAPE_CHUNK_SIZE );
224 if( !r ) {
225 fprintf( stderr, "Problem with iovec_fix_increase_or_free\n" );
226 iovec_free( iovec_entries, iovector );
227 deflateEnd(&strm);
228 return mutex_bucket_unlock( bucket, 0 );
229 }
230 strm.next_out = (uint8_t*)r;
231 strm.avail_out = OT_SCRAPE_CHUNK_SIZE;
203 zres = deflate( &strm, Z_NO_FLUSH ); 232 zres = deflate( &strm, Z_NO_FLUSH );
204 if( ( zres < Z_OK ) && ( zres != Z_BUF_ERROR ) ) 233 if( ( zres < Z_OK ) && ( zres != Z_BUF_ERROR ) )
205 fprintf( stderr, "deflate() failed while in fullscrape_make().\n" ); 234 fprintf( stderr, "deflate() failed while in fullscrape_make().\n" );
206 r = (char*)strm.next_out;
207 } 235 }
208#endif
209
210 /* Check if there still is enough buffer left */
211 while( r >= re )
212 if( fullscrape_increase( iovec_entries, iovector, &r, &re WANT_COMPRESSION_GZIP_PARAM( &strm, mode, Z_NO_FLUSH ) ) )
213 return mutex_bucket_unlock( bucket, 0 );
214
215 IF_COMPRESSION( r = compress_buffer; )
216 } 236 }
217 237
218 /* All torrents done: release lock on current bucket */ 238 /* All torrents done: release lock on current bucket */
@@ -223,27 +243,41 @@ static void fullscrape_make( int *iovec_entries, struct iovec **iovector, ot_tas
223 return; 243 return;
224 } 244 }
225 245
226 if( ( mode & TASK_TASK_MASK ) == TASK_FULLSCRAPE ) 246 if( ( mode & TASK_TASK_MASK ) == TASK_FULLSCRAPE ) {
227 r += sprintf( r, "ee" ); 247 strm.next_in = (uint8_t*)"ee";
248 strm.avail_in = strlen("ee");
249 }
228 250
229#ifdef WANT_COMPRESSION_GZIP 251 if( deflate( &strm, Z_FINISH ) < Z_OK )
230 if( mode & TASK_FLAG_GZIP ) { 252 fprintf( stderr, "deflate() failed while in fullscrape_make()'s endgame.\n" );
231 strm.next_in = (uint8_t*)compress_buffer; 253
232 strm.avail_in = r - compress_buffer; 254 if( !strm.avail_out ) {
255 unsigned int pending;
256 int bits;
257 deflatePending( &strm, &pending, &bits);
258 pending += ( bits ? 1 : 0 );
259
260 /* Allocate a fresh output buffer at the end of our buffers list */
261 r = iovec_fix_increase_or_free( iovec_entries, iovector, strm.next_out, pending );
262 if( !r ) {
263 fprintf( stderr, "Problem with iovec_fix_increase_or_free\n" );
264 deflateEnd(&strm);
265 return mutex_bucket_unlock( bucket, 0 );
266 }
267 strm.next_out = (uint8_t*)r;
268 strm.avail_out = pending;
233 if( deflate( &strm, Z_FINISH ) < Z_OK ) 269 if( deflate( &strm, Z_FINISH ) < Z_OK )
234 fprintf( stderr, "deflate() failed while in fullscrape_make()'s endgame.\n" ); 270 fprintf( stderr, "deflate() failed while in fullscrape_make()'s endgame.\n" );
235 r = (char*)strm.next_out;
236
237 while( r >= re )
238 if( fullscrape_increase( iovec_entries, iovector, &r, &re WANT_COMPRESSION_GZIP_PARAM( &strm, mode, Z_FINISH ) ) )
239 return mutex_bucket_unlock( bucket, 0 );
240 deflateEnd(&strm);
241 } 271 }
242#endif
243 272
244 /* Release unused memory in current output buffer */ 273 /* Release unused memory in current output buffer */
245 iovec_fixlast( iovec_entries, iovector, r ); 274 iovec_fixlast( iovec_entries, iovector, strm.next_out );
275
276 deflateEnd(&strm);
246} 277}
278/* WANT_COMPRESSION_GZIP */
247#endif 279#endif
248 280
281/* WANT_FULLSCRAPE */
282#endif
249const char *g_version_fullscrape_c = "$Source$: $Revision$\n"; 283const char *g_version_fullscrape_c = "$Source$: $Revision$\n";
diff --git a/ot_http.c b/ot_http.c
index 0e3cc29..90b7a4b 100644
--- a/ot_http.c
+++ b/ot_http.c
@@ -31,6 +31,7 @@
31#include "ot_accesslist.h" 31#include "ot_accesslist.h"
32 32
33#define OT_MAXMULTISCRAPE_COUNT 64 33#define OT_MAXMULTISCRAPE_COUNT 64
34#define OT_BATCH_LIMIT (1024*1024*16)
34extern char *g_redirecturl; 35extern char *g_redirecturl;
35 36
36char *g_stats_path; 37char *g_stats_path;
@@ -75,7 +76,13 @@ static void http_senddata( const int64 sock, struct ot_workstruct *ws ) {
75 } 76 }
76 77
77 memcpy( outbuf, ws->reply + written_size, ws->reply_size - written_size ); 78 memcpy( outbuf, ws->reply + written_size, ws->reply_size - written_size );
78 iob_addbuf_free( &cookie->batch, outbuf, ws->reply_size - written_size ); 79 if ( !cookie->batch ) {
80 cookie->batch = malloc( sizeof(io_batch) );
81 memset( cookie->batch, 0, sizeof(io_batch) );
82 cookie->batches = 1;
83 }
84
85 iob_addbuf_free( cookie->batch, outbuf, ws->reply_size - written_size );
79 86
80 /* writeable short data sockets just have a tcp timeout */ 87 /* writeable short data sockets just have a tcp timeout */
81 if( !ws->keep_alive ) { 88 if( !ws->keep_alive ) {
@@ -118,7 +125,7 @@ ssize_t http_sendiovecdata( const int64 sock, struct ot_workstruct *ws, int iove
118 struct http_data *cookie = io_getcookie( sock ); 125 struct http_data *cookie = io_getcookie( sock );
119 char *header; 126 char *header;
120 int i; 127 int i;
121 size_t header_size, size = iovec_length( &iovec_entries, &iovector ); 128 size_t header_size, size = iovec_length( &iovec_entries, (const struct iovec **)&iovector );
122 tai6464 t; 129 tai6464 t;
123 130
124 /* No cookie? Bad socket. Leave. */ 131 /* No cookie? Bad socket. Leave. */
@@ -152,12 +159,29 @@ ssize_t http_sendiovecdata( const int64 sock, struct ot_workstruct *ws, int iove
152 else 159 else
153 header_size = sprintf( header, "HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\nContent-Length: %zd\r\n\r\n", size ); 160 header_size = sprintf( header, "HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\nContent-Length: %zd\r\n\r\n", size );
154 161
155 iob_reset( &cookie->batch ); 162 if (!cookie->batch ) {
156 iob_addbuf_free( &cookie->batch, header, header_size ); 163 cookie->batch = malloc( sizeof(io_batch) );
164 memset( cookie->batch, 0, sizeof(io_batch) );
165 cookie->batches = 1;
166 }
167 iob_addbuf_free( cookie->batch, header, header_size );
168
169 /* Split huge iovectors into separate io_batches */
170 for( i=0; i<iovec_entries; ++i ) {
171 io_batch *current = cookie->batch + cookie->batches - 1;
172
173 /* If the current batch's limit is reached, try to reallocate a new batch to work on */
174 if( current->bytesleft > OT_BATCH_LIMIT ) {
175 io_batch * new_batch = realloc( current, (cookie->batches + 1) * sizeof(io_batch) );
176 if( new_batch ) {
177 cookie->batches++;
178 current = cookie->batch = new_batch;
179 memset( current, 0, sizeof(io_batch) );
180 }
181 }
157 182
158 /* Will move to ot_iovec.c */ 183 iob_addbuf_free( current, iovector[i].iov_base, iovector[i].iov_len );
159 for( i=0; i<iovec_entries; ++i ) 184 }
160 iob_addbuf_munmap( &cookie->batch, iovector[i].iov_base, iovector[i].iov_len );
161 free( iovector ); 185 free( iovector );
162 186
163 /* writeable sockets timeout after 10 minutes */ 187 /* writeable sockets timeout after 10 minutes */
@@ -170,7 +194,7 @@ ssize_t http_sendiovecdata( const int64 sock, struct ot_workstruct *ws, int iove
170 194
171static ssize_t http_handle_stats( const int64 sock, struct ot_workstruct *ws, char *read_ptr ) { 195static ssize_t http_handle_stats( const int64 sock, struct ot_workstruct *ws, char *read_ptr ) {
172static const ot_keywords keywords_main[] = 196static const ot_keywords keywords_main[] =
173 { { "mode", 1 }, {"format", 2 }, { NULL, -3 } }; 197 { { "mode", 1 }, {"format", 2 }, {"info_hash", 3}, { NULL, -3 } };
174static const ot_keywords keywords_mode[] = 198static const ot_keywords keywords_mode[] =
175 { { "peer", TASK_STATS_PEERS }, { "conn", TASK_STATS_CONNS }, { "scrp", TASK_STATS_SCRAPE }, { "udp4", TASK_STATS_UDP }, { "tcp4", TASK_STATS_TCP }, 199 { { "peer", TASK_STATS_PEERS }, { "conn", TASK_STATS_CONNS }, { "scrp", TASK_STATS_SCRAPE }, { "udp4", TASK_STATS_UDP }, { "tcp4", TASK_STATS_TCP },
176 { "busy", TASK_STATS_BUSY_NETWORKS }, { "torr", TASK_STATS_TORRENTS }, { "fscr", TASK_STATS_FULLSCRAPE }, 200 { "busy", TASK_STATS_BUSY_NETWORKS }, { "torr", TASK_STATS_TORRENTS }, { "fscr", TASK_STATS_FULLSCRAPE },
@@ -206,6 +230,7 @@ static const ot_keywords keywords_format[] =
206 case 2: /* matched "format" */ 230 case 2: /* matched "format" */
207 if( ( format = scan_find_keywords( keywords_format, &read_ptr, SCAN_SEARCHPATH_VALUE ) ) <= 0 ) HTTPERROR_400_PARAM; 231 if( ( format = scan_find_keywords( keywords_format, &read_ptr, SCAN_SEARCHPATH_VALUE ) ) <= 0 ) HTTPERROR_400_PARAM;
208 break; 232 break;
233 case 3: HTTPERROR_400_PARAM; /* If the stats URL was mistakenly added as announce URL, return a 400 */
209 } 234 }
210 } 235 }
211 236
@@ -497,8 +522,8 @@ static ssize_t http_handle_announce( const int64 sock, struct ot_workstruct *ws,
497 return ws->reply_size = sprintf( ws->reply, "Successfully added.\n" ); 522 return ws->reply_size = sprintf( ws->reply, "Successfully added.\n" );
498 //} 523 //}
499 } 524 }
500#endif
501 break; 525 break;
526#endif
502 case 9: /* matched "peer_id" */ 527 case 9: /* matched "peer_id" */
503 /* ignore this, when we have less than 20 bytes */ 528 /* ignore this, when we have less than 20 bytes */
504 if( scan_urlencoded_query( &read_ptr, write_ptr = read_ptr, SCAN_SEARCHPATH_VALUE ) != 20 ) HTTPERROR_400_PARAM; 529 if( scan_urlencoded_query( &read_ptr, write_ptr = read_ptr, SCAN_SEARCHPATH_VALUE ) != 20 ) HTTPERROR_400_PARAM;
diff --git a/ot_http.h b/ot_http.h
index b1a60e7..40161d8 100644
--- a/ot_http.h
+++ b/ot_http.h
@@ -14,7 +14,8 @@ typedef enum {
14 14
15struct http_data { 15struct http_data {
16 array request; 16 array request;
17 io_batch batch; 17 io_batch *batch;
18 size_t batches;
18 ot_ip6 ip; 19 ot_ip6 ip;
19 STRUCT_HTTP_FLAG flag; 20 STRUCT_HTTP_FLAG flag;
20}; 21};
diff --git a/ot_iovec.c b/ot_iovec.c
index ec0bd12..fec3912 100644
--- a/ot_iovec.c
+++ b/ot_iovec.c
@@ -5,7 +5,6 @@
5 5
6/* System */ 6/* System */
7#include <sys/types.h> 7#include <sys/types.h>
8#include <sys/mman.h>
9#include <stdlib.h> 8#include <stdlib.h>
10#include <unistd.h> 9#include <unistd.h>
11#include <sys/uio.h> 10#include <sys/uio.h>
@@ -16,56 +15,57 @@
16#include "ot_iovec.h" 15#include "ot_iovec.h"
17 16
18void *iovec_increase( int *iovec_entries, struct iovec **iovector, size_t new_alloc ) { 17void *iovec_increase( int *iovec_entries, struct iovec **iovector, size_t new_alloc ) {
19 void *new_ptr = realloc( *iovector, (1 + *iovec_entries ) * sizeof( struct iovec ) ); 18 void *new_data;
20 if( !new_ptr ) 19 int new_entries = 1 + *iovec_entries;
20 struct iovec *new_vec = realloc( *iovector, new_entries * sizeof( struct iovec ) );
21
22 if( !new_vec )
21 return NULL; 23 return NULL;
22 *iovector = new_ptr; 24
23 new_ptr = mmap( NULL, new_alloc, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0 ); 25 /* Only allocate after we have a place to store the pointer */
24 if( !new_ptr ) 26 new_data = malloc( new_alloc );
27 if( !new_data )
25 return NULL; 28 return NULL;
26 ((*iovector)[*iovec_entries]).iov_base = new_ptr; 29
27 ((*iovector)[*iovec_entries]).iov_len = new_alloc; 30 new_vec[new_entries - 1].iov_base = new_data;
31 new_vec[new_entries - 1].iov_len = new_alloc;
32
33 *iovector = new_vec;
28 ++*iovec_entries; 34 ++*iovec_entries;
29 return new_ptr; 35 return new_data;
30} 36}
31 37
32void iovec_free( int *iovec_entries, struct iovec **iovector ) { 38void iovec_free( int *iovec_entries, struct iovec **iovector ) {
33 int i; 39 int i;
34 for( i=0; i<*iovec_entries; ++i ) 40 for( i=0; i<*iovec_entries; ++i )
35 munmap( ((*iovector)[i]).iov_base, ((*iovector)[i]).iov_len ); 41 free( ((*iovector)[i]).iov_base );
42 *iovector = NULL;
36 *iovec_entries = 0; 43 *iovec_entries = 0;
37} 44}
38 45
39void iovec_fixlast( int *iovec_entries, struct iovec **iovector, void *last_ptr ) { 46void iovec_fixlast( int *iovec_entries, struct iovec **iovector, void *last_ptr ) {
40 int page_size = getpagesize(); 47 if( *iovec_entries ) {
41 size_t old_alloc, new_alloc, old_pages, new_pages; 48 char * base = (char*)((*iovector)[ *iovec_entries - 1 ]).iov_base;
42 char * base = (char*)((*iovector)[ *iovec_entries - 1 ]).iov_base; 49 size_t new_alloc = ((char*)last_ptr) - base;
43
44 if( !*iovec_entries ) return;
45
46 old_alloc = ((*iovector)[ *iovec_entries - 1 ]).iov_len;
47 new_alloc = ((char*)last_ptr) - base;
48 old_pages = 1 + old_alloc / page_size;
49 new_pages = 1 + new_alloc / page_size;
50 50
51 if( old_pages != new_pages ) 51 ((*iovector)[*iovec_entries - 1 ]).iov_base = realloc( base, new_alloc );
52 munmap( base + new_pages * page_size, old_alloc - new_pages * page_size ); 52 ((*iovector)[*iovec_entries - 1 ]).iov_len = new_alloc;
53 ((*iovector)[*iovec_entries - 1 ]).iov_len = new_alloc; 53 }
54} 54}
55 55
56void *iovec_fix_increase_or_free( int *iovec_entries, struct iovec **iovector, void *last_ptr, size_t new_alloc ) { 56void *iovec_fix_increase_or_free( int *iovec_entries, struct iovec **iovector, void *last_ptr, size_t new_alloc ) {
57 void *new_ptr; 57 void *new_data;
58 58
59 iovec_fixlast( iovec_entries, iovector, last_ptr ); 59 iovec_fixlast( iovec_entries, iovector, last_ptr );
60 60
61 if( !( new_ptr = iovec_increase( iovec_entries, iovector, new_alloc ) ) ) 61 if( !( new_data = iovec_increase( iovec_entries, iovector, new_alloc ) ) )
62 iovec_free( iovec_entries, iovector ); 62 iovec_free( iovec_entries, iovector );
63 63
64 return new_ptr; 64 return new_data;
65} 65}
66 66
67 67
68size_t iovec_length( int *iovec_entries, struct iovec **iovector ) { 68size_t iovec_length( const int *iovec_entries, const struct iovec **iovector ) {
69 size_t length = 0; 69 size_t length = 0;
70 int i; 70 int i;
71 for( i=0; i<*iovec_entries; ++i ) 71 for( i=0; i<*iovec_entries; ++i )
diff --git a/ot_iovec.h b/ot_iovec.h
index 5dbe706..e48008e 100644
--- a/ot_iovec.h
+++ b/ot_iovec.h
@@ -12,7 +12,7 @@ void *iovec_increase( int *iovec_entries, struct iovec **iovector, size_t new_a
12void iovec_fixlast( int *iovec_entries, struct iovec **iovector, void *last_ptr ); 12void iovec_fixlast( int *iovec_entries, struct iovec **iovector, void *last_ptr );
13void iovec_free( int *iovec_entries, struct iovec **iovector ); 13void iovec_free( int *iovec_entries, struct iovec **iovector );
14 14
15size_t iovec_length( int *iovec_entries, struct iovec **iovector ); 15size_t iovec_length( const int *iovec_entries, const struct iovec **iovector );
16 16
17void *iovec_fix_increase_or_free( int *iovec_entries, struct iovec **iovector, void *last_ptr, size_t new_alloc ); 17void *iovec_fix_increase_or_free( int *iovec_entries, struct iovec **iovector, void *last_ptr, size_t new_alloc );
18 18
diff --git a/ot_mutex.c b/ot_mutex.c
index 772d936..1212245 100644
--- a/ot_mutex.c
+++ b/ot_mutex.c
@@ -25,65 +25,14 @@
25 25
26/* Our global all torrents list */ 26/* Our global all torrents list */
27static ot_vector all_torrents[OT_BUCKET_COUNT]; 27static ot_vector all_torrents[OT_BUCKET_COUNT];
28static pthread_mutex_t bucket_mutex[OT_BUCKET_COUNT];
28static size_t g_torrent_count; 29static size_t g_torrent_count;
29 30
30/* Bucket Magic */
31static int bucket_locklist[ OT_MAX_THREADS ];
32static int bucket_locklist_count = 0;
33static pthread_mutex_t bucket_mutex;
34static pthread_cond_t bucket_being_unlocked;
35
36/* Self pipe from opentracker.c */ 31/* Self pipe from opentracker.c */
37extern int g_self_pipe[2]; 32extern int g_self_pipe[2];
38 33
39static int bucket_check( int bucket ) {
40 /* C should come with auto-i ;) */
41 int i;
42
43 /* No more space to acquire lock to bucket -- should not happen */
44 if( bucket_locklist_count == OT_MAX_THREADS ) {
45 fprintf( stderr, "More lock requests than mutexes. Consult source code.\n" );
46 return -1;
47 }
48
49 /* See, if bucket is already locked */
50 for( i=0; i<bucket_locklist_count; ++i )
51 if( bucket_locklist[ i ] == bucket ) {
52 stats_issue_event( EVENT_BUCKET_LOCKED, 0, 0 );
53 return -1;
54 }
55
56 return 0;
57}
58
59static void bucket_push( int bucket ) {
60 bucket_locklist[ bucket_locklist_count++ ] = bucket;
61}
62
63static void bucket_remove( int bucket ) {
64 int i = 0;
65
66 while( ( i < bucket_locklist_count ) && ( bucket_locklist[ i ] != bucket ) )
67 ++i;
68
69 if( i == bucket_locklist_count ) {
70 fprintf( stderr, "Request to unlock bucket that was never locked. Consult source code.\n" );
71 return;
72 }
73
74 for( ; i < bucket_locklist_count - 1; ++i )
75 bucket_locklist[ i ] = bucket_locklist[ i + 1 ];
76
77 --bucket_locklist_count;
78}
79
80/* Can block */
81ot_vector *mutex_bucket_lock( int bucket ) { 34ot_vector *mutex_bucket_lock( int bucket ) {
82 pthread_mutex_lock( &bucket_mutex ); 35 pthread_mutex_lock(bucket_mutex + bucket );
83 while( bucket_check( bucket ) )
84 pthread_cond_wait( &bucket_being_unlocked, &bucket_mutex );
85 bucket_push( bucket );
86 pthread_mutex_unlock( &bucket_mutex );
87 return all_torrents + bucket; 36 return all_torrents + bucket;
88} 37}
89 38
@@ -92,11 +41,8 @@ ot_vector *mutex_bucket_lock_by_hash( ot_hash hash ) {
92} 41}
93 42
94void mutex_bucket_unlock( int bucket, int delta_torrentcount ) { 43void mutex_bucket_unlock( int bucket, int delta_torrentcount ) {
95 pthread_mutex_lock( &bucket_mutex ); 44 pthread_mutex_unlock(bucket_mutex + bucket);
96 bucket_remove( bucket );
97 g_torrent_count += delta_torrentcount; 45 g_torrent_count += delta_torrentcount;
98 pthread_cond_broadcast( &bucket_being_unlocked );
99 pthread_mutex_unlock( &bucket_mutex );
100} 46}
101 47
102void mutex_bucket_unlock_by_hash( ot_hash hash, int delta_torrentcount ) { 48void mutex_bucket_unlock_by_hash( ot_hash hash, int delta_torrentcount ) {
@@ -104,11 +50,7 @@ void mutex_bucket_unlock_by_hash( ot_hash hash, int delta_torrentcount ) {
104} 50}
105 51
106size_t mutex_get_torrent_count( ) { 52size_t mutex_get_torrent_count( ) {
107 size_t torrent_count; 53 return g_torrent_count;
108 pthread_mutex_lock( &bucket_mutex );
109 torrent_count = g_torrent_count;
110 pthread_mutex_unlock( &bucket_mutex );
111 return torrent_count;
112} 54}
113 55
114/* TaskQueue Magic */ 56/* TaskQueue Magic */
@@ -130,24 +72,9 @@ static pthread_cond_t tasklist_being_filled;
130int mutex_workqueue_pushtask( int64 sock, ot_tasktype tasktype ) { 72int mutex_workqueue_pushtask( int64 sock, ot_tasktype tasktype ) {
131 struct ot_task ** tmptask, * task; 73 struct ot_task ** tmptask, * task;
132 74
133 /* Want exclusive access to tasklist */
134 MTX_DBG( "pushtask locks.\n" );
135 pthread_mutex_lock( &tasklist_mutex );
136 MTX_DBG( "pushtask locked.\n" );
137
138 task = malloc(sizeof( struct ot_task)); 75 task = malloc(sizeof( struct ot_task));
139 if( !task ) { 76 if( !task )
140 MTX_DBG( "pushtask fail unlocks.\n" );
141 pthread_mutex_unlock( &tasklist_mutex );
142 MTX_DBG( "pushtask fail unlocked.\n" );
143 return -1; 77 return -1;
144 }
145
146 /* Skip to end of list */
147 tmptask = &tasklist;
148 while( *tmptask )
149 tmptask = &(*tmptask)->next;
150 *tmptask = task;
151 78
152 task->taskid = 0; 79 task->taskid = 0;
153 task->tasktype = tasktype; 80 task->tasktype = tasktype;
@@ -156,12 +83,18 @@ int mutex_workqueue_pushtask( int64 sock, ot_tasktype tasktype ) {
156 task->iovec = NULL; 83 task->iovec = NULL;
157 task->next = 0; 84 task->next = 0;
158 85
86 /* Want exclusive access to tasklist */
87 pthread_mutex_lock( &tasklist_mutex );
88
89 /* Skip to end of list */
90 tmptask = &tasklist;
91 while( *tmptask )
92 tmptask = &(*tmptask)->next;
93 *tmptask = task;
94
159 /* Inform waiting workers and release lock */ 95 /* Inform waiting workers and release lock */
160 MTX_DBG( "pushtask broadcasts.\n" );
161 pthread_cond_broadcast( &tasklist_being_filled ); 96 pthread_cond_broadcast( &tasklist_being_filled );
162 MTX_DBG( "pushtask broadcasted, mutex unlocks.\n" );
163 pthread_mutex_unlock( &tasklist_mutex ); 97 pthread_mutex_unlock( &tasklist_mutex );
164 MTX_DBG( "pushtask end mutex unlocked.\n" );
165 return 0; 98 return 0;
166} 99}
167 100
@@ -169,31 +102,25 @@ void mutex_workqueue_canceltask( int64 sock ) {
169 struct ot_task ** task; 102 struct ot_task ** task;
170 103
171 /* Want exclusive access to tasklist */ 104 /* Want exclusive access to tasklist */
172 MTX_DBG( "canceltask locks.\n" );
173 pthread_mutex_lock( &tasklist_mutex ); 105 pthread_mutex_lock( &tasklist_mutex );
174 MTX_DBG( "canceltask locked.\n" );
175
176 task = &tasklist;
177 while( *task && ( (*task)->sock != sock ) )
178 *task = (*task)->next;
179 106
180 if( *task && ( (*task)->sock == sock ) ) { 107 for (task = &tasklist; *task; task = &((*task)->next))
181 struct iovec *iovec = (*task)->iovec; 108 if ((*task)->sock == sock) {
182 struct ot_task *ptask = *task; 109 struct iovec *iovec = (*task)->iovec;
183 int i; 110 struct ot_task *ptask = *task;
111 int i;
184 112
185 /* Free task's iovec */ 113 /* Free task's iovec */
186 for( i=0; i<(*task)->iovec_entries; ++i ) 114 for( i=0; i<(*task)->iovec_entries; ++i )
187 munmap( iovec[i].iov_base, iovec[i].iov_len ); 115 free( iovec[i].iov_base );
188 116
189 *task = (*task)->next; 117 *task = (*task)->next;
190 free( ptask ); 118 free( ptask );
191 } 119 break;
120 }
192 121
193 /* Release lock */ 122 /* Release lock */
194 MTX_DBG( "canceltask unlocks.\n" );
195 pthread_mutex_unlock( &tasklist_mutex ); 123 pthread_mutex_unlock( &tasklist_mutex );
196 MTX_DBG( "canceltask unlocked.\n" );
197} 124}
198 125
199ot_taskid mutex_workqueue_poptask( ot_tasktype *tasktype ) { 126ot_taskid mutex_workqueue_poptask( ot_tasktype *tasktype ) {
@@ -201,33 +128,26 @@ ot_taskid mutex_workqueue_poptask( ot_tasktype *tasktype ) {
201 ot_taskid taskid = 0; 128 ot_taskid taskid = 0;
202 129
203 /* Want exclusive access to tasklist */ 130 /* Want exclusive access to tasklist */
204 MTX_DBG( "poptask mutex locks.\n" );
205 pthread_mutex_lock( &tasklist_mutex ); 131 pthread_mutex_lock( &tasklist_mutex );
206 MTX_DBG( "poptask mutex locked.\n" );
207 132
208 while( !taskid ) { 133 while( !taskid ) {
209 /* Skip to the first unassigned task this worker wants to do */ 134 /* Skip to the first unassigned task this worker wants to do */
210 task = tasklist; 135 for (task = tasklist; task; task = task->next)
211 while( task && ( ( ( TASK_CLASS_MASK & task->tasktype ) != *tasktype ) || task->taskid ) ) 136 if (!task->taskid && ( TASK_CLASS_MASK & task->tasktype ) == *tasktype) {
212 task = task->next; 137 /* If we found an outstanding task, assign a taskid to it
213 138 and leave the loop */
214 /* If we found an outstanding task, assign a taskid to it 139 task->taskid = taskid = ++next_free_taskid;
215 and leave the loop */ 140 *tasktype = task->tasktype;
216 if( task ) { 141 break;
217 task->taskid = taskid = ++next_free_taskid; 142 }
218 *tasktype = task->tasktype; 143
219 } else { 144 /* Wait until the next task is being fed */
220 /* Wait until the next task is being fed */ 145 if (!taskid)
221 MTX_DBG( "poptask cond waits.\n" );
222 pthread_cond_wait( &tasklist_being_filled, &tasklist_mutex ); 146 pthread_cond_wait( &tasklist_being_filled, &tasklist_mutex );
223 MTX_DBG( "poptask cond waited.\n" );
224 }
225 } 147 }
226 148
227 /* Release lock */ 149 /* Release lock */
228 MTX_DBG( "poptask end mutex unlocks.\n" );
229 pthread_mutex_unlock( &tasklist_mutex ); 150 pthread_mutex_unlock( &tasklist_mutex );
230 MTX_DBG( "poptask end mutex unlocked.\n" );
231 151
232 return taskid; 152 return taskid;
233} 153}
@@ -236,24 +156,18 @@ void mutex_workqueue_pushsuccess( ot_taskid taskid ) {
236 struct ot_task ** task; 156 struct ot_task ** task;
237 157
238 /* Want exclusive access to tasklist */ 158 /* Want exclusive access to tasklist */
239 MTX_DBG( "pushsuccess locks.\n" );
240 pthread_mutex_lock( &tasklist_mutex ); 159 pthread_mutex_lock( &tasklist_mutex );
241 MTX_DBG( "pushsuccess locked.\n" );
242 160
243 task = &tasklist; 161 for (task = &tasklist; *task; task = &((*task)->next))
244 while( *task && ( (*task)->taskid != taskid ) ) 162 if ((*task)->taskid == taskid) {
245 *task = (*task)->next; 163 struct ot_task *ptask = *task;
246 164 *task = (*task)->next;
247 if( *task && ( (*task)->taskid == taskid ) ) { 165 free( ptask );
248 struct ot_task *ptask = *task; 166 break;
249 *task = (*task)->next; 167 }
250 free( ptask );
251 }
252 168
253 /* Release lock */ 169 /* Release lock */
254 MTX_DBG( "pushsuccess unlocks.\n" );
255 pthread_mutex_unlock( &tasklist_mutex ); 170 pthread_mutex_unlock( &tasklist_mutex );
256 MTX_DBG( "pushsuccess unlocked.\n" );
257} 171}
258 172
259int mutex_workqueue_pushresult( ot_taskid taskid, int iovec_entries, struct iovec *iovec ) { 173int mutex_workqueue_pushresult( ot_taskid taskid, int iovec_entries, struct iovec *iovec ) {
@@ -261,24 +175,18 @@ int mutex_workqueue_pushresult( ot_taskid taskid, int iovec_entries, struct iove
261 const char byte = 'o'; 175 const char byte = 'o';
262 176
263 /* Want exclusive access to tasklist */ 177 /* Want exclusive access to tasklist */
264 MTX_DBG( "pushresult locks.\n" );
265 pthread_mutex_lock( &tasklist_mutex ); 178 pthread_mutex_lock( &tasklist_mutex );
266 MTX_DBG( "pushresult locked.\n" );
267
268 task = tasklist;
269 while( task && ( task->taskid != taskid ) )
270 task = task->next;
271 179
272 if( task ) { 180 for (task = tasklist; task; task = task->next)
273 task->iovec_entries = iovec_entries; 181 if (task->taskid == taskid) {
274 task->iovec = iovec; 182 task->iovec_entries = iovec_entries;
275 task->tasktype = TASK_DONE; 183 task->iovec = iovec;
276 } 184 task->tasktype = TASK_DONE;
185 break;
186 }
277 187
278 /* Release lock */ 188 /* Release lock */
279 MTX_DBG( "pushresult unlocks.\n" );
280 pthread_mutex_unlock( &tasklist_mutex ); 189 pthread_mutex_unlock( &tasklist_mutex );
281 MTX_DBG( "pushresult unlocked.\n" );
282 190
283 io_trywrite( g_self_pipe[1], &byte, 1 ); 191 io_trywrite( g_self_pipe[1], &byte, 1 );
284 192
@@ -291,43 +199,39 @@ int64 mutex_workqueue_popresult( int *iovec_entries, struct iovec ** iovec ) {
291 int64 sock = -1; 199 int64 sock = -1;
292 200
293 /* Want exclusive access to tasklist */ 201 /* Want exclusive access to tasklist */
294 MTX_DBG( "popresult locks.\n" );
295 pthread_mutex_lock( &tasklist_mutex ); 202 pthread_mutex_lock( &tasklist_mutex );
296 MTX_DBG( "popresult locked.\n" );
297
298 task = &tasklist;
299 while( *task && ( (*task)->tasktype != TASK_DONE ) )
300 task = &(*task)->next;
301 203
302 if( *task && ( (*task)->tasktype == TASK_DONE ) ) { 204 for (task = &tasklist; *task; task = &((*task)->next))
303 struct ot_task *ptask = *task; 205 if ((*task)->tasktype == TASK_DONE) {
206 struct ot_task *ptask = *task;
304 207
305 *iovec_entries = (*task)->iovec_entries; 208 *iovec_entries = (*task)->iovec_entries;
306 *iovec = (*task)->iovec; 209 *iovec = (*task)->iovec;
307 sock = (*task)->sock; 210 sock = (*task)->sock;
308 211
309 *task = (*task)->next; 212 *task = (*task)->next;
310 free( ptask ); 213 free( ptask );
311 } 214 break;
215 }
312 216
313 /* Release lock */ 217 /* Release lock */
314 MTX_DBG( "popresult unlocks.\n" );
315 pthread_mutex_unlock( &tasklist_mutex ); 218 pthread_mutex_unlock( &tasklist_mutex );
316 MTX_DBG( "popresult unlocked.\n" );
317 return sock; 219 return sock;
318} 220}
319 221
320void mutex_init( ) { 222void mutex_init( ) {
223 int i;
321 pthread_mutex_init(&tasklist_mutex, NULL); 224 pthread_mutex_init(&tasklist_mutex, NULL);
322 pthread_cond_init (&tasklist_being_filled, NULL); 225 pthread_cond_init (&tasklist_being_filled, NULL);
323 pthread_mutex_init(&bucket_mutex, NULL); 226 for (i=0; i < OT_BUCKET_COUNT; ++i)
324 pthread_cond_init (&bucket_being_unlocked, NULL); 227 pthread_mutex_init(bucket_mutex + i, NULL);
325 byte_zero( all_torrents, sizeof( all_torrents ) ); 228 byte_zero( all_torrents, sizeof( all_torrents ) );
326} 229}
327 230
328void mutex_deinit( ) { 231void mutex_deinit( ) {
329 pthread_mutex_destroy(&bucket_mutex); 232 int i;
330 pthread_cond_destroy(&bucket_being_unlocked); 233 for (i=0; i < OT_BUCKET_COUNT; ++i)
234 pthread_mutex_destroy(bucket_mutex + i);
331 pthread_mutex_destroy(&tasklist_mutex); 235 pthread_mutex_destroy(&tasklist_mutex);
332 pthread_cond_destroy(&tasklist_being_filled); 236 pthread_cond_destroy(&tasklist_being_filled);
333 byte_zero( all_torrents, sizeof( all_torrents ) ); 237 byte_zero( all_torrents, sizeof( all_torrents ) );
diff --git a/ot_mutex.h b/ot_mutex.h
index bd07009..93c1ecf 100644
--- a/ot_mutex.h
+++ b/ot_mutex.h
@@ -8,8 +8,8 @@
8 8
9#include <sys/uio.h> 9#include <sys/uio.h>
10 10
11void mutex_init( ); 11void mutex_init( void );
12void mutex_deinit( ); 12void mutex_deinit( void );
13 13
14ot_vector *mutex_bucket_lock( int bucket ); 14ot_vector *mutex_bucket_lock( int bucket );
15ot_vector *mutex_bucket_lock_by_hash( ot_hash hash ); 15ot_vector *mutex_bucket_lock_by_hash( ot_hash hash );
@@ -17,7 +17,7 @@ ot_vector *mutex_bucket_lock_by_hash( ot_hash hash );
17void mutex_bucket_unlock( int bucket, int delta_torrentcount ); 17void mutex_bucket_unlock( int bucket, int delta_torrentcount );
18void mutex_bucket_unlock_by_hash( ot_hash hash, int delta_torrentcount ); 18void mutex_bucket_unlock_by_hash( ot_hash hash, int delta_torrentcount );
19 19
20size_t mutex_get_torrent_count(); 20size_t mutex_get_torrent_count(void);
21 21
22typedef enum { 22typedef enum {
23 TASK_STATS_CONNS = 0x0001, 23 TASK_STATS_CONNS = 0x0001,
diff --git a/ot_stats.c b/ot_stats.c
index 83cd058..7d2749f 100644
--- a/ot_stats.c
+++ b/ot_stats.c
@@ -293,7 +293,7 @@ static int torrent_statter( ot_torrent *torrent, uintptr_t data ) {
293/* Converter function from memory to human readable hex strings */ 293/* Converter function from memory to human readable hex strings */
294static char*to_hex(char*d,uint8_t*s){char*m="0123456789ABCDEF";char *t=d;char*e=d+40;while(d<e){*d++=m[*s>>4];*d++=m[*s++&15];}*d=0;return t;} 294static char*to_hex(char*d,uint8_t*s){char*m="0123456789ABCDEF";char *t=d;char*e=d+40;while(d<e){*d++=m[*s>>4];*d++=m[*s++&15];}*d=0;return t;}
295 295
296typedef struct { size_t val; ot_torrent * torrent; } ot_record; 296typedef struct { size_t val; ot_hash hash; } ot_record;
297 297
298/* Fetches stats from tracker */ 298/* Fetches stats from tracker */
299size_t stats_top_txt( char * reply, int amount ) { 299size_t stats_top_txt( char * reply, int amount ) {
@@ -311,18 +311,22 @@ size_t stats_top_txt( char * reply, int amount ) {
311 for( bucket=0; bucket<OT_BUCKET_COUNT; ++bucket ) { 311 for( bucket=0; bucket<OT_BUCKET_COUNT; ++bucket ) {
312 ot_vector *torrents_list = mutex_bucket_lock( bucket ); 312 ot_vector *torrents_list = mutex_bucket_lock( bucket );
313 for( j=0; j<torrents_list->size; ++j ) { 313 for( j=0; j<torrents_list->size; ++j ) {
314 ot_peerlist *peer_list = ( ((ot_torrent*)(torrents_list->data))[j] ).peer_list; 314 ot_torrent *torrent = (ot_torrent*)(torrents_list->data) + j;
315 int idx = amount - 1; while( (idx >= 0) && ( peer_list->peer_count > top100c[idx].val ) ) --idx; 315 idx = amount - 1;
316 while( (idx >= 0) && ( torrent->peer_list->peer_count > top100c[idx].val ) )
317 --idx;
316 if ( idx++ != amount - 1 ) { 318 if ( idx++ != amount - 1 ) {
317 memmove( top100c + idx + 1, top100c + idx, ( amount - 1 - idx ) * sizeof( ot_record ) ); 319 memmove( top100c + idx + 1, top100c + idx, ( amount - 1 - idx ) * sizeof( ot_record ) );
318 top100c[idx].val = peer_list->peer_count; 320 memcpy( &top100c[idx].hash, &torrent->hash, sizeof(ot_hash));
319 top100c[idx].torrent = (ot_torrent*)(torrents_list->data) + j; 321 top100c[idx].val = torrent->peer_list->peer_count;
320 } 322 }
321 idx = amount - 1; while( (idx >= 0) && ( peer_list->seed_count > top100s[idx].val ) ) --idx; 323 idx = amount - 1;
324 while( (idx >= 0) && ( torrent->peer_list->seed_count > top100s[idx].val ) )
325 --idx;
322 if ( idx++ != amount - 1 ) { 326 if ( idx++ != amount - 1 ) {
323 memmove( top100s + idx + 1, top100s + idx, ( amount - 1 - idx ) * sizeof( ot_record ) ); 327 memmove( top100s + idx + 1, top100s + idx, ( amount - 1 - idx ) * sizeof( ot_record ) );
324 top100s[idx].val = peer_list->seed_count; 328 memcpy( &top100s[idx].hash, &torrent->hash, sizeof(ot_hash));
325 top100s[idx].torrent = (ot_torrent*)(torrents_list->data) + j; 329 top100s[idx].val = torrent->peer_list->seed_count;
326 } 330 }
327 } 331 }
328 mutex_bucket_unlock( bucket, 0 ); 332 mutex_bucket_unlock( bucket, 0 );
@@ -332,12 +336,12 @@ size_t stats_top_txt( char * reply, int amount ) {
332 336
333 r += sprintf( r, "Top %d torrents by peers:\n", amount ); 337 r += sprintf( r, "Top %d torrents by peers:\n", amount );
334 for( idx=0; idx<amount; ++idx ) 338 for( idx=0; idx<amount; ++idx )
335 if( top100c[idx].torrent ) 339 if( top100c[idx].val )
336 r += sprintf( r, "\t%zd\t%s\n", top100c[idx].val, to_hex( hex_out, top100c[idx].torrent->hash) ); 340 r += sprintf( r, "\t%zd\t%s\n", top100c[idx].val, to_hex( hex_out, top100c[idx].hash) );
337 r += sprintf( r, "Top %d torrents by seeds:\n", amount ); 341 r += sprintf( r, "Top %d torrents by seeds:\n", amount );
338 for( idx=0; idx<amount; ++idx ) 342 for( idx=0; idx<amount; ++idx )
339 if( top100s[idx].torrent ) 343 if( top100s[idx].val )
340 r += sprintf( r, "\t%zd\t%s\n", top100s[idx].val, to_hex( hex_out, top100s[idx].torrent->hash) ); 344 r += sprintf( r, "\t%zd\t%s\n", top100s[idx].val, to_hex( hex_out, top100s[idx].hash) );
341 345
342 return r - reply; 346 return r - reply;
343} 347}
@@ -691,7 +695,7 @@ void stats_issue_event( ot_status_event event, PROTO_FLAG proto, uintptr_t event
691 int off = snprintf( _debug, sizeof(_debug), "[%08d] scrp: ", (unsigned int)(g_now_seconds - ot_start_time)/60 ); 695 int off = snprintf( _debug, sizeof(_debug), "[%08d] scrp: ", (unsigned int)(g_now_seconds - ot_start_time)/60 );
692 off += fmt_ip6c( _debug+off, *ip ); 696 off += fmt_ip6c( _debug+off, *ip );
693 off += snprintf( _debug+off, sizeof(_debug)-off, " - FULL SCRAPE\n" ); 697 off += snprintf( _debug+off, sizeof(_debug)-off, " - FULL SCRAPE\n" );
694 write( 2, _debug, off ); 698 (void)write( 2, _debug, off );
695 ot_full_scrape_request_count++; 699 ot_full_scrape_request_count++;
696 } 700 }
697 break; 701 break;
@@ -702,7 +706,7 @@ void stats_issue_event( ot_status_event event, PROTO_FLAG proto, uintptr_t event
702 int off = snprintf( _debug, sizeof(_debug), "[%08d] scrp: ", (unsigned int)(g_now_seconds - ot_start_time)/60 ); 706 int off = snprintf( _debug, sizeof(_debug), "[%08d] scrp: ", (unsigned int)(g_now_seconds - ot_start_time)/60 );
703 off += fmt_ip6c(_debug+off, *ip ); 707 off += fmt_ip6c(_debug+off, *ip );
704 off += snprintf( _debug+off, sizeof(_debug)-off, " - FULL SCRAPE\n" ); 708 off += snprintf( _debug+off, sizeof(_debug)-off, " - FULL SCRAPE\n" );
705 write( 2, _debug, off ); 709 (void)write( 2, _debug, off );
706 ot_full_scrape_request_count++; 710 ot_full_scrape_request_count++;
707 } 711 }
708 break; 712 break;
diff --git a/ot_stats.h b/ot_stats.h
index 6a2515b..ed60f68 100644
--- a/ot_stats.h
+++ b/ot_stats.h
@@ -40,11 +40,11 @@ enum {
40 40
41void stats_issue_event( ot_status_event event, PROTO_FLAG proto, uintptr_t event_data ); 41void stats_issue_event( ot_status_event event, PROTO_FLAG proto, uintptr_t event_data );
42void stats_deliver( int64 sock, int tasktype ); 42void stats_deliver( int64 sock, int tasktype );
43void stats_cleanup(); 43void stats_cleanup( void );
44size_t return_stats_for_tracker( char *reply, int mode, int format ); 44size_t return_stats_for_tracker( char *reply, int mode, int format );
45size_t stats_return_tracker_version( char *reply ); 45size_t stats_return_tracker_version( char *reply );
46void stats_init( ); 46void stats_init( void );
47void stats_deinit( ); 47void stats_deinit( void );
48 48
49extern const char *g_version_rijndael_c; 49extern const char *g_version_rijndael_c;
50extern const char *g_version_livesync_c; 50extern const char *g_version_livesync_c;
diff --git a/ot_udp.c b/ot_udp.c
index 3bf311c..edbaca8 100644
--- a/ot_udp.c
+++ b/ot_udp.c
@@ -29,13 +29,21 @@ static ot_time g_hour_of_the_key;
29 29
30static void udp_generate_rijndael_round_key() { 30static void udp_generate_rijndael_round_key() {
31 uint32_t key[16]; 31 uint32_t key[16];
32#ifdef WANT_ARC4RANDOM
33 arc4random_buf(&key[0], sizeof(key));
34#else
32 key[0] = random(); 35 key[0] = random();
33 key[1] = random(); 36 key[1] = random();
34 key[2] = random(); 37 key[2] = random();
35 key[3] = random(); 38 key[3] = random();
39#endif
36 rijndaelKeySetupEnc128( g_rijndael_round_key, (uint8_t*)key ); 40 rijndaelKeySetupEnc128( g_rijndael_round_key, (uint8_t*)key );
37 41
42#ifdef WANT_ARC4RANDOM
43 g_key_of_the_hour[0] = arc4random();
44#else
38 g_key_of_the_hour[0] = random(); 45 g_key_of_the_hour[0] = random();
46#endif
39 g_hour_of_the_key = g_now_minutes; 47 g_hour_of_the_key = g_now_minutes;
40} 48}
41 49
@@ -46,7 +54,11 @@ static void udp_make_connectionid( uint32_t connid[2], const ot_ip6 remoteip, in
46 if( g_now_minutes + 60 > g_hour_of_the_key ) { 54 if( g_now_minutes + 60 > g_hour_of_the_key ) {
47 g_hour_of_the_key = g_now_minutes; 55 g_hour_of_the_key = g_now_minutes;
48 g_key_of_the_hour[1] = g_key_of_the_hour[0]; 56 g_key_of_the_hour[1] = g_key_of_the_hour[0];
49 g_key_of_the_hour[0] = random(); 57#ifdef WANT_ARC4RANDOM
58 g_key_of_the_hour[0] = arc4random();
59#else
60 g_key_of_the_hour[0] = random();
61#endif
50 } 62 }
51 63
52 memcpy( plain, remoteip, sizeof( plain ) ); 64 memcpy( plain, remoteip, sizeof( plain ) );
@@ -206,7 +218,7 @@ void udp_init( int64 sock, unsigned int worker_count ) {
206 if( !g_rijndael_round_key[0] ) 218 if( !g_rijndael_round_key[0] )
207 udp_generate_rijndael_round_key(); 219 udp_generate_rijndael_round_key();
208#ifdef _DEBUG 220#ifdef _DEBUG
209 fprintf( stderr, " installing %d workers on udp socket %ld", worker_count, (unsigned long)sock ); 221 fprintf( stderr, " installing %d workers on udp socket %ld\n", worker_count, (unsigned long)sock );
210#endif 222#endif
211 while( worker_count-- ) 223 while( worker_count-- )
212 pthread_create( &thread_id, NULL, udp_worker, (void *)sock ); 224 pthread_create( &thread_id, NULL, udp_worker, (void *)sock );
diff --git a/proxy.c b/proxy.c
index 1f09777..619b08a 100644
--- a/proxy.c
+++ b/proxy.c
@@ -171,8 +171,8 @@ size_t remove_peer_from_torrent_proxy( ot_hash hash, ot_peer *peer ) {
171 if( exactmatch ) { 171 if( exactmatch ) {
172 ot_peerlist *peer_list = torrent->peer_list; 172 ot_peerlist *peer_list = torrent->peer_list;
173 switch( vector_remove_peer( &peer_list->peers, peer ) ) { 173 switch( vector_remove_peer( &peer_list->peers, peer ) ) {
174 case 2: peer_list->seed_count--; /* Fall throughs intended */ 174 case 2: peer_list->seed_count--; /* Intentional fallthrough */
175 case 1: peer_list->peer_count--; /* Fall throughs intended */ 175 case 1: peer_list->peer_count--; /* Intentional fallthrough */
176 default: break; 176 default: break;
177 } 177 }
178 } 178 }
@@ -553,7 +553,11 @@ int main( int argc, char **argv ) {
553 int scanon = 1, lbound = 0, sbound = 0; 553 int scanon = 1, lbound = 0, sbound = 0;
554 554
555 srandom( time(NULL) ); 555 srandom( time(NULL) );
556#ifdef WANT_ARC4RANDOM
557 g_tracker_id = arc4random();
558#else
556 g_tracker_id = random(); 559 g_tracker_id = random();
560#endif
557 noipv6=1; 561 noipv6=1;
558 562
559 while( scanon ) { 563 while( scanon ) {
diff --git a/trackerlogic.c b/trackerlogic.c
index 310466c..719f8a2 100644
--- a/trackerlogic.c
+++ b/trackerlogic.c
@@ -29,7 +29,7 @@
29#include "ot_livesync.h" 29#include "ot_livesync.h"
30 30
31/* Forward declaration */ 31/* Forward declaration */
32size_t return_peers_for_torrent( ot_torrent *torrent, size_t amount, char *reply, PROTO_FLAG proto ); 32size_t return_peers_for_torrent( struct ot_workstruct * ws, ot_torrent *torrent, size_t amount, char *reply, PROTO_FLAG proto );
33 33
34void free_peerlist( ot_peerlist *peer_list ) { 34void free_peerlist( ot_peerlist *peer_list ) {
35 if( peer_list->peers.data ) { 35 if( peer_list->peers.data ) {
@@ -51,19 +51,19 @@ void add_torrent_from_saved_state( ot_hash hash, ot_time base, size_t down_count
51 51
52 if( !accesslist_hashisvalid( hash ) ) 52 if( !accesslist_hashisvalid( hash ) )
53 return mutex_bucket_unlock_by_hash( hash, 0 ); 53 return mutex_bucket_unlock_by_hash( hash, 0 );
54 54
55 torrent = vector_find_or_insert( torrents_list, (void*)hash, sizeof( ot_torrent ), OT_HASH_COMPARE_SIZE, &exactmatch ); 55 torrent = vector_find_or_insert( torrents_list, (void*)hash, sizeof( ot_torrent ), OT_HASH_COMPARE_SIZE, &exactmatch );
56 if( !torrent || exactmatch ) 56 if( !torrent || exactmatch )
57 return mutex_bucket_unlock_by_hash( hash, 0 ); 57 return mutex_bucket_unlock_by_hash( hash, 0 );
58 58
59 /* Create a new torrent entry, then */ 59 /* Create a new torrent entry, then */
60 memcpy( torrent->hash, hash, sizeof(ot_hash) ); 60 memcpy( torrent->hash, hash, sizeof(ot_hash) );
61 61
62 if( !( torrent->peer_list = malloc( sizeof (ot_peerlist) ) ) ) { 62 if( !( torrent->peer_list = malloc( sizeof (ot_peerlist) ) ) ) {
63 vector_remove_torrent( torrents_list, torrent ); 63 vector_remove_torrent( torrents_list, torrent );
64 return mutex_bucket_unlock_by_hash( hash, 0 ); 64 return mutex_bucket_unlock_by_hash( hash, 0 );
65 } 65 }
66 66
67 byte_zero( torrent->peer_list, sizeof( ot_peerlist ) ); 67 byte_zero( torrent->peer_list, sizeof( ot_peerlist ) );
68 torrent->peer_list->base = base; 68 torrent->peer_list->base = base;
69 torrent->peer_list->down_count = down_count; 69 torrent->peer_list->down_count = down_count;
@@ -178,7 +178,7 @@ size_t add_peer_to_torrent_and_return_peers( PROTO_FLAG proto, struct ot_workstr
178 } 178 }
179#endif 179#endif
180 180
181 ws->reply_size = return_peers_for_torrent( torrent, amount, ws->reply, proto ); 181 ws->reply_size = return_peers_for_torrent( ws, torrent, amount, ws->reply, proto );
182 mutex_bucket_unlock_by_hash( *ws->hash, delta_torrentcount ); 182 mutex_bucket_unlock_by_hash( *ws->hash, delta_torrentcount );
183 return ws->reply_size; 183 return ws->reply_size;
184} 184}
@@ -200,7 +200,7 @@ static size_t return_peers_all( ot_peerlist *peer_list, char *reply ) {
200 while( peer_count-- ) { 200 while( peer_count-- ) {
201 if( OT_PEERFLAG(peers) & PEER_FLAG_SEEDING ) { 201 if( OT_PEERFLAG(peers) & PEER_FLAG_SEEDING ) {
202 r_end-=OT_PEER_COMPARE_SIZE; 202 r_end-=OT_PEER_COMPARE_SIZE;
203 memcpy(r_end,peers++,OT_PEER_COMPARE_SIZE); 203 memcpy(r_end,peers++,OT_PEER_COMPARE_SIZE);
204 } else { 204 } else {
205 memcpy(reply,peers++,OT_PEER_COMPARE_SIZE); 205 memcpy(reply,peers++,OT_PEER_COMPARE_SIZE);
206 reply+=OT_PEER_COMPARE_SIZE; 206 reply+=OT_PEER_COMPARE_SIZE;
@@ -210,7 +210,7 @@ static size_t return_peers_all( ot_peerlist *peer_list, char *reply ) {
210 return result; 210 return result;
211} 211}
212 212
213static size_t return_peers_selection( ot_peerlist *peer_list, size_t amount, char *reply ) { 213static size_t return_peers_selection( struct ot_workstruct *ws, ot_peerlist *peer_list, size_t amount, char *reply ) {
214 unsigned int bucket_offset, bucket_index = 0, num_buckets = 1; 214 unsigned int bucket_offset, bucket_index = 0, num_buckets = 1;
215 ot_vector * bucket_list = &peer_list->peers; 215 ot_vector * bucket_list = &peer_list->peers;
216 unsigned int shifted_pc = peer_list->peer_count; 216 unsigned int shifted_pc = peer_list->peer_count;
@@ -218,7 +218,7 @@ static size_t return_peers_selection( ot_peerlist *peer_list, size_t amount, cha
218 unsigned int shift = 0; 218 unsigned int shift = 0;
219 size_t result = OT_PEER_COMPARE_SIZE * amount; 219 size_t result = OT_PEER_COMPARE_SIZE * amount;
220 char * r_end = reply + result; 220 char * r_end = reply + result;
221 221
222 if( OT_PEERLIST_HASBUCKETS(peer_list) ) { 222 if( OT_PEERLIST_HASBUCKETS(peer_list) ) {
223 num_buckets = bucket_list->size; 223 num_buckets = bucket_list->size;
224 bucket_list = (ot_vector *)bucket_list->data; 224 bucket_list = (ot_vector *)bucket_list->data;
@@ -232,7 +232,7 @@ static size_t return_peers_selection( ot_peerlist *peer_list, size_t amount, cha
232 232
233 /* Initialize somewhere in the middle of peers so that 233 /* Initialize somewhere in the middle of peers so that
234 fixpoint's aliasing doesn't alway miss the same peers */ 234 fixpoint's aliasing doesn't alway miss the same peers */
235 bucket_offset = random() % peer_list->peer_count; 235 bucket_offset = nrand48(ws->rand48_state) % peer_list->peer_count;
236 236
237 while( amount-- ) { 237 while( amount-- ) {
238 ot_peer * peer; 238 ot_peer * peer;
@@ -240,7 +240,7 @@ static size_t return_peers_selection( ot_peerlist *peer_list, size_t amount, cha
240 /* This is the aliased, non shifted range, next value may fall into */ 240 /* This is the aliased, non shifted range, next value may fall into */
241 unsigned int diff = ( ( ( amount + 1 ) * shifted_step ) >> shift ) - 241 unsigned int diff = ( ( ( amount + 1 ) * shifted_step ) >> shift ) -
242 ( ( amount * shifted_step ) >> shift ); 242 ( ( amount * shifted_step ) >> shift );
243 bucket_offset += 1 + random() % diff; 243 bucket_offset += 1 + nrand48(ws->rand48_state) % diff;
244 244
245 while( bucket_offset >= bucket_list[bucket_index].size ) { 245 while( bucket_offset >= bucket_list[bucket_index].size ) {
246 bucket_offset -= bucket_list[bucket_index].size; 246 bucket_offset -= bucket_list[bucket_index].size;
@@ -249,7 +249,7 @@ static size_t return_peers_selection( ot_peerlist *peer_list, size_t amount, cha
249 peer = ((ot_peer*)bucket_list[bucket_index].data) + bucket_offset; 249 peer = ((ot_peer*)bucket_list[bucket_index].data) + bucket_offset;
250 if( OT_PEERFLAG(peer) & PEER_FLAG_SEEDING ) { 250 if( OT_PEERFLAG(peer) & PEER_FLAG_SEEDING ) {
251 r_end-=OT_PEER_COMPARE_SIZE; 251 r_end-=OT_PEER_COMPARE_SIZE;
252 memcpy(r_end,peer,OT_PEER_COMPARE_SIZE); 252 memcpy(r_end,peer,OT_PEER_COMPARE_SIZE);
253 } else { 253 } else {
254 memcpy(reply,peer,OT_PEER_COMPARE_SIZE); 254 memcpy(reply,peer,OT_PEER_COMPARE_SIZE);
255 reply+=OT_PEER_COMPARE_SIZE; 255 reply+=OT_PEER_COMPARE_SIZE;
@@ -262,7 +262,7 @@ static size_t return_peers_selection( ot_peerlist *peer_list, size_t amount, cha
262 * reply must have enough space to hold 92+6*amount bytes 262 * reply must have enough space to hold 92+6*amount bytes
263 * does not yet check not to return self 263 * does not yet check not to return self
264*/ 264*/
265size_t return_peers_for_torrent( ot_torrent *torrent, size_t amount, char *reply, PROTO_FLAG proto ) { 265size_t return_peers_for_torrent( struct ot_workstruct * ws, ot_torrent *torrent, size_t amount, char *reply, PROTO_FLAG proto ) {
266 ot_peerlist *peer_list = torrent->peer_list; 266 ot_peerlist *peer_list = torrent->peer_list;
267 char *r = reply; 267 char *r = reply;
268 268
@@ -283,7 +283,7 @@ size_t return_peers_for_torrent( ot_torrent *torrent, size_t amount, char *reply
283 if( amount == peer_list->peer_count ) 283 if( amount == peer_list->peer_count )
284 r += return_peers_all( peer_list, r ); 284 r += return_peers_all( peer_list, r );
285 else 285 else
286 r += return_peers_selection( peer_list, amount, r ); 286 r += return_peers_selection( ws, peer_list, amount, r );
287 } 287 }
288 288
289 if( proto == FLAG_TCP ) 289 if( proto == FLAG_TCP )
@@ -365,8 +365,8 @@ size_t remove_peer_from_torrent( PROTO_FLAG proto, struct ot_workstruct *ws ) {
365 if( exactmatch ) { 365 if( exactmatch ) {
366 peer_list = torrent->peer_list; 366 peer_list = torrent->peer_list;
367 switch( vector_remove_peer( &peer_list->peers, &ws->peer ) ) { 367 switch( vector_remove_peer( &peer_list->peers, &ws->peer ) ) {
368 case 2: peer_list->seed_count--; /* Fall throughs intended */ 368 case 2: peer_list->seed_count--; /* Intentional fallthrough */
369 case 1: peer_list->peer_count--; /* Fall throughs intended */ 369 case 1: peer_list->peer_count--; /* Intentional fallthrough */
370 default: break; 370 default: break;
371 } 371 }
372 } 372 }
diff --git a/trackerlogic.h b/trackerlogic.h
index 721ba6e..ef59179 100644
--- a/trackerlogic.h
+++ b/trackerlogic.h
@@ -10,6 +10,14 @@
10#include <sys/time.h> 10#include <sys/time.h>
11#include <time.h> 11#include <time.h>
12#include <stdint.h> 12#include <stdint.h>
13#include <stdlib.h>
14
15#if defined(__linux__) && defined(WANT_ARC4RANDOM)
16#include <bsd/stdlib.h>
17#endif
18#ifdef __FreeBSD__
19#define WANT_ARC4RANDOM
20#endif
13 21
14typedef uint8_t ot_hash[20]; 22typedef uint8_t ot_hash[20];
15typedef time_t ot_time; 23typedef time_t ot_time;
@@ -34,7 +42,7 @@ typedef struct { ot_ip6 address; int bits; }
34#define OT_TORRENT_TIMEOUT_HOURS 24 42#define OT_TORRENT_TIMEOUT_HOURS 24
35#define OT_TORRENT_TIMEOUT (60*OT_TORRENT_TIMEOUT_HOURS) 43#define OT_TORRENT_TIMEOUT (60*OT_TORRENT_TIMEOUT_HOURS)
36 44
37#define OT_CLIENT_REQUEST_INTERVAL_RANDOM ( OT_CLIENT_REQUEST_INTERVAL - OT_CLIENT_REQUEST_VARIATION/2 + (int)( random( ) % OT_CLIENT_REQUEST_VARIATION ) ) 45#define OT_CLIENT_REQUEST_INTERVAL_RANDOM ( OT_CLIENT_REQUEST_INTERVAL - OT_CLIENT_REQUEST_VARIATION/2 + (int)( nrand48(ws->rand48_state) % OT_CLIENT_REQUEST_VARIATION ) )
38 46
39/* If WANT_MODEST_FULLSCRAPES is on, ip addresses may not 47/* If WANT_MODEST_FULLSCRAPES is on, ip addresses may not
40 fullscrape more frequently than this amount in seconds */ 48 fullscrape more frequently than this amount in seconds */
@@ -125,12 +133,17 @@ struct ot_workstruct {
125 char *peer_id; 133 char *peer_id;
126 134
127 /* HTTP specific, non static */ 135 /* HTTP specific, non static */
128 int keep_alive;
129 char *request; 136 char *request;
130 ssize_t request_size; 137 ssize_t request_size;
131 ssize_t header_size; 138 ssize_t header_size;
132 char *reply; 139 char *reply;
133 ssize_t reply_size; 140 ssize_t reply_size;
141
142 /* Entropy state for rand48 function so that threads don't need to acquire mutexes for
143 global random() or arc4random() state, which causes heavy load on linuxes */
144 uint16_t rand48_state[3];
145
146 int keep_alive;
134}; 147};
135 148
136/* 149/*
@@ -151,7 +164,7 @@ struct ot_workstruct {
151#error Live logging networks disabled at the moment. 164#error Live logging networks disabled at the moment.
152#endif 165#endif
153 166
154void trackerlogic_init( ); 167void trackerlogic_init( void );
155void trackerlogic_deinit( void ); 168void trackerlogic_deinit( void );
156void exerr( char * message ); 169void exerr( char * message );
157 170