diff options
-rw-r--r-- | Makefile | 11 | ||||
-rw-r--r-- | opentracker.c | 67 | ||||
-rw-r--r-- | opentracker.conf.sample | 47 | ||||
-rw-r--r-- | ot_accesslist.c | 287 | ||||
-rw-r--r-- | ot_accesslist.h | 15 | ||||
-rw-r--r-- | ot_clean.c | 4 | ||||
-rw-r--r-- | ot_fullscrape.c | 286 | ||||
-rw-r--r-- | ot_http.c | 43 | ||||
-rw-r--r-- | ot_http.h | 3 | ||||
-rw-r--r-- | ot_iovec.c | 58 | ||||
-rw-r--r-- | ot_iovec.h | 2 | ||||
-rw-r--r-- | ot_mutex.c | 230 | ||||
-rw-r--r-- | ot_mutex.h | 6 | ||||
-rw-r--r-- | ot_stats.c | 32 | ||||
-rw-r--r-- | ot_stats.h | 6 | ||||
-rw-r--r-- | ot_udp.c | 16 | ||||
-rw-r--r-- | proxy.c | 8 | ||||
-rw-r--r-- | trackerlogic.c | 30 | ||||
-rw-r--r-- | trackerlogic.h | 19 |
19 files changed, 756 insertions, 414 deletions
@@ -18,11 +18,13 @@ LIBOWFAT_HEADERS=$(PREFIX)/libowfat | |||
18 | LIBOWFAT_LIBRARY=$(PREFIX)/libowfat | 18 | LIBOWFAT_LIBRARY=$(PREFIX)/libowfat |
19 | 19 | ||
20 | BINDIR?=$(PREFIX)/bin | 20 | BINDIR?=$(PREFIX)/bin |
21 | STRIP?=strip | ||
21 | 22 | ||
22 | #FEATURES+=-DWANT_V6 | 23 | #FEATURES+=-DWANT_V6 |
23 | 24 | ||
24 | #FEATURES+=-DWANT_ACCESSLIST_BLACK | 25 | #FEATURES+=-DWANT_ACCESSLIST_BLACK |
25 | #FEATURES+=-DWANT_ACCESSLIST_WHITE | 26 | #FEATURES+=-DWANT_ACCESSLIST_WHITE |
27 | #FEATURES+=-DWANT_DYNAMIC_ACCESSLIST | ||
26 | 28 | ||
27 | #FEATURES+=-DWANT_SYNC_LIVE | 29 | #FEATURES+=-DWANT_SYNC_LIVE |
28 | #FEATURES+=-DWANT_IP_FROM_QUERY_STRING | 30 | #FEATURES+=-DWANT_IP_FROM_QUERY_STRING |
@@ -39,6 +41,10 @@ BINDIR?=$(PREFIX)/bin | |||
39 | #FEATURES+=-DWANT_DEV_RANDOM | 41 | #FEATURES+=-DWANT_DEV_RANDOM |
40 | FEATURES+=-DWANT_FULLSCRAPE | 42 | FEATURES+=-DWANT_FULLSCRAPE |
41 | 43 | ||
44 | # Is enabled on BSD systems by default in trackerlogic.h | ||
45 | # on Linux systems you will need -lbds | ||
46 | #FEATURES+=-DWANT_ARC4RANDOM | ||
47 | |||
42 | #FEATURES+=-D_DEBUG_HTTPERROR | 48 | #FEATURES+=-D_DEBUG_HTTPERROR |
43 | 49 | ||
44 | OPTS_debug=-D_DEBUG -g -ggdb # -pg -fprofile-arcs -ftest-coverage | 50 | OPTS_debug=-D_DEBUG -g -ggdb # -pg -fprofile-arcs -ftest-coverage |
@@ -46,6 +52,7 @@ OPTS_production=-O3 | |||
46 | 52 | ||
47 | CFLAGS+=-I$(LIBOWFAT_HEADERS) -Wall -pipe -Wextra #-ansi -pedantic | 53 | CFLAGS+=-I$(LIBOWFAT_HEADERS) -Wall -pipe -Wextra #-ansi -pedantic |
48 | LDFLAGS+=-L$(LIBOWFAT_LIBRARY) -lowfat -pthread -lpthread -lz | 54 | LDFLAGS+=-L$(LIBOWFAT_LIBRARY) -lowfat -pthread -lpthread -lz |
55 | #LDFLAGS+=-lbsd | ||
49 | 56 | ||
50 | BINARY =opentracker | 57 | BINARY =opentracker |
51 | HEADERS=trackerlogic.h scan_urlencoded_query.h ot_mutex.h ot_stats.h ot_vector.h ot_clean.h ot_udp.h ot_iovec.h ot_fullscrape.h ot_accesslist.h ot_http.h ot_livesync.h ot_rijndael.h | 58 | HEADERS=trackerlogic.h scan_urlencoded_query.h ot_mutex.h ot_stats.h ot_vector.h ot_clean.h ot_udp.h ot_iovec.h ot_fullscrape.h ot_accesslist.h ot_http.h ot_livesync.h ot_rijndael.h |
@@ -66,7 +73,7 @@ CFLAGS_debug = $(CFLAGS) $(OPTS_debug) $(FEATURES) | |||
66 | 73 | ||
67 | $(BINARY): $(OBJECTS) $(HEADERS) | 74 | $(BINARY): $(OBJECTS) $(HEADERS) |
68 | $(CC) -o $@ $(OBJECTS) $(LDFLAGS) | 75 | $(CC) -o $@ $(OBJECTS) $(LDFLAGS) |
69 | strip $@ | 76 | $(STRIP) $@ |
70 | $(BINARY).debug: $(OBJECTS_debug) $(HEADERS) | 77 | $(BINARY).debug: $(OBJECTS_debug) $(HEADERS) |
71 | $(CC) -o $@ $(OBJECTS_debug) $(LDFLAGS) | 78 | $(CC) -o $@ $(OBJECTS_debug) $(LDFLAGS) |
72 | proxy: $(OBJECTS_proxy) $(HEADERS) | 79 | proxy: $(OBJECTS_proxy) $(HEADERS) |
@@ -84,4 +91,4 @@ clean: | |||
84 | rm -rf opentracker opentracker.debug *.o *~ | 91 | rm -rf opentracker opentracker.debug *.o *~ |
85 | 92 | ||
86 | install: | 93 | install: |
87 | install -m 755 opentracker $(BINDIR) | 94 | install -m 755 opentracker $(DESTDIR)$(BINDIR) |
diff --git a/opentracker.c b/opentracker.c index 5b9915f..2ca9e06 100644 --- a/opentracker.c +++ b/opentracker.c | |||
@@ -123,7 +123,7 @@ static void help( char *name ) { | |||
123 | HELPLINE("-P port","specify udp port to bind to (default: 6969, you may specify more than one)"); | 123 | HELPLINE("-P port","specify udp port to bind to (default: 6969, you may specify more than one)"); |
124 | HELPLINE("-r redirecturl","specify url where / should be redirected to (default none)"); | 124 | HELPLINE("-r redirecturl","specify url where / should be redirected to (default none)"); |
125 | HELPLINE("-d dir","specify directory to try to chroot to (default: \".\")"); | 125 | HELPLINE("-d dir","specify directory to try to chroot to (default: \".\")"); |
126 | HELPLINE("-u user","specify user under whose priviliges opentracker should run (default: \"nobody\")"); | 126 | HELPLINE("-u user","specify user under whose privileges opentracker should run (default: \"nobody\")"); |
127 | HELPLINE("-A ip","bless an ip address as admin address (e.g. to allow syncs from this address)"); | 127 | HELPLINE("-A ip","bless an ip address as admin address (e.g. to allow syncs from this address)"); |
128 | #ifdef WANT_ACCESSLIST_BLACK | 128 | #ifdef WANT_ACCESSLIST_BLACK |
129 | HELPLINE("-b file","specify blacklist file."); | 129 | HELPLINE("-b file","specify blacklist file."); |
@@ -135,8 +135,8 @@ static void help( char *name ) { | |||
135 | } | 135 | } |
136 | #undef HELPLINE | 136 | #undef HELPLINE |
137 | 137 | ||
138 | static size_t header_complete( char * request, ssize_t byte_count ) { | 138 | static ssize_t header_complete( char * request, ssize_t byte_count ) { |
139 | int i = 0, state = 0; | 139 | ssize_t i = 0, state = 0; |
140 | 140 | ||
141 | for( i=1; i < byte_count; i+=2 ) | 141 | for( i=1; i < byte_count; i+=2 ) |
142 | if( request[i] <= 13 ) { | 142 | if( request[i] <= 13 ) { |
@@ -156,7 +156,10 @@ static size_t header_complete( char * request, ssize_t byte_count ) { | |||
156 | static void handle_dead( const int64 sock ) { | 156 | static void handle_dead( const int64 sock ) { |
157 | struct http_data* cookie=io_getcookie( sock ); | 157 | struct http_data* cookie=io_getcookie( sock ); |
158 | if( cookie ) { | 158 | if( cookie ) { |
159 | iob_reset( &cookie->batch ); | 159 | size_t i; |
160 | for ( i = 0; i < cookie->batches; ++i) | ||
161 | iob_reset( cookie->batch + i ); | ||
162 | free( cookie->batch ); | ||
160 | array_reset( &cookie->request ); | 163 | array_reset( &cookie->request ); |
161 | if( cookie->flag & STRUCT_HTTP_FLAG_WAITINGFORTASK ) | 164 | if( cookie->flag & STRUCT_HTTP_FLAG_WAITINGFORTASK ) |
162 | mutex_workqueue_canceltask( sock ); | 165 | mutex_workqueue_canceltask( sock ); |
@@ -167,13 +170,16 @@ static void handle_dead( const int64 sock ) { | |||
167 | 170 | ||
168 | static void handle_read( const int64 sock, struct ot_workstruct *ws ) { | 171 | static void handle_read( const int64 sock, struct ot_workstruct *ws ) { |
169 | struct http_data* cookie = io_getcookie( sock ); | 172 | struct http_data* cookie = io_getcookie( sock ); |
170 | ssize_t byte_count; | 173 | ssize_t byte_count = io_tryread( sock, ws->inbuf, G_INBUF_SIZE ); |
171 | 174 | ||
172 | if( ( byte_count = io_tryread( sock, ws->inbuf, G_INBUF_SIZE ) ) <= 0 ) { | 175 | if( byte_count == 0 || byte_count == -3 ) { |
173 | handle_dead( sock ); | 176 | handle_dead( sock ); |
174 | return; | 177 | return; |
175 | } | 178 | } |
176 | 179 | ||
180 | if( byte_count == -1) | ||
181 | return; | ||
182 | |||
177 | /* If we get the whole request in one packet, handle it without copying */ | 183 | /* If we get the whole request in one packet, handle it without copying */ |
178 | if( !array_start( &cookie->request ) ) { | 184 | if( !array_start( &cookie->request ) ) { |
179 | if( ( ws->header_size = header_complete( ws->inbuf, byte_count ) ) ) { | 185 | if( ( ws->header_size = header_complete( ws->inbuf, byte_count ) ) ) { |
@@ -181,7 +187,7 @@ static void handle_read( const int64 sock, struct ot_workstruct *ws ) { | |||
181 | ws->request_size = byte_count; | 187 | ws->request_size = byte_count; |
182 | http_handle_request( sock, ws ); | 188 | http_handle_request( sock, ws ); |
183 | } else | 189 | } else |
184 | array_catb( &cookie->request, ws->inbuf, byte_count ); | 190 | array_catb( &cookie->request, ws->inbuf, (size_t)byte_count ); |
185 | return; | 191 | return; |
186 | } | 192 | } |
187 | 193 | ||
@@ -204,8 +210,25 @@ static void handle_read( const int64 sock, struct ot_workstruct *ws ) { | |||
204 | 210 | ||
205 | static void handle_write( const int64 sock ) { | 211 | static void handle_write( const int64 sock ) { |
206 | struct http_data* cookie=io_getcookie( sock ); | 212 | struct http_data* cookie=io_getcookie( sock ); |
207 | if( !cookie || ( iob_send( sock, &cookie->batch ) <= 0 ) ) | 213 | size_t i; |
208 | handle_dead( sock ); | 214 | |
215 | /* Look for the first io_batch still containing bytes to write */ | ||
216 | if( cookie ) | ||
217 | for( i = 0; i < cookie->batches; ++i ) | ||
218 | if( cookie->batch[i].bytesleft ) { | ||
219 | int64 res = iob_send( sock, cookie->batch + i ); | ||
220 | |||
221 | if( res == -3 ) | ||
222 | break; | ||
223 | |||
224 | if( !cookie->batch[i].bytesleft ) | ||
225 | continue; | ||
226 | |||
227 | if( res == -1 || res > 0 || i < cookie->batches - 1 ) | ||
228 | return; | ||
229 | } | ||
230 | |||
231 | handle_dead( sock ); | ||
209 | } | 232 | } |
210 | 233 | ||
211 | static void handle_accept( const int64 serversocket ) { | 234 | static void handle_accept( const int64 serversocket ) { |
@@ -239,6 +262,7 @@ static void handle_accept( const int64 serversocket ) { | |||
239 | tai_unix( &(t.sec), (g_now_seconds + OT_CLIENT_TIMEOUT) ); | 262 | tai_unix( &(t.sec), (g_now_seconds + OT_CLIENT_TIMEOUT) ); |
240 | io_timeout( sock, t ); | 263 | io_timeout( sock, t ); |
241 | } | 264 | } |
265 | io_eagain(serversocket); | ||
242 | } | 266 | } |
243 | 267 | ||
244 | static void * server_mainloop( void * args ) { | 268 | static void * server_mainloop( void * args ) { |
@@ -255,9 +279,18 @@ static void * server_mainloop( void * args ) { | |||
255 | #ifdef _DEBUG_HTTPERROR | 279 | #ifdef _DEBUG_HTTPERROR |
256 | ws.debugbuf= malloc( G_DEBUGBUF_SIZE ); | 280 | ws.debugbuf= malloc( G_DEBUGBUF_SIZE ); |
257 | #endif | 281 | #endif |
282 | |||
258 | if( !ws.inbuf || !ws.outbuf ) | 283 | if( !ws.inbuf || !ws.outbuf ) |
259 | panic( "Initializing worker failed" ); | 284 | panic( "Initializing worker failed" ); |
260 | 285 | ||
286 | #ifdef WANT_ARC4RANDOM | ||
287 | arc4random_buf(&ws.rand48_state[0], 3 * sizeof(uint16_t)); | ||
288 | #else | ||
289 | ws.rand48_state[0] = (uint16_t)random(); | ||
290 | ws.rand48_state[1] = (uint16_t)random(); | ||
291 | ws.rand48_state[2] = (uint16_t)random(); | ||
292 | #endif | ||
293 | |||
261 | for( ; ; ) { | 294 | for( ; ; ) { |
262 | int64 sock; | 295 | int64 sock; |
263 | 296 | ||
@@ -433,6 +466,12 @@ int parse_configfile( char * config_filename ) { | |||
433 | } else if(!byte_diff(p, 16, "access.blacklist" ) && isspace(p[16])) { | 466 | } else if(!byte_diff(p, 16, "access.blacklist" ) && isspace(p[16])) { |
434 | set_config_option( &g_accesslist_filename, p+17 ); | 467 | set_config_option( &g_accesslist_filename, p+17 ); |
435 | #endif | 468 | #endif |
469 | #ifdef WANT_DYNAMIC_ACCESSLIST | ||
470 | } else if(!byte_diff(p, 15, "access.fifo_add" ) && isspace(p[15])) { | ||
471 | set_config_option( &g_accesslist_pipe_add, p+16 ); | ||
472 | } else if(!byte_diff(p, 18, "access.fifo_delete" ) && isspace(p[18])) { | ||
473 | set_config_option( &g_accesslist_pipe_delete, p+19 ); | ||
474 | #endif | ||
436 | #ifdef WANT_RESTRICT_STATS | 475 | #ifdef WANT_RESTRICT_STATS |
437 | } else if(!byte_diff(p, 12, "access.stats" ) && isspace(p[12])) { | 476 | } else if(!byte_diff(p, 12, "access.stats" ) && isspace(p[12])) { |
438 | if( !scan_ip6( p+13, tmpip )) goto parse_error; | 477 | if( !scan_ip6( p+13, tmpip )) goto parse_error; |
@@ -529,12 +568,12 @@ int drop_privileges ( const char * const serveruser, const char * const serverdi | |||
529 | /* If we can't find server user, revert to nobody's default uid */ | 568 | /* If we can't find server user, revert to nobody's default uid */ |
530 | if( !pws ) { | 569 | if( !pws ) { |
531 | fprintf( stderr, "Warning: Could not get password entry for %s. Reverting to uid -2.\n", serveruser ); | 570 | fprintf( stderr, "Warning: Could not get password entry for %s. Reverting to uid -2.\n", serveruser ); |
532 | setegid( (gid_t)-2 ); setgid( (gid_t)-2 ); | 571 | if (setegid( (gid_t)-2 ) || setgid( (gid_t)-2 ) || setuid( (uid_t)-2 ) || seteuid( (uid_t)-2 )) |
533 | setuid( (uid_t)-2 ); seteuid( (uid_t)-2 ); | 572 | panic("Could not set uid to value -2"); |
534 | } | 573 | } |
535 | else { | 574 | else { |
536 | setegid( pws->pw_gid ); setgid( pws->pw_gid ); | 575 | if (setegid( pws->pw_gid ) || setgid( pws->pw_gid ) || setuid( pws->pw_uid ) || seteuid( pws->pw_uid )) |
537 | setuid( pws->pw_uid ); seteuid( pws->pw_uid ); | 576 | panic("Could not set uid to specified value"); |
538 | } | 577 | } |
539 | 578 | ||
540 | if( geteuid() == 0 || getegid() == 0 ) | 579 | if( geteuid() == 0 || getegid() == 0 ) |
@@ -640,6 +679,8 @@ int main( int argc, char **argv ) { | |||
640 | panic( "selfpipe failed: " ); | 679 | panic( "selfpipe failed: " ); |
641 | if( !io_fd( g_self_pipe[0] ) ) | 680 | if( !io_fd( g_self_pipe[0] ) ) |
642 | panic( "selfpipe io_fd failed: " ); | 681 | panic( "selfpipe io_fd failed: " ); |
682 | if( !io_fd( g_self_pipe[1] ) ) | ||
683 | panic( "selfpipe io_fd failed: " ); | ||
643 | io_setcookie( g_self_pipe[0], (void*)FLAG_SELFPIPE ); | 684 | io_setcookie( g_self_pipe[0], (void*)FLAG_SELFPIPE ); |
644 | io_wantread( g_self_pipe[0] ); | 685 | io_wantread( g_self_pipe[0] ); |
645 | 686 | ||
diff --git a/opentracker.conf.sample b/opentracker.conf.sample index db45122..d44f3d4 100644 --- a/opentracker.conf.sample +++ b/opentracker.conf.sample | |||
@@ -44,6 +44,43 @@ | |||
44 | # listing, so choose one of those options at compile time. File format | 44 | # listing, so choose one of those options at compile time. File format |
45 | # is straight forward: "<hex info hash>\n<hex info hash>\n..." | 45 | # is straight forward: "<hex info hash>\n<hex info hash>\n..." |
46 | # | 46 | # |
47 | # IIa) You can enable dynamic changesets to accesslists by enabling | ||
48 | # WANT_DYNAMIC_ACCESSLIST. | ||
49 | # | ||
50 | # The suggested way to work with dynamic changeset lists is to keep a | ||
51 | # main accesslist file that is loaded when opentracker (re)starts and | ||
52 | # reloaded infrequently (hourly or daily). | ||
53 | # | ||
54 | # All changes to the accesslist (e.g. from a web frontend) should be | ||
55 | # both appended to or removed from that file and sent to opentracker. By | ||
56 | # keeping dynamic changeset lists, you can avoid reloading huge | ||
57 | # accesslists whenever just a single entry is added or removed. | ||
58 | # | ||
59 | # Any info_hash (format see above) written to the fifo_add file will be | ||
60 | # kept on a dynamic add-changeset, removed from the dynamic | ||
61 | # delete-changeset and treated as if it was in the main accesslist file. | ||
62 | # The semantic of the respective dynamic changeset depends on whether | ||
63 | # WANT_ACCESSLIST_WHITE or WANT_ACCESSLIST_BLACK is enabled. | ||
64 | # | ||
65 | # access.fifo_add /var/run/opentracker/adder.fifo | ||
66 | # | ||
67 | # Any info_hash (format see above) written to the fifo_delete file will | ||
68 | # be kept on a dynamic delete-changeset, removed from the dynamic | ||
69 | # add-changeset and treated as if it was not in the main accesslist | ||
70 | # file. | ||
71 | # | ||
72 | # access.fifo_delete /var/run/opentracker/deleter.fifo | ||
73 | # | ||
74 | # If you reload the accesslist by sending SIGHUP to the tracker process, | ||
75 | # the dynamic lists are flushed, as opentracker assumes thoses lists are | ||
76 | # merged into the main accesslist. | ||
77 | # | ||
78 | # NOTE: While you can have multiple writers sending lines to the fifos, | ||
79 | # any writes larger than PIPE_BUF (see your limits.h, minimally 512 | ||
80 | # bytes but usually 4096) may be interleaved with data sent by other | ||
81 | # writers. This can lead to unparsable lines of info_hashes. | ||
82 | # | ||
83 | # IIb) | ||
47 | # If you do not want to grant anyone access to your stats, enable the | 84 | # If you do not want to grant anyone access to your stats, enable the |
48 | # WANT_RESTRICT_STATS option in Makefile and bless the ip addresses | 85 | # WANT_RESTRICT_STATS option in Makefile and bless the ip addresses |
49 | # allowed to fetch stats here. | 86 | # allowed to fetch stats here. |
@@ -55,6 +92,16 @@ | |||
55 | # appear anywhere on your tracker. | 92 | # appear anywhere on your tracker. |
56 | # | 93 | # |
57 | # access.stats_path stats | 94 | # access.stats_path stats |
95 | # | ||
96 | # IIc) | ||
97 | # If opentracker lives behind one or multiple reverse proxies, you can | ||
98 | # every http connection appears to come from these proxies. In order to | ||
99 | # take the X-Forwarded-For address instead, compile opentracker with the | ||
100 | # WANT_IP_FROM_PROXY option and set your proxy addresses here. | ||
101 | # | ||
102 | # access.proxy 10.0.1.23 | ||
103 | # access.proxy 10.0.1.24 | ||
104 | # | ||
58 | 105 | ||
59 | # III) Live sync uses udp multicast packets to keep a cluster of opentrackers | 106 | # III) Live sync uses udp multicast packets to keep a cluster of opentrackers |
60 | # synchronized. This option tells opentracker which port to listen for | 107 | # synchronized. This option tells opentracker which port to listen for |
diff --git a/ot_accesslist.c b/ot_accesslist.c index cdb964d..7df503f 100644 --- a/ot_accesslist.c +++ b/ot_accesslist.c | |||
@@ -10,6 +10,11 @@ | |||
10 | #include <stdio.h> | 10 | #include <stdio.h> |
11 | #include <signal.h> | 11 | #include <signal.h> |
12 | #include <unistd.h> | 12 | #include <unistd.h> |
13 | #ifdef WANT_DYNAMIC_ACCESSLIST | ||
14 | #include <sys/types.h> | ||
15 | #include <sys/stat.h> | ||
16 | #include <errno.h> | ||
17 | #endif | ||
13 | 18 | ||
14 | /* Libowfat */ | 19 | /* Libowfat */ |
15 | #include "byte.h" | 20 | #include "byte.h" |
@@ -24,20 +29,83 @@ | |||
24 | 29 | ||
25 | /* GLOBAL VARIABLES */ | 30 | /* GLOBAL VARIABLES */ |
26 | #ifdef WANT_ACCESSLIST | 31 | #ifdef WANT_ACCESSLIST |
27 | char *g_accesslist_filename; | 32 | char *g_accesslist_filename = NULL; |
28 | static ot_hash *g_accesslist; | 33 | #ifdef WANT_DYNAMIC_ACCESSLIST |
29 | static size_t g_accesslist_size; | 34 | char *g_accesslist_pipe_add = NULL; |
35 | char *g_accesslist_pipe_delete = NULL; | ||
36 | #endif | ||
30 | static pthread_mutex_t g_accesslist_mutex; | 37 | static pthread_mutex_t g_accesslist_mutex; |
31 | 38 | ||
39 | /* Accesslists are lock free linked lists. We can not make them locking, because every announce | ||
40 | would try to acquire the mutex, making it the most contested mutex in the whole of opentracker, | ||
41 | basically creating a central performance choke point. | ||
42 | |||
43 | The idea is that updating the list heads happens under the g_accesslist_mutex guard and is | ||
44 | done atomically, while consumers might potentially still hold pointers deeper inside the list. | ||
45 | |||
46 | Consumers (for now only via accesslist_hashisvalid) will always fetch the list head pointer | ||
47 | that is guaranteed to live for at least five minutes. This should be many orders of magnitudes | ||
48 | more than how long it will be needed by the bsearch done on the list. */ | ||
49 | struct ot_accesslist; | ||
50 | typedef struct ot_accesslist ot_accesslist; | ||
51 | struct ot_accesslist { | ||
52 | ot_hash *list; | ||
53 | size_t size; | ||
54 | ot_time base; | ||
55 | ot_accesslist *next; | ||
56 | }; | ||
57 | static ot_accesslist * _Atomic g_accesslist = NULL; | ||
58 | #ifdef WANT_DYNAMIC_ACCESSLIST | ||
59 | static ot_accesslist * _Atomic g_accesslist_add = NULL; | ||
60 | static ot_accesslist * _Atomic g_accesslist_delete = NULL; | ||
61 | #endif | ||
62 | |||
63 | /* Helpers to work on access lists */ | ||
32 | static int vector_compare_hash(const void *hash1, const void *hash2 ) { | 64 | static int vector_compare_hash(const void *hash1, const void *hash2 ) { |
33 | return memcmp( hash1, hash2, OT_HASH_COMPARE_SIZE ); | 65 | return memcmp( hash1, hash2, OT_HASH_COMPARE_SIZE ); |
34 | } | 66 | } |
35 | 67 | ||
68 | static ot_accesslist * accesslist_free(ot_accesslist *accesslist) { | ||
69 | while (accesslist) { | ||
70 | ot_accesslist * this_accesslist = accesslist; | ||
71 | accesslist = this_accesslist->next; | ||
72 | free(this_accesslist->list); | ||
73 | free(this_accesslist); | ||
74 | } | ||
75 | return NULL; | ||
76 | } | ||
77 | |||
78 | static ot_accesslist * accesslist_make(ot_accesslist *next, size_t size) { | ||
79 | ot_accesslist * accesslist_new = malloc(sizeof(ot_accesslist)); | ||
80 | if (accesslist_new) { | ||
81 | accesslist_new->list = size ? malloc(sizeof(ot_hash) * size) : NULL; | ||
82 | accesslist_new->size = size; | ||
83 | accesslist_new->base = g_now_minutes; | ||
84 | accesslist_new->next = next; | ||
85 | if (size && !accesslist_new->list) { | ||
86 | free(accesslist_new); | ||
87 | accesslist_new = NULL; | ||
88 | } | ||
89 | } | ||
90 | return accesslist_new; | ||
91 | } | ||
92 | |||
93 | /* This must be called with g_accesslist_mutex held. | ||
94 | This will never delete head, because that might still be in use. */ | ||
95 | static void accesslist_clean(ot_accesslist *accesslist) { | ||
96 | while (accesslist && accesslist->next) { | ||
97 | if (accesslist->next->base + 5 < g_now_minutes) | ||
98 | accesslist->next = accesslist_free(accesslist->next); | ||
99 | accesslist = accesslist->next; | ||
100 | } | ||
101 | } | ||
102 | |||
36 | /* Read initial access list */ | 103 | /* Read initial access list */ |
37 | static void accesslist_readfile( void ) { | 104 | static void accesslist_readfile( void ) { |
38 | ot_hash *info_hash, *accesslist_new = NULL; | 105 | ot_accesslist * accesslist_new; |
39 | char *map, *map_end, *read_offs; | 106 | ot_hash *info_hash; |
40 | size_t maplen; | 107 | const char *map, *map_end, *read_offs; |
108 | size_t maplen; | ||
41 | 109 | ||
42 | if( ( map = mmap_read( g_accesslist_filename, &maplen ) ) == NULL ) { | 110 | if( ( map = mmap_read( g_accesslist_filename, &maplen ) ) == NULL ) { |
43 | char *wd = getcwd( NULL, 0 ); | 111 | char *wd = getcwd( NULL, 0 ); |
@@ -48,11 +116,13 @@ static void accesslist_readfile( void ) { | |||
48 | 116 | ||
49 | /* You need at least 41 bytes to pass an info_hash, make enough room | 117 | /* You need at least 41 bytes to pass an info_hash, make enough room |
50 | for the maximum amount of them */ | 118 | for the maximum amount of them */ |
51 | info_hash = accesslist_new = malloc( ( maplen / 41 ) * 20 ); | 119 | accesslist_new = accesslist_make(g_accesslist, maplen / 41); |
52 | if( !accesslist_new ) { | 120 | if( !accesslist_new ) { |
53 | fprintf( stderr, "Warning: Not enough memory to allocate %zd bytes for accesslist buffer. May succeed later.\n", ( maplen / 41 ) * 20 ); | 121 | fprintf( stderr, "Warning: Not enough memory to allocate %zd bytes for accesslist buffer. May succeed later.\n", ( maplen / 41 ) * 20 ); |
122 | mmap_unmap( map, maplen); | ||
54 | return; | 123 | return; |
55 | } | 124 | } |
125 | info_hash = accesslist_new->list; | ||
56 | 126 | ||
57 | /* No use to scan if there's not enough room for another full info_hash */ | 127 | /* No use to scan if there's not enough room for another full info_hash */ |
58 | map_end = map + maplen - 40; | 128 | map_end = map + maplen - 40; |
@@ -62,18 +132,18 @@ static void accesslist_readfile( void ) { | |||
62 | while( read_offs <= map_end ) { | 132 | while( read_offs <= map_end ) { |
63 | int i; | 133 | int i; |
64 | for( i=0; i<(int)sizeof(ot_hash); ++i ) { | 134 | for( i=0; i<(int)sizeof(ot_hash); ++i ) { |
65 | int eger1 = scan_fromhex( read_offs[ 2*i ] ); | 135 | int eger1 = scan_fromhex( (unsigned char)read_offs[ 2*i ] ); |
66 | int eger2 = scan_fromhex( read_offs[ 1 + 2*i ] ); | 136 | int eger2 = scan_fromhex( (unsigned char)read_offs[ 1 + 2*i ] ); |
67 | if( eger1 < 0 || eger2 < 0 ) | 137 | if( eger1 < 0 || eger2 < 0 ) |
68 | break; | 138 | break; |
69 | (*info_hash)[i] = eger1 * 16 + eger2; | 139 | (*info_hash)[i] = (uint8_t)(eger1 * 16 + eger2); |
70 | } | 140 | } |
71 | 141 | ||
72 | if( i == sizeof(ot_hash) ) { | 142 | if( i == sizeof(ot_hash) ) { |
73 | read_offs += 40; | 143 | read_offs += 40; |
74 | 144 | ||
75 | /* Append accesslist to accesslist vector */ | 145 | /* Append accesslist to accesslist vector */ |
76 | if( read_offs == map_end || scan_fromhex( *read_offs ) < 0 ) | 146 | if( read_offs == map_end || scan_fromhex( (unsigned char)*read_offs ) < 0 ) |
77 | ++info_hash; | 147 | ++info_hash; |
78 | } | 148 | } |
79 | 149 | ||
@@ -81,28 +151,55 @@ static void accesslist_readfile( void ) { | |||
81 | while( read_offs <= map_end && *(read_offs++) != '\n' ); | 151 | while( read_offs <= map_end && *(read_offs++) != '\n' ); |
82 | } | 152 | } |
83 | #ifdef _DEBUG | 153 | #ifdef _DEBUG |
84 | fprintf( stderr, "Added %zd info_hashes to accesslist\n", (size_t)(info_hash - accesslist_new) ); | 154 | fprintf( stderr, "Added %zd info_hashes to accesslist\n", (size_t)(info_hash - accesslist_new->list) ); |
85 | #endif | 155 | #endif |
86 | 156 | ||
87 | mmap_unmap( map, maplen); | 157 | mmap_unmap( map, maplen); |
88 | 158 | ||
89 | qsort( accesslist_new, info_hash - accesslist_new, sizeof( *info_hash ), vector_compare_hash ); | 159 | qsort( accesslist_new->list, info_hash - accesslist_new->list, sizeof( *info_hash ), vector_compare_hash ); |
160 | accesslist_new->size = info_hash - accesslist_new->list; | ||
90 | 161 | ||
91 | /* Now exchange the accesslist vector in the least race condition prone way */ | 162 | /* Now exchange the accesslist vector in the least race condition prone way */ |
92 | pthread_mutex_lock(&g_accesslist_mutex); | 163 | pthread_mutex_lock(&g_accesslist_mutex); |
93 | free( g_accesslist ); | 164 | accesslist_new->next = g_accesslist; |
94 | g_accesslist = accesslist_new; | 165 | g_accesslist = accesslist_new; /* Only now set a new list */ |
95 | g_accesslist_size = info_hash - accesslist_new; | 166 | |
167 | #ifdef WANT_DYNAMIC_ACCESSLIST | ||
168 | /* If we have dynamic accesslists, reloading a new one will always void the add/delete lists. | ||
169 | Insert empty ones at the list head */ | ||
170 | if (g_accesslist_add && (accesslist_new = accesslist_make(g_accesslist_add, 0)) != NULL) | ||
171 | g_accesslist_add = accesslist_new; | ||
172 | if (g_accesslist_delete && (accesslist_new = accesslist_make(g_accesslist_delete, 0)) != NULL) | ||
173 | g_accesslist_delete = accesslist_new; | ||
174 | #endif | ||
175 | |||
176 | accesslist_clean(g_accesslist); | ||
177 | |||
96 | pthread_mutex_unlock(&g_accesslist_mutex); | 178 | pthread_mutex_unlock(&g_accesslist_mutex); |
97 | } | 179 | } |
98 | 180 | ||
99 | int accesslist_hashisvalid( ot_hash hash ) { | 181 | int accesslist_hashisvalid( ot_hash hash ) { |
100 | void *exactmatch; | 182 | /* Get working copy of current access list */ |
183 | ot_accesslist * accesslist = g_accesslist; | ||
184 | #ifdef WANT_DYNAMIC_ACCESSLIST | ||
185 | ot_accesslist * accesslist_add, * accesslist_delete; | ||
186 | #endif | ||
187 | void * exactmatch = NULL; | ||
101 | 188 | ||
102 | /* Lock should hardly ever be contended */ | 189 | if (accesslist) |
103 | pthread_mutex_lock(&g_accesslist_mutex); | 190 | exactmatch = bsearch( hash, accesslist->list, accesslist->size, OT_HASH_COMPARE_SIZE, vector_compare_hash ); |
104 | exactmatch = bsearch( hash, g_accesslist, g_accesslist_size, OT_HASH_COMPARE_SIZE, vector_compare_hash ); | 191 | |
105 | pthread_mutex_unlock(&g_accesslist_mutex); | 192 | #ifdef WANT_DYNAMIC_ACCESSLIST |
193 | /* If we had no match on the main list, scan the list of dynamically added hashes */ | ||
194 | accesslist_add = g_accesslist_add; | ||
195 | if ((exactmatch == NULL) && accesslist_add) | ||
196 | exactmatch = bsearch( hash, accesslist_add->list, accesslist_add->size, OT_HASH_COMPARE_SIZE, vector_compare_hash ); | ||
197 | |||
198 | /* If we found a matching hash on the main list, scan the list of dynamically deleted hashes */ | ||
199 | accesslist_delete = g_accesslist_delete; | ||
200 | if ((exactmatch != NULL) && accesslist_delete && bsearch( hash, accesslist_add->list, accesslist_add->size, OT_HASH_COMPARE_SIZE, vector_compare_hash )) | ||
201 | exactmatch = NULL; | ||
202 | #endif | ||
106 | 203 | ||
107 | #ifdef WANT_ACCESSLIST_BLACK | 204 | #ifdef WANT_ACCESSLIST_BLACK |
108 | return exactmatch == NULL; | 205 | return exactmatch == NULL; |
@@ -121,6 +218,8 @@ static void * accesslist_worker( void * args ) { | |||
121 | (void)args; | 218 | (void)args; |
122 | 219 | ||
123 | while( 1 ) { | 220 | while( 1 ) { |
221 | if (!g_opentracker_running) | ||
222 | return NULL; | ||
124 | 223 | ||
125 | /* Initial attempt to read accesslist */ | 224 | /* Initial attempt to read accesslist */ |
126 | accesslist_readfile( ); | 225 | accesslist_readfile( ); |
@@ -131,18 +230,156 @@ static void * accesslist_worker( void * args ) { | |||
131 | return NULL; | 230 | return NULL; |
132 | } | 231 | } |
133 | 232 | ||
233 | #ifdef WANT_DYNAMIC_ACCESSLIST | ||
234 | static pthread_t thread_adder_id, thread_deleter_id; | ||
235 | static void * accesslist_adddel_worker(char * fifoname, ot_accesslist * _Atomic * adding_to, ot_accesslist * _Atomic * removing_from) { | ||
236 | struct stat st; | ||
237 | |||
238 | if (!stat(fifoname, &st)) { | ||
239 | if (!S_ISFIFO(st.st_mode)) { | ||
240 | fprintf(stderr, "Error when starting dynamic accesslists: Found Non-FIFO file at %s.\nPlease remove it and restart opentracker.\n", fifoname); | ||
241 | return NULL; | ||
242 | } | ||
243 | } else { | ||
244 | int error = mkfifo(fifoname, 0755); | ||
245 | if (error && error != EEXIST) { | ||
246 | fprintf(stderr, "Error when starting dynamic accesslists: Couldn't create FIFO at %s, error: %s\n", fifoname, strerror(errno)); | ||
247 | return NULL; | ||
248 | } | ||
249 | } | ||
250 | |||
251 | while (g_opentracker_running) { | ||
252 | FILE * fifo = fopen(fifoname, "r"); | ||
253 | char *line = NULL; | ||
254 | size_t linecap = 0; | ||
255 | ssize_t linelen; | ||
256 | |||
257 | if (!fifo) { | ||
258 | fprintf(stderr, "Error when reading dynamic accesslists: Couldn't open FIFO at %s, error: %s\n", fifoname, strerror(errno)); | ||
259 | return NULL; | ||
260 | } | ||
261 | |||
262 | while ((linelen = getline(&line, &linecap, fifo)) > 0) { | ||
263 | ot_hash info_hash; | ||
264 | int i; | ||
265 | |||
266 | printf("Got line %*s", (int)linelen, line); | ||
267 | /* We do ignore anything that is not of the form "^[:xdigit:]{40}[^:xdigit:].*" | ||
268 | If there's not enough characters for an info_hash in the line, skip it. */ | ||
269 | if (linelen < 41) | ||
270 | continue; | ||
271 | |||
272 | for( i=0; i<(int)sizeof(ot_hash); ++i ) { | ||
273 | int eger1 = scan_fromhex( (unsigned char)line[ 2*i ] ); | ||
274 | int eger2 = scan_fromhex( (unsigned char)line[ 1 + 2*i ] ); | ||
275 | if( eger1 < 0 || eger2 < 0 ) | ||
276 | break; | ||
277 | ((uint8_t*)info_hash)[i] = (uint8_t)(eger1 * 16 + eger2); | ||
278 | } | ||
279 | printf("parsed info_hash %20s\n", info_hash); | ||
280 | if( i != sizeof(ot_hash) ) | ||
281 | continue; | ||
282 | |||
283 | /* From now on we modify g_accesslist_add and g_accesslist_delete, so prevent the | ||
284 | other worker threads from doing the same */ | ||
285 | pthread_mutex_lock(&g_accesslist_mutex); | ||
286 | |||
287 | /* If the info hash is in the removing_from list, create a new head without that entry */ | ||
288 | if (*removing_from && (*removing_from)->list) { | ||
289 | ot_hash * exactmatch = bsearch( info_hash, (*removing_from)->list, (*removing_from)->size, OT_HASH_COMPARE_SIZE, vector_compare_hash ); | ||
290 | if (exactmatch) { | ||
291 | ptrdiff_t off = exactmatch - (*removing_from)->list; | ||
292 | ot_accesslist * accesslist_new = accesslist_make(*removing_from, (*removing_from)->size - 1); | ||
293 | if (accesslist_new) { | ||
294 | memcpy(accesslist_new->list, (*removing_from)->list, sizeof(ot_hash) * off); | ||
295 | memcpy(accesslist_new->list + off, (*removing_from)->list + off + 1, (*removing_from)->size - off - 1); | ||
296 | *removing_from = accesslist_new; | ||
297 | } | ||
298 | } | ||
299 | } | ||
300 | |||
301 | /* Simple case: there's no adding_to list yet, create one with one member */ | ||
302 | if (!*adding_to) { | ||
303 | ot_accesslist * accesslist_new = accesslist_make(NULL, 1); | ||
304 | if (accesslist_new) { | ||
305 | memcpy(accesslist_new->list, info_hash, sizeof(ot_hash)); | ||
306 | *adding_to = accesslist_new; | ||
307 | } | ||
308 | } else { | ||
309 | int exactmatch = 0; | ||
310 | ot_hash * insert_point = binary_search( info_hash, (*adding_to)->list, (*adding_to)->size, OT_HASH_COMPARE_SIZE, sizeof(ot_hash), &exactmatch ); | ||
311 | |||
312 | /* Only if the info hash is not in the adding_to list, create a new head with that entry */ | ||
313 | if (!exactmatch) { | ||
314 | ot_accesslist * accesslist_new = accesslist_make(*adding_to, (*adding_to)->size + 1); | ||
315 | ptrdiff_t off = insert_point - (*adding_to)->list; | ||
316 | if (accesslist_new) { | ||
317 | memcpy(accesslist_new->list, (*adding_to)->list, sizeof(ot_hash) * off); | ||
318 | memcpy(accesslist_new->list + off, info_hash, sizeof(info_hash)); | ||
319 | memcpy(accesslist_new->list + off + 1, (*adding_to)->list + off, (*adding_to)->size - off); | ||
320 | *adding_to = accesslist_new; | ||
321 | } | ||
322 | } | ||
323 | } | ||
324 | |||
325 | pthread_mutex_unlock(&g_accesslist_mutex); | ||
326 | } | ||
327 | |||
328 | fclose(fifo); | ||
329 | } | ||
330 | return NULL; | ||
331 | } | ||
332 | |||
333 | static void * accesslist_adder_worker( void * args ) { | ||
334 | (void)args; | ||
335 | return accesslist_adddel_worker(g_accesslist_pipe_add, &g_accesslist_add, &g_accesslist_delete); | ||
336 | } | ||
337 | static void * accesslist_deleter_worker( void * args ) { | ||
338 | (void)args; | ||
339 | return accesslist_adddel_worker(g_accesslist_pipe_delete, &g_accesslist_delete, &g_accesslist_add); | ||
340 | } | ||
341 | #endif | ||
342 | |||
134 | static pthread_t thread_id; | 343 | static pthread_t thread_id; |
135 | void accesslist_init( ) { | 344 | void accesslist_init( ) { |
136 | pthread_mutex_init(&g_accesslist_mutex, NULL); | 345 | pthread_mutex_init(&g_accesslist_mutex, NULL); |
137 | pthread_create( &thread_id, NULL, accesslist_worker, NULL ); | 346 | pthread_create( &thread_id, NULL, accesslist_worker, NULL ); |
347 | #ifdef WANT_DYNAMIC_ACCESSLIST | ||
348 | if (g_accesslist_pipe_add) | ||
349 | pthread_create( &thread_adder_id, NULL, accesslist_adder_worker, NULL ); | ||
350 | if (g_accesslist_pipe_delete) | ||
351 | pthread_create( &thread_deleter_id, NULL, accesslist_deleter_worker, NULL ); | ||
352 | #endif | ||
138 | } | 353 | } |
139 | 354 | ||
140 | void accesslist_deinit( void ) { | 355 | void accesslist_deinit( void ) { |
356 | /* Wake up sleeping worker */ | ||
357 | pthread_kill(thread_id, SIGHUP); | ||
358 | |||
359 | pthread_mutex_lock(&g_accesslist_mutex); | ||
360 | |||
361 | g_accesslist = accesslist_free(g_accesslist); | ||
362 | |||
363 | #ifdef WANT_DYNAMIC_ACCESSLIST | ||
364 | g_accesslist_add = accesslist_free(g_accesslist_add); | ||
365 | g_accesslist_delete = accesslist_free(g_accesslist_delete); | ||
366 | #endif | ||
367 | |||
368 | pthread_mutex_unlock(&g_accesslist_mutex); | ||
141 | pthread_cancel( thread_id ); | 369 | pthread_cancel( thread_id ); |
142 | pthread_mutex_destroy(&g_accesslist_mutex); | 370 | pthread_mutex_destroy(&g_accesslist_mutex); |
143 | free( g_accesslist ); | 371 | } |
144 | g_accesslist = 0; | 372 | |
145 | g_accesslist_size = 0; | 373 | void accesslist_cleanup( void ) { |
374 | pthread_mutex_lock(&g_accesslist_mutex); | ||
375 | |||
376 | accesslist_clean(g_accesslist); | ||
377 | #if WANT_DYNAMIC_ACCESSLIST | ||
378 | accesslist_clean(g_accesslist_add); | ||
379 | accesslist_clean(g_accesslist_delete); | ||
380 | #endif | ||
381 | |||
382 | pthread_mutex_unlock(&g_accesslist_mutex); | ||
146 | } | 383 | } |
147 | #endif | 384 | #endif |
148 | 385 | ||
@@ -295,7 +532,7 @@ int accesslist_blessip( ot_ip6 ip, ot_permissions permissions ) { | |||
295 | if( permissions & OT_PERMISSION_MAY_PROXY ) off += snprintf( _debug+off, 512-off, " may_proxy" ); | 532 | if( permissions & OT_PERMISSION_MAY_PROXY ) off += snprintf( _debug+off, 512-off, " may_proxy" ); |
296 | if( !permissions ) off += snprintf( _debug+off, sizeof(_debug)-off, " nothing\n" ); | 533 | if( !permissions ) off += snprintf( _debug+off, sizeof(_debug)-off, " nothing\n" ); |
297 | _debug[off++] = '.'; | 534 | _debug[off++] = '.'; |
298 | write( 2, _debug, off ); | 535 | (void)write( 2, _debug, off ); |
299 | } | 536 | } |
300 | #endif | 537 | #endif |
301 | 538 | ||
diff --git a/ot_accesslist.h b/ot_accesslist.h index b38b91a..281f61b 100644 --- a/ot_accesslist.h +++ b/ot_accesslist.h | |||
@@ -6,19 +6,28 @@ | |||
6 | #ifndef OT_ACCESSLIST_H__ | 6 | #ifndef OT_ACCESSLIST_H__ |
7 | #define OT_ACCESSLIST_H__ | 7 | #define OT_ACCESSLIST_H__ |
8 | 8 | ||
9 | #if defined ( WANT_ACCESSLIST_BLACK ) && defined (WANT_ACCESSLIST_WHITE ) | 9 | #if defined ( WANT_ACCESSLIST_BLACK ) && defined ( WANT_ACCESSLIST_WHITE ) |
10 | # error WANT_ACCESSLIST_BLACK and WANT_ACCESSLIST_WHITE are exclusive. | 10 | # error WANT_ACCESSLIST_BLACK and WANT_ACCESSLIST_WHITE are exclusive. |
11 | #endif | 11 | #endif |
12 | 12 | ||
13 | #if defined ( WANT_ACCESSLIST_BLACK ) || defined (WANT_ACCESSLIST_WHITE ) | 13 | #if defined ( WANT_ACCESSLIST_BLACK ) || defined (WANT_ACCESSLIST_WHITE ) |
14 | #define WANT_ACCESSLIST | 14 | #define WANT_ACCESSLIST |
15 | void accesslist_init( ); | 15 | void accesslist_init( void ); |
16 | void accesslist_deinit( ); | 16 | void accesslist_deinit( void ); |
17 | int accesslist_hashisvalid( ot_hash hash ); | 17 | int accesslist_hashisvalid( ot_hash hash ); |
18 | void accesslist_cleanup( void ); | ||
18 | 19 | ||
19 | extern char *g_accesslist_filename; | 20 | extern char *g_accesslist_filename; |
21 | #ifdef WANT_DYNAMIC_ACCESSLIST | ||
22 | extern char *g_accesslist_pipe_add; | ||
23 | extern char *g_accesslist_pipe_delete; | ||
24 | #endif | ||
20 | 25 | ||
21 | #else | 26 | #else |
27 | #ifdef WANT_DYNAMIC_ACCESSLIST | ||
28 | # error WANT_DYNAMIC_ACCESSLIST needs either WANT_ACCESSLIST_BLACK or WANT_ACCESSLIST_WHITE | ||
29 | #endif | ||
30 | |||
22 | #define accesslist_init( accesslist_filename ) | 31 | #define accesslist_init( accesslist_filename ) |
23 | #define accesslist_deinit( ) | 32 | #define accesslist_deinit( ) |
24 | #define accesslist_hashisvalid( hash ) 1 | 33 | #define accesslist_hashisvalid( hash ) 1 |
@@ -17,6 +17,7 @@ | |||
17 | #include "ot_vector.h" | 17 | #include "ot_vector.h" |
18 | #include "ot_clean.h" | 18 | #include "ot_clean.h" |
19 | #include "ot_stats.h" | 19 | #include "ot_stats.h" |
20 | #include "ot_accesslist.h" | ||
20 | 21 | ||
21 | /* Returns amount of removed peers */ | 22 | /* Returns amount of removed peers */ |
22 | static ssize_t clean_single_bucket( ot_peer *peers, size_t peer_count, time_t timedout, int *removed_seeders ) { | 23 | static ssize_t clean_single_bucket( ot_peer *peers, size_t peer_count, time_t timedout, int *removed_seeders ) { |
@@ -123,6 +124,9 @@ static void * clean_worker( void * args ) { | |||
123 | usleep( OT_CLEAN_SLEEP ); | 124 | usleep( OT_CLEAN_SLEEP ); |
124 | } | 125 | } |
125 | stats_cleanup(); | 126 | stats_cleanup(); |
127 | #ifdef WANT_ACCESSLIST | ||
128 | accesslist_cleanup(); | ||
129 | #endif | ||
126 | } | 130 | } |
127 | return NULL; | 131 | return NULL; |
128 | } | 132 | } |
diff --git a/ot_fullscrape.c b/ot_fullscrape.c index faea4b9..5d115dc 100644 --- a/ot_fullscrape.c +++ b/ot_fullscrape.c | |||
@@ -30,21 +30,16 @@ | |||
30 | Full scrapes usually are huge and one does not want to | 30 | Full scrapes usually are huge and one does not want to |
31 | allocate more memory. So lets get them in 512k units | 31 | allocate more memory. So lets get them in 512k units |
32 | */ | 32 | */ |
33 | #define OT_SCRAPE_CHUNK_SIZE (512*1024) | 33 | #define OT_SCRAPE_CHUNK_SIZE (1024*1024) |
34 | 34 | ||
35 | /* "d8:completei%zde10:downloadedi%zde10:incompletei%zdee" */ | 35 | /* "d8:completei%zde10:downloadedi%zde10:incompletei%zdee" */ |
36 | #define OT_SCRAPE_MAXENTRYLEN 256 | 36 | #define OT_SCRAPE_MAXENTRYLEN 256 |
37 | 37 | ||
38 | #ifdef WANT_COMPRESSION_GZIP | ||
39 | #define IF_COMPRESSION( TASK ) if( mode & TASK_FLAG_GZIP ) TASK | ||
40 | #define WANT_COMPRESSION_GZIP_PARAM( param1, param2, param3 ) , param1, param2, param3 | ||
41 | #else | ||
42 | #define IF_COMPRESSION( TASK ) | ||
43 | #define WANT_COMPRESSION_GZIP_PARAM( param1, param2, param3 ) | ||
44 | #endif | ||
45 | |||
46 | /* Forward declaration */ | 38 | /* Forward declaration */ |
47 | static void fullscrape_make( int *iovec_entries, struct iovec **iovector, ot_tasktype mode ); | 39 | static void fullscrape_make( int *iovec_entries, struct iovec **iovector, ot_tasktype mode ); |
40 | #ifdef WANT_COMPRESSION_GZIP | ||
41 | static void fullscrape_make_gzip( int *iovec_entries, struct iovec **iovector, ot_tasktype mode ); | ||
42 | #endif | ||
48 | 43 | ||
49 | /* Converter function from memory to human readable hex strings | 44 | /* Converter function from memory to human readable hex strings |
50 | XXX - Duplicated from ot_stats. Needs fix. */ | 45 | XXX - Duplicated from ot_stats. Needs fix. */ |
@@ -59,14 +54,17 @@ static void * fullscrape_worker( void * args ) { | |||
59 | 54 | ||
60 | (void) args; | 55 | (void) args; |
61 | 56 | ||
62 | while( 1 ) { | 57 | while( g_opentracker_running ) { |
63 | ot_tasktype tasktype = TASK_FULLSCRAPE; | 58 | ot_tasktype tasktype = TASK_FULLSCRAPE; |
64 | ot_taskid taskid = mutex_workqueue_poptask( &tasktype ); | 59 | ot_taskid taskid = mutex_workqueue_poptask( &tasktype ); |
65 | fullscrape_make( &iovec_entries, &iovector, tasktype ); | 60 | #ifdef WANT_COMPRESSION_GZIP |
61 | if (tasktype & TASK_FLAG_GZIP) | ||
62 | fullscrape_make_gzip( &iovec_entries, &iovector, tasktype ); | ||
63 | else | ||
64 | #endif | ||
65 | fullscrape_make( &iovec_entries, &iovector, tasktype ); | ||
66 | if( mutex_workqueue_pushresult( taskid, iovec_entries, iovector ) ) | 66 | if( mutex_workqueue_pushresult( taskid, iovec_entries, iovector ) ) |
67 | iovec_free( &iovec_entries, &iovector ); | 67 | iovec_free( &iovec_entries, &iovector ); |
68 | if( !g_opentracker_running ) | ||
69 | return NULL; | ||
70 | } | 68 | } |
71 | return NULL; | 69 | return NULL; |
72 | } | 70 | } |
@@ -84,68 +82,57 @@ void fullscrape_deliver( int64 sock, ot_tasktype tasktype ) { | |||
84 | mutex_workqueue_pushtask( sock, tasktype ); | 82 | mutex_workqueue_pushtask( sock, tasktype ); |
85 | } | 83 | } |
86 | 84 | ||
87 | static int fullscrape_increase( int *iovec_entries, struct iovec **iovector, | 85 | static char * fullscrape_write_one( ot_tasktype mode, char *r, ot_peerlist *peer_list, ot_hash *hash ) { |
88 | char **r, char **re WANT_COMPRESSION_GZIP_PARAM( z_stream *strm, ot_tasktype mode, int zaction ) ) { | 86 | switch( mode & TASK_TASK_MASK ) { |
89 | /* Allocate a fresh output buffer at the end of our buffers list */ | 87 | case TASK_FULLSCRAPE: |
90 | if( !( *r = iovec_fix_increase_or_free( iovec_entries, iovector, *r, OT_SCRAPE_CHUNK_SIZE ) ) ) { | 88 | default: |
91 | 89 | /* push hash as bencoded string */ | |
92 | /* Deallocate gzip buffers */ | 90 | *r++='2'; *r++='0'; *r++=':'; |
93 | IF_COMPRESSION( deflateEnd(strm); ) | 91 | memcpy( r, hash, sizeof(ot_hash) ); r += sizeof(ot_hash); |
94 | 92 | /* push rest of the scrape string */ | |
95 | /* Release lock on current bucket and return */ | 93 | r += sprintf( r, "d8:completei%zde10:downloadedi%zde10:incompletei%zdee", peer_list->seed_count, peer_list->down_count, peer_list->peer_count-peer_list->seed_count ); |
96 | return -1; | 94 | |
97 | } | 95 | break; |
98 | 96 | case TASK_FULLSCRAPE_TPB_ASCII: | |
99 | /* Adjust new end of output buffer */ | 97 | to_hex( r, *hash ); r+= 2 * sizeof(ot_hash); |
100 | *re = *r + OT_SCRAPE_CHUNK_SIZE - OT_SCRAPE_MAXENTRYLEN; | 98 | r += sprintf( r, ":%zd:%zd\n", peer_list->seed_count, peer_list->peer_count-peer_list->seed_count ); |
101 | 99 | break; | |
102 | /* When compressing, we have all the bytes in output buffer */ | 100 | case TASK_FULLSCRAPE_TPB_ASCII_PLUS: |
103 | #ifdef WANT_COMPRESSION_GZIP | 101 | to_hex( r, *hash ); r+= 2 * sizeof(ot_hash); |
104 | if( mode & TASK_FLAG_GZIP ) { | 102 | r += sprintf( r, ":%zd:%zd:%zd\n", peer_list->seed_count, peer_list->peer_count-peer_list->seed_count, peer_list->down_count ); |
105 | int zres; | 103 | break; |
106 | *re -= OT_SCRAPE_MAXENTRYLEN; | 104 | case TASK_FULLSCRAPE_TPB_BINARY: |
107 | strm->next_out = (uint8_t*)*r; | 105 | memcpy( r, *hash, sizeof(ot_hash) ); r += sizeof(ot_hash); |
108 | strm->avail_out = OT_SCRAPE_CHUNK_SIZE; | 106 | *(uint32_t*)(r+0) = htonl( (uint32_t) peer_list->seed_count ); |
109 | zres = deflate( strm, zaction ); | 107 | *(uint32_t*)(r+4) = htonl( (uint32_t)( peer_list->peer_count-peer_list->seed_count) ); |
110 | if( ( zres < Z_OK ) && ( zres != Z_BUF_ERROR ) ) | 108 | r+=8; |
111 | fprintf( stderr, "deflate() failed while in fullscrape_increase(%d).\n", zaction ); | 109 | break; |
112 | *r = (char*)strm->next_out; | 110 | case TASK_FULLSCRAPE_TPB_URLENCODED: |
113 | } | 111 | r += fmt_urlencoded( r, (char *)*hash, 20 ); |
114 | #endif | 112 | r += sprintf( r, ":%zd:%zd\n", peer_list->seed_count, peer_list->peer_count-peer_list->seed_count ); |
115 | 113 | break; | |
116 | return 0; | 114 | case TASK_FULLSCRAPE_TRACKERSTATE: |
115 | to_hex( r, *hash ); r+= 2 * sizeof(ot_hash); | ||
116 | r += sprintf( r, ":%zd:%zd\n", peer_list->base, peer_list->down_count ); | ||
117 | break; | ||
118 | } | ||
119 | return r; | ||
117 | } | 120 | } |
118 | 121 | ||
119 | static void fullscrape_make( int *iovec_entries, struct iovec **iovector, ot_tasktype mode ) { | 122 | static void fullscrape_make( int *iovec_entries, struct iovec **iovector, ot_tasktype mode ) { |
120 | int bucket; | 123 | int bucket; |
121 | char *r, *re; | 124 | char *r, *re; |
122 | #ifdef WANT_COMPRESSION_GZIP | ||
123 | char compress_buffer[OT_SCRAPE_MAXENTRYLEN]; | ||
124 | z_stream strm; | ||
125 | #endif | ||
126 | 125 | ||
127 | /* Setup return vector... */ | 126 | /* Setup return vector... */ |
128 | *iovec_entries = 0; | 127 | *iovec_entries = 0; |
129 | *iovector = NULL; | 128 | *iovector = NULL; |
130 | if( !( r = iovec_increase( iovec_entries, iovector, OT_SCRAPE_CHUNK_SIZE ) ) ) | 129 | r = iovec_increase( iovec_entries, iovector, OT_SCRAPE_CHUNK_SIZE ); |
130 | if( !r ) | ||
131 | return; | 131 | return; |
132 | 132 | ||
133 | /* re points to low watermark */ | 133 | /* re points to low watermark */ |
134 | re = r + OT_SCRAPE_CHUNK_SIZE - OT_SCRAPE_MAXENTRYLEN; | 134 | re = r + OT_SCRAPE_CHUNK_SIZE - OT_SCRAPE_MAXENTRYLEN; |
135 | 135 | ||
136 | #ifdef WANT_COMPRESSION_GZIP | ||
137 | if( mode & TASK_FLAG_GZIP ) { | ||
138 | re += OT_SCRAPE_MAXENTRYLEN; | ||
139 | byte_zero( &strm, sizeof(strm) ); | ||
140 | strm.next_in = (uint8_t*)compress_buffer; | ||
141 | strm.next_out = (uint8_t*)r; | ||
142 | strm.avail_out = OT_SCRAPE_CHUNK_SIZE; | ||
143 | if( deflateInit2(&strm,7,Z_DEFLATED,31,8,Z_DEFAULT_STRATEGY) != Z_OK ) | ||
144 | fprintf( stderr, "not ok.\n" ); | ||
145 | r = compress_buffer; | ||
146 | } | ||
147 | #endif | ||
148 | |||
149 | if( ( mode & TASK_TASK_MASK ) == TASK_FULLSCRAPE ) | 136 | if( ( mode & TASK_TASK_MASK ) == TASK_FULLSCRAPE ) |
150 | r += sprintf( r, "d5:filesd" ); | 137 | r += sprintf( r, "d5:filesd" ); |
151 | 138 | ||
@@ -153,66 +140,99 @@ static void fullscrape_make( int *iovec_entries, struct iovec **iovector, ot_tas | |||
153 | for( bucket=0; bucket<OT_BUCKET_COUNT; ++bucket ) { | 140 | for( bucket=0; bucket<OT_BUCKET_COUNT; ++bucket ) { |
154 | /* Get exclusive access to that bucket */ | 141 | /* Get exclusive access to that bucket */ |
155 | ot_vector *torrents_list = mutex_bucket_lock( bucket ); | 142 | ot_vector *torrents_list = mutex_bucket_lock( bucket ); |
156 | size_t tor_offset; | 143 | ot_torrent *torrents = (ot_torrent*)(torrents_list->data); |
144 | size_t i; | ||
157 | 145 | ||
158 | /* For each torrent in this bucket.. */ | 146 | /* For each torrent in this bucket.. */ |
159 | for( tor_offset=0; tor_offset<torrents_list->size; ++tor_offset ) { | 147 | for( i=0; i<torrents_list->size; ++i ) { |
160 | /* Address torrents members */ | 148 | r = fullscrape_write_one( mode, r, torrents[i].peer_list, &torrents[i].hash ); |
161 | ot_peerlist *peer_list = ( ((ot_torrent*)(torrents_list->data))[tor_offset] ).peer_list; | 149 | |
162 | ot_hash *hash =&( ((ot_torrent*)(torrents_list->data))[tor_offset] ).hash; | 150 | if( r > re) { |
163 | 151 | /* Allocate a fresh output buffer at the end of our buffers list */ | |
164 | switch( mode & TASK_TASK_MASK ) { | 152 | r = iovec_fix_increase_or_free( iovec_entries, iovector, r, OT_SCRAPE_CHUNK_SIZE ); |
165 | case TASK_FULLSCRAPE: | 153 | if( !r ) |
166 | default: | 154 | return mutex_bucket_unlock( bucket, 0 ); |
167 | /* push hash as bencoded string */ | 155 | |
168 | *r++='2'; *r++='0'; *r++=':'; | 156 | /* re points to low watermark */ |
169 | memcpy( r, hash, sizeof(ot_hash) ); r += sizeof(ot_hash); | 157 | re = r + OT_SCRAPE_CHUNK_SIZE - OT_SCRAPE_MAXENTRYLEN; |
170 | /* push rest of the scrape string */ | ||
171 | r += sprintf( r, "d8:completei%zde10:downloadedi%zde10:incompletei%zdee", peer_list->seed_count, peer_list->down_count, peer_list->peer_count-peer_list->seed_count ); | ||
172 | |||
173 | break; | ||
174 | case TASK_FULLSCRAPE_TPB_ASCII: | ||
175 | to_hex( r, *hash ); r+= 2 * sizeof(ot_hash); | ||
176 | r += sprintf( r, ":%zd:%zd\n", peer_list->seed_count, peer_list->peer_count-peer_list->seed_count ); | ||
177 | break; | ||
178 | case TASK_FULLSCRAPE_TPB_ASCII_PLUS: | ||
179 | to_hex( r, *hash ); r+= 2 * sizeof(ot_hash); | ||
180 | r += sprintf( r, ":%zd:%zd:%zd\n", peer_list->seed_count, peer_list->peer_count-peer_list->seed_count, peer_list->down_count ); | ||
181 | break; | ||
182 | case TASK_FULLSCRAPE_TPB_BINARY: | ||
183 | memcpy( r, *hash, sizeof(ot_hash) ); r += sizeof(ot_hash); | ||
184 | *(uint32_t*)(r+0) = htonl( (uint32_t) peer_list->seed_count ); | ||
185 | *(uint32_t*)(r+4) = htonl( (uint32_t)( peer_list->peer_count-peer_list->seed_count) ); | ||
186 | r+=8; | ||
187 | break; | ||
188 | case TASK_FULLSCRAPE_TPB_URLENCODED: | ||
189 | r += fmt_urlencoded( r, (char *)*hash, 20 ); | ||
190 | r += sprintf( r, ":%zd:%zd\n", peer_list->seed_count, peer_list->peer_count-peer_list->seed_count ); | ||
191 | break; | ||
192 | case TASK_FULLSCRAPE_TRACKERSTATE: | ||
193 | to_hex( r, *hash ); r+= 2 * sizeof(ot_hash); | ||
194 | r += sprintf( r, ":%zd:%zd\n", peer_list->base, peer_list->down_count ); | ||
195 | break; | ||
196 | } | 158 | } |
159 | } | ||
160 | |||
161 | /* All torrents done: release lock on current bucket */ | ||
162 | mutex_bucket_unlock( bucket, 0 ); | ||
163 | |||
164 | /* Parent thread died? */ | ||
165 | if( !g_opentracker_running ) | ||
166 | return; | ||
167 | } | ||
168 | |||
169 | if( ( mode & TASK_TASK_MASK ) == TASK_FULLSCRAPE ) | ||
170 | r += sprintf( r, "ee" ); | ||
171 | |||
172 | /* Release unused memory in current output buffer */ | ||
173 | iovec_fixlast( iovec_entries, iovector, r ); | ||
174 | } | ||
197 | 175 | ||
198 | #ifdef WANT_COMPRESSION_GZIP | 176 | #ifdef WANT_COMPRESSION_GZIP |
199 | if( mode & TASK_FLAG_GZIP ) { | 177 | |
200 | int zres; | 178 | static void fullscrape_make_gzip( int *iovec_entries, struct iovec **iovector, ot_tasktype mode ) { |
201 | strm.next_in = (uint8_t*)compress_buffer; | 179 | int bucket; |
202 | strm.avail_in = r - compress_buffer; | 180 | char *r; |
181 | int zres; | ||
182 | z_stream strm; | ||
183 | |||
184 | /* Setup return vector... */ | ||
185 | *iovec_entries = 0; | ||
186 | *iovector = NULL; | ||
187 | r = iovec_increase( iovec_entries, iovector, OT_SCRAPE_CHUNK_SIZE ); | ||
188 | if( !r ) | ||
189 | return; | ||
190 | |||
191 | byte_zero( &strm, sizeof(strm) ); | ||
192 | strm.next_out = (uint8_t*)r; | ||
193 | strm.avail_out = OT_SCRAPE_CHUNK_SIZE; | ||
194 | if( deflateInit2(&strm,7,Z_DEFLATED,31,9,Z_DEFAULT_STRATEGY) != Z_OK ) | ||
195 | fprintf( stderr, "not ok.\n" ); | ||
196 | |||
197 | if( ( mode & TASK_TASK_MASK ) == TASK_FULLSCRAPE ) { | ||
198 | strm.next_in = (uint8_t*)"d5:filesd"; | ||
199 | strm.avail_in = strlen("d5:filesd"); | ||
200 | zres = deflate( &strm, Z_NO_FLUSH ); | ||
201 | } | ||
202 | |||
203 | /* For each bucket... */ | ||
204 | for( bucket=0; bucket<OT_BUCKET_COUNT; ++bucket ) { | ||
205 | /* Get exclusive access to that bucket */ | ||
206 | ot_vector *torrents_list = mutex_bucket_lock( bucket ); | ||
207 | ot_torrent *torrents = (ot_torrent*)(torrents_list->data); | ||
208 | size_t i; | ||
209 | |||
210 | /* For each torrent in this bucket.. */ | ||
211 | for( i=0; i<torrents_list->size; ++i ) { | ||
212 | char compress_buffer[OT_SCRAPE_MAXENTRYLEN]; | ||
213 | r = fullscrape_write_one( mode, compress_buffer, torrents[i].peer_list, &torrents[i].hash ); | ||
214 | strm.next_in = (uint8_t*)compress_buffer; | ||
215 | strm.avail_in = r - compress_buffer; | ||
216 | zres = deflate( &strm, Z_NO_FLUSH ); | ||
217 | if( ( zres < Z_OK ) && ( zres != Z_BUF_ERROR ) ) | ||
218 | fprintf( stderr, "deflate() failed while in fullscrape_make().\n" ); | ||
219 | |||
220 | /* Check if there still is enough buffer left */ | ||
221 | while( !strm.avail_out ) { | ||
222 | /* Allocate a fresh output buffer at the end of our buffers list */ | ||
223 | r = iovec_increase( iovec_entries, iovector, OT_SCRAPE_CHUNK_SIZE ); | ||
224 | if( !r ) { | ||
225 | fprintf( stderr, "Problem with iovec_fix_increase_or_free\n" ); | ||
226 | iovec_free( iovec_entries, iovector ); | ||
227 | deflateEnd(&strm); | ||
228 | return mutex_bucket_unlock( bucket, 0 ); | ||
229 | } | ||
230 | strm.next_out = (uint8_t*)r; | ||
231 | strm.avail_out = OT_SCRAPE_CHUNK_SIZE; | ||
203 | zres = deflate( &strm, Z_NO_FLUSH ); | 232 | zres = deflate( &strm, Z_NO_FLUSH ); |
204 | if( ( zres < Z_OK ) && ( zres != Z_BUF_ERROR ) ) | 233 | if( ( zres < Z_OK ) && ( zres != Z_BUF_ERROR ) ) |
205 | fprintf( stderr, "deflate() failed while in fullscrape_make().\n" ); | 234 | fprintf( stderr, "deflate() failed while in fullscrape_make().\n" ); |
206 | r = (char*)strm.next_out; | ||
207 | } | 235 | } |
208 | #endif | ||
209 | |||
210 | /* Check if there still is enough buffer left */ | ||
211 | while( r >= re ) | ||
212 | if( fullscrape_increase( iovec_entries, iovector, &r, &re WANT_COMPRESSION_GZIP_PARAM( &strm, mode, Z_NO_FLUSH ) ) ) | ||
213 | return mutex_bucket_unlock( bucket, 0 ); | ||
214 | |||
215 | IF_COMPRESSION( r = compress_buffer; ) | ||
216 | } | 236 | } |
217 | 237 | ||
218 | /* All torrents done: release lock on current bucket */ | 238 | /* All torrents done: release lock on current bucket */ |
@@ -223,27 +243,41 @@ static void fullscrape_make( int *iovec_entries, struct iovec **iovector, ot_tas | |||
223 | return; | 243 | return; |
224 | } | 244 | } |
225 | 245 | ||
226 | if( ( mode & TASK_TASK_MASK ) == TASK_FULLSCRAPE ) | 246 | if( ( mode & TASK_TASK_MASK ) == TASK_FULLSCRAPE ) { |
227 | r += sprintf( r, "ee" ); | 247 | strm.next_in = (uint8_t*)"ee"; |
248 | strm.avail_in = strlen("ee"); | ||
249 | } | ||
228 | 250 | ||
229 | #ifdef WANT_COMPRESSION_GZIP | 251 | if( deflate( &strm, Z_FINISH ) < Z_OK ) |
230 | if( mode & TASK_FLAG_GZIP ) { | 252 | fprintf( stderr, "deflate() failed while in fullscrape_make()'s endgame.\n" ); |
231 | strm.next_in = (uint8_t*)compress_buffer; | 253 | |
232 | strm.avail_in = r - compress_buffer; | 254 | if( !strm.avail_out ) { |
255 | unsigned int pending; | ||
256 | int bits; | ||
257 | deflatePending( &strm, &pending, &bits); | ||
258 | pending += ( bits ? 1 : 0 ); | ||
259 | |||
260 | /* Allocate a fresh output buffer at the end of our buffers list */ | ||
261 | r = iovec_fix_increase_or_free( iovec_entries, iovector, strm.next_out, pending ); | ||
262 | if( !r ) { | ||
263 | fprintf( stderr, "Problem with iovec_fix_increase_or_free\n" ); | ||
264 | deflateEnd(&strm); | ||
265 | return mutex_bucket_unlock( bucket, 0 ); | ||
266 | } | ||
267 | strm.next_out = (uint8_t*)r; | ||
268 | strm.avail_out = pending; | ||
233 | if( deflate( &strm, Z_FINISH ) < Z_OK ) | 269 | if( deflate( &strm, Z_FINISH ) < Z_OK ) |
234 | fprintf( stderr, "deflate() failed while in fullscrape_make()'s endgame.\n" ); | 270 | fprintf( stderr, "deflate() failed while in fullscrape_make()'s endgame.\n" ); |
235 | r = (char*)strm.next_out; | ||
236 | |||
237 | while( r >= re ) | ||
238 | if( fullscrape_increase( iovec_entries, iovector, &r, &re WANT_COMPRESSION_GZIP_PARAM( &strm, mode, Z_FINISH ) ) ) | ||
239 | return mutex_bucket_unlock( bucket, 0 ); | ||
240 | deflateEnd(&strm); | ||
241 | } | 271 | } |
242 | #endif | ||
243 | 272 | ||
244 | /* Release unused memory in current output buffer */ | 273 | /* Release unused memory in current output buffer */ |
245 | iovec_fixlast( iovec_entries, iovector, r ); | 274 | iovec_fixlast( iovec_entries, iovector, strm.next_out ); |
275 | |||
276 | deflateEnd(&strm); | ||
246 | } | 277 | } |
278 | /* WANT_COMPRESSION_GZIP */ | ||
247 | #endif | 279 | #endif |
248 | 280 | ||
281 | /* WANT_FULLSCRAPE */ | ||
282 | #endif | ||
249 | const char *g_version_fullscrape_c = "$Source$: $Revision$\n"; | 283 | const char *g_version_fullscrape_c = "$Source$: $Revision$\n"; |
@@ -31,6 +31,7 @@ | |||
31 | #include "ot_accesslist.h" | 31 | #include "ot_accesslist.h" |
32 | 32 | ||
33 | #define OT_MAXMULTISCRAPE_COUNT 64 | 33 | #define OT_MAXMULTISCRAPE_COUNT 64 |
34 | #define OT_BATCH_LIMIT (1024*1024*16) | ||
34 | extern char *g_redirecturl; | 35 | extern char *g_redirecturl; |
35 | 36 | ||
36 | char *g_stats_path; | 37 | char *g_stats_path; |
@@ -75,7 +76,13 @@ static void http_senddata( const int64 sock, struct ot_workstruct *ws ) { | |||
75 | } | 76 | } |
76 | 77 | ||
77 | memcpy( outbuf, ws->reply + written_size, ws->reply_size - written_size ); | 78 | memcpy( outbuf, ws->reply + written_size, ws->reply_size - written_size ); |
78 | iob_addbuf_free( &cookie->batch, outbuf, ws->reply_size - written_size ); | 79 | if ( !cookie->batch ) { |
80 | cookie->batch = malloc( sizeof(io_batch) ); | ||
81 | memset( cookie->batch, 0, sizeof(io_batch) ); | ||
82 | cookie->batches = 1; | ||
83 | } | ||
84 | |||
85 | iob_addbuf_free( cookie->batch, outbuf, ws->reply_size - written_size ); | ||
79 | 86 | ||
80 | /* writeable short data sockets just have a tcp timeout */ | 87 | /* writeable short data sockets just have a tcp timeout */ |
81 | if( !ws->keep_alive ) { | 88 | if( !ws->keep_alive ) { |
@@ -118,7 +125,7 @@ ssize_t http_sendiovecdata( const int64 sock, struct ot_workstruct *ws, int iove | |||
118 | struct http_data *cookie = io_getcookie( sock ); | 125 | struct http_data *cookie = io_getcookie( sock ); |
119 | char *header; | 126 | char *header; |
120 | int i; | 127 | int i; |
121 | size_t header_size, size = iovec_length( &iovec_entries, &iovector ); | 128 | size_t header_size, size = iovec_length( &iovec_entries, (const struct iovec **)&iovector ); |
122 | tai6464 t; | 129 | tai6464 t; |
123 | 130 | ||
124 | /* No cookie? Bad socket. Leave. */ | 131 | /* No cookie? Bad socket. Leave. */ |
@@ -152,12 +159,29 @@ ssize_t http_sendiovecdata( const int64 sock, struct ot_workstruct *ws, int iove | |||
152 | else | 159 | else |
153 | header_size = sprintf( header, "HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\nContent-Length: %zd\r\n\r\n", size ); | 160 | header_size = sprintf( header, "HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\nContent-Length: %zd\r\n\r\n", size ); |
154 | 161 | ||
155 | iob_reset( &cookie->batch ); | 162 | if (!cookie->batch ) { |
156 | iob_addbuf_free( &cookie->batch, header, header_size ); | 163 | cookie->batch = malloc( sizeof(io_batch) ); |
164 | memset( cookie->batch, 0, sizeof(io_batch) ); | ||
165 | cookie->batches = 1; | ||
166 | } | ||
167 | iob_addbuf_free( cookie->batch, header, header_size ); | ||
168 | |||
169 | /* Split huge iovectors into separate io_batches */ | ||
170 | for( i=0; i<iovec_entries; ++i ) { | ||
171 | io_batch *current = cookie->batch + cookie->batches - 1; | ||
172 | |||
173 | /* If the current batch's limit is reached, try to reallocate a new batch to work on */ | ||
174 | if( current->bytesleft > OT_BATCH_LIMIT ) { | ||
175 | io_batch * new_batch = realloc( current, (cookie->batches + 1) * sizeof(io_batch) ); | ||
176 | if( new_batch ) { | ||
177 | cookie->batches++; | ||
178 | current = cookie->batch = new_batch; | ||
179 | memset( current, 0, sizeof(io_batch) ); | ||
180 | } | ||
181 | } | ||
157 | 182 | ||
158 | /* Will move to ot_iovec.c */ | 183 | iob_addbuf_free( current, iovector[i].iov_base, iovector[i].iov_len ); |
159 | for( i=0; i<iovec_entries; ++i ) | 184 | } |
160 | iob_addbuf_munmap( &cookie->batch, iovector[i].iov_base, iovector[i].iov_len ); | ||
161 | free( iovector ); | 185 | free( iovector ); |
162 | 186 | ||
163 | /* writeable sockets timeout after 10 minutes */ | 187 | /* writeable sockets timeout after 10 minutes */ |
@@ -170,7 +194,7 @@ ssize_t http_sendiovecdata( const int64 sock, struct ot_workstruct *ws, int iove | |||
170 | 194 | ||
171 | static ssize_t http_handle_stats( const int64 sock, struct ot_workstruct *ws, char *read_ptr ) { | 195 | static ssize_t http_handle_stats( const int64 sock, struct ot_workstruct *ws, char *read_ptr ) { |
172 | static const ot_keywords keywords_main[] = | 196 | static const ot_keywords keywords_main[] = |
173 | { { "mode", 1 }, {"format", 2 }, { NULL, -3 } }; | 197 | { { "mode", 1 }, {"format", 2 }, {"info_hash", 3}, { NULL, -3 } }; |
174 | static const ot_keywords keywords_mode[] = | 198 | static const ot_keywords keywords_mode[] = |
175 | { { "peer", TASK_STATS_PEERS }, { "conn", TASK_STATS_CONNS }, { "scrp", TASK_STATS_SCRAPE }, { "udp4", TASK_STATS_UDP }, { "tcp4", TASK_STATS_TCP }, | 199 | { { "peer", TASK_STATS_PEERS }, { "conn", TASK_STATS_CONNS }, { "scrp", TASK_STATS_SCRAPE }, { "udp4", TASK_STATS_UDP }, { "tcp4", TASK_STATS_TCP }, |
176 | { "busy", TASK_STATS_BUSY_NETWORKS }, { "torr", TASK_STATS_TORRENTS }, { "fscr", TASK_STATS_FULLSCRAPE }, | 200 | { "busy", TASK_STATS_BUSY_NETWORKS }, { "torr", TASK_STATS_TORRENTS }, { "fscr", TASK_STATS_FULLSCRAPE }, |
@@ -206,6 +230,7 @@ static const ot_keywords keywords_format[] = | |||
206 | case 2: /* matched "format" */ | 230 | case 2: /* matched "format" */ |
207 | if( ( format = scan_find_keywords( keywords_format, &read_ptr, SCAN_SEARCHPATH_VALUE ) ) <= 0 ) HTTPERROR_400_PARAM; | 231 | if( ( format = scan_find_keywords( keywords_format, &read_ptr, SCAN_SEARCHPATH_VALUE ) ) <= 0 ) HTTPERROR_400_PARAM; |
208 | break; | 232 | break; |
233 | case 3: HTTPERROR_400_PARAM; /* If the stats URL was mistakenly added as announce URL, return a 400 */ | ||
209 | } | 234 | } |
210 | } | 235 | } |
211 | 236 | ||
@@ -497,8 +522,8 @@ static ssize_t http_handle_announce( const int64 sock, struct ot_workstruct *ws, | |||
497 | return ws->reply_size = sprintf( ws->reply, "Successfully added.\n" ); | 522 | return ws->reply_size = sprintf( ws->reply, "Successfully added.\n" ); |
498 | //} | 523 | //} |
499 | } | 524 | } |
500 | #endif | ||
501 | break; | 525 | break; |
526 | #endif | ||
502 | case 9: /* matched "peer_id" */ | 527 | case 9: /* matched "peer_id" */ |
503 | /* ignore this, when we have less than 20 bytes */ | 528 | /* ignore this, when we have less than 20 bytes */ |
504 | if( scan_urlencoded_query( &read_ptr, write_ptr = read_ptr, SCAN_SEARCHPATH_VALUE ) != 20 ) HTTPERROR_400_PARAM; | 529 | if( scan_urlencoded_query( &read_ptr, write_ptr = read_ptr, SCAN_SEARCHPATH_VALUE ) != 20 ) HTTPERROR_400_PARAM; |
@@ -14,7 +14,8 @@ typedef enum { | |||
14 | 14 | ||
15 | struct http_data { | 15 | struct http_data { |
16 | array request; | 16 | array request; |
17 | io_batch batch; | 17 | io_batch *batch; |
18 | size_t batches; | ||
18 | ot_ip6 ip; | 19 | ot_ip6 ip; |
19 | STRUCT_HTTP_FLAG flag; | 20 | STRUCT_HTTP_FLAG flag; |
20 | }; | 21 | }; |
@@ -5,7 +5,6 @@ | |||
5 | 5 | ||
6 | /* System */ | 6 | /* System */ |
7 | #include <sys/types.h> | 7 | #include <sys/types.h> |
8 | #include <sys/mman.h> | ||
9 | #include <stdlib.h> | 8 | #include <stdlib.h> |
10 | #include <unistd.h> | 9 | #include <unistd.h> |
11 | #include <sys/uio.h> | 10 | #include <sys/uio.h> |
@@ -16,56 +15,57 @@ | |||
16 | #include "ot_iovec.h" | 15 | #include "ot_iovec.h" |
17 | 16 | ||
18 | void *iovec_increase( int *iovec_entries, struct iovec **iovector, size_t new_alloc ) { | 17 | void *iovec_increase( int *iovec_entries, struct iovec **iovector, size_t new_alloc ) { |
19 | void *new_ptr = realloc( *iovector, (1 + *iovec_entries ) * sizeof( struct iovec ) ); | 18 | void *new_data; |
20 | if( !new_ptr ) | 19 | int new_entries = 1 + *iovec_entries; |
20 | struct iovec *new_vec = realloc( *iovector, new_entries * sizeof( struct iovec ) ); | ||
21 | |||
22 | if( !new_vec ) | ||
21 | return NULL; | 23 | return NULL; |
22 | *iovector = new_ptr; | 24 | |
23 | new_ptr = mmap( NULL, new_alloc, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0 ); | 25 | /* Only allocate after we have a place to store the pointer */ |
24 | if( !new_ptr ) | 26 | new_data = malloc( new_alloc ); |
27 | if( !new_data ) | ||
25 | return NULL; | 28 | return NULL; |
26 | ((*iovector)[*iovec_entries]).iov_base = new_ptr; | 29 | |
27 | ((*iovector)[*iovec_entries]).iov_len = new_alloc; | 30 | new_vec[new_entries - 1].iov_base = new_data; |
31 | new_vec[new_entries - 1].iov_len = new_alloc; | ||
32 | |||
33 | *iovector = new_vec; | ||
28 | ++*iovec_entries; | 34 | ++*iovec_entries; |
29 | return new_ptr; | 35 | return new_data; |
30 | } | 36 | } |
31 | 37 | ||
32 | void iovec_free( int *iovec_entries, struct iovec **iovector ) { | 38 | void iovec_free( int *iovec_entries, struct iovec **iovector ) { |
33 | int i; | 39 | int i; |
34 | for( i=0; i<*iovec_entries; ++i ) | 40 | for( i=0; i<*iovec_entries; ++i ) |
35 | munmap( ((*iovector)[i]).iov_base, ((*iovector)[i]).iov_len ); | 41 | free( ((*iovector)[i]).iov_base ); |
42 | *iovector = NULL; | ||
36 | *iovec_entries = 0; | 43 | *iovec_entries = 0; |
37 | } | 44 | } |
38 | 45 | ||
39 | void iovec_fixlast( int *iovec_entries, struct iovec **iovector, void *last_ptr ) { | 46 | void iovec_fixlast( int *iovec_entries, struct iovec **iovector, void *last_ptr ) { |
40 | int page_size = getpagesize(); | 47 | if( *iovec_entries ) { |
41 | size_t old_alloc, new_alloc, old_pages, new_pages; | 48 | char * base = (char*)((*iovector)[ *iovec_entries - 1 ]).iov_base; |
42 | char * base = (char*)((*iovector)[ *iovec_entries - 1 ]).iov_base; | 49 | size_t new_alloc = ((char*)last_ptr) - base; |
43 | |||
44 | if( !*iovec_entries ) return; | ||
45 | |||
46 | old_alloc = ((*iovector)[ *iovec_entries - 1 ]).iov_len; | ||
47 | new_alloc = ((char*)last_ptr) - base; | ||
48 | old_pages = 1 + old_alloc / page_size; | ||
49 | new_pages = 1 + new_alloc / page_size; | ||
50 | 50 | ||
51 | if( old_pages != new_pages ) | 51 | ((*iovector)[*iovec_entries - 1 ]).iov_base = realloc( base, new_alloc ); |
52 | munmap( base + new_pages * page_size, old_alloc - new_pages * page_size ); | 52 | ((*iovector)[*iovec_entries - 1 ]).iov_len = new_alloc; |
53 | ((*iovector)[*iovec_entries - 1 ]).iov_len = new_alloc; | 53 | } |
54 | } | 54 | } |
55 | 55 | ||
56 | void *iovec_fix_increase_or_free( int *iovec_entries, struct iovec **iovector, void *last_ptr, size_t new_alloc ) { | 56 | void *iovec_fix_increase_or_free( int *iovec_entries, struct iovec **iovector, void *last_ptr, size_t new_alloc ) { |
57 | void *new_ptr; | 57 | void *new_data; |
58 | 58 | ||
59 | iovec_fixlast( iovec_entries, iovector, last_ptr ); | 59 | iovec_fixlast( iovec_entries, iovector, last_ptr ); |
60 | 60 | ||
61 | if( !( new_ptr = iovec_increase( iovec_entries, iovector, new_alloc ) ) ) | 61 | if( !( new_data = iovec_increase( iovec_entries, iovector, new_alloc ) ) ) |
62 | iovec_free( iovec_entries, iovector ); | 62 | iovec_free( iovec_entries, iovector ); |
63 | 63 | ||
64 | return new_ptr; | 64 | return new_data; |
65 | } | 65 | } |
66 | 66 | ||
67 | 67 | ||
68 | size_t iovec_length( int *iovec_entries, struct iovec **iovector ) { | 68 | size_t iovec_length( const int *iovec_entries, const struct iovec **iovector ) { |
69 | size_t length = 0; | 69 | size_t length = 0; |
70 | int i; | 70 | int i; |
71 | for( i=0; i<*iovec_entries; ++i ) | 71 | for( i=0; i<*iovec_entries; ++i ) |
@@ -12,7 +12,7 @@ void *iovec_increase( int *iovec_entries, struct iovec **iovector, size_t new_a | |||
12 | void iovec_fixlast( int *iovec_entries, struct iovec **iovector, void *last_ptr ); | 12 | void iovec_fixlast( int *iovec_entries, struct iovec **iovector, void *last_ptr ); |
13 | void iovec_free( int *iovec_entries, struct iovec **iovector ); | 13 | void iovec_free( int *iovec_entries, struct iovec **iovector ); |
14 | 14 | ||
15 | size_t iovec_length( int *iovec_entries, struct iovec **iovector ); | 15 | size_t iovec_length( const int *iovec_entries, const struct iovec **iovector ); |
16 | 16 | ||
17 | void *iovec_fix_increase_or_free( int *iovec_entries, struct iovec **iovector, void *last_ptr, size_t new_alloc ); | 17 | void *iovec_fix_increase_or_free( int *iovec_entries, struct iovec **iovector, void *last_ptr, size_t new_alloc ); |
18 | 18 | ||
@@ -25,65 +25,14 @@ | |||
25 | 25 | ||
26 | /* Our global all torrents list */ | 26 | /* Our global all torrents list */ |
27 | static ot_vector all_torrents[OT_BUCKET_COUNT]; | 27 | static ot_vector all_torrents[OT_BUCKET_COUNT]; |
28 | static pthread_mutex_t bucket_mutex[OT_BUCKET_COUNT]; | ||
28 | static size_t g_torrent_count; | 29 | static size_t g_torrent_count; |
29 | 30 | ||
30 | /* Bucket Magic */ | ||
31 | static int bucket_locklist[ OT_MAX_THREADS ]; | ||
32 | static int bucket_locklist_count = 0; | ||
33 | static pthread_mutex_t bucket_mutex; | ||
34 | static pthread_cond_t bucket_being_unlocked; | ||
35 | |||
36 | /* Self pipe from opentracker.c */ | 31 | /* Self pipe from opentracker.c */ |
37 | extern int g_self_pipe[2]; | 32 | extern int g_self_pipe[2]; |
38 | 33 | ||
39 | static int bucket_check( int bucket ) { | ||
40 | /* C should come with auto-i ;) */ | ||
41 | int i; | ||
42 | |||
43 | /* No more space to acquire lock to bucket -- should not happen */ | ||
44 | if( bucket_locklist_count == OT_MAX_THREADS ) { | ||
45 | fprintf( stderr, "More lock requests than mutexes. Consult source code.\n" ); | ||
46 | return -1; | ||
47 | } | ||
48 | |||
49 | /* See, if bucket is already locked */ | ||
50 | for( i=0; i<bucket_locklist_count; ++i ) | ||
51 | if( bucket_locklist[ i ] == bucket ) { | ||
52 | stats_issue_event( EVENT_BUCKET_LOCKED, 0, 0 ); | ||
53 | return -1; | ||
54 | } | ||
55 | |||
56 | return 0; | ||
57 | } | ||
58 | |||
59 | static void bucket_push( int bucket ) { | ||
60 | bucket_locklist[ bucket_locklist_count++ ] = bucket; | ||
61 | } | ||
62 | |||
63 | static void bucket_remove( int bucket ) { | ||
64 | int i = 0; | ||
65 | |||
66 | while( ( i < bucket_locklist_count ) && ( bucket_locklist[ i ] != bucket ) ) | ||
67 | ++i; | ||
68 | |||
69 | if( i == bucket_locklist_count ) { | ||
70 | fprintf( stderr, "Request to unlock bucket that was never locked. Consult source code.\n" ); | ||
71 | return; | ||
72 | } | ||
73 | |||
74 | for( ; i < bucket_locklist_count - 1; ++i ) | ||
75 | bucket_locklist[ i ] = bucket_locklist[ i + 1 ]; | ||
76 | |||
77 | --bucket_locklist_count; | ||
78 | } | ||
79 | |||
80 | /* Can block */ | ||
81 | ot_vector *mutex_bucket_lock( int bucket ) { | 34 | ot_vector *mutex_bucket_lock( int bucket ) { |
82 | pthread_mutex_lock( &bucket_mutex ); | 35 | pthread_mutex_lock(bucket_mutex + bucket ); |
83 | while( bucket_check( bucket ) ) | ||
84 | pthread_cond_wait( &bucket_being_unlocked, &bucket_mutex ); | ||
85 | bucket_push( bucket ); | ||
86 | pthread_mutex_unlock( &bucket_mutex ); | ||
87 | return all_torrents + bucket; | 36 | return all_torrents + bucket; |
88 | } | 37 | } |
89 | 38 | ||
@@ -92,11 +41,8 @@ ot_vector *mutex_bucket_lock_by_hash( ot_hash hash ) { | |||
92 | } | 41 | } |
93 | 42 | ||
94 | void mutex_bucket_unlock( int bucket, int delta_torrentcount ) { | 43 | void mutex_bucket_unlock( int bucket, int delta_torrentcount ) { |
95 | pthread_mutex_lock( &bucket_mutex ); | 44 | pthread_mutex_unlock(bucket_mutex + bucket); |
96 | bucket_remove( bucket ); | ||
97 | g_torrent_count += delta_torrentcount; | 45 | g_torrent_count += delta_torrentcount; |
98 | pthread_cond_broadcast( &bucket_being_unlocked ); | ||
99 | pthread_mutex_unlock( &bucket_mutex ); | ||
100 | } | 46 | } |
101 | 47 | ||
102 | void mutex_bucket_unlock_by_hash( ot_hash hash, int delta_torrentcount ) { | 48 | void mutex_bucket_unlock_by_hash( ot_hash hash, int delta_torrentcount ) { |
@@ -104,11 +50,7 @@ void mutex_bucket_unlock_by_hash( ot_hash hash, int delta_torrentcount ) { | |||
104 | } | 50 | } |
105 | 51 | ||
106 | size_t mutex_get_torrent_count( ) { | 52 | size_t mutex_get_torrent_count( ) { |
107 | size_t torrent_count; | 53 | return g_torrent_count; |
108 | pthread_mutex_lock( &bucket_mutex ); | ||
109 | torrent_count = g_torrent_count; | ||
110 | pthread_mutex_unlock( &bucket_mutex ); | ||
111 | return torrent_count; | ||
112 | } | 54 | } |
113 | 55 | ||
114 | /* TaskQueue Magic */ | 56 | /* TaskQueue Magic */ |
@@ -130,24 +72,9 @@ static pthread_cond_t tasklist_being_filled; | |||
130 | int mutex_workqueue_pushtask( int64 sock, ot_tasktype tasktype ) { | 72 | int mutex_workqueue_pushtask( int64 sock, ot_tasktype tasktype ) { |
131 | struct ot_task ** tmptask, * task; | 73 | struct ot_task ** tmptask, * task; |
132 | 74 | ||
133 | /* Want exclusive access to tasklist */ | ||
134 | MTX_DBG( "pushtask locks.\n" ); | ||
135 | pthread_mutex_lock( &tasklist_mutex ); | ||
136 | MTX_DBG( "pushtask locked.\n" ); | ||
137 | |||
138 | task = malloc(sizeof( struct ot_task)); | 75 | task = malloc(sizeof( struct ot_task)); |
139 | if( !task ) { | 76 | if( !task ) |
140 | MTX_DBG( "pushtask fail unlocks.\n" ); | ||
141 | pthread_mutex_unlock( &tasklist_mutex ); | ||
142 | MTX_DBG( "pushtask fail unlocked.\n" ); | ||
143 | return -1; | 77 | return -1; |
144 | } | ||
145 | |||
146 | /* Skip to end of list */ | ||
147 | tmptask = &tasklist; | ||
148 | while( *tmptask ) | ||
149 | tmptask = &(*tmptask)->next; | ||
150 | *tmptask = task; | ||
151 | 78 | ||
152 | task->taskid = 0; | 79 | task->taskid = 0; |
153 | task->tasktype = tasktype; | 80 | task->tasktype = tasktype; |
@@ -156,12 +83,18 @@ int mutex_workqueue_pushtask( int64 sock, ot_tasktype tasktype ) { | |||
156 | task->iovec = NULL; | 83 | task->iovec = NULL; |
157 | task->next = 0; | 84 | task->next = 0; |
158 | 85 | ||
86 | /* Want exclusive access to tasklist */ | ||
87 | pthread_mutex_lock( &tasklist_mutex ); | ||
88 | |||
89 | /* Skip to end of list */ | ||
90 | tmptask = &tasklist; | ||
91 | while( *tmptask ) | ||
92 | tmptask = &(*tmptask)->next; | ||
93 | *tmptask = task; | ||
94 | |||
159 | /* Inform waiting workers and release lock */ | 95 | /* Inform waiting workers and release lock */ |
160 | MTX_DBG( "pushtask broadcasts.\n" ); | ||
161 | pthread_cond_broadcast( &tasklist_being_filled ); | 96 | pthread_cond_broadcast( &tasklist_being_filled ); |
162 | MTX_DBG( "pushtask broadcasted, mutex unlocks.\n" ); | ||
163 | pthread_mutex_unlock( &tasklist_mutex ); | 97 | pthread_mutex_unlock( &tasklist_mutex ); |
164 | MTX_DBG( "pushtask end mutex unlocked.\n" ); | ||
165 | return 0; | 98 | return 0; |
166 | } | 99 | } |
167 | 100 | ||
@@ -169,31 +102,25 @@ void mutex_workqueue_canceltask( int64 sock ) { | |||
169 | struct ot_task ** task; | 102 | struct ot_task ** task; |
170 | 103 | ||
171 | /* Want exclusive access to tasklist */ | 104 | /* Want exclusive access to tasklist */ |
172 | MTX_DBG( "canceltask locks.\n" ); | ||
173 | pthread_mutex_lock( &tasklist_mutex ); | 105 | pthread_mutex_lock( &tasklist_mutex ); |
174 | MTX_DBG( "canceltask locked.\n" ); | ||
175 | |||
176 | task = &tasklist; | ||
177 | while( *task && ( (*task)->sock != sock ) ) | ||
178 | *task = (*task)->next; | ||
179 | 106 | ||
180 | if( *task && ( (*task)->sock == sock ) ) { | 107 | for (task = &tasklist; *task; task = &((*task)->next)) |
181 | struct iovec *iovec = (*task)->iovec; | 108 | if ((*task)->sock == sock) { |
182 | struct ot_task *ptask = *task; | 109 | struct iovec *iovec = (*task)->iovec; |
183 | int i; | 110 | struct ot_task *ptask = *task; |
111 | int i; | ||
184 | 112 | ||
185 | /* Free task's iovec */ | 113 | /* Free task's iovec */ |
186 | for( i=0; i<(*task)->iovec_entries; ++i ) | 114 | for( i=0; i<(*task)->iovec_entries; ++i ) |
187 | munmap( iovec[i].iov_base, iovec[i].iov_len ); | 115 | free( iovec[i].iov_base ); |
188 | 116 | ||
189 | *task = (*task)->next; | 117 | *task = (*task)->next; |
190 | free( ptask ); | 118 | free( ptask ); |
191 | } | 119 | break; |
120 | } | ||
192 | 121 | ||
193 | /* Release lock */ | 122 | /* Release lock */ |
194 | MTX_DBG( "canceltask unlocks.\n" ); | ||
195 | pthread_mutex_unlock( &tasklist_mutex ); | 123 | pthread_mutex_unlock( &tasklist_mutex ); |
196 | MTX_DBG( "canceltask unlocked.\n" ); | ||
197 | } | 124 | } |
198 | 125 | ||
199 | ot_taskid mutex_workqueue_poptask( ot_tasktype *tasktype ) { | 126 | ot_taskid mutex_workqueue_poptask( ot_tasktype *tasktype ) { |
@@ -201,33 +128,26 @@ ot_taskid mutex_workqueue_poptask( ot_tasktype *tasktype ) { | |||
201 | ot_taskid taskid = 0; | 128 | ot_taskid taskid = 0; |
202 | 129 | ||
203 | /* Want exclusive access to tasklist */ | 130 | /* Want exclusive access to tasklist */ |
204 | MTX_DBG( "poptask mutex locks.\n" ); | ||
205 | pthread_mutex_lock( &tasklist_mutex ); | 131 | pthread_mutex_lock( &tasklist_mutex ); |
206 | MTX_DBG( "poptask mutex locked.\n" ); | ||
207 | 132 | ||
208 | while( !taskid ) { | 133 | while( !taskid ) { |
209 | /* Skip to the first unassigned task this worker wants to do */ | 134 | /* Skip to the first unassigned task this worker wants to do */ |
210 | task = tasklist; | 135 | for (task = tasklist; task; task = task->next) |
211 | while( task && ( ( ( TASK_CLASS_MASK & task->tasktype ) != *tasktype ) || task->taskid ) ) | 136 | if (!task->taskid && ( TASK_CLASS_MASK & task->tasktype ) == *tasktype) { |
212 | task = task->next; | 137 | /* If we found an outstanding task, assign a taskid to it |
213 | 138 | and leave the loop */ | |
214 | /* If we found an outstanding task, assign a taskid to it | 139 | task->taskid = taskid = ++next_free_taskid; |
215 | and leave the loop */ | 140 | *tasktype = task->tasktype; |
216 | if( task ) { | 141 | break; |
217 | task->taskid = taskid = ++next_free_taskid; | 142 | } |
218 | *tasktype = task->tasktype; | 143 | |
219 | } else { | 144 | /* Wait until the next task is being fed */ |
220 | /* Wait until the next task is being fed */ | 145 | if (!taskid) |
221 | MTX_DBG( "poptask cond waits.\n" ); | ||
222 | pthread_cond_wait( &tasklist_being_filled, &tasklist_mutex ); | 146 | pthread_cond_wait( &tasklist_being_filled, &tasklist_mutex ); |
223 | MTX_DBG( "poptask cond waited.\n" ); | ||
224 | } | ||
225 | } | 147 | } |
226 | 148 | ||
227 | /* Release lock */ | 149 | /* Release lock */ |
228 | MTX_DBG( "poptask end mutex unlocks.\n" ); | ||
229 | pthread_mutex_unlock( &tasklist_mutex ); | 150 | pthread_mutex_unlock( &tasklist_mutex ); |
230 | MTX_DBG( "poptask end mutex unlocked.\n" ); | ||
231 | 151 | ||
232 | return taskid; | 152 | return taskid; |
233 | } | 153 | } |
@@ -236,24 +156,18 @@ void mutex_workqueue_pushsuccess( ot_taskid taskid ) { | |||
236 | struct ot_task ** task; | 156 | struct ot_task ** task; |
237 | 157 | ||
238 | /* Want exclusive access to tasklist */ | 158 | /* Want exclusive access to tasklist */ |
239 | MTX_DBG( "pushsuccess locks.\n" ); | ||
240 | pthread_mutex_lock( &tasklist_mutex ); | 159 | pthread_mutex_lock( &tasklist_mutex ); |
241 | MTX_DBG( "pushsuccess locked.\n" ); | ||
242 | 160 | ||
243 | task = &tasklist; | 161 | for (task = &tasklist; *task; task = &((*task)->next)) |
244 | while( *task && ( (*task)->taskid != taskid ) ) | 162 | if ((*task)->taskid == taskid) { |
245 | *task = (*task)->next; | 163 | struct ot_task *ptask = *task; |
246 | 164 | *task = (*task)->next; | |
247 | if( *task && ( (*task)->taskid == taskid ) ) { | 165 | free( ptask ); |
248 | struct ot_task *ptask = *task; | 166 | break; |
249 | *task = (*task)->next; | 167 | } |
250 | free( ptask ); | ||
251 | } | ||
252 | 168 | ||
253 | /* Release lock */ | 169 | /* Release lock */ |
254 | MTX_DBG( "pushsuccess unlocks.\n" ); | ||
255 | pthread_mutex_unlock( &tasklist_mutex ); | 170 | pthread_mutex_unlock( &tasklist_mutex ); |
256 | MTX_DBG( "pushsuccess unlocked.\n" ); | ||
257 | } | 171 | } |
258 | 172 | ||
259 | int mutex_workqueue_pushresult( ot_taskid taskid, int iovec_entries, struct iovec *iovec ) { | 173 | int mutex_workqueue_pushresult( ot_taskid taskid, int iovec_entries, struct iovec *iovec ) { |
@@ -261,24 +175,18 @@ int mutex_workqueue_pushresult( ot_taskid taskid, int iovec_entries, struct iove | |||
261 | const char byte = 'o'; | 175 | const char byte = 'o'; |
262 | 176 | ||
263 | /* Want exclusive access to tasklist */ | 177 | /* Want exclusive access to tasklist */ |
264 | MTX_DBG( "pushresult locks.\n" ); | ||
265 | pthread_mutex_lock( &tasklist_mutex ); | 178 | pthread_mutex_lock( &tasklist_mutex ); |
266 | MTX_DBG( "pushresult locked.\n" ); | ||
267 | |||
268 | task = tasklist; | ||
269 | while( task && ( task->taskid != taskid ) ) | ||
270 | task = task->next; | ||
271 | 179 | ||
272 | if( task ) { | 180 | for (task = tasklist; task; task = task->next) |
273 | task->iovec_entries = iovec_entries; | 181 | if (task->taskid == taskid) { |
274 | task->iovec = iovec; | 182 | task->iovec_entries = iovec_entries; |
275 | task->tasktype = TASK_DONE; | 183 | task->iovec = iovec; |
276 | } | 184 | task->tasktype = TASK_DONE; |
185 | break; | ||
186 | } | ||
277 | 187 | ||
278 | /* Release lock */ | 188 | /* Release lock */ |
279 | MTX_DBG( "pushresult unlocks.\n" ); | ||
280 | pthread_mutex_unlock( &tasklist_mutex ); | 189 | pthread_mutex_unlock( &tasklist_mutex ); |
281 | MTX_DBG( "pushresult unlocked.\n" ); | ||
282 | 190 | ||
283 | io_trywrite( g_self_pipe[1], &byte, 1 ); | 191 | io_trywrite( g_self_pipe[1], &byte, 1 ); |
284 | 192 | ||
@@ -291,43 +199,39 @@ int64 mutex_workqueue_popresult( int *iovec_entries, struct iovec ** iovec ) { | |||
291 | int64 sock = -1; | 199 | int64 sock = -1; |
292 | 200 | ||
293 | /* Want exclusive access to tasklist */ | 201 | /* Want exclusive access to tasklist */ |
294 | MTX_DBG( "popresult locks.\n" ); | ||
295 | pthread_mutex_lock( &tasklist_mutex ); | 202 | pthread_mutex_lock( &tasklist_mutex ); |
296 | MTX_DBG( "popresult locked.\n" ); | ||
297 | |||
298 | task = &tasklist; | ||
299 | while( *task && ( (*task)->tasktype != TASK_DONE ) ) | ||
300 | task = &(*task)->next; | ||
301 | 203 | ||
302 | if( *task && ( (*task)->tasktype == TASK_DONE ) ) { | 204 | for (task = &tasklist; *task; task = &((*task)->next)) |
303 | struct ot_task *ptask = *task; | 205 | if ((*task)->tasktype == TASK_DONE) { |
206 | struct ot_task *ptask = *task; | ||
304 | 207 | ||
305 | *iovec_entries = (*task)->iovec_entries; | 208 | *iovec_entries = (*task)->iovec_entries; |
306 | *iovec = (*task)->iovec; | 209 | *iovec = (*task)->iovec; |
307 | sock = (*task)->sock; | 210 | sock = (*task)->sock; |
308 | 211 | ||
309 | *task = (*task)->next; | 212 | *task = (*task)->next; |
310 | free( ptask ); | 213 | free( ptask ); |
311 | } | 214 | break; |
215 | } | ||
312 | 216 | ||
313 | /* Release lock */ | 217 | /* Release lock */ |
314 | MTX_DBG( "popresult unlocks.\n" ); | ||
315 | pthread_mutex_unlock( &tasklist_mutex ); | 218 | pthread_mutex_unlock( &tasklist_mutex ); |
316 | MTX_DBG( "popresult unlocked.\n" ); | ||
317 | return sock; | 219 | return sock; |
318 | } | 220 | } |
319 | 221 | ||
320 | void mutex_init( ) { | 222 | void mutex_init( ) { |
223 | int i; | ||
321 | pthread_mutex_init(&tasklist_mutex, NULL); | 224 | pthread_mutex_init(&tasklist_mutex, NULL); |
322 | pthread_cond_init (&tasklist_being_filled, NULL); | 225 | pthread_cond_init (&tasklist_being_filled, NULL); |
323 | pthread_mutex_init(&bucket_mutex, NULL); | 226 | for (i=0; i < OT_BUCKET_COUNT; ++i) |
324 | pthread_cond_init (&bucket_being_unlocked, NULL); | 227 | pthread_mutex_init(bucket_mutex + i, NULL); |
325 | byte_zero( all_torrents, sizeof( all_torrents ) ); | 228 | byte_zero( all_torrents, sizeof( all_torrents ) ); |
326 | } | 229 | } |
327 | 230 | ||
328 | void mutex_deinit( ) { | 231 | void mutex_deinit( ) { |
329 | pthread_mutex_destroy(&bucket_mutex); | 232 | int i; |
330 | pthread_cond_destroy(&bucket_being_unlocked); | 233 | for (i=0; i < OT_BUCKET_COUNT; ++i) |
234 | pthread_mutex_destroy(bucket_mutex + i); | ||
331 | pthread_mutex_destroy(&tasklist_mutex); | 235 | pthread_mutex_destroy(&tasklist_mutex); |
332 | pthread_cond_destroy(&tasklist_being_filled); | 236 | pthread_cond_destroy(&tasklist_being_filled); |
333 | byte_zero( all_torrents, sizeof( all_torrents ) ); | 237 | byte_zero( all_torrents, sizeof( all_torrents ) ); |
@@ -8,8 +8,8 @@ | |||
8 | 8 | ||
9 | #include <sys/uio.h> | 9 | #include <sys/uio.h> |
10 | 10 | ||
11 | void mutex_init( ); | 11 | void mutex_init( void ); |
12 | void mutex_deinit( ); | 12 | void mutex_deinit( void ); |
13 | 13 | ||
14 | ot_vector *mutex_bucket_lock( int bucket ); | 14 | ot_vector *mutex_bucket_lock( int bucket ); |
15 | ot_vector *mutex_bucket_lock_by_hash( ot_hash hash ); | 15 | ot_vector *mutex_bucket_lock_by_hash( ot_hash hash ); |
@@ -17,7 +17,7 @@ ot_vector *mutex_bucket_lock_by_hash( ot_hash hash ); | |||
17 | void mutex_bucket_unlock( int bucket, int delta_torrentcount ); | 17 | void mutex_bucket_unlock( int bucket, int delta_torrentcount ); |
18 | void mutex_bucket_unlock_by_hash( ot_hash hash, int delta_torrentcount ); | 18 | void mutex_bucket_unlock_by_hash( ot_hash hash, int delta_torrentcount ); |
19 | 19 | ||
20 | size_t mutex_get_torrent_count(); | 20 | size_t mutex_get_torrent_count(void); |
21 | 21 | ||
22 | typedef enum { | 22 | typedef enum { |
23 | TASK_STATS_CONNS = 0x0001, | 23 | TASK_STATS_CONNS = 0x0001, |
@@ -293,7 +293,7 @@ static int torrent_statter( ot_torrent *torrent, uintptr_t data ) { | |||
293 | /* Converter function from memory to human readable hex strings */ | 293 | /* Converter function from memory to human readable hex strings */ |
294 | static char*to_hex(char*d,uint8_t*s){char*m="0123456789ABCDEF";char *t=d;char*e=d+40;while(d<e){*d++=m[*s>>4];*d++=m[*s++&15];}*d=0;return t;} | 294 | static char*to_hex(char*d,uint8_t*s){char*m="0123456789ABCDEF";char *t=d;char*e=d+40;while(d<e){*d++=m[*s>>4];*d++=m[*s++&15];}*d=0;return t;} |
295 | 295 | ||
296 | typedef struct { size_t val; ot_torrent * torrent; } ot_record; | 296 | typedef struct { size_t val; ot_hash hash; } ot_record; |
297 | 297 | ||
298 | /* Fetches stats from tracker */ | 298 | /* Fetches stats from tracker */ |
299 | size_t stats_top_txt( char * reply, int amount ) { | 299 | size_t stats_top_txt( char * reply, int amount ) { |
@@ -311,18 +311,22 @@ size_t stats_top_txt( char * reply, int amount ) { | |||
311 | for( bucket=0; bucket<OT_BUCKET_COUNT; ++bucket ) { | 311 | for( bucket=0; bucket<OT_BUCKET_COUNT; ++bucket ) { |
312 | ot_vector *torrents_list = mutex_bucket_lock( bucket ); | 312 | ot_vector *torrents_list = mutex_bucket_lock( bucket ); |
313 | for( j=0; j<torrents_list->size; ++j ) { | 313 | for( j=0; j<torrents_list->size; ++j ) { |
314 | ot_peerlist *peer_list = ( ((ot_torrent*)(torrents_list->data))[j] ).peer_list; | 314 | ot_torrent *torrent = (ot_torrent*)(torrents_list->data) + j; |
315 | int idx = amount - 1; while( (idx >= 0) && ( peer_list->peer_count > top100c[idx].val ) ) --idx; | 315 | idx = amount - 1; |
316 | while( (idx >= 0) && ( torrent->peer_list->peer_count > top100c[idx].val ) ) | ||
317 | --idx; | ||
316 | if ( idx++ != amount - 1 ) { | 318 | if ( idx++ != amount - 1 ) { |
317 | memmove( top100c + idx + 1, top100c + idx, ( amount - 1 - idx ) * sizeof( ot_record ) ); | 319 | memmove( top100c + idx + 1, top100c + idx, ( amount - 1 - idx ) * sizeof( ot_record ) ); |
318 | top100c[idx].val = peer_list->peer_count; | 320 | memcpy( &top100c[idx].hash, &torrent->hash, sizeof(ot_hash)); |
319 | top100c[idx].torrent = (ot_torrent*)(torrents_list->data) + j; | 321 | top100c[idx].val = torrent->peer_list->peer_count; |
320 | } | 322 | } |
321 | idx = amount - 1; while( (idx >= 0) && ( peer_list->seed_count > top100s[idx].val ) ) --idx; | 323 | idx = amount - 1; |
324 | while( (idx >= 0) && ( torrent->peer_list->seed_count > top100s[idx].val ) ) | ||
325 | --idx; | ||
322 | if ( idx++ != amount - 1 ) { | 326 | if ( idx++ != amount - 1 ) { |
323 | memmove( top100s + idx + 1, top100s + idx, ( amount - 1 - idx ) * sizeof( ot_record ) ); | 327 | memmove( top100s + idx + 1, top100s + idx, ( amount - 1 - idx ) * sizeof( ot_record ) ); |
324 | top100s[idx].val = peer_list->seed_count; | 328 | memcpy( &top100s[idx].hash, &torrent->hash, sizeof(ot_hash)); |
325 | top100s[idx].torrent = (ot_torrent*)(torrents_list->data) + j; | 329 | top100s[idx].val = torrent->peer_list->seed_count; |
326 | } | 330 | } |
327 | } | 331 | } |
328 | mutex_bucket_unlock( bucket, 0 ); | 332 | mutex_bucket_unlock( bucket, 0 ); |
@@ -332,12 +336,12 @@ size_t stats_top_txt( char * reply, int amount ) { | |||
332 | 336 | ||
333 | r += sprintf( r, "Top %d torrents by peers:\n", amount ); | 337 | r += sprintf( r, "Top %d torrents by peers:\n", amount ); |
334 | for( idx=0; idx<amount; ++idx ) | 338 | for( idx=0; idx<amount; ++idx ) |
335 | if( top100c[idx].torrent ) | 339 | if( top100c[idx].val ) |
336 | r += sprintf( r, "\t%zd\t%s\n", top100c[idx].val, to_hex( hex_out, top100c[idx].torrent->hash) ); | 340 | r += sprintf( r, "\t%zd\t%s\n", top100c[idx].val, to_hex( hex_out, top100c[idx].hash) ); |
337 | r += sprintf( r, "Top %d torrents by seeds:\n", amount ); | 341 | r += sprintf( r, "Top %d torrents by seeds:\n", amount ); |
338 | for( idx=0; idx<amount; ++idx ) | 342 | for( idx=0; idx<amount; ++idx ) |
339 | if( top100s[idx].torrent ) | 343 | if( top100s[idx].val ) |
340 | r += sprintf( r, "\t%zd\t%s\n", top100s[idx].val, to_hex( hex_out, top100s[idx].torrent->hash) ); | 344 | r += sprintf( r, "\t%zd\t%s\n", top100s[idx].val, to_hex( hex_out, top100s[idx].hash) ); |
341 | 345 | ||
342 | return r - reply; | 346 | return r - reply; |
343 | } | 347 | } |
@@ -691,7 +695,7 @@ void stats_issue_event( ot_status_event event, PROTO_FLAG proto, uintptr_t event | |||
691 | int off = snprintf( _debug, sizeof(_debug), "[%08d] scrp: ", (unsigned int)(g_now_seconds - ot_start_time)/60 ); | 695 | int off = snprintf( _debug, sizeof(_debug), "[%08d] scrp: ", (unsigned int)(g_now_seconds - ot_start_time)/60 ); |
692 | off += fmt_ip6c( _debug+off, *ip ); | 696 | off += fmt_ip6c( _debug+off, *ip ); |
693 | off += snprintf( _debug+off, sizeof(_debug)-off, " - FULL SCRAPE\n" ); | 697 | off += snprintf( _debug+off, sizeof(_debug)-off, " - FULL SCRAPE\n" ); |
694 | write( 2, _debug, off ); | 698 | (void)write( 2, _debug, off ); |
695 | ot_full_scrape_request_count++; | 699 | ot_full_scrape_request_count++; |
696 | } | 700 | } |
697 | break; | 701 | break; |
@@ -702,7 +706,7 @@ void stats_issue_event( ot_status_event event, PROTO_FLAG proto, uintptr_t event | |||
702 | int off = snprintf( _debug, sizeof(_debug), "[%08d] scrp: ", (unsigned int)(g_now_seconds - ot_start_time)/60 ); | 706 | int off = snprintf( _debug, sizeof(_debug), "[%08d] scrp: ", (unsigned int)(g_now_seconds - ot_start_time)/60 ); |
703 | off += fmt_ip6c(_debug+off, *ip ); | 707 | off += fmt_ip6c(_debug+off, *ip ); |
704 | off += snprintf( _debug+off, sizeof(_debug)-off, " - FULL SCRAPE\n" ); | 708 | off += snprintf( _debug+off, sizeof(_debug)-off, " - FULL SCRAPE\n" ); |
705 | write( 2, _debug, off ); | 709 | (void)write( 2, _debug, off ); |
706 | ot_full_scrape_request_count++; | 710 | ot_full_scrape_request_count++; |
707 | } | 711 | } |
708 | break; | 712 | break; |
@@ -40,11 +40,11 @@ enum { | |||
40 | 40 | ||
41 | void stats_issue_event( ot_status_event event, PROTO_FLAG proto, uintptr_t event_data ); | 41 | void stats_issue_event( ot_status_event event, PROTO_FLAG proto, uintptr_t event_data ); |
42 | void stats_deliver( int64 sock, int tasktype ); | 42 | void stats_deliver( int64 sock, int tasktype ); |
43 | void stats_cleanup(); | 43 | void stats_cleanup( void ); |
44 | size_t return_stats_for_tracker( char *reply, int mode, int format ); | 44 | size_t return_stats_for_tracker( char *reply, int mode, int format ); |
45 | size_t stats_return_tracker_version( char *reply ); | 45 | size_t stats_return_tracker_version( char *reply ); |
46 | void stats_init( ); | 46 | void stats_init( void ); |
47 | void stats_deinit( ); | 47 | void stats_deinit( void ); |
48 | 48 | ||
49 | extern const char *g_version_rijndael_c; | 49 | extern const char *g_version_rijndael_c; |
50 | extern const char *g_version_livesync_c; | 50 | extern const char *g_version_livesync_c; |
@@ -29,13 +29,21 @@ static ot_time g_hour_of_the_key; | |||
29 | 29 | ||
30 | static void udp_generate_rijndael_round_key() { | 30 | static void udp_generate_rijndael_round_key() { |
31 | uint32_t key[16]; | 31 | uint32_t key[16]; |
32 | #ifdef WANT_ARC4RANDOM | ||
33 | arc4random_buf(&key[0], sizeof(key)); | ||
34 | #else | ||
32 | key[0] = random(); | 35 | key[0] = random(); |
33 | key[1] = random(); | 36 | key[1] = random(); |
34 | key[2] = random(); | 37 | key[2] = random(); |
35 | key[3] = random(); | 38 | key[3] = random(); |
39 | #endif | ||
36 | rijndaelKeySetupEnc128( g_rijndael_round_key, (uint8_t*)key ); | 40 | rijndaelKeySetupEnc128( g_rijndael_round_key, (uint8_t*)key ); |
37 | 41 | ||
42 | #ifdef WANT_ARC4RANDOM | ||
43 | g_key_of_the_hour[0] = arc4random(); | ||
44 | #else | ||
38 | g_key_of_the_hour[0] = random(); | 45 | g_key_of_the_hour[0] = random(); |
46 | #endif | ||
39 | g_hour_of_the_key = g_now_minutes; | 47 | g_hour_of_the_key = g_now_minutes; |
40 | } | 48 | } |
41 | 49 | ||
@@ -46,7 +54,11 @@ static void udp_make_connectionid( uint32_t connid[2], const ot_ip6 remoteip, in | |||
46 | if( g_now_minutes + 60 > g_hour_of_the_key ) { | 54 | if( g_now_minutes + 60 > g_hour_of_the_key ) { |
47 | g_hour_of_the_key = g_now_minutes; | 55 | g_hour_of_the_key = g_now_minutes; |
48 | g_key_of_the_hour[1] = g_key_of_the_hour[0]; | 56 | g_key_of_the_hour[1] = g_key_of_the_hour[0]; |
49 | g_key_of_the_hour[0] = random(); | 57 | #ifdef WANT_ARC4RANDOM |
58 | g_key_of_the_hour[0] = arc4random(); | ||
59 | #else | ||
60 | g_key_of_the_hour[0] = random(); | ||
61 | #endif | ||
50 | } | 62 | } |
51 | 63 | ||
52 | memcpy( plain, remoteip, sizeof( plain ) ); | 64 | memcpy( plain, remoteip, sizeof( plain ) ); |
@@ -206,7 +218,7 @@ void udp_init( int64 sock, unsigned int worker_count ) { | |||
206 | if( !g_rijndael_round_key[0] ) | 218 | if( !g_rijndael_round_key[0] ) |
207 | udp_generate_rijndael_round_key(); | 219 | udp_generate_rijndael_round_key(); |
208 | #ifdef _DEBUG | 220 | #ifdef _DEBUG |
209 | fprintf( stderr, " installing %d workers on udp socket %ld", worker_count, (unsigned long)sock ); | 221 | fprintf( stderr, " installing %d workers on udp socket %ld\n", worker_count, (unsigned long)sock ); |
210 | #endif | 222 | #endif |
211 | while( worker_count-- ) | 223 | while( worker_count-- ) |
212 | pthread_create( &thread_id, NULL, udp_worker, (void *)sock ); | 224 | pthread_create( &thread_id, NULL, udp_worker, (void *)sock ); |
@@ -171,8 +171,8 @@ size_t remove_peer_from_torrent_proxy( ot_hash hash, ot_peer *peer ) { | |||
171 | if( exactmatch ) { | 171 | if( exactmatch ) { |
172 | ot_peerlist *peer_list = torrent->peer_list; | 172 | ot_peerlist *peer_list = torrent->peer_list; |
173 | switch( vector_remove_peer( &peer_list->peers, peer ) ) { | 173 | switch( vector_remove_peer( &peer_list->peers, peer ) ) { |
174 | case 2: peer_list->seed_count--; /* Fall throughs intended */ | 174 | case 2: peer_list->seed_count--; /* Intentional fallthrough */ |
175 | case 1: peer_list->peer_count--; /* Fall throughs intended */ | 175 | case 1: peer_list->peer_count--; /* Intentional fallthrough */ |
176 | default: break; | 176 | default: break; |
177 | } | 177 | } |
178 | } | 178 | } |
@@ -553,7 +553,11 @@ int main( int argc, char **argv ) { | |||
553 | int scanon = 1, lbound = 0, sbound = 0; | 553 | int scanon = 1, lbound = 0, sbound = 0; |
554 | 554 | ||
555 | srandom( time(NULL) ); | 555 | srandom( time(NULL) ); |
556 | #ifdef WANT_ARC4RANDOM | ||
557 | g_tracker_id = arc4random(); | ||
558 | #else | ||
556 | g_tracker_id = random(); | 559 | g_tracker_id = random(); |
560 | #endif | ||
557 | noipv6=1; | 561 | noipv6=1; |
558 | 562 | ||
559 | while( scanon ) { | 563 | while( scanon ) { |
diff --git a/trackerlogic.c b/trackerlogic.c index 310466c..719f8a2 100644 --- a/trackerlogic.c +++ b/trackerlogic.c | |||
@@ -29,7 +29,7 @@ | |||
29 | #include "ot_livesync.h" | 29 | #include "ot_livesync.h" |
30 | 30 | ||
31 | /* Forward declaration */ | 31 | /* Forward declaration */ |
32 | size_t return_peers_for_torrent( ot_torrent *torrent, size_t amount, char *reply, PROTO_FLAG proto ); | 32 | size_t return_peers_for_torrent( struct ot_workstruct * ws, ot_torrent *torrent, size_t amount, char *reply, PROTO_FLAG proto ); |
33 | 33 | ||
34 | void free_peerlist( ot_peerlist *peer_list ) { | 34 | void free_peerlist( ot_peerlist *peer_list ) { |
35 | if( peer_list->peers.data ) { | 35 | if( peer_list->peers.data ) { |
@@ -51,19 +51,19 @@ void add_torrent_from_saved_state( ot_hash hash, ot_time base, size_t down_count | |||
51 | 51 | ||
52 | if( !accesslist_hashisvalid( hash ) ) | 52 | if( !accesslist_hashisvalid( hash ) ) |
53 | return mutex_bucket_unlock_by_hash( hash, 0 ); | 53 | return mutex_bucket_unlock_by_hash( hash, 0 ); |
54 | 54 | ||
55 | torrent = vector_find_or_insert( torrents_list, (void*)hash, sizeof( ot_torrent ), OT_HASH_COMPARE_SIZE, &exactmatch ); | 55 | torrent = vector_find_or_insert( torrents_list, (void*)hash, sizeof( ot_torrent ), OT_HASH_COMPARE_SIZE, &exactmatch ); |
56 | if( !torrent || exactmatch ) | 56 | if( !torrent || exactmatch ) |
57 | return mutex_bucket_unlock_by_hash( hash, 0 ); | 57 | return mutex_bucket_unlock_by_hash( hash, 0 ); |
58 | 58 | ||
59 | /* Create a new torrent entry, then */ | 59 | /* Create a new torrent entry, then */ |
60 | memcpy( torrent->hash, hash, sizeof(ot_hash) ); | 60 | memcpy( torrent->hash, hash, sizeof(ot_hash) ); |
61 | 61 | ||
62 | if( !( torrent->peer_list = malloc( sizeof (ot_peerlist) ) ) ) { | 62 | if( !( torrent->peer_list = malloc( sizeof (ot_peerlist) ) ) ) { |
63 | vector_remove_torrent( torrents_list, torrent ); | 63 | vector_remove_torrent( torrents_list, torrent ); |
64 | return mutex_bucket_unlock_by_hash( hash, 0 ); | 64 | return mutex_bucket_unlock_by_hash( hash, 0 ); |
65 | } | 65 | } |
66 | 66 | ||
67 | byte_zero( torrent->peer_list, sizeof( ot_peerlist ) ); | 67 | byte_zero( torrent->peer_list, sizeof( ot_peerlist ) ); |
68 | torrent->peer_list->base = base; | 68 | torrent->peer_list->base = base; |
69 | torrent->peer_list->down_count = down_count; | 69 | torrent->peer_list->down_count = down_count; |
@@ -178,7 +178,7 @@ size_t add_peer_to_torrent_and_return_peers( PROTO_FLAG proto, struct ot_workstr | |||
178 | } | 178 | } |
179 | #endif | 179 | #endif |
180 | 180 | ||
181 | ws->reply_size = return_peers_for_torrent( torrent, amount, ws->reply, proto ); | 181 | ws->reply_size = return_peers_for_torrent( ws, torrent, amount, ws->reply, proto ); |
182 | mutex_bucket_unlock_by_hash( *ws->hash, delta_torrentcount ); | 182 | mutex_bucket_unlock_by_hash( *ws->hash, delta_torrentcount ); |
183 | return ws->reply_size; | 183 | return ws->reply_size; |
184 | } | 184 | } |
@@ -200,7 +200,7 @@ static size_t return_peers_all( ot_peerlist *peer_list, char *reply ) { | |||
200 | while( peer_count-- ) { | 200 | while( peer_count-- ) { |
201 | if( OT_PEERFLAG(peers) & PEER_FLAG_SEEDING ) { | 201 | if( OT_PEERFLAG(peers) & PEER_FLAG_SEEDING ) { |
202 | r_end-=OT_PEER_COMPARE_SIZE; | 202 | r_end-=OT_PEER_COMPARE_SIZE; |
203 | memcpy(r_end,peers++,OT_PEER_COMPARE_SIZE); | 203 | memcpy(r_end,peers++,OT_PEER_COMPARE_SIZE); |
204 | } else { | 204 | } else { |
205 | memcpy(reply,peers++,OT_PEER_COMPARE_SIZE); | 205 | memcpy(reply,peers++,OT_PEER_COMPARE_SIZE); |
206 | reply+=OT_PEER_COMPARE_SIZE; | 206 | reply+=OT_PEER_COMPARE_SIZE; |
@@ -210,7 +210,7 @@ static size_t return_peers_all( ot_peerlist *peer_list, char *reply ) { | |||
210 | return result; | 210 | return result; |
211 | } | 211 | } |
212 | 212 | ||
213 | static size_t return_peers_selection( ot_peerlist *peer_list, size_t amount, char *reply ) { | 213 | static size_t return_peers_selection( struct ot_workstruct *ws, ot_peerlist *peer_list, size_t amount, char *reply ) { |
214 | unsigned int bucket_offset, bucket_index = 0, num_buckets = 1; | 214 | unsigned int bucket_offset, bucket_index = 0, num_buckets = 1; |
215 | ot_vector * bucket_list = &peer_list->peers; | 215 | ot_vector * bucket_list = &peer_list->peers; |
216 | unsigned int shifted_pc = peer_list->peer_count; | 216 | unsigned int shifted_pc = peer_list->peer_count; |
@@ -218,7 +218,7 @@ static size_t return_peers_selection( ot_peerlist *peer_list, size_t amount, cha | |||
218 | unsigned int shift = 0; | 218 | unsigned int shift = 0; |
219 | size_t result = OT_PEER_COMPARE_SIZE * amount; | 219 | size_t result = OT_PEER_COMPARE_SIZE * amount; |
220 | char * r_end = reply + result; | 220 | char * r_end = reply + result; |
221 | 221 | ||
222 | if( OT_PEERLIST_HASBUCKETS(peer_list) ) { | 222 | if( OT_PEERLIST_HASBUCKETS(peer_list) ) { |
223 | num_buckets = bucket_list->size; | 223 | num_buckets = bucket_list->size; |
224 | bucket_list = (ot_vector *)bucket_list->data; | 224 | bucket_list = (ot_vector *)bucket_list->data; |
@@ -232,7 +232,7 @@ static size_t return_peers_selection( ot_peerlist *peer_list, size_t amount, cha | |||
232 | 232 | ||
233 | /* Initialize somewhere in the middle of peers so that | 233 | /* Initialize somewhere in the middle of peers so that |
234 | fixpoint's aliasing doesn't alway miss the same peers */ | 234 | fixpoint's aliasing doesn't alway miss the same peers */ |
235 | bucket_offset = random() % peer_list->peer_count; | 235 | bucket_offset = nrand48(ws->rand48_state) % peer_list->peer_count; |
236 | 236 | ||
237 | while( amount-- ) { | 237 | while( amount-- ) { |
238 | ot_peer * peer; | 238 | ot_peer * peer; |
@@ -240,7 +240,7 @@ static size_t return_peers_selection( ot_peerlist *peer_list, size_t amount, cha | |||
240 | /* This is the aliased, non shifted range, next value may fall into */ | 240 | /* This is the aliased, non shifted range, next value may fall into */ |
241 | unsigned int diff = ( ( ( amount + 1 ) * shifted_step ) >> shift ) - | 241 | unsigned int diff = ( ( ( amount + 1 ) * shifted_step ) >> shift ) - |
242 | ( ( amount * shifted_step ) >> shift ); | 242 | ( ( amount * shifted_step ) >> shift ); |
243 | bucket_offset += 1 + random() % diff; | 243 | bucket_offset += 1 + nrand48(ws->rand48_state) % diff; |
244 | 244 | ||
245 | while( bucket_offset >= bucket_list[bucket_index].size ) { | 245 | while( bucket_offset >= bucket_list[bucket_index].size ) { |
246 | bucket_offset -= bucket_list[bucket_index].size; | 246 | bucket_offset -= bucket_list[bucket_index].size; |
@@ -249,7 +249,7 @@ static size_t return_peers_selection( ot_peerlist *peer_list, size_t amount, cha | |||
249 | peer = ((ot_peer*)bucket_list[bucket_index].data) + bucket_offset; | 249 | peer = ((ot_peer*)bucket_list[bucket_index].data) + bucket_offset; |
250 | if( OT_PEERFLAG(peer) & PEER_FLAG_SEEDING ) { | 250 | if( OT_PEERFLAG(peer) & PEER_FLAG_SEEDING ) { |
251 | r_end-=OT_PEER_COMPARE_SIZE; | 251 | r_end-=OT_PEER_COMPARE_SIZE; |
252 | memcpy(r_end,peer,OT_PEER_COMPARE_SIZE); | 252 | memcpy(r_end,peer,OT_PEER_COMPARE_SIZE); |
253 | } else { | 253 | } else { |
254 | memcpy(reply,peer,OT_PEER_COMPARE_SIZE); | 254 | memcpy(reply,peer,OT_PEER_COMPARE_SIZE); |
255 | reply+=OT_PEER_COMPARE_SIZE; | 255 | reply+=OT_PEER_COMPARE_SIZE; |
@@ -262,7 +262,7 @@ static size_t return_peers_selection( ot_peerlist *peer_list, size_t amount, cha | |||
262 | * reply must have enough space to hold 92+6*amount bytes | 262 | * reply must have enough space to hold 92+6*amount bytes |
263 | * does not yet check not to return self | 263 | * does not yet check not to return self |
264 | */ | 264 | */ |
265 | size_t return_peers_for_torrent( ot_torrent *torrent, size_t amount, char *reply, PROTO_FLAG proto ) { | 265 | size_t return_peers_for_torrent( struct ot_workstruct * ws, ot_torrent *torrent, size_t amount, char *reply, PROTO_FLAG proto ) { |
266 | ot_peerlist *peer_list = torrent->peer_list; | 266 | ot_peerlist *peer_list = torrent->peer_list; |
267 | char *r = reply; | 267 | char *r = reply; |
268 | 268 | ||
@@ -283,7 +283,7 @@ size_t return_peers_for_torrent( ot_torrent *torrent, size_t amount, char *reply | |||
283 | if( amount == peer_list->peer_count ) | 283 | if( amount == peer_list->peer_count ) |
284 | r += return_peers_all( peer_list, r ); | 284 | r += return_peers_all( peer_list, r ); |
285 | else | 285 | else |
286 | r += return_peers_selection( peer_list, amount, r ); | 286 | r += return_peers_selection( ws, peer_list, amount, r ); |
287 | } | 287 | } |
288 | 288 | ||
289 | if( proto == FLAG_TCP ) | 289 | if( proto == FLAG_TCP ) |
@@ -365,8 +365,8 @@ size_t remove_peer_from_torrent( PROTO_FLAG proto, struct ot_workstruct *ws ) { | |||
365 | if( exactmatch ) { | 365 | if( exactmatch ) { |
366 | peer_list = torrent->peer_list; | 366 | peer_list = torrent->peer_list; |
367 | switch( vector_remove_peer( &peer_list->peers, &ws->peer ) ) { | 367 | switch( vector_remove_peer( &peer_list->peers, &ws->peer ) ) { |
368 | case 2: peer_list->seed_count--; /* Fall throughs intended */ | 368 | case 2: peer_list->seed_count--; /* Intentional fallthrough */ |
369 | case 1: peer_list->peer_count--; /* Fall throughs intended */ | 369 | case 1: peer_list->peer_count--; /* Intentional fallthrough */ |
370 | default: break; | 370 | default: break; |
371 | } | 371 | } |
372 | } | 372 | } |
diff --git a/trackerlogic.h b/trackerlogic.h index 721ba6e..ef59179 100644 --- a/trackerlogic.h +++ b/trackerlogic.h | |||
@@ -10,6 +10,14 @@ | |||
10 | #include <sys/time.h> | 10 | #include <sys/time.h> |
11 | #include <time.h> | 11 | #include <time.h> |
12 | #include <stdint.h> | 12 | #include <stdint.h> |
13 | #include <stdlib.h> | ||
14 | |||
15 | #if defined(__linux__) && defined(WANT_ARC4RANDOM) | ||
16 | #include <bsd/stdlib.h> | ||
17 | #endif | ||
18 | #ifdef __FreeBSD__ | ||
19 | #define WANT_ARC4RANDOM | ||
20 | #endif | ||
13 | 21 | ||
14 | typedef uint8_t ot_hash[20]; | 22 | typedef uint8_t ot_hash[20]; |
15 | typedef time_t ot_time; | 23 | typedef time_t ot_time; |
@@ -34,7 +42,7 @@ typedef struct { ot_ip6 address; int bits; } | |||
34 | #define OT_TORRENT_TIMEOUT_HOURS 24 | 42 | #define OT_TORRENT_TIMEOUT_HOURS 24 |
35 | #define OT_TORRENT_TIMEOUT (60*OT_TORRENT_TIMEOUT_HOURS) | 43 | #define OT_TORRENT_TIMEOUT (60*OT_TORRENT_TIMEOUT_HOURS) |
36 | 44 | ||
37 | #define OT_CLIENT_REQUEST_INTERVAL_RANDOM ( OT_CLIENT_REQUEST_INTERVAL - OT_CLIENT_REQUEST_VARIATION/2 + (int)( random( ) % OT_CLIENT_REQUEST_VARIATION ) ) | 45 | #define OT_CLIENT_REQUEST_INTERVAL_RANDOM ( OT_CLIENT_REQUEST_INTERVAL - OT_CLIENT_REQUEST_VARIATION/2 + (int)( nrand48(ws->rand48_state) % OT_CLIENT_REQUEST_VARIATION ) ) |
38 | 46 | ||
39 | /* If WANT_MODEST_FULLSCRAPES is on, ip addresses may not | 47 | /* If WANT_MODEST_FULLSCRAPES is on, ip addresses may not |
40 | fullscrape more frequently than this amount in seconds */ | 48 | fullscrape more frequently than this amount in seconds */ |
@@ -125,12 +133,17 @@ struct ot_workstruct { | |||
125 | char *peer_id; | 133 | char *peer_id; |
126 | 134 | ||
127 | /* HTTP specific, non static */ | 135 | /* HTTP specific, non static */ |
128 | int keep_alive; | ||
129 | char *request; | 136 | char *request; |
130 | ssize_t request_size; | 137 | ssize_t request_size; |
131 | ssize_t header_size; | 138 | ssize_t header_size; |
132 | char *reply; | 139 | char *reply; |
133 | ssize_t reply_size; | 140 | ssize_t reply_size; |
141 | |||
142 | /* Entropy state for rand48 function so that threads don't need to acquire mutexes for | ||
143 | global random() or arc4random() state, which causes heavy load on linuxes */ | ||
144 | uint16_t rand48_state[3]; | ||
145 | |||
146 | int keep_alive; | ||
134 | }; | 147 | }; |
135 | 148 | ||
136 | /* | 149 | /* |
@@ -151,7 +164,7 @@ struct ot_workstruct { | |||
151 | #error Live logging networks disabled at the moment. | 164 | #error Live logging networks disabled at the moment. |
152 | #endif | 165 | #endif |
153 | 166 | ||
154 | void trackerlogic_init( ); | 167 | void trackerlogic_init( void ); |
155 | void trackerlogic_deinit( void ); | 168 | void trackerlogic_deinit( void ); |
156 | void exerr( char * message ); | 169 | void exerr( char * message ); |
157 | 170 | ||