summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorerdgeist <>2009-06-15 14:41:47 +0000
committererdgeist <>2009-06-15 14:41:47 +0000
commit6c19143bc1279889ded58df7ee4ab9f452d58bfa (patch)
tree581fbb28104f48fa796759742c3c13e09daa537f
parentd1139121010cd91b4adf3fd61cfa04d8c806552f (diff)
Major rewrite of live bad network logging. You can now limit logging to any netmask. HTTP interface for that coming soon.
-rw-r--r--ot_stats.c297
1 files changed, 155 insertions, 142 deletions
diff --git a/ot_stats.c b/ot_stats.c
index a8b442f..8192d27 100644
--- a/ot_stats.c
+++ b/ot_stats.c
@@ -17,6 +17,7 @@
17/* Libowfat */ 17/* Libowfat */
18#include "byte.h" 18#include "byte.h"
19#include "io.h" 19#include "io.h"
20#include "ip4.h"
20#include "ip6.h" 21#include "ip6.h"
21 22
22/* Opentracker */ 23/* Opentracker */
@@ -56,28 +57,36 @@ static unsigned long long ot_overall_stall_count;
56 57
57static time_t ot_start_time; 58static time_t ot_start_time;
58 59
59#ifdef WANT_LOG_NETWORKS 60#define STATS_NETWORK_NODE_BITWIDTH 4
60#define STATS_NETWORK_NODE_BITWIDTH 8 61#define STATS_NETWORK_NODE_COUNT (1<<STATS_NETWORK_NODE_BITWIDTH)
61#define STATS_NETWORK_NODE_COUNT (1<<STATS_NETWORK_NODE_BITWIDTH) 62
63#define __BYTE(P,D) (((uint8_t*)P)[D/8])
64#define __MSK (STATS_NETWORK_NODE_COUNT-1)
65#define __SHFT(D) ((D^STATS_NETWORK_NODE_BITWIDTH)&STATS_NETWORK_NODE_BITWIDTH)
66
67#define __LDR(P,D) ((__BYTE((P),(D))>>__SHFT((D)))&__MSK)
68#define __STR(P,D,V) __BYTE((P),(D))=(__BYTE((P),(D))&~(__MSK<<__SHFT((D))))|((V)<<__SHFT((D)))
62 69
63#ifdef WANT_V6 70#ifdef WANT_V6
64#define STATS_NETWORK_NODE_MAXDEPTH (48/8-1) 71#define STATS_NETWORK_NODE_MAXDEPTH (68-STATS_NETWORK_NODE_BITWIDTH)
72#define STATS_NETWORK_NODE_LIMIT (48-STATS_NETWORK_NODE_BITWIDTH)
65#else 73#else
66#define STATS_NETWORK_NODE_MAXDEPTH (12+24/8-1) 74#define STATS_NETWORK_NODE_MAXDEPTH (28-STATS_NETWORK_NODE_BITWIDTH)
75#define STATS_NETWORK_NODE_LIMIT (24-STATS_NETWORK_NODE_BITWIDTH)
67#endif 76#endif
68 77
69
70typedef union stats_network_node stats_network_node; 78typedef union stats_network_node stats_network_node;
71union stats_network_node { 79union stats_network_node {
72 int counters[STATS_NETWORK_NODE_COUNT]; 80 int counters[STATS_NETWORK_NODE_COUNT];
73 stats_network_node *children[STATS_NETWORK_NODE_COUNT]; 81 stats_network_node *children[STATS_NETWORK_NODE_COUNT];
74}; 82};
75 83
84#ifdef WANT_LOG_NETWORKS
76static stats_network_node *stats_network_counters_root = NULL; 85static stats_network_node *stats_network_counters_root = NULL;
86#endif
77 87
78static int stat_increase_network_count( stats_network_node **node, int depth, uintptr_t ip ) { 88static int stat_increase_network_count( stats_network_node **node, int depth, uintptr_t ip ) {
79 uint8_t *_ip = (uint8_t*)ip; 89 int foo = __LDR(ip,depth);
80 int foo = _ip[depth];
81 90
82 if( !*node ) { 91 if( !*node ) {
83 *node = malloc( sizeof( stats_network_node ) ); 92 *node = malloc( sizeof( stats_network_node ) );
@@ -87,7 +96,7 @@ static int stat_increase_network_count( stats_network_node **node, int depth, ui
87 } 96 }
88 97
89 if( depth < STATS_NETWORK_NODE_MAXDEPTH ) 98 if( depth < STATS_NETWORK_NODE_MAXDEPTH )
90 return stat_increase_network_count( &(*node)->children[ foo ], depth+1, ip ); 99 return stat_increase_network_count( &(*node)->children[ foo ], depth+STATS_NETWORK_NODE_BITWIDTH, ip );
91 100
92 (*node)->counters[ foo ]++; 101 (*node)->counters[ foo ]++;
93 return 0; 102 return 0;
@@ -97,11 +106,12 @@ static int stats_shift_down_network_count( stats_network_node **node, int depth,
97 int i, rest = 0; 106 int i, rest = 0;
98 if( !*node ) return 0; 107 if( !*node ) return 0;
99 108
100 if( ++depth == STATS_NETWORK_NODE_MAXDEPTH ) 109 depth += STATS_NETWORK_NODE_BITWIDTH;
101 for( i=0; i<STATS_NETWORK_NODE_COUNT; ++i ) { 110 if( depth == STATS_NETWORK_NODE_MAXDEPTH ) {
111 for( i=0; i<STATS_NETWORK_NODE_COUNT; ++i )
102 rest += ((*node)->counters[i]>>=shift); 112 rest += ((*node)->counters[i]>>=shift);
103 return rest; 113 return rest;
104 } 114 }
105 115
106 for( i=0; i<STATS_NETWORK_NODE_COUNT; ++i ) { 116 for( i=0; i<STATS_NETWORK_NODE_COUNT; ++i ) {
107 stats_network_node **childnode = &(*node)->children[i]; 117 stats_network_node **childnode = &(*node)->children[i];
@@ -120,56 +130,155 @@ static int stats_shift_down_network_count( stats_network_node **node, int depth,
120 return rest; 130 return rest;
121} 131}
122 132
123static void stats_get_highscore_networks( stats_network_node *node, int depth, ot_ip6 node_value, int *scores, ot_ip6 *networks, int network_count ) { 133static size_t stats_get_highscore_networks( stats_network_node *node, int depth, ot_ip6 node_value, size_t *scores, ot_ip6 *networks, int network_count, int limit ) {
124 uint8_t *_node_value = (uint8_t*)node_value; 134 size_t score = 0;
125 int i; 135 int i;
126 136
127 if( !node ) return; 137 if( !node ) return 0;
128 138
129 if( depth < STATS_NETWORK_NODE_MAXDEPTH ) { 139 if( depth < limit ) {
130 for( i=0; i<STATS_NETWORK_NODE_COUNT; ++i ) 140 for( i=0; i<STATS_NETWORK_NODE_COUNT; ++i )
131 if( node->children[i] ) { 141 if( node->children[i] ) {
132 _node_value[depth] = i; 142 __STR(node_value,depth,i);
133 stats_get_highscore_networks( node->children[i], depth+1, node_value, scores, networks, network_count ); 143 score += stats_get_highscore_networks( node->children[i], depth+STATS_NETWORK_NODE_BITWIDTH, node_value, scores, networks, network_count, limit );
134 }
135 } else
136 for( i=0; i<STATS_NETWORK_NODE_COUNT; ++i ) {
137 int j=1;
138 if( node->counters[i] <= scores[0] ) continue;
139
140 _node_value[depth] = i;
141 while( (j<network_count) && (node->counters[i]>scores[j] ) ) ++j;
142 --j;
143
144 memcpy( scores, scores + 1, j * sizeof( *scores ) );
145 memcpy( networks, networks + 1, j * sizeof( *networks ) );
146 scores[ j ] = node->counters[ i ];
147 memcpy( networks + j, _node_value, sizeof( *networks ) );
148 } 144 }
145 return score;
146 }
147
148 if( depth > limit && depth < STATS_NETWORK_NODE_MAXDEPTH ) {
149 for( i=0; i<STATS_NETWORK_NODE_COUNT; ++i )
150 if( node->children[i] )
151 score += stats_get_highscore_networks( node->children[i], depth+STATS_NETWORK_NODE_BITWIDTH, node_value, scores, networks, network_count, limit );
152 return score;
153 }
154
155 if( depth > limit && depth == STATS_NETWORK_NODE_MAXDEPTH ) {
156 for( i=0; i<STATS_NETWORK_NODE_COUNT; ++i )
157 score += node->counters[i];
158 return score;
159 }
160
161 /* if( depth == limit ) */
162 for( i=0; i<STATS_NETWORK_NODE_COUNT; ++i ) {
163 int j=1;
164 size_t node_score;
165
166 if( depth == STATS_NETWORK_NODE_MAXDEPTH )
167 node_score = node->counters[i];
168 else
169 node_score = stats_get_highscore_networks( node->children[i], depth+STATS_NETWORK_NODE_BITWIDTH, node_value, scores, networks, network_count, limit );
170
171 score += node_score;
172
173 if( node_score <= scores[0] ) continue;
174
175 __STR(node_value,depth,i);
176 while( j < network_count && node_score > scores[j] ) ++j;
177 --j;
178
179 memcpy( scores, scores + 1, j * sizeof( *scores ) );
180 memcpy( networks, networks + 1, j * sizeof( *networks ) );
181 scores[ j ] = node_score;
182 memcpy( networks + j, node_value, sizeof( *networks ) );
183 }
184
185 return score;
149} 186}
150 187
151static size_t stats_return_busy_networks( char * reply ) { 188static size_t stats_return_busy_networks( char * reply, stats_network_node *tree, int amount ) {
152 ot_ip6 networks[256]; 189 ot_ip6 networks[amount];
153 ot_ip6 node_value; 190 ot_ip6 node_value;
154 int scores[256]; 191 size_t scores[amount];
155 int i; 192 int i;
156 char * r = reply; 193 char * r = reply;
157 194
158 memset( scores, 0, sizeof( *scores ) * 256 ); 195 memset( scores, 0, sizeof( scores ) );
159 memset( networks, 0, sizeof( *networks ) * 256 ); 196 memset( networks, 0, sizeof( networks ) );
197 memset( node_value, 0, sizeof( node_value ) );
160 198
161 stats_get_highscore_networks( stats_network_counters_root, 0, node_value, scores, networks, 256 ); 199 stats_get_highscore_networks( tree, 0, node_value, scores, networks, amount, STATS_NETWORK_NODE_MAXDEPTH );
162 200
163 for( i=255; i>=0; --i) { 201 r += sprintf( r, "Networks, limit /%d:\n", STATS_NETWORK_NODE_MAXDEPTH+STATS_NETWORK_NODE_BITWIDTH );
164 r += sprintf( r, "%08i: ", scores[i] ); 202 for( i=amount-1; i>=0; --i) {
165 r += fmt_ip6c( r, networks[i] ); 203 if( scores[i] ) {
166 *r++ = '\n'; 204 r += sprintf( r, "%08zd: ", scores[i] );
205#ifdef WANT_V6
206 r += fmt_ip6c( r, networks[i] );
207#else
208 r += fmt_ip4( r, networks[i]);
209#endif
210 *r++ = '\n';
211 }
167 } 212 }
168 213
214 memset( scores, 0, sizeof( scores ) );
215 memset( networks, 0, sizeof( networks ) );
216 memset( node_value, 0, sizeof( node_value ) );
217
218 stats_get_highscore_networks( tree, 0, node_value, scores, networks, amount, STATS_NETWORK_NODE_LIMIT );
219
220 r += sprintf( r, "\nNetworks, limit /%d:\n", STATS_NETWORK_NODE_LIMIT+STATS_NETWORK_NODE_BITWIDTH );
221 for( i=amount-1; i>=0; --i) {
222 if( scores[i] ) {
223 r += sprintf( r, "%08zd: ", scores[i] );
224#ifdef WANT_V6
225 r += fmt_ip6c( r, networks[i] );
226#else
227 r += fmt_ip4( r, networks[i] );
228#endif
229 *r++ = '\n';
230 }
231 }
232
169 return r - reply; 233 return r - reply;
170} 234}
171 235
172#endif 236static size_t stats_slash24s_txt( char *reply, size_t amount ) {
237 stats_network_node *slash24s_network_counters_root = NULL;
238 char *r=reply;
239 int bucket;
240 size_t i;
241
242 for( bucket=0; bucket<OT_BUCKET_COUNT; ++bucket ) {
243 ot_vector *torrents_list = mutex_bucket_lock( bucket );
244 for( i=0; i<torrents_list->size; ++i ) {
245 ot_peerlist *peer_list = ( ((ot_torrent*)(torrents_list->data))[i] ).peer_list;
246 ot_vector *bucket_list = &peer_list->peers;
247 int num_buckets = 1;
248
249 if( OT_PEERLIST_HASBUCKETS( peer_list ) ) {
250 num_buckets = bucket_list->size;
251 bucket_list = (ot_vector *)bucket_list->data;
252 }
253
254 while( num_buckets-- ) {
255 ot_peer *peers = (ot_peer*)bucket_list->data;
256 size_t numpeers = bucket_list->size;
257 while( numpeers-- )
258 if( stat_increase_network_count( &slash24s_network_counters_root, 0, (uintptr_t)(peers++) ) )
259 goto bailout_unlock;
260 ++bucket_list;
261 }
262 }
263 mutex_bucket_unlock( bucket, 0 );
264 if( !g_opentracker_running )
265 goto bailout_error;
266 }
267
268 /* The tree is built. Now analyze */
269 r += stats_return_busy_networks( r, slash24s_network_counters_root, amount );
270 goto success;
271
272bailout_unlock:
273 mutex_bucket_unlock( bucket, 0 );
274bailout_error:
275 r = reply;
276success:
277 stats_shift_down_network_count( &slash24s_network_counters_root, 0, STATS_NETWORK_NODE_MAXDEPTH*STATS_NETWORK_NODE_BITWIDTH );
278 if( slash24s_network_counters_root )
279 free( slash24s_network_counters_root );
280 return r-reply;
281}
173 282
174typedef struct { 283typedef struct {
175 unsigned long long torrent_count; 284 unsigned long long torrent_count;
@@ -234,98 +343,6 @@ size_t stats_top10_txt( char * reply ) {
234 return r - reply; 343 return r - reply;
235} 344}
236 345
237/* This function collects 4096 /24s in 4096 possible
238 malloc blocks
239 */
240static size_t stats_slash24s_txt( char * reply, size_t amount, uint32_t thresh ) {
241
242#define NUM_TOPBITS 12
243#define NUM_LOWBITS (24-NUM_TOPBITS)
244#define NUM_BUFS (1<<NUM_TOPBITS)
245#define NUM_S24S (1<<NUM_LOWBITS)
246#define MSK_S24S (NUM_S24S-1)
247
248 uint32_t *counts[ NUM_BUFS ];
249 uint32_t slash24s[amount*2]; /* first dword amount, second dword subnet */
250 size_t i, j, k, l;
251 char *r = reply;
252
253 byte_zero( counts, sizeof( counts ) );
254 byte_zero( slash24s, amount * 2 * sizeof(uint32_t) );
255
256 r += sprintf( r, "Stats for all /24s with more than %u announced torrents:\n\n", thresh );
257
258#if 0
259 /* XXX: TOOD: Doesn't work yet with new peer storage model */
260 for( bucket=0; bucket<OT_BUCKET_COUNT; ++bucket ) {
261 ot_vector *torrents_list = mutex_bucket_lock( bucket );
262 for( j=0; j<torrents_list->size; ++j ) {
263 ot_peerlist *peer_list = ( ((ot_torrent*)(torrents_list->data))[j] ).peer_list;
264 for( k=0; k<OT_POOLS_COUNT; ++k ) {
265 ot_peer *peers = peer_list->peers[k].data;
266 size_t numpeers = peer_list->peers[k].size;
267 for( l=0; l<numpeers; ++l ) {
268 uint32_t s24 = ntohl(*(uint32_t*)(peers+l)) >> 8;
269 uint32_t *count = counts[ s24 >> NUM_LOWBITS ];
270 if( !count ) {
271 count = malloc( sizeof(uint32_t) * NUM_S24S );
272 if( !count ) {
273 mutex_bucket_unlock( bucket, 0 );
274 goto bailout_cleanup;
275 }
276 byte_zero( count, sizeof( uint32_t ) * NUM_S24S );
277 counts[ s24 >> NUM_LOWBITS ] = count;
278 }
279 count[ s24 & MSK_S24S ]++;
280 }
281 }
282 }
283 mutex_bucket_unlock( bucket, 0 );
284 if( !g_opentracker_running )
285 goto bailout_cleanup;
286 }
287#endif
288
289 k = l = 0; /* Debug: count allocated bufs */
290 for( i=0; i < NUM_BUFS; ++i ) {
291 uint32_t *count = counts[i];
292 if( !counts[i] )
293 continue;
294 ++k; /* Debug: count allocated bufs */
295 for( j=0; j < NUM_S24S; ++j ) {
296 if( count[j] > thresh ) {
297 /* This subnet seems to announce more torrents than the last in our list */
298 int insert_pos = amount - 1;
299 while( ( insert_pos >= 0 ) && ( count[j] > slash24s[ 2 * insert_pos ] ) )
300 --insert_pos;
301 ++insert_pos;
302 memcpy( slash24s + 2 * ( insert_pos + 1 ), slash24s + 2 * ( insert_pos ), 2 * sizeof( uint32_t ) * ( amount - insert_pos - 1 ) );
303 slash24s[ 2 * insert_pos ] = count[j];
304 slash24s[ 2 * insert_pos + 1 ] = ( i << NUM_TOPBITS ) + j;
305 if( slash24s[ 2 * amount - 2 ] > thresh )
306 thresh = slash24s[ 2 * amount - 2 ];
307 }
308 if( count[j] ) ++l;
309 }
310 free( count );
311 }
312
313 r += sprintf( r, "Allocated bufs: %zd, used s24s: %zd\n", k, l );
314
315 for( i=0; i < amount; ++i )
316 if( slash24s[ 2*i ] >= thresh ) {
317 uint32_t ip = slash24s[ 2*i +1 ];
318 r += sprintf( r, "% 10ld %d.%d.%d.0/24\n", (long)slash24s[ 2*i ], (int)(ip >> 16), (int)(255 & ( ip >> 8 )), (int)(ip & 255) );
319 }
320
321 return r - reply;
322
323 for( i=0; i < NUM_BUFS; ++i )
324 free( counts[i] );
325
326 return 0;
327}
328
329static unsigned long events_per_time( unsigned long long events, time_t t ) { 346static unsigned long events_per_time( unsigned long long events, time_t t ) {
330 return events / ( (unsigned int)t ? (unsigned int)t : 1 ); 347 return events / ( (unsigned int)t ? (unsigned int)t : 1 );
331} 348}
@@ -532,10 +549,6 @@ size_t return_stats_for_tracker( char *reply, int mode, int format ) {
532 return stats_return_renew_bucket( reply ); 549 return stats_return_renew_bucket( reply );
533 case TASK_STATS_SYNCS: 550 case TASK_STATS_SYNCS:
534 return stats_return_sync_mrtg( reply ); 551 return stats_return_sync_mrtg( reply );
535#ifdef WANT_LOG_NETWORKS
536 case TASK_STATS_BUSY_NETWORKS:
537 return stats_return_busy_networks( reply );
538#endif
539 default: 552 default:
540 return 0; 553 return 0;
541 } 554 }
@@ -552,7 +565,7 @@ static void stats_make( int *iovec_entries, struct iovec **iovector, ot_tasktype
552 switch( mode & TASK_TASK_MASK ) { 565 switch( mode & TASK_TASK_MASK ) {
553 case TASK_STATS_TORRENTS: r += stats_torrents_mrtg( r ); break; 566 case TASK_STATS_TORRENTS: r += stats_torrents_mrtg( r ); break;
554 case TASK_STATS_PEERS: r += stats_peers_mrtg( r ); break; 567 case TASK_STATS_PEERS: r += stats_peers_mrtg( r ); break;
555 case TASK_STATS_SLASH24S: r += stats_slash24s_txt( r, 25, 16 ); break; 568 case TASK_STATS_SLASH24S: r += stats_slash24s_txt( r, 128 ); break;
556 case TASK_STATS_TOP10: r += stats_top10_txt( r ); break; 569 case TASK_STATS_TOP10: r += stats_top10_txt( r ); break;
557 case TASK_STATS_EVERYTHING: r += stats_return_everything( r ); break; 570 case TASK_STATS_EVERYTHING: r += stats_return_everything( r ); break;
558 default: 571 default: