From 6c19143bc1279889ded58df7ee4ab9f452d58bfa Mon Sep 17 00:00:00 2001 From: erdgeist <> Date: Mon, 15 Jun 2009 14:41:47 +0000 Subject: Major rewrite of live bad network logging. You can now limit logging to any netmask. HTTP interface for that coming soon. --- ot_stats.c | 297 ++++++++++++++++++++++++++++++++----------------------------- 1 file changed, 155 insertions(+), 142 deletions(-) diff --git a/ot_stats.c b/ot_stats.c index a8b442f..8192d27 100644 --- a/ot_stats.c +++ b/ot_stats.c @@ -17,6 +17,7 @@ /* Libowfat */ #include "byte.h" #include "io.h" +#include "ip4.h" #include "ip6.h" /* Opentracker */ @@ -56,28 +57,36 @@ static unsigned long long ot_overall_stall_count; static time_t ot_start_time; -#ifdef WANT_LOG_NETWORKS -#define STATS_NETWORK_NODE_BITWIDTH 8 -#define STATS_NETWORK_NODE_COUNT (1<>__SHFT((D)))&__MSK) +#define __STR(P,D,V) __BYTE((P),(D))=(__BYTE((P),(D))&~(__MSK<<__SHFT((D))))|((V)<<__SHFT((D))) #ifdef WANT_V6 -#define STATS_NETWORK_NODE_MAXDEPTH (48/8-1) +#define STATS_NETWORK_NODE_MAXDEPTH (68-STATS_NETWORK_NODE_BITWIDTH) +#define STATS_NETWORK_NODE_LIMIT (48-STATS_NETWORK_NODE_BITWIDTH) #else -#define STATS_NETWORK_NODE_MAXDEPTH (12+24/8-1) +#define STATS_NETWORK_NODE_MAXDEPTH (28-STATS_NETWORK_NODE_BITWIDTH) +#define STATS_NETWORK_NODE_LIMIT (24-STATS_NETWORK_NODE_BITWIDTH) #endif - typedef union stats_network_node stats_network_node; union stats_network_node { int counters[STATS_NETWORK_NODE_COUNT]; stats_network_node *children[STATS_NETWORK_NODE_COUNT]; }; +#ifdef WANT_LOG_NETWORKS static stats_network_node *stats_network_counters_root = NULL; +#endif static int stat_increase_network_count( stats_network_node **node, int depth, uintptr_t ip ) { - uint8_t *_ip = (uint8_t*)ip; - int foo = _ip[depth]; + int foo = __LDR(ip,depth); if( !*node ) { *node = malloc( sizeof( stats_network_node ) ); @@ -87,7 +96,7 @@ static int stat_increase_network_count( stats_network_node **node, int depth, ui } if( depth < STATS_NETWORK_NODE_MAXDEPTH ) - return stat_increase_network_count( &(*node)->children[ foo ], depth+1, ip ); + return stat_increase_network_count( &(*node)->children[ foo ], depth+STATS_NETWORK_NODE_BITWIDTH, ip ); (*node)->counters[ foo ]++; return 0; @@ -97,11 +106,12 @@ static int stats_shift_down_network_count( stats_network_node **node, int depth, int i, rest = 0; if( !*node ) return 0; - if( ++depth == STATS_NETWORK_NODE_MAXDEPTH ) - for( i=0; icounters[i]>>=shift); - return rest; - } + return rest; + } for( i=0; ichildren[i]; @@ -120,56 +130,155 @@ static int stats_shift_down_network_count( stats_network_node **node, int depth, return rest; } -static void stats_get_highscore_networks( stats_network_node *node, int depth, ot_ip6 node_value, int *scores, ot_ip6 *networks, int network_count ) { - uint8_t *_node_value = (uint8_t*)node_value; +static size_t stats_get_highscore_networks( stats_network_node *node, int depth, ot_ip6 node_value, size_t *scores, ot_ip6 *networks, int network_count, int limit ) { + size_t score = 0; int i; - if( !node ) return; + if( !node ) return 0; - if( depth < STATS_NETWORK_NODE_MAXDEPTH ) { + if( depth < limit ) { for( i=0; ichildren[i] ) { - _node_value[depth] = i; - stats_get_highscore_networks( node->children[i], depth+1, node_value, scores, networks, network_count ); - } - } else - for( i=0; icounters[i] <= scores[0] ) continue; - - _node_value[depth] = i; - while( (jcounters[i]>scores[j] ) ) ++j; - --j; - - memcpy( scores, scores + 1, j * sizeof( *scores ) ); - memcpy( networks, networks + 1, j * sizeof( *networks ) ); - scores[ j ] = node->counters[ i ]; - memcpy( networks + j, _node_value, sizeof( *networks ) ); + if( node->children[i] ) { + __STR(node_value,depth,i); + score += stats_get_highscore_networks( node->children[i], depth+STATS_NETWORK_NODE_BITWIDTH, node_value, scores, networks, network_count, limit ); } + return score; + } + + if( depth > limit && depth < STATS_NETWORK_NODE_MAXDEPTH ) { + for( i=0; ichildren[i] ) + score += stats_get_highscore_networks( node->children[i], depth+STATS_NETWORK_NODE_BITWIDTH, node_value, scores, networks, network_count, limit ); + return score; + } + + if( depth > limit && depth == STATS_NETWORK_NODE_MAXDEPTH ) { + for( i=0; icounters[i]; + return score; + } + + /* if( depth == limit ) */ + for( i=0; icounters[i]; + else + node_score = stats_get_highscore_networks( node->children[i], depth+STATS_NETWORK_NODE_BITWIDTH, node_value, scores, networks, network_count, limit ); + + score += node_score; + + if( node_score <= scores[0] ) continue; + + __STR(node_value,depth,i); + while( j < network_count && node_score > scores[j] ) ++j; + --j; + + memcpy( scores, scores + 1, j * sizeof( *scores ) ); + memcpy( networks, networks + 1, j * sizeof( *networks ) ); + scores[ j ] = node_score; + memcpy( networks + j, node_value, sizeof( *networks ) ); + } + + return score; } -static size_t stats_return_busy_networks( char * reply ) { - ot_ip6 networks[256]; +static size_t stats_return_busy_networks( char * reply, stats_network_node *tree, int amount ) { + ot_ip6 networks[amount]; ot_ip6 node_value; - int scores[256]; + size_t scores[amount]; int i; char * r = reply; - memset( scores, 0, sizeof( *scores ) * 256 ); - memset( networks, 0, sizeof( *networks ) * 256 ); + memset( scores, 0, sizeof( scores ) ); + memset( networks, 0, sizeof( networks ) ); + memset( node_value, 0, sizeof( node_value ) ); - stats_get_highscore_networks( stats_network_counters_root, 0, node_value, scores, networks, 256 ); + stats_get_highscore_networks( tree, 0, node_value, scores, networks, amount, STATS_NETWORK_NODE_MAXDEPTH ); - for( i=255; i>=0; --i) { - r += sprintf( r, "%08i: ", scores[i] ); - r += fmt_ip6c( r, networks[i] ); - *r++ = '\n'; + r += sprintf( r, "Networks, limit /%d:\n", STATS_NETWORK_NODE_MAXDEPTH+STATS_NETWORK_NODE_BITWIDTH ); + for( i=amount-1; i>=0; --i) { + if( scores[i] ) { + r += sprintf( r, "%08zd: ", scores[i] ); +#ifdef WANT_V6 + r += fmt_ip6c( r, networks[i] ); +#else + r += fmt_ip4( r, networks[i]); +#endif + *r++ = '\n'; + } } + memset( scores, 0, sizeof( scores ) ); + memset( networks, 0, sizeof( networks ) ); + memset( node_value, 0, sizeof( node_value ) ); + + stats_get_highscore_networks( tree, 0, node_value, scores, networks, amount, STATS_NETWORK_NODE_LIMIT ); + + r += sprintf( r, "\nNetworks, limit /%d:\n", STATS_NETWORK_NODE_LIMIT+STATS_NETWORK_NODE_BITWIDTH ); + for( i=amount-1; i>=0; --i) { + if( scores[i] ) { + r += sprintf( r, "%08zd: ", scores[i] ); +#ifdef WANT_V6 + r += fmt_ip6c( r, networks[i] ); +#else + r += fmt_ip4( r, networks[i] ); +#endif + *r++ = '\n'; + } + } + return r - reply; } -#endif +static size_t stats_slash24s_txt( char *reply, size_t amount ) { + stats_network_node *slash24s_network_counters_root = NULL; + char *r=reply; + int bucket; + size_t i; + + for( bucket=0; bucketsize; ++i ) { + ot_peerlist *peer_list = ( ((ot_torrent*)(torrents_list->data))[i] ).peer_list; + ot_vector *bucket_list = &peer_list->peers; + int num_buckets = 1; + + if( OT_PEERLIST_HASBUCKETS( peer_list ) ) { + num_buckets = bucket_list->size; + bucket_list = (ot_vector *)bucket_list->data; + } + + while( num_buckets-- ) { + ot_peer *peers = (ot_peer*)bucket_list->data; + size_t numpeers = bucket_list->size; + while( numpeers-- ) + if( stat_increase_network_count( &slash24s_network_counters_root, 0, (uintptr_t)(peers++) ) ) + goto bailout_unlock; + ++bucket_list; + } + } + mutex_bucket_unlock( bucket, 0 ); + if( !g_opentracker_running ) + goto bailout_error; + } + + /* The tree is built. Now analyze */ + r += stats_return_busy_networks( r, slash24s_network_counters_root, amount ); + goto success; + +bailout_unlock: + mutex_bucket_unlock( bucket, 0 ); +bailout_error: + r = reply; +success: + stats_shift_down_network_count( &slash24s_network_counters_root, 0, STATS_NETWORK_NODE_MAXDEPTH*STATS_NETWORK_NODE_BITWIDTH ); + if( slash24s_network_counters_root ) + free( slash24s_network_counters_root ); + return r-reply; +} typedef struct { unsigned long long torrent_count; @@ -234,98 +343,6 @@ size_t stats_top10_txt( char * reply ) { return r - reply; } -/* This function collects 4096 /24s in 4096 possible - malloc blocks - */ -static size_t stats_slash24s_txt( char * reply, size_t amount, uint32_t thresh ) { - -#define NUM_TOPBITS 12 -#define NUM_LOWBITS (24-NUM_TOPBITS) -#define NUM_BUFS (1<size; ++j ) { - ot_peerlist *peer_list = ( ((ot_torrent*)(torrents_list->data))[j] ).peer_list; - for( k=0; kpeers[k].data; - size_t numpeers = peer_list->peers[k].size; - for( l=0; l> 8; - uint32_t *count = counts[ s24 >> NUM_LOWBITS ]; - if( !count ) { - count = malloc( sizeof(uint32_t) * NUM_S24S ); - if( !count ) { - mutex_bucket_unlock( bucket, 0 ); - goto bailout_cleanup; - } - byte_zero( count, sizeof( uint32_t ) * NUM_S24S ); - counts[ s24 >> NUM_LOWBITS ] = count; - } - count[ s24 & MSK_S24S ]++; - } - } - } - mutex_bucket_unlock( bucket, 0 ); - if( !g_opentracker_running ) - goto bailout_cleanup; - } -#endif - - k = l = 0; /* Debug: count allocated bufs */ - for( i=0; i < NUM_BUFS; ++i ) { - uint32_t *count = counts[i]; - if( !counts[i] ) - continue; - ++k; /* Debug: count allocated bufs */ - for( j=0; j < NUM_S24S; ++j ) { - if( count[j] > thresh ) { - /* This subnet seems to announce more torrents than the last in our list */ - int insert_pos = amount - 1; - while( ( insert_pos >= 0 ) && ( count[j] > slash24s[ 2 * insert_pos ] ) ) - --insert_pos; - ++insert_pos; - memcpy( slash24s + 2 * ( insert_pos + 1 ), slash24s + 2 * ( insert_pos ), 2 * sizeof( uint32_t ) * ( amount - insert_pos - 1 ) ); - slash24s[ 2 * insert_pos ] = count[j]; - slash24s[ 2 * insert_pos + 1 ] = ( i << NUM_TOPBITS ) + j; - if( slash24s[ 2 * amount - 2 ] > thresh ) - thresh = slash24s[ 2 * amount - 2 ]; - } - if( count[j] ) ++l; - } - free( count ); - } - - r += sprintf( r, "Allocated bufs: %zd, used s24s: %zd\n", k, l ); - - for( i=0; i < amount; ++i ) - if( slash24s[ 2*i ] >= thresh ) { - uint32_t ip = slash24s[ 2*i +1 ]; - r += sprintf( r, "% 10ld %d.%d.%d.0/24\n", (long)slash24s[ 2*i ], (int)(ip >> 16), (int)(255 & ( ip >> 8 )), (int)(ip & 255) ); - } - - return r - reply; - - for( i=0; i < NUM_BUFS; ++i ) - free( counts[i] ); - - return 0; -} - static unsigned long events_per_time( unsigned long long events, time_t t ) { return events / ( (unsigned int)t ? (unsigned int)t : 1 ); } @@ -532,10 +549,6 @@ size_t return_stats_for_tracker( char *reply, int mode, int format ) { return stats_return_renew_bucket( reply ); case TASK_STATS_SYNCS: return stats_return_sync_mrtg( reply ); -#ifdef WANT_LOG_NETWORKS - case TASK_STATS_BUSY_NETWORKS: - return stats_return_busy_networks( reply ); -#endif default: return 0; } @@ -552,7 +565,7 @@ static void stats_make( int *iovec_entries, struct iovec **iovector, ot_tasktype switch( mode & TASK_TASK_MASK ) { case TASK_STATS_TORRENTS: r += stats_torrents_mrtg( r ); break; case TASK_STATS_PEERS: r += stats_peers_mrtg( r ); break; - case TASK_STATS_SLASH24S: r += stats_slash24s_txt( r, 25, 16 ); break; + case TASK_STATS_SLASH24S: r += stats_slash24s_txt( r, 128 ); break; case TASK_STATS_TOP10: r += stats_top10_txt( r ); break; case TASK_STATS_EVERYTHING: r += stats_return_everything( r ); break; default: -- cgit v1.2.3