From 9297967f8523f1ac22bbac1823529b970a07cf56 Mon Sep 17 00:00:00 2001 From: erdgeist <> Date: Wed, 4 Mar 2009 14:35:21 +0000 Subject: Add a stat option the count bucket stalls Add an iterator for all torrents. It's slower but for stats it's okay. Move some stats collection stuff to the new iterator. More to come. Start a "report all stats" page. Start fixing the code to identify "busy" networks. Add the concept of hosts allowed to proxy. Add a parser for the X-Forwarded-For: HTTP header. Clean up HTTP Header handling code. (Remove some left overs of now vanished sync code). --- ot_stats.c | 341 ++++++++++++++++++++++++++++++++----------------------------- 1 file changed, 180 insertions(+), 161 deletions(-) (limited to 'ot_stats.c') diff --git a/ot_stats.c b/ot_stats.c index a53ac3b..518dbb5 100644 --- a/ot_stats.c +++ b/ot_stats.c @@ -1,7 +1,7 @@ /* This software was written by Dirk Engling - It is considered beerware. Prost. Skol. Cheers or whatever. - - $id$ */ + It is considered beerware. Prost. Skol. Cheers or whatever. + + $id$ */ /* System */ #include @@ -50,12 +50,13 @@ static unsigned long long ot_full_scrape_size = 0; static unsigned long long ot_failed_request_counts[CODE_HTTPERROR_COUNT]; static unsigned long long ot_renewed[OT_PEER_TIMEOUT]; static unsigned long long ot_overall_sync_count; +static unsigned long long ot_overall_stall_count; static time_t ot_start_time; #ifdef WANT_LOG_NETWORKS #define STATS_NETWORK_NODE_BITWIDTH 8 -#define STATS_NETWORK_NODE_MAXDEPTH 3 +#define STATS_NETWORK_NODE_MAXDEPTH 16 #define STATS_NETWORK_NODE_BITMASK ((1<> ( 32 - STATS_NETWORK_NODE_BITWIDTH * ( ++depth ) ) ) & STATS_NETWORK_NODE_BITMASK; - +static int stat_increase_network_count( stats_network_node **node, int depth, uintptr_t ip ) { + ot_ip6 *_ip = (ot_ip6*)ip; + int foo = (*_ip)[depth]; + if( !*node ) { *node = malloc( sizeof( stats_network_node ) ); if( !*node ) return -1; memset( *node, 0, sizeof( stats_network_node ) ); } - + if( depth < STATS_NETWORK_NODE_MAXDEPTH ) return stat_increase_network_count( &(*node)->children[ foo ], depth, ip ); - + (*node)->counters[ foo ]++; return 0; } @@ -88,40 +90,40 @@ static int stat_increase_network_count( stats_network_node **node, int depth, ui static int stats_shift_down_network_count( stats_network_node **node, int depth, int shift ) { int i, rest = 0; if( !*node ) return 0; - + if( ++depth == STATS_NETWORK_NODE_MAXDEPTH ) for( i=0; icounters[i]>>=shift); - return rest; - } - + return rest; + } + for( i=0; ichildren[i]; int rest_val; - + if( !*childnode ) continue; - + rest += rest_val = stats_shift_down_network_count( childnode, depth, shift ); - + if( rest_val ) continue; - + free( (*node)->children[i] ); (*node)->children[i] = NULL; } - + return rest; } static void stats_get_highscore_networks( stats_network_node *node, int depth, uint32_t node_value, int *scores, uint32_t *networks, int network_count ) { int i; - + if( !node ) return; - + if( !depth++ ) { memset( scores, 0, sizeof( *scores ) * network_count ); memset( networks, 0, sizeof( *networks ) * network_count ); } - + if( depth < STATS_NETWORK_NODE_MAXDEPTH ) { for( i=0; ichildren[i] ) @@ -130,15 +132,15 @@ static void stats_get_highscore_networks( stats_network_node *node, int depth, u for( i=0; icounters[i] <= scores[0] ) continue; - + while( (jcounters[i]>scores[j] ) ) ++j; --j; - + memcpy( scores, scores + 1, j * sizeof( *scores ) ); memcpy( networks, networks + 1, j * sizeof( *networks ) ); scores[ j ] = node->counters[ i ]; networks[ j ] = node_value | ( i << ( 32 - depth * STATS_NETWORK_NODE_BITWIDTH ) ); - } + } } static size_t stats_return_busy_networks( char * reply ) { @@ -146,17 +148,31 @@ static size_t stats_return_busy_networks( char * reply ) { int scores[16]; int i; char * r = reply; - + stats_get_highscore_networks( stats_network_counters_root, 0, 0, scores, networks, 16 ); - + for( i=15; i>=0; --i) r += sprintf( r, "%08i: %d.%d.%d.0/24\n", scores[i], (networks[i]>>24)&0xff, (networks[i]>>16)&0xff, (networks[i]>>8)&0xff ); - + return r - reply; } #endif +typedef struct { + unsigned long long torrent_count; + unsigned long long peer_count; + unsigned long long seed_count; +} torrent_stats; + +static int torrent_statter( ot_torrent *torrent, uintptr_t data ) { + torrent_stats *stats = (torrent_stats*)data; + stats->torrent_count++; + stats->peer_count += torrent->peer_list->peer_count; + stats->seed_count += torrent->peer_list->seed_count; + return 0; +} + /* Converter function from memory to human readable hex strings */ static char*to_hex(char*d,uint8_t*s){char*m="0123456789ABCDEF";char *t=d;char*e=d+40;while(d>4];*d++=m[*s++&15];}*d=0;return t;} @@ -168,10 +184,10 @@ size_t stats_top10_txt( char * reply ) { ot_record top10s[10], top10c[10]; char *r = reply, hex_out[42]; int idx, bucket; - + byte_zero( top10s, sizeof( top10s ) ); byte_zero( top10c, sizeof( top10c ) ); - + for( bucket=0; bucketsize; ++j ) { @@ -193,7 +209,7 @@ size_t stats_top10_txt( char * reply ) { if( !g_opentracker_running ) return 0; } - + r += sprintf( r, "Top 10 torrents by peers:\n" ); for( idx=0; idx<10; ++idx ) if( top10c[idx].torrent ) @@ -202,31 +218,31 @@ size_t stats_top10_txt( char * reply ) { for( idx=0; idx<10; ++idx ) if( top10s[idx].torrent ) r += sprintf( r, "\t%zd\t%s\n", top10s[idx].val, to_hex( hex_out, top10s[idx].torrent->hash) ); - + return r - reply; } /* This function collects 4096 /24s in 4096 possible - malloc blocks -*/ + malloc blocks + */ static size_t stats_slash24s_txt( char * reply, size_t amount, uint32_t thresh ) { - + #define NUM_TOPBITS 12 #define NUM_LOWBITS (24-NUM_TOPBITS) #define NUM_BUFS (1<= thresh ) { uint32_t ip = slash24s[ 2*i +1 ]; r += sprintf( r, "% 10ld %d.%d.%d.0/24\n", (long)slash24s[ 2*i ], (int)(ip >> 16), (int)(255 & ( ip >> 8 )), (int)(ip & 255) ); } - + return r - reply; - + for( i=0; i < NUM_BUFS; ++i ) free( counts[i] ); - + return 0; } /* struct { - size_t size - size_t space - size_t count + size_t size + size_t space + size_t count } */ @@ -313,105 +329,95 @@ static unsigned long events_per_time( unsigned long long events, time_t t ) { static size_t stats_connections_mrtg( char * reply ) { ot_time t = time( NULL ) - ot_start_time; return sprintf( reply, - "%llu\n%llu\n%i seconds (%i hours)\nopentracker connections, %lu conns/s :: %lu success/s.", - ot_overall_tcp_connections+ot_overall_udp_connections, - ot_overall_tcp_successfulannounces+ot_overall_udp_successfulannounces+ot_overall_udp_connects, - (int)t, - (int)(t / 3600), - events_per_time( ot_overall_tcp_connections+ot_overall_udp_connections, t ), - events_per_time( ot_overall_tcp_successfulannounces+ot_overall_udp_successfulannounces+ot_overall_udp_connects, t ) - ); + "%llu\n%llu\n%i seconds (%i hours)\nopentracker connections, %lu conns/s :: %lu success/s.", + ot_overall_tcp_connections+ot_overall_udp_connections, + ot_overall_tcp_successfulannounces+ot_overall_udp_successfulannounces+ot_overall_udp_connects, + (int)t, + (int)(t / 3600), + events_per_time( ot_overall_tcp_connections+ot_overall_udp_connections, t ), + events_per_time( ot_overall_tcp_successfulannounces+ot_overall_udp_successfulannounces+ot_overall_udp_connects, t ) + ); } static size_t stats_udpconnections_mrtg( char * reply ) { ot_time t = time( NULL ) - ot_start_time; return sprintf( reply, - "%llu\n%llu\n%i seconds (%i hours)\nopentracker udp4 stats, %lu conns/s :: %lu success/s.", - ot_overall_udp_connections, - ot_overall_udp_successfulannounces+ot_overall_udp_connects, - (int)t, - (int)(t / 3600), - events_per_time( ot_overall_udp_connections, t ), - events_per_time( ot_overall_udp_successfulannounces+ot_overall_udp_connects, t ) - ); + "%llu\n%llu\n%i seconds (%i hours)\nopentracker udp4 stats, %lu conns/s :: %lu success/s.", + ot_overall_udp_connections, + ot_overall_udp_successfulannounces+ot_overall_udp_connects, + (int)t, + (int)(t / 3600), + events_per_time( ot_overall_udp_connections, t ), + events_per_time( ot_overall_udp_successfulannounces+ot_overall_udp_connects, t ) + ); } static size_t stats_tcpconnections_mrtg( char * reply ) { time_t t = time( NULL ) - ot_start_time; return sprintf( reply, - "%llu\n%llu\n%i seconds (%i hours)\nopentracker tcp4 stats, %lu conns/s :: %lu success/s.", - ot_overall_tcp_connections, - ot_overall_tcp_successfulannounces, - (int)t, - (int)(t / 3600), - events_per_time( ot_overall_tcp_connections, t ), - events_per_time( ot_overall_tcp_successfulannounces, t ) - ); + "%llu\n%llu\n%i seconds (%i hours)\nopentracker tcp4 stats, %lu conns/s :: %lu success/s.", + ot_overall_tcp_connections, + ot_overall_tcp_successfulannounces, + (int)t, + (int)(t / 3600), + events_per_time( ot_overall_tcp_connections, t ), + events_per_time( ot_overall_tcp_successfulannounces, t ) + ); } static size_t stats_scrape_mrtg( char * reply ) { time_t t = time( NULL ) - ot_start_time; return sprintf( reply, - "%llu\n%llu\n%i seconds (%i hours)\nopentracker scrape stats, %lu scrape/s (tcp and udp)", - ot_overall_tcp_successfulscrapes, - ot_overall_udp_successfulscrapes, - (int)t, - (int)(t / 3600), - events_per_time( (ot_overall_tcp_successfulscrapes+ot_overall_udp_successfulscrapes), t ) - ); + "%llu\n%llu\n%i seconds (%i hours)\nopentracker scrape stats, %lu scrape/s (tcp and udp)", + ot_overall_tcp_successfulscrapes, + ot_overall_udp_successfulscrapes, + (int)t, + (int)(t / 3600), + events_per_time( (ot_overall_tcp_successfulscrapes+ot_overall_udp_successfulscrapes), t ) + ); } static size_t stats_fullscrapes_mrtg( char * reply ) { ot_time t = time( NULL ) - ot_start_time; return sprintf( reply, - "%llu\n%llu\n%i seconds (%i hours)\nopentracker full scrape stats, %lu conns/s :: %lu bytes/s.", - ot_full_scrape_count * 1000, - ot_full_scrape_size, - (int)t, - (int)(t / 3600), - events_per_time( ot_full_scrape_count, t ), - events_per_time( ot_full_scrape_size, t ) - ); + "%llu\n%llu\n%i seconds (%i hours)\nopentracker full scrape stats, %lu conns/s :: %lu bytes/s.", + ot_full_scrape_count * 1000, + ot_full_scrape_size, + (int)t, + (int)(t / 3600), + events_per_time( ot_full_scrape_count, t ), + events_per_time( ot_full_scrape_size, t ) + ); } static size_t stats_peers_mrtg( char * reply ) { - size_t torrent_count = 0, peer_count = 0, seed_count = 0, j; - int bucket; - - for( bucket=0; bucketsize; - for( j=0; jsize; ++j ) { - ot_peerlist *peer_list = ( ((ot_torrent*)(torrents_list->data))[j] ).peer_list; - peer_count += peer_list->peer_count; seed_count += peer_list->seed_count; - } - mutex_bucket_unlock( bucket, 0 ); - if( !g_opentracker_running ) - return 0; - } - return sprintf( reply, "%zd\n%zd\nopentracker serving %zd torrents\nopentracker", - peer_count, - seed_count, - torrent_count - ); + torrent_stats stats = {0,0,0}; + + iterate_all_torrents( torrent_statter, (uintptr_t)&stats ); + + return sprintf( reply, "%llu\n%llu\nopentracker serving %llu torrents\nopentracker", + stats.peer_count, + stats.seed_count, + stats.torrent_count + ); } static size_t stats_startstop_mrtg( char * reply ) { size_t torrent_count = mutex_get_torrent_count(); - + return sprintf( reply, "%zd\n%zd\nopentracker handling %zd torrents\nopentracker", - (size_t)0, - (size_t)0, - torrent_count - ); + (size_t)0, + (size_t)0, + torrent_count + ); } static size_t stats_toraddrem_mrtg( char * reply ) { size_t peer_count = 0, j; int bucket; - + for( bucket=0; bucket\n" ); + r += sprintf( r, " %llu\n", (unsigned long long)(time( NULL ) - ot_start_time) ); + r += sprintf( r, " %zd\n", mutex_get_torrent_count() ); + /* r += sprintf( r, " %llu\n", ); */ + + r += sprintf( reply, "" ); + return r - reply; } extern const char @@ -480,9 +496,9 @@ extern const char size_t stats_return_tracker_version( char *reply ) { return sprintf( reply, "%s%s%s%s%s%s%s%s%s%s%s%s%s", - g_version_opentracker_c, g_version_accesslist_c, g_version_clean_c, g_version_fullscrape_c, g_version_http_c, - g_version_iovec_c, g_version_mutex_c, g_version_stats_c, g_version_udp_c, g_version_vector_c, - g_version_scan_urlencoded_query_c, g_version_trackerlogic_c, g_version_livesync_c ); + g_version_opentracker_c, g_version_accesslist_c, g_version_clean_c, g_version_fullscrape_c, g_version_http_c, + g_version_iovec_c, g_version_mutex_c, g_version_stats_c, g_version_udp_c, g_version_vector_c, + g_version_scan_urlencoded_query_c, g_version_trackerlogic_c, g_version_livesync_c ); } size_t return_stats_for_tracker( char *reply, int mode, int format ) { @@ -509,7 +525,7 @@ size_t return_stats_for_tracker( char *reply, int mode, int format ) { case TASK_STATS_RENEW: return stats_return_renew_bucket( reply ); case TASK_STATS_SYNCS: - return stats_return_sync_mrtg( reply ); + return stats_return_sync_mrtg( reply ); #ifdef WANT_LOG_NETWORKS case TASK_STATS_BUSY_NETWORKS: return stats_return_busy_networks( reply ); @@ -521,12 +537,12 @@ size_t return_stats_for_tracker( char *reply, int mode, int format ) { static void stats_make( int *iovec_entries, struct iovec **iovector, ot_tasktype mode ) { char *r; - + *iovec_entries = 0; *iovector = NULL; if( !( r = iovec_increase( iovec_entries, iovector, OT_STATS_TMPSIZE ) ) ) return; - + switch( mode & TASK_TASK_MASK ) { case TASK_STATS_TORRENTS: r += stats_torrents_mrtg( r ); break; case TASK_STATS_PEERS: r += stats_peers_mrtg( r ); break; @@ -560,26 +576,26 @@ void stats_issue_event( ot_status_event event, PROTO_FLAG proto, uintptr_t event ot_full_scrape_size += event_data; break; case EVENT_FULLSCRAPE_REQUEST: - { - ot_ip6 *ip = (ot_ip6*)event_data; /* ugly hack to transfer ip to stats */ - char _debug[512]; - int off = snprintf( _debug, sizeof(_debug), "[%08d] scrp: ", (unsigned int)(g_now_seconds - ot_start_time)/60 ); - off += fmt_ip6( _debug+off, *ip ); - off += snprintf( _debug+off, sizeof(_debug)-off, " - FULL SCRAPE\n" ); - write( 2, _debug, off ); - ot_full_scrape_request_count++; - } + { + ot_ip6 *ip = (ot_ip6*)event_data; /* ugly hack to transfer ip to stats */ + char _debug[512]; + int off = snprintf( _debug, sizeof(_debug), "[%08d] scrp: ", (unsigned int)(g_now_seconds - ot_start_time)/60 ); + off += fmt_ip6( _debug+off, *ip ); + off += snprintf( _debug+off, sizeof(_debug)-off, " - FULL SCRAPE\n" ); + write( 2, _debug, off ); + ot_full_scrape_request_count++; + } break; case EVENT_FULLSCRAPE_REQUEST_GZIP: - { - ot_ip6 *ip = (ot_ip6*)event_data; /* ugly hack to transfer ip to stats */ - char _debug[512]; - int off = snprintf( _debug, sizeof(_debug), "[%08d] scrp: ", (unsigned int)(g_now_seconds - ot_start_time)/60 ); - off += fmt_ip6(_debug+off, *ip ); - off += snprintf( _debug+off, sizeof(_debug)-off, " - FULL SCRAPE\n" ); - write( 2, _debug, off ); - ot_full_scrape_request_count++; - } + { + ot_ip6 *ip = (ot_ip6*)event_data; /* ugly hack to transfer ip to stats */ + char _debug[512]; + int off = snprintf( _debug, sizeof(_debug), "[%08d] scrp: ", (unsigned int)(g_now_seconds - ot_start_time)/60 ); + off += fmt_ip6(_debug+off, *ip ); + off += snprintf( _debug+off, sizeof(_debug)-off, " - FULL SCRAPE\n" ); + write( 2, _debug, off ); + ot_full_scrape_request_count++; + } break; case EVENT_FAILED: ot_failed_request_counts[event_data]++; @@ -590,6 +606,9 @@ void stats_issue_event( ot_status_event event, PROTO_FLAG proto, uintptr_t event case EVENT_SYNC: ot_overall_sync_count+=event_data; break; + case EVENT_BUCKET_LOCKED: + ot_overall_stall_count++; + break; default: break; } @@ -598,9 +617,9 @@ void stats_issue_event( ot_status_event event, PROTO_FLAG proto, uintptr_t event static void * stats_worker( void * args ) { int iovec_entries; struct iovec *iovector; - + args = args; - + while( 1 ) { ot_tasktype tasktype = TASK_STATS; ot_taskid taskid = mutex_workqueue_poptask( &tasktype ); -- cgit v1.2.3