From bbfd674e08c6b1fa4081f9c678fd220bead749d7 Mon Sep 17 00:00:00 2001 From: Dirk Engling Date: Sat, 7 Nov 2026 03:55:27 +0100 Subject: Start to opentracker's memory requirements * peer lists for v4 and v6 are now lazily allocated and, more importantly, will be freed once there are no more peers * The downloaded count is now per torrent and not per peer list anymore, moving in what was alignment loss, anyway * The (now) per torrent downloaded and the per peer list peer and seed count shrunk from size_t to uint32_t * A helper guards accesses to the peer and seed count to prevent null pointer dereferences * The base time also shrank from time_t to uint32_t, minutes since the epoch currently fits in 25 bits --- src/ot_clean.c | 77 +++++++++++++++++++++++-------------- src/ot_fullscrape.c | 8 ++-- src/ot_stats.c | 8 ++-- src/trackerlogic.c | 109 ++++++++++++++++++++++++++-------------------------- src/trackerlogic.h | 15 +++++--- 5 files changed, 120 insertions(+), 97 deletions(-) (limited to 'src') diff --git a/src/ot_clean.c b/src/ot_clean.c index eaf069a..77cb4ba 100644 --- a/src/ot_clean.c +++ b/src/ot_clean.c @@ -49,26 +49,17 @@ static ssize_t clean_single_bucket(ot_peer *peers, size_t peer_count, size_t pee return (peers - insert_point) / peer_size; } -int clean_single_peer_list(ot_peerlist *peer_list, size_t peer_size) { - ot_vector *peer_vector = &peer_list->peers; - ot_time timedout = time_since_minutes(peer_list->base); - size_t num_buckets = 1, removed_seeders = 0; +/* Age one family's peers by timedout minutes. Frees the list and clears the + slot once its last peer is gone, so an absent family costs nothing. */ +static void clean_peer_list_family(ot_peerlist **slot, size_t peer_size, ot_time timedout) { + ot_peerlist *peer_list = *slot; + ot_vector *peer_vector; + size_t num_buckets = 1, removed_seeders = 0; - /* No need to clean empty torrent */ - if (!timedout) - return 0; - - /* Torrent has idled out */ - if (timedout > OT_TORRENT_TIMEOUT) - return 1; - - /* Nothing to be cleaned here? Test if torrent is worth keeping */ - if (timedout > OT_PEER_TIMEOUT) { - if (!peer_list->peer_count) - return peer_list->down_count ? 0 : 1; - timedout = OT_PEER_TIMEOUT; - } + if (!peer_list) + return; + peer_vector = &peer_list->peers; if (OT_PEERLIST_HASBUCKETS(peer_list)) { num_buckets = peer_vector->size; peer_vector = (ot_vector *)peer_vector->data; @@ -87,25 +78,53 @@ int clean_single_peer_list(ot_peerlist *peer_list, size_t peer_size) { peer_list->seed_count -= removed_seeders; - /* See if we need to convert a torrent from simple vector to bucket list */ + if (!peer_list->peer_count) { + free_peerlist(peer_list); + *slot = NULL; + return; + } + + /* See if we need to convert the list from simple vector to bucket list */ if ((peer_list->peer_count > OT_PEER_BUCKET_MINCOUNT) || OT_PEERLIST_HASBUCKETS(peer_list)) vector_redistribute_buckets(peer_list, peer_size); - - if (peer_list->peer_count) - peer_list->base = time_now_minutes(); - else { - /* When we got here, the last time that torrent - has been touched is OT_PEER_TIMEOUT Minutes before */ - peer_list->base = time_now_minutes() - OT_PEER_TIMEOUT; - } - return 0; } /* Clean a single torrent return 1 if torrent timed out */ int clean_single_torrent(ot_torrent *torrent) { - return clean_single_peer_list(torrent->peer_list6, OT_PEER_SIZE6) * clean_single_peer_list(torrent->peer_list4, OT_PEER_SIZE4); + ot_time timedout = time_since_minutes(torrent->base); + + /* Touched since the last sweep: nothing to do */ + if (!timedout) + return 0; + + /* Torrent has idled out */ + if (timedout > OT_TORRENT_TIMEOUT) + return 1; + + /* Already empty and nothing arrived before the peer timeout: keep it only + while a down count is worth remembering */ + if (timedout > OT_PEER_TIMEOUT && !torrent->peer_list6 && !torrent->peer_list4) + return torrent->down_count ? 0 : 1; + + /* Peers past the peer timeout are gone regardless, which caps how far the + lists are aged in this pass */ + if (timedout > OT_PEER_TIMEOUT) + timedout = OT_PEER_TIMEOUT; + + clean_peer_list_family(&torrent->peer_list6, OT_PEER_SIZE6, timedout); + clean_peer_list_family(&torrent->peer_list4, OT_PEER_SIZE4, timedout); + + if (torrent->peer_list6 || torrent->peer_list4) + torrent->base = time_now_minutes(); /* still carrying peers */ + else { + /* Fell empty: hold base a peer timeout back so a later idle sweep makes + the keep-or-drop decision above */ + ot_time now = time_now_minutes(); + torrent->base = now > OT_PEER_TIMEOUT ? (uint32_t)(now - OT_PEER_TIMEOUT) : 0; + } + return 0; } /* Written to once, by clean_deinit, and never read: the byte stays in and diff --git a/src/ot_fullscrape.c b/src/ot_fullscrape.c index 52ec480..9fae444 100644 --- a/src/ot_fullscrape.c +++ b/src/ot_fullscrape.c @@ -90,9 +90,9 @@ static void fullscrape_write_one( ot_tasktype mode, char **cursor, ot_torrent *t exactly that size */ char *r = *cursor; const char *r_end = r + OT_SCRAPE_MAXENTRYLEN; - size_t seed_count = torrent->peer_list6->seed_count + torrent->peer_list4->seed_count; - size_t peer_count = torrent->peer_list6->peer_count + torrent->peer_list4->peer_count; - size_t down_count = torrent->peer_list6->down_count + torrent->peer_list4->down_count; + size_t seed_count = peerlist_seed_count(torrent->peer_list6) + peerlist_seed_count(torrent->peer_list4); + size_t peer_count = peerlist_peer_count(torrent->peer_list6) + peerlist_peer_count(torrent->peer_list4); + size_t down_count = torrent->down_count; switch (mode & TASK_TASK_MASK) { case TASK_FULLSCRAPE: @@ -131,7 +131,7 @@ static void fullscrape_write_one( ot_tasktype mode, char **cursor, ot_torrent *t case TASK_FULLSCRAPE_TRACKERSTATE: hex_from_hash(r, *hash); r += 2 * sizeof(ot_hash); - fmt_append(&r, r_end, ":%zd:%zd\n", (ssize_t)(time_wall_minutes() - time_since_minutes(torrent->peer_list6->base)), down_count); + fmt_append(&r, r_end, ":%zd:%zd\n", (ssize_t)(time_wall_minutes() - time_since_minutes(torrent->base)), down_count); break; } *cursor = r; diff --git a/src/ot_stats.c b/src/ot_stats.c index 9901081..c50d4e3 100644 --- a/src/ot_stats.c +++ b/src/ot_stats.c @@ -299,8 +299,8 @@ typedef struct { static int torrent_statter(ot_torrent *torrent, uintptr_t data) { torrent_stats *stats = (torrent_stats *)data; stats->torrent_count++; - stats->peer_count += torrent->peer_list6->peer_count + torrent->peer_list4->peer_count; - stats->seed_count += torrent->peer_list6->seed_count + torrent->peer_list4->seed_count; + stats->peer_count += peerlist_peer_count(torrent->peer_list6) + peerlist_peer_count(torrent->peer_list4); + stats->seed_count += peerlist_seed_count(torrent->peer_list6) + peerlist_seed_count(torrent->peer_list4); return 0; } @@ -343,8 +343,8 @@ void stats_top_txt(char **reply, const char *reply_end, size_t amount) { ot_vector *torrents_list = mutex_bucket_lock(bucket); for (j = 0; j < torrents_list->size; ++j) { ot_torrent *torrent = (ot_torrent *)(torrents_list->data) + j; - size_t peer_count = torrent->peer_list6->peer_count + torrent->peer_list4->peer_count; - size_t seed_count = torrent->peer_list6->seed_count + torrent->peer_list4->seed_count; + size_t peer_count = peerlist_peer_count(torrent->peer_list6) + peerlist_peer_count(torrent->peer_list4); + size_t seed_count = peerlist_seed_count(torrent->peer_list6) + peerlist_seed_count(torrent->peer_list4); size_t leech_count = peer_count - seed_count; stats_insert_into_highscore(peer_count, top100c, amount, &torrent->hash); diff --git a/src/trackerlogic.c b/src/trackerlogic.c index 2313379..6b0659c 100644 --- a/src/trackerlogic.c +++ b/src/trackerlogic.c @@ -42,6 +42,7 @@ void free_peerlist(ot_peerlist *peer_list) { what a state dump records. */ void add_torrent_from_saved_state(ot_hash const hash, ot_time age, size_t down_count) { int exactmatch; + ot_time now = time_now_minutes(); ot_torrent *torrent; ot_vector *torrents_list; @@ -61,19 +62,12 @@ void add_torrent_from_saved_state(ot_hash const hash, ot_time age, size_t down_c return; } - /* Create a new torrent entry, then */ + /* A restored torrent starts with no peers, carrying only its base and its + download count until the first announce brings a family list to life */ memset(torrent, 0, sizeof(ot_torrent)); memcpy(torrent->hash, hash, sizeof(ot_hash)); - - if (!(torrent->peer_list6 = calloc(1, sizeof(ot_peerlist))) || !(torrent->peer_list4 = calloc(1, sizeof(ot_peerlist)))) { - vector_remove_torrent(torrents_list, torrent); - mutex_bucket_unlock_by_hash(hash, 0); - return; - } - - torrent->peer_list6->base = time_now_minutes() - age; - torrent->peer_list4->base = time_now_minutes() - age; - torrent->peer_list6->down_count = down_count; + torrent->base = now > age ? (uint32_t)(now - age) : 0; + torrent->down_count = (uint32_t)down_count; mutex_bucket_unlock_by_hash(hash, 1); } @@ -113,21 +107,23 @@ size_t add_peer_to_torrent_and_return_peers(PROTO_FLAG proto, struct ot_workstru /* Create a new torrent entry, then */ memset(torrent, 0, sizeof(ot_torrent)); memcpy(torrent->hash, *ws->hash, sizeof(ot_hash)); - - if (!(torrent->peer_list6 = calloc(1, sizeof(ot_peerlist))) || !(torrent->peer_list4 = calloc(1, sizeof(ot_peerlist)))) { - vector_remove_torrent(torrents_list, torrent); - mutex_bucket_unlock_by_hash(*ws->hash, 0); - return 0; - } - delta_torrentcount = 1; } else clean_single_torrent(torrent); - torrent->peer_list6->base = time_now_minutes(); - torrent->peer_list4->base = time_now_minutes(); + torrent->base = time_now_minutes(); - peer_list = peer_size == OT_PEER_SIZE6 ? torrent->peer_list6 : torrent->peer_list4; + /* The announcing peer's family list is created on first use */ + { + ot_peerlist **slot = peer_size == OT_PEER_SIZE6 ? &torrent->peer_list6 : &torrent->peer_list4; + if (!*slot && !(*slot = calloc(1, sizeof(ot_peerlist)))) { + if (delta_torrentcount) + vector_remove_torrent(torrents_list, torrent); + mutex_bucket_unlock_by_hash(*ws->hash, 0); + return 0; + } + peer_list = *slot; + } /* Check for peer in torrent */ peer_dest = vector_find_or_insert_peer(&(peer_list->peers), peer_src, peer_size, &exactmatch); @@ -155,7 +151,7 @@ size_t add_peer_to_torrent_and_return_peers(PROTO_FLAG proto, struct ot_workstru peer_list->peer_count++; if (OT_PEERFLAG(ws->peer) & PEER_FLAG_COMPLETED) { - peer_list->down_count++; + torrent->down_count++; stats_issue_event(ws, EVENT_COMPLETED, 0, 0); } if (OT_PEERFLAG(ws->peer) & PEER_FLAG_SEEDING) @@ -182,7 +178,7 @@ size_t add_peer_to_torrent_and_return_peers(PROTO_FLAG proto, struct ot_workstru if (!(OT_PEERFLAG_D(peer_dest, peer_size) & PEER_FLAG_SEEDING) && (OT_PEERFLAG(ws->peer) & PEER_FLAG_SEEDING)) peer_list->seed_count++; if (!(OT_PEERFLAG_D(peer_dest, peer_size) & PEER_FLAG_COMPLETED) && (OT_PEERFLAG(ws->peer) & PEER_FLAG_COMPLETED)) { - peer_list->down_count++; + torrent->down_count++; stats_issue_event(ws, EVENT_COMPLETED, 0, 0); } if (OT_PEERFLAG_D(peer_dest, peer_size) & PEER_FLAG_COMPLETED) @@ -286,11 +282,12 @@ static size_t return_peers_for_torrent_udp(struct ot_workstruct *ws, ot_torrent char *r = reply; size_t peer_size = peer_size_from_peer6(&ws->peer); ot_peerlist *peer_list = peer_size == OT_PEER_SIZE6 ? torrent->peer_list6 : torrent->peer_list4; - size_t peer_count = torrent->peer_list6->peer_count + torrent->peer_list4->peer_count; - size_t seed_count = torrent->peer_list6->seed_count + torrent->peer_list4->seed_count; + size_t list_count = peerlist_peer_count(peer_list); + size_t peer_count = peerlist_peer_count(torrent->peer_list6) + peerlist_peer_count(torrent->peer_list4); + size_t seed_count = peerlist_seed_count(torrent->peer_list6) + peerlist_seed_count(torrent->peer_list4); - if (amount > peer_list->peer_count) - amount = peer_list->peer_count; + if (amount > list_count) + amount = list_count; *(uint32_t *)(r + 0) = htonl(OT_CLIENT_REQUEST_INTERVAL_RANDOM); *(uint32_t *)(r + 4) = htonl(peer_count - seed_count); @@ -298,7 +295,7 @@ static size_t return_peers_for_torrent_udp(struct ot_workstruct *ws, ot_torrent r += 12; if (amount) { - if (amount == peer_list->peer_count) + if (amount == list_count) r += return_peers_all(peer_list, peer_size, r); else r += return_peers_selection(ws, peer_list, peer_size, amount, r); @@ -310,13 +307,15 @@ static size_t return_peers_for_torrent_tcp(struct ot_workstruct *ws, ot_torrent char *r = reply; const char *r_end = ws->outbuf + G_OUTBUF_SIZE; int erval = OT_CLIENT_REQUEST_INTERVAL_RANDOM; - size_t seed_count = torrent->peer_list6->seed_count + torrent->peer_list4->seed_count; - size_t down_count = torrent->peer_list6->down_count + torrent->peer_list4->down_count; - size_t peer_count = torrent->peer_list6->peer_count + torrent->peer_list4->peer_count - seed_count; + size_t count6 = peerlist_peer_count(torrent->peer_list6); + size_t count4 = peerlist_peer_count(torrent->peer_list4); + size_t seed_count = peerlist_seed_count(torrent->peer_list6) + peerlist_seed_count(torrent->peer_list4); + size_t down_count = torrent->down_count; + size_t peer_count = count6 + count4 - seed_count; /* Simple case: amount of peers in both lists is less than requested, here we return all results */ - size_t amount_v4 = torrent->peer_list4->peer_count; - size_t amount_v6 = torrent->peer_list6->peer_count; + size_t amount_v4 = count4; + size_t amount_v6 = count6; /* Complex case: both lists have more than enough entries and we need to split between v4 and v6 clients */ if (amount_v4 + amount_v6 > amount) { @@ -332,8 +331,8 @@ static size_t return_peers_for_torrent_tcp(struct ot_workstruct *ws, ot_torrent /* Fill the rest according to which family's pool provides more peers */ amount_left = amount - (amount_v4 + amount_v6); - left_v4 = torrent->peer_list4->peer_count - amount_v4; - left_v6 = torrent->peer_list6->peer_count - amount_v6; + left_v4 = count4 - amount_v4; + left_v6 = count6 - amount_v6; if (left_v4 + left_v6) { percent_v4 = (SCALE * left_v4) / (left_v4 + left_v6); @@ -344,9 +343,9 @@ static size_t return_peers_for_torrent_tcp(struct ot_workstruct *ws, ot_torrent amount_v6 += (amount_left * percent_v6) / SCALE; /* Integer division rounding can leave out a peer */ - if (amount_v4 + amount_v6 < amount && amount_v6 < torrent->peer_list6->peer_count) + if (amount_v4 + amount_v6 < amount && amount_v6 < count6) ++amount_v6; - if (amount_v4 + amount_v6 < amount && amount_v4 < torrent->peer_list4->peer_count) + if (amount_v4 + amount_v6 < amount && amount_v4 < count4) ++amount_v4; } @@ -355,7 +354,7 @@ static size_t return_peers_for_torrent_tcp(struct ot_workstruct *ws, ot_torrent if (amount_v4) { fmt_append(&r, r_end, PEERS_BENCODED4 "%zd:", OT_PEER_COMPARE_SIZE4 * amount_v4); - if (amount_v4 == torrent->peer_list4->peer_count) + if (amount_v4 == count4) r += return_peers_all(torrent->peer_list4, OT_PEER_SIZE4, r); else r += return_peers_selection(ws, torrent->peer_list4, OT_PEER_SIZE4, amount_v4, r); @@ -363,7 +362,7 @@ static size_t return_peers_for_torrent_tcp(struct ot_workstruct *ws, ot_torrent if (amount_v6) { fmt_append(&r, r_end, PEERS_BENCODED6 "%zd:", OT_PEER_COMPARE_SIZE6 * amount_v6); - if (amount_v6 == torrent->peer_list6->peer_count) + if (amount_v6 == count6) r += return_peers_all(torrent->peer_list6, OT_PEER_SIZE6, r); else r += return_peers_selection(ws, torrent->peer_list6, OT_PEER_SIZE6, amount_v6, r); @@ -402,9 +401,9 @@ size_t return_udp_scrape_for_torrent(ot_hash const hash, char *reply) { memset(reply, 0, 12); delta_torrentcount = -1; } else { - r[0] = htonl(torrent->peer_list6->seed_count + torrent->peer_list4->seed_count); - r[1] = htonl(torrent->peer_list6->down_count + torrent->peer_list4->down_count); - r[2] = htonl(torrent->peer_list6->peer_count + torrent->peer_list4->peer_count - torrent->peer_list6->seed_count - torrent->peer_list4->seed_count); + r[0] = htonl(peerlist_seed_count(torrent->peer_list6) + peerlist_seed_count(torrent->peer_list4)); + r[1] = htonl(torrent->down_count); + r[2] = htonl(peerlist_peer_count(torrent->peer_list6) + peerlist_peer_count(torrent->peer_list4) - peerlist_seed_count(torrent->peer_list6) - peerlist_seed_count(torrent->peer_list4)); } } mutex_bucket_unlock_by_hash(hash, delta_torrentcount); @@ -431,9 +430,9 @@ void return_tcp_scrape_for_torrent(ot_hash const *hash_list, size_t amount, char } else { fmt_append(reply, reply_end, "20:"); fmt_bytes(reply, reply_end, hash, sizeof(ot_hash)); - fmt_append(reply, reply_end, "d8:completei%zde10:downloadedi%zde10:incompletei%zdee", - torrent->peer_list6->seed_count + torrent->peer_list4->seed_count, torrent->peer_list6->down_count + torrent->peer_list4->down_count, - torrent->peer_list6->peer_count + torrent->peer_list4->peer_count - torrent->peer_list6->seed_count - torrent->peer_list4->seed_count); + size_t seed_count = peerlist_seed_count(torrent->peer_list6) + peerlist_seed_count(torrent->peer_list4); + size_t peer_count = peerlist_peer_count(torrent->peer_list6) + peerlist_peer_count(torrent->peer_list4); + fmt_append(reply, reply_end, "d8:completei%zde10:downloadedi%zde10:incompletei%zdee", seed_count, (size_t)torrent->down_count, peer_count - seed_count); } } mutex_bucket_unlock_by_hash(*hash, delta_torrentcount); @@ -461,17 +460,19 @@ size_t remove_peer_from_torrent(PROTO_FLAG proto, struct ot_workstruct *ws) { if (exactmatch) { peer_list = peer_size == OT_PEER_SIZE6 ? torrent->peer_list6 : torrent->peer_list4; - switch (vector_remove_peer(&peer_list->peers, peer_src, peer_size)) { - case 2: - peer_list->seed_count--; /* Intentional fallthrough */ - case 1: - peer_list->peer_count--; /* Intentional fallthrough */ - default: - break; + if (peer_list) { + switch (vector_remove_peer(&peer_list->peers, peer_src, peer_size)) { + case 2: + peer_list->seed_count--; /* Intentional fallthrough */ + case 1: + peer_list->peer_count--; /* Intentional fallthrough */ + default: + break; + } } - peer_count = torrent->peer_list6->peer_count + torrent->peer_list4->peer_count; - seed_count = torrent->peer_list6->seed_count + torrent->peer_list4->seed_count; + peer_count = peerlist_peer_count(torrent->peer_list6) + peerlist_peer_count(torrent->peer_list4); + seed_count = peerlist_seed_count(torrent->peer_list6) + peerlist_seed_count(torrent->peer_list4); } if (proto == FLAG_TCP) { diff --git a/src/trackerlogic.h b/src/trackerlogic.h index 241b6a7..6b53406 100644 --- a/src/trackerlogic.h +++ b/src/trackerlogic.h @@ -165,17 +165,17 @@ struct ot_peerlist; typedef struct ot_peerlist ot_peerlist; typedef struct { ot_hash hash; - ot_peerlist *peer_list6; - ot_peerlist *peer_list4; + ot_peerlist *peer_list6; /* NULL until the torrent has a v6 peer */ + ot_peerlist *peer_list4; /* NULL until the torrent has a v4 peer */ + uint32_t base; /* minutes, when the torrent was last cleaned */ + uint32_t down_count; /* completed events, summed over both families */ } ot_torrent; #include "ot_vector.h" struct ot_peerlist { - ot_time base; - size_t seed_count; - size_t peer_count; - size_t down_count; + uint32_t seed_count; + uint32_t peer_count; /* normal peers vector or pointer to ot_vector[32] buckets if data != NULL and space == 0 */ @@ -183,6 +183,9 @@ struct ot_peerlist { }; #define OT_PEERLIST_HASBUCKETS(peer_list) ((peer_list)->peers.size > (peer_list)->peers.space) +static inline uint32_t peerlist_peer_count(const ot_peerlist *peer_list) { return peer_list ? peer_list->peer_count : 0; } +static inline uint32_t peerlist_seed_count(const ot_peerlist *peer_list) { return peer_list ? peer_list->seed_count : 0; } + /* The http replies the tracker counts when it has to refuse a request. */ enum { CODE_HTTPERROR_302, -- cgit v1.3