Start to opentracker's memory requirements

* peer lists for v4 and v6 are now lazily allocated and, more importantly, will be freed once there are no more peers
* The downloaded count is now per torrent and not per peer list anymore, moving in what was alignment loss, anyway
* The (now) per torrent downloaded and the per peer list peer and seed count shrunk from size_t to uint32_t
* A helper guards accesses to the peer and seed count to prevent null pointer dereferences
* The base time also shrank from time_t to uint32_t, minutes since the epoch currently fits in 25 bits

1 files changed, 55 insertions, 54 deletions

diff --git a/src/trackerlogic.c b/src/trackerlogic.c
index 2313379..6b0659c 100644
--- a/src/trackerlogic.c
+++ b/src/trackerlogic.c
@@ -42,6 +42,7 @@ void free_peerlist(ot_peerlist *peer_list) {
    what a state dump records. */
 void add_torrent_from_saved_state(ot_hash const hash, ot_time age, size_t down_count) {
   int         exactmatch;
+  ot_time     now = time_now_minutes();
   ot_torrent *torrent;
   ot_vector  *torrents_list;
 
@@ -61,19 +62,12 @@ void add_torrent_from_saved_state(ot_hash const hash, ot_time age, size_t down_c
     return;
   }
 
-  /* Create a new torrent entry, then */
+  /* A restored torrent starts with no peers, carrying only its base and its
+     download count until the first announce brings a family list to life */
   memset(torrent, 0, sizeof(ot_torrent));
   memcpy(torrent->hash, hash, sizeof(ot_hash));
-
-  if (!(torrent->peer_list6 = calloc(1, sizeof(ot_peerlist))) || !(torrent->peer_list4 = calloc(1, sizeof(ot_peerlist)))) {
-    vector_remove_torrent(torrents_list, torrent);
-    mutex_bucket_unlock_by_hash(hash, 0);
-    return;
-  }
-
-  torrent->peer_list6->base       = time_now_minutes() - age;
-  torrent->peer_list4->base       = time_now_minutes() - age;
-  torrent->peer_list6->down_count = down_count;
+  torrent->base       = now > age ? (uint32_t)(now - age) : 0;
+  torrent->down_count = (uint32_t)down_count;
 
   mutex_bucket_unlock_by_hash(hash, 1);
 }
@@ -113,21 +107,23 @@ size_t add_peer_to_torrent_and_return_peers(PROTO_FLAG proto, struct ot_workstru
     /* Create a new torrent entry, then */
     memset(torrent, 0, sizeof(ot_torrent));
     memcpy(torrent->hash, *ws->hash, sizeof(ot_hash));
-
-    if (!(torrent->peer_list6 = calloc(1, sizeof(ot_peerlist))) || !(torrent->peer_list4 = calloc(1, sizeof(ot_peerlist)))) {
-      vector_remove_torrent(torrents_list, torrent);
-      mutex_bucket_unlock_by_hash(*ws->hash, 0);
-      return 0;
-    }
-
     delta_torrentcount = 1;
   } else
     clean_single_torrent(torrent);
 
-  torrent->peer_list6->base = time_now_minutes();
-  torrent->peer_list4->base = time_now_minutes();
+  torrent->base = time_now_minutes();
 
-  peer_list = peer_size == OT_PEER_SIZE6 ? torrent->peer_list6 : torrent->peer_list4;
+  /* The announcing peer's family list is created on first use */
+  {
+    ot_peerlist **slot = peer_size == OT_PEER_SIZE6 ? &torrent->peer_list6 : &torrent->peer_list4;
+    if (!*slot && !(*slot = calloc(1, sizeof(ot_peerlist)))) {
+      if (delta_torrentcount)
+        vector_remove_torrent(torrents_list, torrent);
+      mutex_bucket_unlock_by_hash(*ws->hash, 0);
+      return 0;
+    }
+    peer_list = *slot;
+  }
 
   /* Check for peer in torrent */
   peer_dest = vector_find_or_insert_peer(&(peer_list->peers), peer_src, peer_size, &exactmatch);
@@ -155,7 +151,7 @@ size_t add_peer_to_torrent_and_return_peers(PROTO_FLAG proto, struct ot_workstru
 
     peer_list->peer_count++;
     if (OT_PEERFLAG(ws->peer) & PEER_FLAG_COMPLETED) {
-      peer_list->down_count++;
+      torrent->down_count++;
       stats_issue_event(ws, EVENT_COMPLETED, 0, 0);
     }
     if (OT_PEERFLAG(ws->peer) & PEER_FLAG_SEEDING)
@@ -182,7 +178,7 @@ size_t add_peer_to_torrent_and_return_peers(PROTO_FLAG proto, struct ot_workstru
     if (!(OT_PEERFLAG_D(peer_dest, peer_size) & PEER_FLAG_SEEDING) && (OT_PEERFLAG(ws->peer) & PEER_FLAG_SEEDING))
       peer_list->seed_count++;
     if (!(OT_PEERFLAG_D(peer_dest, peer_size) & PEER_FLAG_COMPLETED) && (OT_PEERFLAG(ws->peer) & PEER_FLAG_COMPLETED)) {
-      peer_list->down_count++;
+      torrent->down_count++;
       stats_issue_event(ws, EVENT_COMPLETED, 0, 0);
     }
     if (OT_PEERFLAG_D(peer_dest, peer_size) & PEER_FLAG_COMPLETED)
@@ -286,11 +282,12 @@ static size_t return_peers_for_torrent_udp(struct ot_workstruct *ws, ot_torrent
   char        *r          = reply;
   size_t       peer_size  = peer_size_from_peer6(&ws->peer);
   ot_peerlist *peer_list  = peer_size == OT_PEER_SIZE6 ? torrent->peer_list6 : torrent->peer_list4;
-  size_t       peer_count = torrent->peer_list6->peer_count + torrent->peer_list4->peer_count;
-  size_t       seed_count = torrent->peer_list6->seed_count + torrent->peer_list4->seed_count;
+  size_t       list_count = peerlist_peer_count(peer_list);
+  size_t       peer_count = peerlist_peer_count(torrent->peer_list6) + peerlist_peer_count(torrent->peer_list4);
+  size_t       seed_count = peerlist_seed_count(torrent->peer_list6) + peerlist_seed_count(torrent->peer_list4);
 
-  if (amount > peer_list->peer_count)
-    amount = peer_list->peer_count;
+  if (amount > list_count)
+    amount = list_count;
 
   *(uint32_t *)(r + 0)  = htonl(OT_CLIENT_REQUEST_INTERVAL_RANDOM);
   *(uint32_t *)(r + 4)  = htonl(peer_count - seed_count);
@@ -298,7 +295,7 @@ static size_t return_peers_for_torrent_udp(struct ot_workstruct *ws, ot_torrent
   r                    += 12;
 
   if (amount) {
-    if (amount == peer_list->peer_count)
+    if (amount == list_count)
       r += return_peers_all(peer_list, peer_size, r);
     else
       r += return_peers_selection(ws, peer_list, peer_size, amount, r);
@@ -310,13 +307,15 @@ static size_t return_peers_for_torrent_tcp(struct ot_workstruct *ws, ot_torrent
   char       *r          = reply;
   const char *r_end      = ws->outbuf + G_OUTBUF_SIZE;
   int         erval      = OT_CLIENT_REQUEST_INTERVAL_RANDOM;
-  size_t      seed_count = torrent->peer_list6->seed_count + torrent->peer_list4->seed_count;
-  size_t      down_count = torrent->peer_list6->down_count + torrent->peer_list4->down_count;
-  size_t      peer_count = torrent->peer_list6->peer_count + torrent->peer_list4->peer_count - seed_count;
+  size_t      count6     = peerlist_peer_count(torrent->peer_list6);
+  size_t      count4     = peerlist_peer_count(torrent->peer_list4);
+  size_t      seed_count = peerlist_seed_count(torrent->peer_list6) + peerlist_seed_count(torrent->peer_list4);
+  size_t      down_count = torrent->down_count;
+  size_t      peer_count = count6 + count4 - seed_count;
 
   /* Simple case: amount of peers in both lists is less than requested, here we return all results */
-  size_t amount_v4  = torrent->peer_list4->peer_count;
-  size_t amount_v6  = torrent->peer_list6->peer_count;
+  size_t amount_v4  = count4;
+  size_t amount_v6  = count6;
 
   /* Complex case: both lists have more than enough entries and we need to split between v4 and v6 clients */
   if (amount_v4 + amount_v6 > amount) {
@@ -332,8 +331,8 @@ static size_t return_peers_for_torrent_tcp(struct ot_workstruct *ws, ot_torrent
     /* Fill the rest according to which family's pool provides more peers */
     amount_left = amount - (amount_v4 + amount_v6);
 
-    left_v4     = torrent->peer_list4->peer_count - amount_v4;
-    left_v6     = torrent->peer_list6->peer_count - amount_v6;
+    left_v4     = count4 - amount_v4;
+    left_v6     = count6 - amount_v6;
 
     if (left_v4 + left_v6) {
       percent_v4 = (SCALE * left_v4) / (left_v4 + left_v6);
@@ -344,9 +343,9 @@ static size_t return_peers_for_torrent_tcp(struct ot_workstruct *ws, ot_torrent
     amount_v6 += (amount_left * percent_v6) / SCALE;
 
     /* Integer division rounding can leave out a peer */
-    if (amount_v4 + amount_v6 < amount && amount_v6 < torrent->peer_list6->peer_count)
+    if (amount_v4 + amount_v6 < amount && amount_v6 < count6)
       ++amount_v6;
-    if (amount_v4 + amount_v6 < amount && amount_v4 < torrent->peer_list4->peer_count)
+    if (amount_v4 + amount_v6 < amount && amount_v4 < count4)
       ++amount_v4;
   }
 
@@ -355,7 +354,7 @@ static size_t return_peers_for_torrent_tcp(struct ot_workstruct *ws, ot_torrent
 
   if (amount_v4) {
     fmt_append(&r, r_end, PEERS_BENCODED4 "%zd:", OT_PEER_COMPARE_SIZE4 * amount_v4);
-    if (amount_v4 == torrent->peer_list4->peer_count)
+    if (amount_v4 == count4)
       r += return_peers_all(torrent->peer_list4, OT_PEER_SIZE4, r);
     else
       r += return_peers_selection(ws, torrent->peer_list4, OT_PEER_SIZE4, amount_v4, r);
@@ -363,7 +362,7 @@ static size_t return_peers_for_torrent_tcp(struct ot_workstruct *ws, ot_torrent
 
   if (amount_v6) {
     fmt_append(&r, r_end, PEERS_BENCODED6 "%zd:", OT_PEER_COMPARE_SIZE6 * amount_v6);
-    if (amount_v6 == torrent->peer_list6->peer_count)
+    if (amount_v6 == count6)
       r += return_peers_all(torrent->peer_list6, OT_PEER_SIZE6, r);
     else
       r += return_peers_selection(ws, torrent->peer_list6, OT_PEER_SIZE6, amount_v6, r);
@@ -402,9 +401,9 @@ size_t return_udp_scrape_for_torrent(ot_hash const hash, char *reply) {
       memset(reply, 0, 12);
       delta_torrentcount = -1;
     } else {
-      r[0] = htonl(torrent->peer_list6->seed_count + torrent->peer_list4->seed_count);
-      r[1] = htonl(torrent->peer_list6->down_count + torrent->peer_list4->down_count);
-      r[2] = htonl(torrent->peer_list6->peer_count + torrent->peer_list4->peer_count - torrent->peer_list6->seed_count - torrent->peer_list4->seed_count);
+      r[0] = htonl(peerlist_seed_count(torrent->peer_list6) + peerlist_seed_count(torrent->peer_list4));
+      r[1] = htonl(torrent->down_count);
+      r[2] = htonl(peerlist_peer_count(torrent->peer_list6) + peerlist_peer_count(torrent->peer_list4) - peerlist_seed_count(torrent->peer_list6) - peerlist_seed_count(torrent->peer_list4));
     }
   }
   mutex_bucket_unlock_by_hash(hash, delta_torrentcount);
@@ -431,9 +430,9 @@ void return_tcp_scrape_for_torrent(ot_hash const *hash_list, size_t amount, char
       } else {
         fmt_append(reply, reply_end, "20:");
         fmt_bytes(reply, reply_end, hash, sizeof(ot_hash));
-        fmt_append(reply, reply_end, "d8:completei%zde10:downloadedi%zde10:incompletei%zdee",
-                   torrent->peer_list6->seed_count + torrent->peer_list4->seed_count, torrent->peer_list6->down_count + torrent->peer_list4->down_count,
-                   torrent->peer_list6->peer_count + torrent->peer_list4->peer_count - torrent->peer_list6->seed_count - torrent->peer_list4->seed_count);
+        size_t seed_count = peerlist_seed_count(torrent->peer_list6) + peerlist_seed_count(torrent->peer_list4);
+        size_t peer_count = peerlist_peer_count(torrent->peer_list6) + peerlist_peer_count(torrent->peer_list4);
+        fmt_append(reply, reply_end, "d8:completei%zde10:downloadedi%zde10:incompletei%zdee", seed_count, (size_t)torrent->down_count, peer_count - seed_count);
       }
     }
     mutex_bucket_unlock_by_hash(*hash, delta_torrentcount);
@@ -461,17 +460,19 @@ size_t remove_peer_from_torrent(PROTO_FLAG proto, struct ot_workstruct *ws) {
 
   if (exactmatch) {
     peer_list = peer_size == OT_PEER_SIZE6 ? torrent->peer_list6 : torrent->peer_list4;
-    switch (vector_remove_peer(&peer_list->peers, peer_src, peer_size)) {
-    case 2:
-      peer_list->seed_count--; /* Intentional fallthrough */
-    case 1:
-      peer_list->peer_count--; /* Intentional fallthrough */
-    default:
-      break;
+    if (peer_list) {
+      switch (vector_remove_peer(&peer_list->peers, peer_src, peer_size)) {
+      case 2:
+        peer_list->seed_count--; /* Intentional fallthrough */
+      case 1:
+        peer_list->peer_count--; /* Intentional fallthrough */
+      default:
+        break;
+      }
     }
 
-    peer_count = torrent->peer_list6->peer_count + torrent->peer_list4->peer_count;
-    seed_count = torrent->peer_list6->seed_count + torrent->peer_list4->seed_count;
+    peer_count = peerlist_peer_count(torrent->peer_list6) + peerlist_peer_count(torrent->peer_list4);
+    seed_count = peerlist_seed_count(torrent->peer_list6) + peerlist_seed_count(torrent->peer_list4);
   }
 
   if (proto == FLAG_TCP) {