14 files changed, 463 insertions, 41 deletions

diff --git a/Makefile b/Makefile
index a224845..706b742 100644
--- a/Makefile
+++ b/Makefile
@@ -18,15 +18,24 @@ LIBOWFAT_LIBRARY=$(PREFIX)/libowfat
 BINDIR?=$(PREFIX)/bin
 STRIP?=strip
 
-#FEATURES+=-DWAND_V4_ONLY
+#FEATURES+=-DWANT_V4_ONLY
 #FEATURES+=-DWANT_ACCESSLIST_BLACK
 #FEATURES+=-DWANT_ACCESSLIST_WHITE
 #FEATURES+=-DWANT_DYNAMIC_ACCESSLIST
 
 #FEATURES+=-DWANT_SYNC_LIVE
 #FEATURES+=-DWANT_IP_FROM_QUERY_STRING
-FEATURES+=-DWANT_COMPRESSION_GZIP
-FEATURES+=-DWANT_COMPRESSION_GZIP_ALWAYS
+
+# If you want gzip support to be compiled in, uncomment the next include.
+# You can further modify the behaviour by setting DWANT_COMPRESSION_GZIP_ALWAYS
+# in Makefile.gzip
+include Makefile.gzip
+
+# If you want zstd support to be compiled in, uncomment the next include.
+# You can further modify the behaviour by setting DWANT_COMPRESSION_ZSTD_ALWAYS
+# in Makefile.zstd
+#include Makefile.zstd
+
 #FEATURES+=-DWANT_LOG_NETWORKS
 #FEATURES+=-DWANT_RESTRICT_STATS
 #FEATURES+=-DWANT_IP_FROM_PROXY
@@ -43,8 +52,8 @@ FEATURES+=-DWANT_FULLSCRAPE
 #FEATURES+=-DWANT_NO_AUTO_FREE
 
 # Is enabled on BSD systems by default in trackerlogic.h
-# on Linux systems you will need -lbds
-#FEATURES+=-DWANT_ARC4RANDOM
+# on Linux systems the include Makefile adds -lbsd
+#include Makefile.arc4random
 
 #FEATURES+=-D_DEBUG_HTTPERROR
 #FEATURES+=-D_DEBUG_RANDOMTORRENTS
@@ -55,8 +64,7 @@ OPTS_debug=-D_DEBUG -g -ggdb # -pg -fprofile-arcs -ftest-coverage
 OPTS_production=-O3
 
 CFLAGS+=-I$(LIBOWFAT_HEADERS) -DGIT_VERSION=$(GIT_VERSION) -Wall -pipe -pthread -Wextra #-ansi -pedantic
-LDFLAGS+=-L$(LIBOWFAT_LIBRARY) -lowfat -pthread -lz
-#LDFLAGS+=-lbsd
+LDFLAGS+=-L$(LIBOWFAT_LIBRARY) -lowfat -pthread
 
 BINARY =opentracker
 HEADERS=trackerlogic.h scan_urlencoded_query.h ot_mutex.h ot_stats.h ot_vector.h ot_clean.h ot_udp.h ot_iovec.h ot_fullscrape.h ot_accesslist.h ot_http.h ot_livesync.h ot_rijndael.h
diff --git a/Makefile.arc4random b/Makefile.arc4random
new file mode 100644
index 0000000..1488408
--- /dev/null
+++ b/Makefile.arc4random
@@ -0,0 +1,3 @@
+FEATURES+=-DWANT_ARC4RANDOM
+LDFLAGS+=-lbsd
+
diff --git a/Makefile.gzip b/Makefile.gzip
new file mode 100644
index 0000000..70d6d62
--- /dev/null
+++ b/Makefile.gzip
@@ -0,0 +1,4 @@
+FEATURES+=-DWANT_COMPRESSION_GZIP
+#FEATURES+=-DWANT_COMPRESSION_GZIP_ALWAYS
+
+LDFLAGS+=-lz
diff --git a/Makefile.zstd b/Makefile.zstd
new file mode 100644
index 0000000..2bb56be
--- /dev/null
+++ b/Makefile.zstd
@@ -0,0 +1,3 @@
+FEATURES+=-DWANT_COMPRESSION_ZSTD
+#FEATURES+=-DWANT_COMPRESSION_ZSTD_ALWAYS
+LDFLAGS+=-lzstd
diff --git a/man1/opentracker.1 b/man1/opentracker.1
new file mode 100644
index 0000000..0103ebe
--- /dev/null
+++ b/man1/opentracker.1
@@ -0,0 +1,130 @@
+.Dd 15/4/2024
+.Dt opentracker 1
+.Os Unix
+.Sh opentracker
+.Nm opentracker
+.Nd a free and open bittorrent tracker
+.Sh SYNOPSIS
+.Nm
+.Op Fl f Ar config
+.Op Fl i Ar ip-select
+.Op Fl p Ar port-bind-tcp
+.Op Fl P Ar port-bind-udp
+.Op Fl A Ar blessed-ip
+.Op Fl r Ar redirect-url
+.Op Fl d Ar chdir
+.Op Fl u Ar user
+.Op Fl w| Fl b accesslist
+.Sh DESCRIPTION
+.Nm
+is a bittorrent tracker that implements announce and scrape actions over the
+UDP and the plain http protocol, aiming for minimal resource usage.
+.Pp
+When invoked with parameters, it binds to TCP and UDP port 6969 on all
+interfaces. The recommended way to configure opentracker is by providing a
+config file using the
+.Op Fl f Ar config
+option. See
+.Xr opentracker.conf 4
+for details.
+.Pp
+.Sh OPTIONS
+The following options are available:
+.Bl -tag -width -indent=8
+.It Fl f Ar config
+Parse a config file with a list of options. Consecutive command options
+will override options from the config file. See
+.Xr opentracker.conf 4
+for details.
+.It Fl i Ar ip-select
+Select an ip address that will be used with the next
+.Op Fl p
+or
+.Op Fl P
+command to actually bind to this address. Setting this option without any bind
+options in the config file or
+.Op Fl p
+or
+.Op Fl P
+commands will limit opentracker to only bind to this address.
+.It Fl p Ar port-bind-tcp
+Bind to the TCP port on the last preceding ip address set with the
+.Op Fl i ip-select
+option or to all available addresses if none has been set. Can be given multiple
+times.
+.It Fl P Ar port-bind-udp
+Bind to the UDP port on the last preceding ip address set with the
+.Op Fl i ip-select
+option or to all available addresses if none has been set. Can be given multiple
+times.
+.It Fl A Ar blessed-ip
+Set an ip address in IPv4 or IPv6 or a net in CIDR notation to bless the network
+for access to restricted resources.
+.It Fl r Ar redirect-url
+Set the URL that
+.Nm
+will redirect users to when the / address is requested via HTTP.
+.It Fl d Ar chdir
+Sets the directory
+.Nm
+will
+.Xr chroot 2
+to if ran as root or
+.Xr chdir 2
+to if ran as unprivileged user. Note that any accesslist files need to be
+relative to and within that directory.
+.It Fl u Ar user
+User to run
+.Nm
+under after all operations that need privileges have finished.
+.It Fl w Ar accesslist | Fl b Ar accesslist
+If
+.Nm
+has been compiled with the
+.B WANT_ACCESSLIST_BLACK
+or
+.Br WANT_ACCESSLIST_WHITE
+options, this option sets the location of the accesslist.
+.El
+.Sh EXAMPLES
+Start
+.Nm
+bound on UDP and TCP ports 6969 on IPv6 localhost.
+.Dl # ./opentracker -i ::1 -p 6969 -P 6969
+.Pp
+Start
+.Nm
+bound on UDP port 6868 and TCP port 6868 on IPv4 localhost and allow
+privileged access from the network 192.168/16 while redirecting
+HTTP clients accessing the root directory, which is not covered by the
+bittorrent tracker protocol, to https://my-trackersite.com/.
+.Dl # ./opentracker -i 192.168.0.4 -p 6868 -P 6969 -A 192.168/16 -r https://my-trackersite.com/
+The announce URLs are http://192.168.0.4:6868/announce and
+udp://192.168.0.4:6868/announce respectively.
+.Sh FILES
+.Bl -tag -width indent
+.It Pa opentracker.conf
+The
+.Nm
+config file.
+.El
+.Sh SEE ALSO
+.Xr opentracker.conf 4
+.Pp
+opentracker documentation
+.Lk https://erdgeist.org/arts/software/opentracker
+.Pp
+Bittorrent tracker protocol
+.Lk http://www.bittorrent.org/beps/bep_0015.html
+.Sh AUTHOR
+.An Dirk Engling
+.Aq Mt erdgeist@erdgeist.org .
+.Sh LICENSE
+This software is released under the Beerware License:
+.Pp
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software
+and associated documentation files (the "Software"), to deal in the Software with the following
+terms and conditions:
+.Pp
+If you meet the author(s) someday, and you think this software is worth it, you can buy them
+a beer in return.
diff --git a/man4/opentracker.conf.4 b/man4/opentracker.conf.4
new file mode 100644
index 0000000..2bc1389
--- /dev/null
+++ b/man4/opentracker.conf.4
@@ -0,0 +1,86 @@
+.Dd 2024-04-18
+.Dt opentracker.conf 5
+.Os Unix
+.Sh NAME
+.Nm opentracker.conf
+.Nd configuration file for opentracker
+.Sh SYNOPSIS
+.Nm
+.Sh DESCRIPTION
+The
+.Nm
+configuration file specifies various options for configuring the behavior of the opentracker program.
+.Pp
+Lines starting with '#' are comments and are ignored. Options are specified as 'keyword value' pairs.
+.Pp
+The following options are available:
+.Pp
+.Bl -tag -width ".It access.proxy" -compact
+.It listen.tcp_udp Ar address
+Specifies an address opentracker will listen on for both TCP and UDP connections. If none are specified, opentracker listens on 0.0.0.0:6969 by default. Can be added more than once.
+.Pp
+.It listen.tcp Ar address
+Specifies the address opentracker will listen on for TCP connections. Can be added more than once.
+.Pp
+.It listen.udp Ar address
+Specifies the address opentracker will listen on for UDP connections. Can be added more than once.
+.Pp
+.It listen.udp.workers Ar threads
+Specifies how many threads will be spawned to handle UDP connections. Defaults to 4.
+.Pp
+.It access.whitelist Ar path/to/whitelist
+Specifies the path to the whitelist file containing all torrent hashes that opentracker will serve. Use this option if opentracker runs in a non-open mode.
+.Pp
+.It access.blacklist Ar path/to/blacklist
+Specifies the path to the blacklist file containing all torrent hashes that opentracker will not serve. Use this option if opentracker was compiled to allow blacklisting.
+.Pp
+.It access.fifo_add Ar path/to/adder.fifo
+Specifies the path to the FIFO (named pipe) used for dynamic changesets to accesslists. Info hashes written to this FIFO will be added to the main accesslist file.
+.Pp
+.It access.fifo_delete Ar path/to/deleter.fifo
+Specifies the path to the FIFO (named pipe) used for dynamic changesets to accesslists. Info hashes written to this FIFO will be removed from the main accesslist file.
+.Pp
+.It access.stats Ar ip_address_or_network
+Specifies the IP address or network in CIDR notation allowed to fetch stats from opentracker.
+.Pp
+.It access.stats_path Ar path
+Specifies the path to the stats location. You can configure opentracker to appear anywhere on your tracker. Defaults to /stats.
+.Pp
+.It access.proxy Ar ip_address_or_network
+Specifies the IP address or network of the reverse proxies. Opentracker will take the X-Forwarded-For address instead of the source IP address. Can be added more than once.
+.Pp
+.It livesync.cluster.listen Ar ip_address:port
+Specifies the IP address and port opentracker will listen on for incoming live sync packets to keep a cluster of opentrackers synchronized.
+.Pp
+.It livesync.cluster.node_ip Ar ip_address
+Specifies one trusted IP address for sync between trackers running in a cluster. Can be added more than once.
+.Pp
+.It batchsync.cluster.admin_ip Ar ip_address
+Specifies the admin IP address for old-style (HTTP-based) asynchronous tracker syncing.
+.Pp
+.It tracker.rootdir Ar path
+Specifies the directory opentracker will chroot/chdir to. All black/white list files must be located in this directory.
+.Pp
+.It tracker.user Ar username
+Specifies the user opentracker will setuid to after binding to potentially privileged ports.
+.Pp
+.It tracker.redirect_url Ar URL
+Specifies the URL opentracker will redirect to in response to a "GET / HTTP" request.
+.El
+.Sh EXAMPLES
+To specify the address opentracker will listen on for both TCP and UDP connections:
+.Dl listen.tcp_udp 0.0.0.0:6969
+.Pp
+To specify the address opentracker will listen on for TCP connections:
+.Dl listen.tcp 0.0.0.0
+.Pp
+To specify the address opentracker will listen on for UDP connections:
+.Dl listen.udp 0.0.0.0:6969
+.Pp
+.Sh SEE ALSO
+.Xr opentracker 1
+.Pp
+.Sh AUTHOR
+.An Dirk Engling
+.Aq Mt erdgeist@erdgeist.org
+.Pp
diff --git a/opentracker.conf.sample b/opentracker.conf.sample
index f8bef43..054e405 100644
--- a/opentracker.conf.sample
+++ b/opentracker.conf.sample
@@ -2,7 +2,7 @@
 #
 
 # I)   Address opentracker will listen on, using both, tcp AND udp family
-#      (note, that port 6969 is implicite if ommitted).
+#      (note, that port 6969 is implicit if omitted).
 #
 #      If no listen option is given (here or on the command line), opentracker
 #      listens on 0.0.0.0:6969 tcp and udp.
@@ -83,9 +83,10 @@
 # IIb)
 #      If you do not want to grant anyone access to your stats, enable the
 #      WANT_RESTRICT_STATS option in Makefile and bless the ip addresses
-#      allowed to fetch stats here.
+#      or network allowed to fetch stats here.
 #
 # access.stats 192.168.0.23
+# access.stats 10.1.1.23
 #
 #      There is another way of hiding your stats. You can obfuscate the path
 #      to them. Normally it is located at /stats but you can configure it to
diff --git a/ot_fullscrape.c b/ot_fullscrape.c
index aed2ad9..6fd6d1c 100644
--- a/ot_fullscrape.c
+++ b/ot_fullscrape.c
@@ -14,6 +14,10 @@
 #ifdef WANT_COMPRESSION_GZIP
 #include <zlib.h>
 #endif
+#ifdef WANT_COMPRESSION_ZSTD
+#include <zstd.h>
+#endif
+
 
 /* Libowfat */
 #include "byte.h"
@@ -40,6 +44,9 @@ static void fullscrape_make(int taskid, ot_tasktype mode);
 #ifdef WANT_COMPRESSION_GZIP
 static void fullscrape_make_gzip(int taskid, ot_tasktype mode);
 #endif
+#ifdef WANT_COMPRESSION_ZSTD
+static void fullscrape_make_zstd(int taskid, ot_tasktype mode);
+#endif
 
 /* Converter function from memory to human readable hex strings
    XXX - Duplicated from ot_stats. Needs fix. */
@@ -64,6 +71,11 @@ static void *fullscrape_worker(void *args) {
   while (g_opentracker_running) {
     ot_tasktype tasktype = TASK_FULLSCRAPE;
     ot_taskid   taskid   = mutex_workqueue_poptask(&tasktype);
+#ifdef WANT_COMPRESSION_ZSTD
+    if (tasktype & TASK_FLAG_ZSTD)
+      fullscrape_make_zstd(taskid, tasktype);
+    else
+#endif
 #ifdef WANT_COMPRESSION_GZIP
     if (tasktype & TASK_FLAG_GZIP)
       fullscrape_make_gzip(taskid, tasktype);
@@ -205,7 +217,6 @@ static void fullscrape_make_gzip(int taskid, ot_tasktype mode) {
   struct iovec iovector = {NULL, 0};
   int          zres;
   z_stream     strm;
-  fprintf(stderr, "GZIP path\n");
   /* Setup return vector... */
   iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE);
   if (!iovector.iov_base)
@@ -267,8 +278,10 @@ static void fullscrape_make_gzip(int taskid, ot_tasktype mode) {
     mutex_bucket_unlock(bucket, 0);
 
     /* Parent thread died? */
-    if (!g_opentracker_running)
+    if (!g_opentracker_running) {
+      deflateEnd(&strm);
       return;
+    }
   }
 
   if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) {
@@ -282,7 +295,8 @@ static void fullscrape_make_gzip(int taskid, ot_tasktype mode) {
   iovector.iov_len = (char *)strm.next_out - (char *)iovector.iov_base;
   if (mutex_workqueue_pushchunked(taskid, &iovector)) {
     free(iovector.iov_base);
-    return mutex_bucket_unlock(bucket, 0);
+    deflateEnd(&strm);
+    return;
   }
 
   /* Check if there's a last batch of data in the zlib buffer */
@@ -293,7 +307,7 @@ static void fullscrape_make_gzip(int taskid, ot_tasktype mode) {
       if (!iovector.iov_base) {
         fprintf(stderr, "Problem with iovec_fix_increase_or_free\n");
         deflateEnd(&strm);
-        return mutex_bucket_unlock(bucket, 0);
+        return;
       }
       strm.next_out  = iovector.iov_base;
       strm.avail_out = OT_SCRAPE_CHUNK_SIZE;
@@ -311,5 +325,133 @@ static void fullscrape_make_gzip(int taskid, ot_tasktype mode) {
 /* WANT_COMPRESSION_GZIP */
 #endif
 
+#ifdef WANT_COMPRESSION_ZSTD
+
+static void fullscrape_make_zstd(int taskid, ot_tasktype mode) {
+  int            bucket;
+  char          *r;
+  struct iovec   iovector = {NULL, 0};
+  ZSTD_CCtx     *zstream = ZSTD_createCCtx();
+  ZSTD_inBuffer  inbuf;
+  ZSTD_outBuffer outbuf;
+  size_t         more_bytes;
+
+  if (!zstream)
+    return;
+
+  /* Setup return vector... */
+  iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE);
+  if (!iovector.iov_base) {
+    ZSTD_freeCCtx(zstream);
+    return;
+  }
+
+  /* Working with a compression level 6 is half as fast as level 3, but
+     seems to be the last reasonable bump that's worth extra cpu */
+  ZSTD_CCtx_setParameter(zstream, ZSTD_c_compressionLevel, 6);
+
+  outbuf.dst  = iovector.iov_base;
+  outbuf.size = OT_SCRAPE_CHUNK_SIZE;
+  outbuf.pos  = 0;
+
+  if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) {
+    inbuf.src  = (const void *)"d5:filesd";
+    inbuf.size = strlen("d5:filesd");
+    inbuf.pos  = 0;
+    ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_continue);
+  }
+
+  /* For each bucket... */
+  for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) {
+    /* Get exclusive access to that bucket */
+    ot_vector  *torrents_list = mutex_bucket_lock(bucket);
+    ot_torrent *torrents      = (ot_torrent *)(torrents_list->data);
+    size_t      i;
+
+    /* For each torrent in this bucket.. */
+    for (i = 0; i < torrents_list->size; ++i) {
+      char compress_buffer[OT_SCRAPE_MAXENTRYLEN];
+      r             = fullscrape_write_one(mode, compress_buffer, torrents + i, &torrents[i].hash);
+      inbuf.src     = compress_buffer;
+      inbuf.size    = r - compress_buffer;
+      inbuf.pos     = 0;
+      ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_continue);
+
+      /* Check if there still is enough buffer left */
+      while (outbuf.pos + OT_SCRAPE_MAXENTRYLEN > outbuf.size) {
+        iovector.iov_len = outbuf.size;
+
+        if (mutex_workqueue_pushchunked(taskid, &iovector)) {
+          free(iovector.iov_base);
+          ZSTD_freeCCtx(zstream);
+          return mutex_bucket_unlock(bucket, 0);
+        }
+        /* Allocate a fresh output buffer */
+        iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE);
+        if (!iovector.iov_base) {
+          fprintf(stderr, "Out of memory trying to claim ouput buffer\n");
+          ZSTD_freeCCtx(zstream);
+          return mutex_bucket_unlock(bucket, 0);
+        }
+
+        outbuf.dst  = iovector.iov_base;
+        outbuf.size = OT_SCRAPE_CHUNK_SIZE;
+        outbuf.pos  = 0;
+
+        ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_continue);
+      }
+    }
+
+    /* All torrents done: release lock on current bucket */
+    mutex_bucket_unlock(bucket, 0);
+
+    /* Parent thread died? */
+    if (!g_opentracker_running)
+      return;
+  }
+
+  if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) {
+    inbuf.src  = (const void *)"ee";
+    inbuf.size = strlen("ee");
+    inbuf.pos  = 0;
+  }
+
+  more_bytes = ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_end);
+
+  iovector.iov_len = outbuf.pos;
+  if (mutex_workqueue_pushchunked(taskid, &iovector)) {
+    free(iovector.iov_base);
+    ZSTD_freeCCtx(zstream);
+    return;
+  }
+
+  /* Check if there's a last batch of data in the zlib buffer */
+  if (more_bytes) {
+      /* Allocate a fresh output buffer */
+      iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE);
+
+      if (!iovector.iov_base) {
+        fprintf(stderr, "Problem with iovec_fix_increase_or_free\n");
+        ZSTD_freeCCtx(zstream);
+        return;
+      }
+
+      outbuf.dst  = iovector.iov_base;
+      outbuf.size = OT_SCRAPE_CHUNK_SIZE;
+      outbuf.pos  = 0;
+
+      ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_end);
+
+      /* Only pass the new buffer if there actually was some data left in the buffer */
+      iovector.iov_len = outbuf.pos;
+      if (!iovector.iov_len || mutex_workqueue_pushchunked(taskid, &iovector))
+        free(iovector.iov_base);
+  }
+
+  ZSTD_freeCCtx(zstream);
+}
+/* WANT_COMPRESSION_ZSTD */
+#endif
+
 /* WANT_FULLSCRAPE */
 #endif
diff --git a/ot_http.c b/ot_http.c
index cd2dfc1..a87a02e 100644
--- a/ot_http.c
+++ b/ot_http.c
@@ -4,9 +4,9 @@
    $id$ */
 
 /* System */
+#define _GNU_SOURCE
 #include <arpa/inet.h>
 #include <pthread.h>
-#define _GNU_SOURCE
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -32,7 +32,7 @@
 #include "trackerlogic.h"
 
 #ifdef WANT_NO_AUTO_FREE
-#define OT_IOB_INIT(B) bzero(B, sizeof(io_batch))
+#define OT_IOB_INIT(B) (bzero(B, sizeof(io_batch)), 0)
 #else
 #define OT_IOB_INIT(B) iob_init_autofree(B, 0)
 #endif
@@ -88,7 +88,15 @@ static void http_senddata(const int64 sock, struct ot_workstruct *ws) {
     memcpy(outbuf, ws->reply + written_size, ws->reply_size - written_size);
     if (!cookie->batch) {
       cookie->batch = malloc(sizeof(io_batch));
-      OT_IOB_INIT(cookie->batch);
+      if (!cookie->batch || OT_IOB_INIT(cookie->batch) == -1) {
+        free(cookie->batch);
+        free(outbuf);
+        array_reset(&cookie->request);
+        free(cookie);
+        io_close(sock);
+        return;
+      }
+
       cookie->batches = 1;
     }
 
@@ -159,7 +167,9 @@ ssize_t http_sendiovecdata(const int64 sock, struct ot_workstruct *ws, int iovec
 
   if (iovec_entries) {
 
-    if (cookie->flag & STRUCT_HTTP_FLAG_GZIP)
+    if (cookie->flag & STRUCT_HTTP_FLAG_ZSTD)
+      encoding = "Content-Encoding: zstd\r\n";
+    else if (cookie->flag & STRUCT_HTTP_FLAG_GZIP)
       encoding = "Content-Encoding: gzip\r\n";
     else if (cookie->flag & STRUCT_HTTP_FLAG_BZIP2)
       encoding = "Content-Encoding: bzip2\r\n";
@@ -181,12 +191,12 @@ ssize_t http_sendiovecdata(const int64 sock, struct ot_workstruct *ws, int iovec
 
     if (!cookie->batch) {
       cookie->batch = malloc(sizeof(io_batch));
-      if (!cookie->batch) {
+      if (!cookie->batch || OT_IOB_INIT(cookie->batch) == -1) {
+        free(cookie->batch);
         free(header);
         iovec_free(&iovec_entries, &iovector);
         HTTPERROR_500;
       }
-      OT_IOB_INIT(cookie->batch);
       cookie->batches = 1;
     }
     current = cookie->batch + cookie->batches - 1;
@@ -199,8 +209,8 @@ ssize_t http_sendiovecdata(const int64 sock, struct ot_workstruct *ws, int iovec
         io_batch *new_batch = realloc(cookie->batch, (cookie->batches + 1) * sizeof(io_batch));
         if (new_batch) {
           cookie->batch = new_batch;
-          current       = cookie->batch + cookie->batches++;
-          OT_IOB_INIT(current);
+          if (OT_IOB_INIT(current) != -1)
+            current = cookie->batch + cookie->batches++;
         }
       }
       iob_addbuf_free(current, iovector[i].iov_base, iovector[i].iov_len);
@@ -369,19 +379,34 @@ static ssize_t http_handle_fullscrape(const int64 sock, struct ot_workstruct *ws
   }
 #endif
 
-#ifdef WANT_COMPRESSION_GZIP
+
+#if defined(WANT_COMPRESSION_GZIP) || defined(WANT_COMPRESSION_ZSTD)
   ws->request[ws->request_size - 1] = 0;
-#ifndef WANT_COMPRESSION_GZIP_ALWAYS
+#ifdef WANT_COMPRESSION_GZIP
   if (strstr(ws->request, "gzip")) {
-#endif
     cookie->flag |= STRUCT_HTTP_FLAG_GZIP;
-    format        = TASK_FLAG_GZIP;
-    stats_issue_event(EVENT_FULLSCRAPE_REQUEST_GZIP, 0, (uintptr_t)cookie->ip);
-#ifndef WANT_COMPRESSION_GZIP_ALWAYS
-  } else
+    format       |= TASK_FLAG_GZIP;
+  }
 #endif
+#ifdef WANT_COMPRESSION_ZSTD
+  if (strstr(ws->request, "zstd")) {
+    cookie->flag |= STRUCT_HTTP_FLAG_ZSTD;
+    format       |= TASK_FLAG_ZSTD;
+  }
+#endif
+
+#if defined(WANT_COMPRESSION_ZSTD) && defined(WANT_COMPRESSION_ZSTD_ALWAYS)
+  cookie->flag |= STRUCT_HTTP_FLAG_ZSTD;
+  format       |= TASK_FLAG_ZSTD;
 #endif
-    stats_issue_event(EVENT_FULLSCRAPE_REQUEST, 0, (uintptr_t)cookie->ip);
+
+#if defined(WANT_COMPRESSION_GZIP) && defined(WANT_COMPRESSION_GZIP_ALWAYS)
+  cookie->flag |= STRUCT_HTTP_FLAG_GZIP;
+  format       |= TASK_FLAG_GZIP;
+#endif
+#endif
+
+  stats_issue_event(EVENT_FULLSCRAPE_REQUEST, 0, (uintptr_t)cookie->ip);
 
 #ifdef _DEBUG_HTTPERROR
   fprintf(stderr, "%s", ws->debugbuf);
diff --git a/ot_http.h b/ot_http.h
index fecb4eb..b5ae9ff 100644
--- a/ot_http.h
+++ b/ot_http.h
@@ -10,8 +10,9 @@ typedef enum {
   STRUCT_HTTP_FLAG_WAITINGFORTASK      = 1,
   STRUCT_HTTP_FLAG_GZIP                = 2,
   STRUCT_HTTP_FLAG_BZIP2               = 4,
-  STRUCT_HTTP_FLAG_CHUNKED             = 8,
-  STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER = 16
+  STRUCT_HTTP_FLAG_ZSTD                = 8,
+  STRUCT_HTTP_FLAG_CHUNKED             = 16,
+  STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER = 32
 } STRUCT_HTTP_FLAG;
 
 struct http_data {
diff --git a/ot_mutex.h b/ot_mutex.h
index 66b627f..cdfabc9 100644
--- a/ot_mutex.h
+++ b/ot_mutex.h
@@ -59,7 +59,8 @@ typedef enum {
 
   TASK_FLAG_GZIP                 = 0x1000,
   TASK_FLAG_BZIP2                = 0x2000,
-  TASK_FLAG_CHUNKED              = 0x4000,
+  TASK_FLAG_ZSTD                 = 0x4000,
+  TASK_FLAG_CHUNKED              = 0x8000,
 
   TASK_TASK_MASK                 = 0x0fff,
   TASK_CLASS_MASK                = 0x0f00,
diff --git a/ot_stats.c b/ot_stats.c
index 158884f..b2eaec9 100644
--- a/ot_stats.c
+++ b/ot_stats.c
@@ -320,7 +320,7 @@ typedef struct {
 /* Fetches stats from tracker */
 size_t stats_top_txt(char *reply, int amount) {
   size_t    j;
-  ot_record top100s[100], top100c[100];
+  ot_record top100s[100], top100c[100], top100l[100];
   char     *r = reply, hex_out[42];
   int       idx, bucket;
 
@@ -329,14 +329,16 @@ size_t stats_top_txt(char *reply, int amount) {
 
   byte_zero(top100s, sizeof(top100s));
   byte_zero(top100c, sizeof(top100c));
+  byte_zero(top100l, sizeof(top100l));
 
   for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) {
-    ot_vector *torrents_list = mutex_bucket_lock(bucket);
+    ot_vector *torrents_list  = mutex_bucket_lock(bucket);
     for (j = 0; j < torrents_list->size; ++j) {
-      ot_torrent *torrent    = (ot_torrent *)(torrents_list->data) + j;
-      size_t      peer_count = torrent->peer_list6->peer_count + torrent->peer_list4->peer_count;
-      size_t      seed_count = torrent->peer_list6->seed_count + torrent->peer_list4->seed_count;
-      idx                    = amount - 1;
+      ot_torrent *torrent     = (ot_torrent *)(torrents_list->data) + j;
+      size_t      peer_count  = torrent->peer_list6->peer_count + torrent->peer_list4->peer_count;
+      size_t      seed_count  = torrent->peer_list6->seed_count + torrent->peer_list4->seed_count;
+      size_t      leech_count = peer_count - seed_count;
+      idx                     = amount - 1;
       while ((idx >= 0) && (peer_count > top100c[idx].val))
         --idx;
       if (idx++ != amount - 1) {
@@ -352,6 +354,14 @@ size_t stats_top_txt(char *reply, int amount) {
         memcpy(&top100s[idx].hash, &torrent->hash, sizeof(ot_hash));
         top100s[idx].val = seed_count;
       }
+      idx = amount - 1;
+      while ((idx >= 0) && (leech_count > top100l[idx].val))
+        --idx;
+      if (idx++ != amount - 1) {
+        memmove(top100l + idx + 1, top100l + idx, (amount - 1 - idx) * sizeof(ot_record));
+        memcpy(&top100l[idx].hash, &torrent->hash, sizeof(ot_hash));
+        top100l[idx].val = leech_count;
+      }
     }
     mutex_bucket_unlock(bucket, 0);
     if (!g_opentracker_running)
@@ -366,6 +376,10 @@ size_t stats_top_txt(char *reply, int amount) {
   for (idx = 0; idx < amount; ++idx)
     if (top100s[idx].val)
       r += sprintf(r, "\t%zd\t%s\n", top100s[idx].val, to_hex(hex_out, top100s[idx].hash));
+  r += sprintf(r, "Top %d torrents by leechers:\n", amount);
+  for (idx = 0; idx < amount; ++idx)
+    if (top100l[idx].val)
+      r += sprintf(r, "\t%zd\t%s\n", top100l[idx].val, to_hex(hex_out, top100l[idx].hash));
 
   return r - reply;
 }
diff --git a/ot_stats.h b/ot_stats.h
index 4f75049..8ed2b1e 100644
--- a/ot_stats.h
+++ b/ot_stats.h
@@ -19,6 +19,7 @@ typedef enum {
   EVENT_SCRAPE,
   EVENT_FULLSCRAPE_REQUEST,
   EVENT_FULLSCRAPE_REQUEST_GZIP,
+  EVENT_FULLSCRAPE_REQUEST_ZSTD,
   EVENT_FULLSCRAPE, /* TCP only */
   EVENT_FAILED,
   EVENT_BUCKET_LOCKED,
diff --git a/ot_vector.c b/ot_vector.c
index 2bc07b5..2acfbef 100644
--- a/ot_vector.c
+++ b/ot_vector.c
@@ -67,12 +67,15 @@ void *vector_find_or_insert(ot_vector *vector, void *key, size_t member_size, si
     return match;
 
   if (vector->size + 1 > vector->space) {
-    size_t   new_space = vector->space ? OT_VECTOR_GROW_RATIO * vector->space : OT_VECTOR_MIN_MEMBERS;
-    uint8_t *new_data  = realloc(vector->data, new_space * member_size);
+    size_t    new_space = vector->space ? OT_VECTOR_GROW_RATIO * vector->space : OT_VECTOR_MIN_MEMBERS;
+    ptrdiff_t match_off = match - (uint8_t *)vector->data;
+    uint8_t  *new_data  = realloc(vector->data, new_space * member_size);
+
     if (!new_data)
       return NULL;
+
     /* Adjust pointer if it moved by realloc */
-    match         = new_data + (match - (uint8_t *)vector->data);
+    match         = new_data + match_off;
 
     vector->data  = new_data;
     vector->space = new_space;