6 files changed, 156 insertions, 136 deletions

diff --git a/Makefile b/Makefile
index d363e2c..688a2c4 100644
--- a/Makefile
+++ b/Makefile
@@ -10,16 +10,6 @@ STRIP?=strip
 #FEATURES+=-DWANT_SYNC_LIVE
 #FEATURES+=-DWANT_IP_FROM_QUERY_STRING
 
-# If you want gzip support to be compiled in, uncomment the next include.
-# You can further modify the behaviour by setting DWANT_COMPRESSION_GZIP_ALWAYS
-# in Makefile.gzip
-include Makefile.gzip
-
-# If you want zstd support to be compiled in, uncomment the next include.
-# You can further modify the behaviour by setting DWANT_COMPRESSION_ZSTD_ALWAYS
-# in Makefile.zstd
-#include Makefile.zstd
-
 #FEATURES+=-DWANT_RESTRICT_STATS
 #FEATURES+=-DWANT_IP_FROM_PROXY
 #FEATURES+=-DWANT_LOG_NUMWANT
@@ -29,155 +19,61 @@ include Makefile.gzip
 #FEATURES+=-DWANT_DEV_RANDOM
 FEATURES+=-DWANT_FULLSCRAPE
 
-# opentracker takes its randomness from arc4random, which is part of libc on
-# the BSDs, on macOS and on glibc 2.36 and newer. On an older glibc or on musl
-# it comes from libbsd instead, so uncomment the next include to link it.
-#LDFLAGS+=-lbsd
-
 #FEATURES+=-D_DEBUG_HTTPERROR
 #FEATURES+=-D_DEBUG_RANDOMTORRENTS
 
-VERSION=1.0.0
-TARBALL_VERSION=test -r .tarball-version && read v < .tarball-version && echo "$$v"
-HAVE_GIT=command -v git >/dev/null && test -d .git
-GIT_VERSION=`sh -c '$(TARBALL_VERSION) || { $(HAVE_GIT) && git describe --tags --match "v*" --always --dirty 2>/dev/null; }'`
-OT_TAG=`sh -c '$(TARBALL_VERSION) || { $(HAVE_GIT) && git describe --tags --match "v*" --exact-match 2>/dev/null; }'`
+# Compression support, off unless included. Each file carries a second switch
+# to compress even where the client did not ask.
+include Mk/gzip.mk
+#include Mk/zstd.mk
 
-OPTS_debug=-D_DEBUG -g -ggdb # -pg -fprofile-arcs -ftest-coverage
-OPTS_production=-O3
+# arc4random is part of libc on the BSDs, on macOS and on glibc 2.36 and
+# newer. On an older glibc or on musl it comes from libbsd instead.
+#LDFLAGS+=-lbsd
 
-CFLAGS+=-DOT_VERSION=\"$(VERSION)\" -DGIT_VERSION=\"$(GIT_VERSION)\" -DOT_TAG=\"$(OT_TAG)\" -Wall -pipe -pthread -Wextra -Wwrite-strings #-ansi -pedantic
+CFLAGS+=-Wall -Wextra -Wwrite-strings -pipe -pthread
 LDFLAGS+=-pthread
 
-# Record feature flags as a dep to allow rebuild when they change
-BUILDSTAMP=.buildstamp
-BUILDDEPS=Makefile $(BUILDSTAMP)
-
-$(BUILDSTAMP): FORCE
-        @echo "$(CFLAGS) $(FEATURES) $(LDFLAGS)" | cmp -s - $@ 2>/dev/null || echo "$(CFLAGS) $(FEATURES) $(LDFLAGS)" > $@
-
-FORCE:
+OPTS_production=-O3
+OPTS_debug=-D_DEBUG -g -ggdb
 
-BINARY =opentracker
+BINARY=opentracker
 HEADERS=trackerlogic.h ot_urlencoded.h ot_batch.h ot_buf.h ot_event.h ot_fdstate.h ot_fmt.h ot_hex.h ot_ip.h ot_config.h ot_scan.h ot_socket.h ot_tcp.h ot_worker.h ot_workqueue.h ot_mutex.h ot_random.h ot_stats.h ot_vector.h ot_clean.h ot_udp.h ot_iovec.h ot_fullscrape.h ot_accesslist.h ot_http.h ot_livesync.h ot_rijndael.h
 SOURCES=opentracker.c trackerlogic.c ot_urlencoded.c ot_batch.c ot_buf.c ot_event.c ot_fdstate.c ot_fmt.c ot_hex.c ot_ip.c ot_config.c ot_scan.c ot_socket.c ot_tcp.c ot_worker.c ot_workqueue.c ot_mutex.c ot_random.c ot_stats.c ot_vector.c ot_clean.c ot_udp.c ot_iovec.c ot_fullscrape.c ot_accesslist.c ot_http.c ot_livesync.c ot_rijndael.c
+BINARIES=$(BINARY) $(BINARY).debug
 
-all: $(BINARY) $(BINARY).debug
+# Everything is built in one go from sources, so there are no object files to
+# carry a flag change over into the next build. The stamp holds the flags the
+# last build used, which covers FEATURES passed in the environment and a moved
+# HEAD. It is only written when they differ.
+BUILDSTAMP=.buildstamp
+BUILDDEPS=Makefile $(BUILDSTAMP)
+
+CFLAGS_production=$(CFLAGS) $(OPTS_production) $(FEATURES)
+CFLAGS_debug=$(CFLAGS) $(OPTS_debug) $(FEATURES)
 
-CFLAGS_production = $(CFLAGS) $(OPTS_production) $(FEATURES)
-CFLAGS_debug = $(CFLAGS) $(OPTS_debug) $(FEATURES)
+all: $(BINARY) $(BINARY).debug
 
 $(BINARY): $(SOURCES) $(HEADERS) $(BUILDDEPS)
         $(CC) -o $@ $(CFLAGS_production) $(SOURCES) $(LDFLAGS)
         $(STRIP) $@
+
 $(BINARY).debug: $(SOURCES) $(HEADERS) $(BUILDDEPS)
         $(CC) -o $@ $(CFLAGS_debug) $(SOURCES) $(LDFLAGS)
 
-# The proxy is out of service
-#
-# SOURCES_proxy=proxy.c ot_batch.c ot_fdstate.c ot_vector.c ot_mutex.c ot_random.c ot_scan.c ot_ip.c ot_socket.c
-# CFLAGS_proxy = $(CFLAGS_production) -DWANT_SYNC_LIVE
-# CFLAGS_proxy_debug = $(CFLAGS_debug) -DWANT_SYNC_LIVE
-#
-# proxy: $(SOURCES_proxy) $(HEADERS) $(BUILDDEPS)
-#       $(CC) -o $@ $(CFLAGS_proxy) $(SOURCES_proxy) $(LDFLAGS)
-# proxy.debug: $(SOURCES_proxy) $(HEADERS) $(BUILDDEPS)
-#       $(CC) -o $@ $(CFLAGS_proxy_debug) $(SOURCES_proxy) $(LDFLAGS)
-
-# Unit tests. The modules under test are compiled straight into the test
-# binary, so a test may reach a static function by including the source if it
-# has to. tests/stubs.c supplies the few symbols those modules reference but
-# the tests never reach.
-TESTBIN=tests/runner
-TEST_SOURCES=tests/runner.c tests/test.c tests/stubs.c tests/t_urlencoded.c tests/t_iovec.c tests/t_vector.c tests/t_random.c tests/t_hex.c tests/t_accesslist.c tests/t_scan.c tests/t_buf.c tests/t_ip.c tests/t_socket.c tests/t_mutex.c tests/t_trackerlogic.c tests/t_workqueue.c tests/t_batch.c tests/t_fdstate.c tests/t_event.c tests/t_fmt.c
-TEST_UNITS=ot_urlencoded.c ot_fmt.c ot_event.c ot_batch.c ot_fdstate.c ot_buf.c ot_ip.c ot_socket.c ot_iovec.c ot_vector.c ot_random.c ot_hex.c ot_scan.c ot_accesslist.c ot_mutex.c ot_workqueue.c trackerlogic.c
-TEST_HEADERS=tests/test.h $(HEADERS)
-# Tests reach code that sits behind feature flags, so the units under test are
-# built with those features enabled. A test binary is not a production binary.
-FEATURES_test=$(FEATURES) -DWANT_ACCESSLIST_BLACK -DWANT_SYNC_LIVE
-CFLAGS_test = $(CFLAGS) -I. -g -O1 $(FEATURES_test)
-
-$(TESTBIN): $(TEST_SOURCES) $(TEST_UNITS) $(TEST_HEADERS) $(BUILDDEPS)
-        $(CC) -o $@ $(CFLAGS_test) $(TEST_SOURCES) $(TEST_UNITS)
-
-check: $(TESTBIN) $(BINARY)
-        ./$(TESTBIN)
-
-$(TESTBIN).poll: $(TEST_SOURCES) $(TEST_UNITS) $(TEST_HEADERS) $(BUILDDEPS)
-        $(CC) -o $@ $(CFLAGS_test) -DOT_EVENT_FORCE_POLL $(TEST_SOURCES) $(TEST_UNITS)
-
-check-poll: $(TESTBIN).poll
-        ./$(TESTBIN).poll ot_event
-
-# The same unit tests under the sanitizers. Use clang over $(CC) because the
-# two toolchains do not check the same things
-# Override CC_sanitize to use another compiler.
-CC_sanitize?=clang
-CFLAGS_sanitize=$(CFLAGS_test) -fsanitize=address,undefined -fno-omit-frame-pointer
-LDFLAGS_sanitize=$(LDFLAGS) -fsanitize=address,undefined
-
-$(TESTBIN).sanitize: $(TEST_SOURCES) $(TEST_UNITS) $(TEST_HEADERS) $(BUILDDEPS)
-        $(CC_sanitize) -o $@ $(CFLAGS_sanitize) $(TEST_SOURCES) $(TEST_UNITS) $(LDFLAGS_sanitize)
-
-check-sanitize: $(TESTBIN).sanitize
-        ./$(TESTBIN).sanitize
-
-# Black box tests: these drive the real binary over the wire, so they cover
-# the event loop and the parsers working together, which the unit tests
-# cannot reach. Built with the sanitizers on, so the tracker reports its own
-# memory errors and the harness turns those into failures.
-PYTHON?=python3
-# The client timeout is cut right down so that tests can reach the expiry
-# sweep. tests/integration/test_expiry.py mirrors these two numbers.
-FEATURES_integration=$(FEATURES) -DWANT_IP_FROM_QUERY_STRING -DWANT_ACCESSLIST_BLACK -DWANT_DYNAMIC_ACCESSLIST -DWANT_KEEPALIVE -DOT_CLIENT_TIMEOUT=5 -DOT_CLIENT_TIMEOUT_CHECKINTERVAL=1
-CFLAGS_integration=$(CFLAGS) $(OPTS_debug) $(FEATURES_integration) -fsanitize=address,undefined -fno-omit-frame-pointer
-LDFLAGS_integration=$(LDFLAGS) -fsanitize=address,undefined
-
-opentracker.integration: $(SOURCES) $(HEADERS) $(BUILDDEPS)
-        $(CC) -o $@ $(CFLAGS_integration) $(SOURCES) $(LDFLAGS_integration)
-
-check-integration: opentracker.integration
-        OPENTRACKER=./opentracker.integration $(PYTHON) -m unittest discover -s tests/integration
-
-# The same black box tests against a ThreadSanitizer build
-CFLAGS_tsan=$(CFLAGS) $(OPTS_debug) $(FEATURES_integration) -fsanitize=thread -fno-omit-frame-pointer
-LDFLAGS_tsan=$(LDFLAGS) -fsanitize=thread
-
-opentracker.tsan: $(SOURCES) $(HEADERS) $(BUILDDEPS)
-        $(CC) -o $@ $(CFLAGS_tsan) $(SOURCES) $(LDFLAGS_tsan)
-
-check-tsan: opentracker.tsan
-        OPENTRACKER=./opentracker.tsan $(PYTHON) -m unittest discover -s tests/integration
-
-# Everything, cheapest first, so a quick failure comes back quickly.
-check-all:
-        $(MAKE) check
-        $(MAKE) check-poll
-        $(MAKE) check-sanitize
-        $(MAKE) check-integration
-        $(MAKE) check-tsan
-
-.PHONY: all FORCE dist check check-poll check-sanitize check-integration check-tsan check-all clean install
-
-BINARIES=opentracker opentracker.debug opentracker.integration opentracker.tsan $(TESTBIN) $(TESTBIN).sanitize $(TESTBIN).poll
+$(BUILDSTAMP): FORCE
+        @echo "$(CFLAGS) $(FEATURES) $(LDFLAGS)" | cmp -s - $@ 2>/dev/null || echo "$(CFLAGS) $(FEATURES) $(LDFLAGS)" > $@
 
 clean:
         rm -rf $(BINARIES) $(BINARIES:=.dSYM) $(BUILDSTAMP) *.o *~
 
 install:
-        install -m 755 opentracker $(DESTDIR)$(BINDIR)
+        install -m 755 $(BINARY) $(DESTDIR)$(BINDIR)
+
+FORCE:
 
-# Roll a release: tag it v<something>, then make dist. The commit is written
-# into the tarball, so a build from it reports the same version this checkout
-# would, without a git to ask. The tarball is HEAD by construction, so the
-# working tree it was rolled from does not enter into the version.
-DISTNAME=opentracker-$(VERSION)
+.PHONY: all clean install FORCE
 
-dist:
-        @test "`git describe --tags --match 'v*' --exact-match 2>/dev/null`" = "v$(VERSION)" || \
-                { echo "HEAD must carry the tag v$(VERSION). Tag the release, or correct VERSION."; exit 1; }
-        rm -rf $(DISTNAME) $(DISTNAME).tar.gz
-        git archive --format=tar --prefix=$(DISTNAME)/ HEAD | tar -xf -
-        git describe --tags --match "v*" --always > $(DISTNAME)/.tarball-version
-        tar -czf $(DISTNAME).tar.gz $(DISTNAME)
-        rm -rf $(DISTNAME)
+include Mk/tests.mk
+include Mk/release.mk
+include Mk/proxy.mk
diff --git a/Makefile.gzip b/Mk/gzip.mk
index 70d6d62..70d6d62 100644
--- a/Makefile.gzip
+++ b/Mk/gzip.mk
diff --git a/Mk/proxy.mk b/Mk/proxy.mk
new file mode 100644
index 0000000..2e876e9
--- /dev/null
+++ b/Mk/proxy.mk
@@ -0,0 +1,25 @@
+# The proxy relays sync traffic between trackers and between multicast domains
+# that cannot reach each other. It does not build; see the note at the top of
+# proxy.c. Scheduled for v1.1.0, when these rules come back.
+#
+# LIBOWFAT_HEADERS=$(PREFIX)/libowfat
+# LIBOWFAT_LIBRARY=$(PREFIX)/libowfat
+# CFLAGS_libowfat=-I$(LIBOWFAT_HEADERS)
+# LDFLAGS_libowfat=-L$(LIBOWFAT_LIBRARY) -lowfat
+#
+# SOURCES_proxy=proxy.c ot_batch.c ot_fdstate.c ot_vector.c ot_mutex.c ot_random.c ot_scan.c ot_ip.c ot_socket.c
+# CFLAGS_proxy=$(CFLAGS_production) $(CFLAGS_libowfat) -DWANT_SYNC_LIVE
+# CFLAGS_proxy_debug=$(CFLAGS_debug) $(CFLAGS_libowfat) -DWANT_SYNC_LIVE
+#
+# BINARIES+=proxy proxy.debug
+#
+# proxy: $(SOURCES_proxy) $(HEADERS) $(BUILDDEPS)
+#       $(CC) -o $@ $(CFLAGS_proxy) $(SOURCES_proxy) $(LDFLAGS) $(LDFLAGS_libowfat)
+# proxy.debug: $(SOURCES_proxy) $(HEADERS) $(BUILDDEPS)
+#       $(CC) -o $@ $(CFLAGS_proxy_debug) $(SOURCES_proxy) $(LDFLAGS) $(LDFLAGS_libowfat)
+
+proxy proxy.debug:
+        @echo "$@ is out of service until v1.1.0, see the note at the top of proxy.c"
+        @false
+
+.PHONY: proxy proxy.debug
diff --git a/Mk/release.mk b/Mk/release.mk
new file mode 100644
index 0000000..63429ad
--- /dev/null
+++ b/Mk/release.mk
@@ -0,0 +1,27 @@
+# .tarball-version is what dist wrote into the tarball
+# In a checkout it is absent and git answers instead. A release is always
+# tagged, so in a tarball both of these are that tag, and the gitweb link
+# is to it. Between releases there is no tag and the version is the commit.
+VERSION=1.0.0
+
+TARBALL_VERSION=test -r .tarball-version && read v < .tarball-version && echo "$$v"
+HAVE_GIT=command -v git >/dev/null && test -d .git
+GIT_VERSION=`sh -c '$(TARBALL_VERSION) || { $(HAVE_GIT) && git describe --tags --match "v*" --always --dirty 2>/dev/null; }'`
+OT_TAG=`sh -c '$(TARBALL_VERSION) || { $(HAVE_GIT) && git describe --tags --match "v*" --exact-match 2>/dev/null; }'`
+
+CFLAGS+=-DOT_VERSION=\"$(VERSION)\" -DGIT_VERSION=\"$(GIT_VERSION)\" -DOT_TAG=\"$(OT_TAG)\"
+
+# Roll a release: tag it v$(VERSION), then make dist. The tarball is HEAD by
+# construction, so the working tree it was rolled from does not enter into it.
+DISTNAME=opentracker-$(VERSION)
+
+dist:
+        @test "`git describe --tags --match 'v*' --exact-match 2>/dev/null`" = "v$(VERSION)" || \
+                { echo "HEAD must carry the tag v$(VERSION). Tag the release, or correct VERSION."; exit 1; }
+        rm -rf $(DISTNAME) $(DISTNAME).tar.gz
+        git archive --format=tar --prefix=$(DISTNAME)/ HEAD | tar -xf -
+        git describe --tags --match "v*" --always > $(DISTNAME)/.tarball-version
+        tar -czf $(DISTNAME).tar.gz $(DISTNAME)
+        rm -rf $(DISTNAME)
+
+.PHONY: dist
diff --git a/Mk/tests.mk b/Mk/tests.mk
new file mode 100644
index 0000000..d542044
--- /dev/null
+++ b/Mk/tests.mk
@@ -0,0 +1,72 @@
+BUILDDEPS+=Mk/tests.mk
+
+# The modules under test are compiled straight into the test binary, so a test
+# may reach a static function by including the source. tests/stubs.c supplies
+# the few symbols those modules reference but the tests never reach.
+TESTBIN=tests/runner
+TEST_SOURCES=tests/runner.c tests/test.c tests/stubs.c tests/t_urlencoded.c tests/t_iovec.c tests/t_vector.c tests/t_random.c tests/t_hex.c tests/t_accesslist.c tests/t_scan.c tests/t_buf.c tests/t_ip.c tests/t_socket.c tests/t_mutex.c tests/t_trackerlogic.c tests/t_workqueue.c tests/t_batch.c tests/t_fdstate.c tests/t_event.c tests/t_fmt.c
+TEST_UNITS=ot_urlencoded.c ot_fmt.c ot_event.c ot_batch.c ot_fdstate.c ot_buf.c ot_ip.c ot_socket.c ot_iovec.c ot_vector.c ot_random.c ot_hex.c ot_scan.c ot_accesslist.c ot_mutex.c ot_workqueue.c trackerlogic.c
+TEST_HEADERS=tests/test.h $(HEADERS)
+
+# A test binary is not a production binary, so it is built with the features
+# the tests need to reach.
+FEATURES_test=$(FEATURES) -DWANT_ACCESSLIST_BLACK -DWANT_SYNC_LIVE
+CFLAGS_test=$(CFLAGS) -I. -g -O1 $(FEATURES_test)
+
+# clang over $(CC) because the two toolchains do not check the same things.
+CC_sanitize?=clang
+CFLAGS_sanitize=$(CFLAGS_test) -fsanitize=address,undefined -fno-omit-frame-pointer
+LDFLAGS_sanitize=$(LDFLAGS) -fsanitize=address,undefined
+
+# Black box tests drive the real binary over the wire, so they cover the event
+# loop and the parsers working together. The client timeout is cut right down
+# so a test can reach the expiry sweep; tests/integration/test_expiry.py
+# mirrors these two numbers.
+PYTHON?=python3
+FEATURES_integration=$(FEATURES) -DWANT_IP_FROM_QUERY_STRING -DWANT_ACCESSLIST_BLACK -DWANT_DYNAMIC_ACCESSLIST -DWANT_KEEPALIVE -DOT_CLIENT_TIMEOUT=5 -DOT_CLIENT_TIMEOUT_CHECKINTERVAL=1
+CFLAGS_integration=$(CFLAGS) $(OPTS_debug) $(FEATURES_integration) -fsanitize=address,undefined -fno-omit-frame-pointer
+LDFLAGS_integration=$(LDFLAGS) -fsanitize=address,undefined
+CFLAGS_tsan=$(CFLAGS) $(OPTS_debug) $(FEATURES_integration) -fsanitize=thread -fno-omit-frame-pointer
+LDFLAGS_tsan=$(LDFLAGS) -fsanitize=thread
+
+BINARIES+=$(TESTBIN) $(TESTBIN).poll $(TESTBIN).sanitize opentracker.integration opentracker.tsan
+
+$(TESTBIN): $(TEST_SOURCES) $(TEST_UNITS) $(TEST_HEADERS) $(BUILDDEPS)
+        $(CC) -o $@ $(CFLAGS_test) $(TEST_SOURCES) $(TEST_UNITS)
+
+$(TESTBIN).poll: $(TEST_SOURCES) $(TEST_UNITS) $(TEST_HEADERS) $(BUILDDEPS)
+        $(CC) -o $@ $(CFLAGS_test) -DOT_EVENT_FORCE_POLL $(TEST_SOURCES) $(TEST_UNITS)
+
+$(TESTBIN).sanitize: $(TEST_SOURCES) $(TEST_UNITS) $(TEST_HEADERS) $(BUILDDEPS)
+        $(CC_sanitize) -o $@ $(CFLAGS_sanitize) $(TEST_SOURCES) $(TEST_UNITS) $(LDFLAGS_sanitize)
+
+opentracker.integration: $(SOURCES) $(HEADERS) $(BUILDDEPS)
+        $(CC) -o $@ $(CFLAGS_integration) $(SOURCES) $(LDFLAGS_integration)
+
+opentracker.tsan: $(SOURCES) $(HEADERS) $(BUILDDEPS)
+        $(CC) -o $@ $(CFLAGS_tsan) $(SOURCES) $(LDFLAGS_tsan)
+
+check: $(TESTBIN) $(BINARY)
+        ./$(TESTBIN)
+
+check-poll: $(TESTBIN).poll
+        ./$(TESTBIN).poll ot_event
+
+check-sanitize: $(TESTBIN).sanitize
+        ./$(TESTBIN).sanitize
+
+check-integration: opentracker.integration
+        OPENTRACKER=./opentracker.integration $(PYTHON) -m unittest discover -s tests/integration
+
+check-tsan: opentracker.tsan
+        OPENTRACKER=./opentracker.tsan $(PYTHON) -m unittest discover -s tests/integration
+
+# Cheapest first, so a quick failure comes back quickly.
+check-all:
+        $(MAKE) check
+        $(MAKE) check-poll
+        $(MAKE) check-sanitize
+        $(MAKE) check-integration
+        $(MAKE) check-tsan
+
+.PHONY: check check-poll check-sanitize check-integration check-tsan check-all
diff --git a/Makefile.zstd b/Mk/zstd.mk
index 2bb56be..2bb56be 100644
--- a/Makefile.zstd
+++ b/Mk/zstd.mk