From 9d21cb476d07ab5583c99b1418d4ffb54fbb2a10 Mon Sep 17 00:00:00 2001 From: Dirk Engling Date: Tue, 27 Oct 2026 01:16:37 +0100 Subject: Adding the actual tests to do aforementioned 'Massively increase coverage by pulling fullscrapes and sending udp packets' --- tests/integration/test_accesslist.py | 102 +++++++++++++++++++++++ tests/integration/test_fullscrape.py | 114 ++++++++++++++++++++++++++ tests/integration/test_stats.py | 146 +++++++++++++++++++++++++++++++++ tests/integration/test_udp.py | 153 +++++++++++++++++++++++++++++++++++ 4 files changed, 515 insertions(+) create mode 100644 tests/integration/test_accesslist.py create mode 100644 tests/integration/test_fullscrape.py create mode 100644 tests/integration/test_stats.py create mode 100644 tests/integration/test_udp.py (limited to 'tests') diff --git a/tests/integration/test_accesslist.py b/tests/integration/test_accesslist.py new file mode 100644 index 0000000..b4e134e --- /dev/null +++ b/tests/integration/test_accesslist.py @@ -0,0 +1,102 @@ +"""The blacklist, loaded from a file at startup. + +A blacklisted info hash must be refused at both the announce and the scrape +entry, and the file parser has to cope with the shapes an operator actually +produces: an empty file, and a last line with no newline after it. +""" + +import struct +import unittest + +from harness import TrackerTestCase + +BLOCKED = b"\xb1" * 20 +ALLOWED = b"\xa1" * 20 + + +def listing(*hashes): + return b"".join(h.hex().upper().encode() + b"\n" for h in hashes) + + +class Blacklist(TrackerTestCase): + config = "access.blacklist {dir}/blacklist\n" + files = {"blacklist": listing(BLOCKED)} + + def test_a_blacklisted_hash_is_refused(self): + answer = self.tracker.announce(BLOCKED, port=6881, left=0) + self.assertIn(b"failure reason", answer) + self.assertIn(b"not authorized", answer[b"failure reason"]) + + def test_a_blacklisted_hash_never_becomes_a_torrent(self): + before = self.tracker.torrent_count() + self.tracker.announce(BLOCKED, port=6881, left=0) + self.assertEqual(self.tracker.torrent_count(), before, "the refused hash was stored anyway") + + def test_any_other_hash_is_served(self): + answer = self.tracker.announce(ALLOWED, port=6881, left=0) + self.assertNotIn(b"failure reason", answer) + self.assertEqual(answer[b"complete"], 1) + + def test_the_block_holds_over_udp_but_the_answer_is_malformed(self): + """The UDP path has no spelling for "not authorized": the announce + writes no body, so the tracker sends the eight byte header alone + where BEP 15 calls for at least twenty. Pinned rather than blessed — + an error packet or silence would both be defensible, a short announce + is neither.""" + sock, connection_id = self.tracker.udp_connect() + sock.close() + request = struct.pack( + ">QII20s20sQQQIIIiH", connection_id, 1, 7, BLOCKED, b"p" * 20, 0, 0, 0, 0, 0, 0, -1, 6881 + ) + answer = self.tracker.udp_raw(request) + self.assertIsNotNone(answer) + self.assertEqual(len(answer), 8, "header only, no interval or counts") + action, transaction = struct.unpack(">II", answer) + self.assertEqual(action, 1, "it still claims to be an announce") + self.assertEqual(transaction, 7) + + def test_a_blocked_hash_is_not_stored_when_it_arrives_over_udp(self): + sock, connection_id = self.tracker.udp_connect() + sock.close() + self.tracker.udp_raw( + struct.pack(">QII20s20sQQQIIIiH", connection_id, 1, 8, BLOCKED, b"p" * 20, 0, 0, 0, 0, 0, 0, -1, 6881) + ) + self.assertEqual(self.tracker.torrent_count(), 0) + + +class BlacklistWithoutTrailingNewline(TrackerTestCase): + """A single hash with no newline after it used to size the list at zero + and then write into it, which took the tracker down before it listened.""" + + config = "access.blacklist {dir}/blacklist\n" + files = {"blacklist": BLOCKED.hex().upper().encode()} + + def test_the_last_line_counts_even_without_its_newline(self): + answer = self.tracker.announce(BLOCKED, port=6881, left=0) + self.assertIn(b"failure reason", answer) + + def test_and_the_tracker_still_serves_everything_else(self): + self.assertEqual(self.tracker.announce(ALLOWED, port=6881, left=0)[b"complete"], 1) + + +class EmptyBlacklist(TrackerTestCase): + config = "access.blacklist {dir}/blacklist\n" + files = {"blacklist": b""} + + def test_an_empty_file_blocks_nothing_and_starts_cleanly(self): + self.assertEqual(self.tracker.announce(BLOCKED, port=6881, left=0)[b"complete"], 1) + self.assertEqual(self.tracker.announce(ALLOWED, port=6882, left=0)[b"complete"], 1) + + +class MissingBlacklist(TrackerTestCase): + """The file is named but absent: the tracker warns and serves everything, + because it may be created later.""" + + config = "access.blacklist {dir}/absent\n" + + def test_a_missing_file_blocks_nothing(self): + self.assertEqual(self.tracker.announce(BLOCKED, port=6881, left=0)[b"complete"], 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/integration/test_fullscrape.py b/tests/integration/test_fullscrape.py new file mode 100644 index 0000000..4059656 --- /dev/null +++ b/tests/integration/test_fullscrape.py @@ -0,0 +1,114 @@ +"""The full scrape, in every format the tracker offers. + +Each format is a separate writer, and they differ in more than cosmetics: +one is binary, one is bencoded, three are line based and one dumps the +tracker's own state for a restart. A worker thread produces them in chunks, +so these also exercise the chunked transfer the ordinary replies never take. +""" + +import gzip +import struct +import unittest + +from harness import TrackerTestCase, bdecode + +HEX = b"0123456789ABCDEF" + + +class FullScrape(TrackerTestCase): + def setUp(self): + TrackerTestCase.setUp(self) + self.seeding = self.hash_for("fsA") + self.leeching = self.hash_for("fsB") + self.tracker.announce(self.seeding, port=6001, left=0) + self.tracker.announce(self.leeching, port=6002, left=4096) + + def scrape_all(self, query, headers=None): + body, head = self.tracker.get_with_headers(query, headers) + if head.get("Content-Encoding") == "gzip": + body = gzip.decompress(body) + return body + + # --------------------------------------------------------- bencoded + + def test_a_bare_scrape_returns_every_torrent(self): + answer = bdecode(self.scrape_all("/scrape")) + self.assertIn(self.seeding, answer[b"files"]) + self.assertIn(self.leeching, answer[b"files"]) + self.assertEqual(answer[b"files"][self.seeding][b"complete"], 1) + self.assertEqual(answer[b"files"][self.leeching][b"incomplete"], 1) + + def test_the_bencoded_format_matches_the_bare_scrape(self): + self.assertEqual(bdecode(self.scrape_all("/stats?mode=tpbs&format=ben")), bdecode(self.scrape_all("/scrape"))) + + def test_a_full_scrape_can_be_gzipped(self): + plain = self.scrape_all("/scrape") + body, head = self.tracker.get_with_headers("/scrape", {"Accept-Encoding": "gzip"}) + self.assertEqual(head.get("Content-Encoding"), "gzip") + self.assertEqual(gzip.decompress(body), plain) + + # ----------------------------------------------------------- binary + + def test_the_binary_format_is_a_hash_and_two_counts(self): + body = self.scrape_all("/stats?mode=tpbs&format=bin") + self.assertEqual(len(body) % 28, 0, "twenty byte hash plus two counts") + entries = {body[i : i + 20]: struct.unpack(">II", body[i + 20 : i + 28]) for i in range(0, len(body), 28)} + self.assertEqual(entries[self.seeding], (1, 0), "one seeder, no leechers") + self.assertEqual(entries[self.leeching], (0, 1), "no seeders, one leecher") + + # ------------------------------------------------------- line based + + def entries_from_lines(self, body): + return {line.split(b":")[0]: line.split(b":")[1:] for line in body.splitlines() if line} + + def test_the_text_format_is_hex_colon_seeders_colon_leechers(self): + entries = self.entries_from_lines(self.scrape_all("/stats?mode=tpbs&format=txt")) + self.assertEqual(entries[self.seeding.hex().upper().encode()], [b"1", b"0"]) + self.assertEqual(entries[self.leeching.hex().upper().encode()], [b"0", b"1"]) + + def test_the_text_plus_format_adds_the_completed_count(self): + entries = self.entries_from_lines(self.scrape_all("/stats?mode=tpbs&format=txtp")) + self.assertEqual(entries[self.seeding.hex().upper().encode()], [b"1", b"0", b"0"]) + + def test_the_urlencoded_format_escapes_the_hash(self): + body = self.scrape_all("/stats?mode=tpbs&format=url") + lines = [line for line in body.splitlines() if line] + self.assertEqual(len(lines), 2) + for line in lines: + escaped = line.rsplit(b":", 2)[0] + self.assertIn(b"%", escaped, "these hashes are mostly unprintable bytes") + + def test_the_state_dump_carries_a_base_time_rather_than_a_peer_count(self): + """What a restart reads back, so its second column is the torrent's + own clock and not a count of anybody.""" + entries = self.entries_from_lines(self.scrape_all("/stats?mode=statedump")) + self.assertIn(self.seeding.hex().upper().encode(), entries) + base, completed = entries[self.seeding.hex().upper().encode()] + self.assertGreater(int(base), 0) + self.assertEqual(int(completed), 0) + + # ------------------------------------------------------------ shape + + def test_every_format_sees_the_same_torrents(self): + counts = { + "ben": len(bdecode(self.scrape_all("/stats?mode=tpbs&format=ben"))[b"files"]), + "bin": len(self.scrape_all("/stats?mode=tpbs&format=bin")) // 28, + "txt": len(self.entries_from_lines(self.scrape_all("/stats?mode=tpbs&format=txt"))), + "txtp": len(self.entries_from_lines(self.scrape_all("/stats?mode=tpbs&format=txtp"))), + "url": len([l for l in self.scrape_all("/stats?mode=tpbs&format=url").splitlines() if l]), + } + self.assertEqual(set(counts.values()), {2}, "all formats, two torrents: %r" % counts) + + +class FullScrapeChunking(TrackerTestCase): + def test_a_scrape_larger_than_one_chunk_arrives_whole(self): + """The writer hands the socket a chunk at a time, so a scrape that + spans several is the case where the seams show.""" + for i in range(400): + self.tracker.announce(struct.pack(">I", i) + b"\0" * 16, port=7000 + i, left=0) + answer = bdecode(self.tracker.get("/scrape")) + self.assertEqual(len(answer[b"files"]), 400) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/integration/test_stats.py b/tests/integration/test_stats.py new file mode 100644 index 0000000..920d199 --- /dev/null +++ b/tests/integration/test_stats.py @@ -0,0 +1,146 @@ +"""The /stats endpoint. + +Each mode is its own writer into a fixed buffer, so these pin the shape of +every one of them and, where the number means something, that it moves when +the thing it counts happens. +""" + +import re +import unittest +import urllib.error +import xml.etree.ElementTree + +from harness import TrackerTestCase + +# The modes a default build answers. "tpbs" and "statedump" are full scrapes +# and belong to test_fullscrape; "fulllog" and "woodpeckers" sit behind +# feature flags this build does not set; "busy" is advertised but unbuilt, +# which the next case pins. +MRTG_MODES = ["peer", "conn", "scrp", "udp4", "tcp4", "torr", "fscr", "completed", "syncs"] +TEXT_MODES = ["herr", "s24s", "top10", "top100", "renew", "version"] + + +class StatsShape(TrackerTestCase): + def test_every_mode_answers_something(self): + for mode in MRTG_MODES + TEXT_MODES + ["everything"]: + self.assertTrue(self.tracker.stats(mode), "mode %s answered empty" % mode) + + def test_an_mrtg_mode_opens_with_two_numbers(self): + """MRTG reads the first two lines as counters and the rest as labels, + so the order matters to something outside this project.""" + for mode in MRTG_MODES: + lines = self.tracker.stats(mode).splitlines() + self.assertGreaterEqual(len(lines), 2, "mode %s" % mode) + for line in lines[:2]: + self.assertRegex(line, rb"^\d+$", "mode %s line %r" % (mode, line)) + + def test_the_busy_mode_is_advertised_but_not_built(self): + """TASK_STATS_BUSY_NETWORKS is in the enum and in the mode table, and + no switch anywhere handles it, so the worker frees the empty answer + and the request ends as a server error. Pinned rather than blessed: + either the report gets written or the mode leaves the table.""" + with self.assertRaises(urllib.error.HTTPError) as caught: + self.tracker.stats("busy") + self.assertEqual(caught.exception.code, 500) + + def test_an_unknown_mode_is_refused(self): + with self.assertRaises(urllib.error.HTTPError) as caught: + self.tracker.stats("nosuchmode") + self.assertEqual(caught.exception.code, 400) + + def test_everything_is_well_formed_xml(self): + root = xml.etree.ElementTree.fromstring(self.tracker.stats("everything").decode()) + self.assertEqual(root.tag, "stats") + self.assertIsNotNone(root.find("torrents/count_mutex")) + self.assertIsNotNone(root.find("connections/tcp/announce")) + + def test_version_names_a_commit(self): + self.assertIn(b"opentracker", self.tracker.stats("version").lower() + b"opentracker") + self.assertRegex(self.tracker.stats("version"), rb"https?://") + + +class StatsCount(TrackerTestCase): + def test_the_torrent_count_follows_the_torrents(self): + before = self.tracker.torrent_count() + self.tracker.announce(self.hash_for("statA"), port=6001, left=0) + self.assertEqual(self.tracker.torrent_count(), before + 1) + self.tracker.announce(self.hash_for("statB"), port=6002, left=0) + self.assertEqual(self.tracker.torrent_count(), before + 2) + + def test_the_peer_count_separates_seeders_from_the_rest(self): + self.tracker.announce(self.hash_for("statpeers"), port=6001, left=0) + self.tracker.announce(self.hash_for("statpeers"), port=6002, left=4096) + peers, seeds = self.tracker.stats("peer").splitlines()[:2] + self.assertEqual(int(peers), 2) + self.assertEqual(int(seeds), 1) + + def test_the_completed_count_follows_the_completed_event(self): + before = int(self.tracker.stats("completed").splitlines()[0]) + self.tracker.announce(self.hash_for("statdone"), port=6001, left=0, event="completed") + self.assertEqual(int(self.tracker.stats("completed").splitlines()[0]), before + 1) + + def test_a_bad_request_is_counted_as_an_http_error(self): + before = self.tracker.stats("herr") + try: + self.tracker.stats("nosuchmode") + except urllib.error.HTTPError: + pass + after = self.tracker.stats("herr") + self.assertNotEqual(before, after, "no error counter moved") + + def test_the_tcp_counter_follows_tcp_requests(self): + before = int(self.tracker.stats("tcp4").splitlines()[0]) + for _ in range(3): + self.tracker.announce(self.hash_for("stattcp"), port=6001, left=0) + self.assertGreaterEqual(int(self.tracker.stats("tcp4").splitlines()[0]), before + 3) + + def test_the_udp_counter_follows_udp_requests(self): + before = int(self.tracker.stats("udp4").splitlines()[0]) + sock, connection_id = self.tracker.udp_connect() + try: + self.tracker.udp_announce(sock, connection_id, self.hash_for("statudp"), left=0) + finally: + sock.close() + self.assertGreater(int(self.tracker.stats("udp4").splitlines()[0]), before) + + def test_the_scrape_counter_follows_scrapes(self): + before = int(self.tracker.stats("scrp").splitlines()[0]) + self.tracker.scrape(self.hash_for("statscrape")) + self.assertEqual(int(self.tracker.stats("scrp").splitlines()[0]), before + 1) + + +class StatsTop(TrackerTestCase): + def test_the_top_list_ranks_by_peer_count(self): + busy = self.hash_for("topbusy") + quiet = self.hash_for("topquiet") + for port in range(6001, 6006): + self.tracker.announce(busy, port=port, left=0) + self.tracker.announce(quiet, port=6100, left=0) + + body = self.tracker.stats("top10") + self.assertIn(b"Top 10 torrents by peers", body) + ranked = re.findall(rb"\t(\d+)\t([0-9A-F]{40})", body.split(b"by seeds")[0]) + self.assertEqual(ranked[0][1], busy.hex().upper().encode(), "the busy torrent ranks first") + self.assertEqual(int(ranked[0][0]), 5) + + def test_the_top_list_has_three_sections(self): + self.tracker.announce(self.hash_for("topsections"), port=6001, left=0) + body = self.tracker.stats("top100") + for heading in (b"by peers", b"by seeds", b"by leechers"): + self.assertIn(heading, body) + + +class StatsNetworks(TrackerTestCase): + def test_the_slash24_report_names_the_loopback_network(self): + self.tracker.announce(self.hash_for("s24"), port=6001, left=0) + body = self.tracker.stats("s24s") + self.assertIn(b"Networks, limit /28:", body) + self.assertIn(b"Networks, limit /24:", body) + # The prefix is masked out of a v4 peer's four address bytes and then + # printed through the v6 formatter, so 127.0.0.0/24 reads as "7f00::" + # rather than as an address anyone would recognise. Pinned as it is. + self.assertIn(b"7f00::", body) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/integration/test_udp.py b/tests/integration/test_udp.py new file mode 100644 index 0000000..1dfa21b --- /dev/null +++ b/tests/integration/test_udp.py @@ -0,0 +1,153 @@ +"""The UDP tracker protocol, BEP 15. + +The HTTP tests cover the torrent store; these cover the packet handling in +front of it, which has its own framing, its own connection id and its own +rules about what to answer and what to ignore. +""" + +import socket +import struct +import unittest + +from harness import TrackerTestCase + +MAGIC = 0x41727101980 + + +class UdpProtocol(TrackerTestCase): + def test_connect_echoes_the_transaction_id(self): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.settimeout(5) + try: + sock.sendto(struct.pack(">QII", MAGIC, 0, 0xDEADBEEF), ("127.0.0.1", self.tracker.port)) + action, transaction, connection_id = struct.unpack(">IIQ", sock.recv(2048)[:16]) + self.assertEqual(action, 0) + self.assertEqual(transaction, 0xDEADBEEF) + self.assertNotEqual(connection_id, 0) + finally: + sock.close() + + def test_the_connection_id_is_stable_for_one_peer(self): + first, id_one = self.tracker.udp_connect() + second, id_two = self.tracker.udp_connect() + try: + self.assertEqual(id_one, id_two) + finally: + first.close() + second.close() + + def test_a_wrong_connection_id_is_refused(self): + payload = struct.pack(">QII20s20sQQQIIIiH", 0, 1, 2, self.hash_for("udpbad"), b"p" * 20, 0, 0, 0, 0, 0, 0, -1, 6881) + answer = self.tracker.udp_raw(payload) + self.assertIsNotNone(answer, "expected an error packet") + action, transaction = struct.unpack(">II", answer[:8]) + self.assertEqual(action, 3, "action 3 is the error packet") + self.assertIn(b"Connection ID missmatch", answer[8:]) + + def test_a_packet_shorter_than_the_header_is_ignored(self): + self.assertIsNone(self.tracker.udp_raw(b"\x00" * 15)) + + def test_an_unknown_action_is_ignored(self): + self.assertIsNone(self.tracker.udp_raw(struct.pack(">QII", MAGIC, 9, 1))) + + def test_connect_requires_the_protocol_magic(self): + self.assertIsNone(self.tracker.udp_raw(struct.pack(">QII", MAGIC + 1, 0, 1))) + + def test_an_announce_cut_short_is_ignored(self): + sock, connection_id = self.tracker.udp_connect() + try: + self.assertIsNone(self.tracker.udp_raw(struct.pack(">QII", connection_id, 1, 1) + b"\0" * 40)) + finally: + sock.close() + + +class UdpAnnounce(TrackerTestCase): + def test_a_seeder_is_counted_as_a_seeder(self): + sock, connection_id = self.tracker.udp_connect() + try: + answer = self.tracker.udp_announce(sock, connection_id, self.hash_for("udpseed"), left=0) + self.assertEqual(answer["seeders"], 1) + self.assertEqual(answer["leechers"], 0) + self.assertGreater(answer["interval"], 0) + finally: + sock.close() + + def test_a_leecher_is_counted_as_a_leecher(self): + sock, connection_id = self.tracker.udp_connect() + try: + answer = self.tracker.udp_announce(sock, connection_id, self.hash_for("udpleech"), left=1024) + self.assertEqual(answer["seeders"], 0) + self.assertEqual(answer["leechers"], 1) + finally: + sock.close() + + def test_stopped_removes_the_peer(self): + info_hash = self.hash_for("udpstop") + sock, connection_id = self.tracker.udp_connect() + try: + self.tracker.udp_announce(sock, connection_id, info_hash, left=0) + self.tracker.udp_announce(sock, connection_id, info_hash, left=0, event=3) + answer = self.tracker.udp_announce(sock, connection_id, info_hash, left=1024) + self.assertEqual(answer["seeders"], 0, "the seeder announced stopped") + finally: + sock.close() + + def test_an_announce_over_udp_is_visible_over_http(self): + """One torrent store behind both bearers, not two.""" + info_hash = self.hash_for("udpcross") + sock, connection_id = self.tracker.udp_connect() + try: + self.tracker.udp_announce(sock, connection_id, info_hash, left=0) + finally: + sock.close() + scrape = self.tracker.scrape(info_hash) + self.assertEqual(scrape[b"files"][info_hash][b"complete"], 1) + + def test_the_peer_list_carries_the_announcing_peer(self): + info_hash = self.hash_for("udppeers") + sock, connection_id = self.tracker.udp_connect() + try: + answer = self.tracker.udp_announce(sock, connection_id, info_hash, left=0, port=6881) + self.assertEqual(len(answer["peers"]) % 6, 0, "v4 peers are six bytes each") + self.assertGreaterEqual(len(answer["peers"]), 6) + self.assertEqual(struct.unpack(">H", answer["peers"][4:6])[0], 6881) + finally: + sock.close() + + +class UdpScrape(TrackerTestCase): + def test_scrape_reports_the_swarm(self): + info_hash = self.hash_for("udpscrape") + sock, connection_id = self.tracker.udp_connect() + try: + self.tracker.udp_announce(sock, connection_id, info_hash, left=0) + seeders, completed, leechers = self.tracker.udp_scrape(sock, connection_id, [info_hash])[0] + self.assertEqual(seeders, 1) + self.assertEqual(leechers, 0) + self.assertEqual(completed, 0) + finally: + sock.close() + + def test_an_unknown_hash_scrapes_as_empty(self): + sock, connection_id = self.tracker.udp_connect() + try: + self.assertEqual(self.tracker.udp_scrape(sock, connection_id, [self.hash_for("udpnone")])[0], (0, 0, 0)) + finally: + sock.close() + + def test_several_hashes_answer_in_order(self): + seeding = self.hash_for("udpmultiA") + leeching = self.hash_for("udpmultiB") + sock, connection_id = self.tracker.udp_connect() + try: + self.tracker.udp_announce(sock, connection_id, seeding, left=0) + self.tracker.udp_announce(sock, connection_id, leeching, left=1024) + answers = self.tracker.udp_scrape(sock, connection_id, [seeding, leeching]) + self.assertEqual(answers[0][0], 1, "first hash is the seeded one") + self.assertEqual(answers[1][2], 1, "second hash is the leeched one") + finally: + sock.close() + + +if __name__ == "__main__": + unittest.main() -- cgit v1.3