diff --git a/README.md b/README.md index 37fa5b0..13e2523 100644 --- a/README.md +++ b/README.md @@ -243,6 +243,16 @@ What the response deliberately does **not** contain: paid reputation feed's verdicts on an open endpoint would be redistributing their data rather than ours. An entry asserts only that the block exists. +Entries report the **network** that was blocked (ipapi.is's `asn.route` for +the signup IP, which is what `register_ip_block` registers), not the address +that signed up. A `/32` or `/128` entry means a single address really is what +was blocked — either the classification returned no usable route CIDR, or the +signup predates range-blocking (2026-07-15). The `block_cidr` column recording +this arrived with the API itself, so existing rows were backfilled once from +the ipapi range cache by `bin/backfill-block-cidr.py`; ten rows with no cache +entry left are shown as their `/32`, since a narrower entry is a safer error +than claiming a range is blocked when it isn't. + Scope: this is the **bot's own record**, not Mastodon's full blocklist — blocks a moderator added by hand in the admin UI are not in it. Entries appear only once the block actually landed on the Mastodon side (a failed @@ -258,6 +268,7 @@ only once the block actually landed on the Mastodon side (a failed | `.env.example` | Config template (copy to `.env`, never commit) | | `nginx/hooks.yttrx.com.conf` | nginx site for admin.yttrx.com (webhook, healthz, blocklist API) | | `bin/welcomebot-logs`, `bin/welcomebot-signups`, `bin/welcomebot-suspicious` | Ops CLIs installed to `/root/bin` on admin.yttrx.com | +| `bin/backfill-block-cidr.py` | One-shot: reconstruct `block_cidr` for rows predating it (run 2026-09-13) | | `test_local.py` | Offline smoke test (no network) | ## Configuration diff --git a/bin/backfill-block-cidr.py b/bin/backfill-block-cidr.py new file mode 100755 index 0000000..0772502 --- /dev/null +++ b/bin/backfill-block-cidr.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""One-shot backfill of signup_ip.block_cidr, run once on admin.yttrx.com +2026-09-13 when the blocklist API shipped. + +block_cidr records the network register_ip_block actually blocked. It didn't +exist before the API did, so every pre-existing blocked row had nothing to +report and /api/blocks fell back to its signup IP's /32 — narrower than what +is really blocked, and not what the endpoint is for. The range is recoverable +from the bot's own ipapi range cache, which stores the CIDR each +classification was cached under (the same value that was handed to +register_ip_block). + +Run inside the container, dry-run first: + + docker cp bin/backfill-block-cidr.py yttrx-welcomebot:/tmp/ + docker exec -e PYTHONPATH=/app yttrx-welcomebot python /tmp/backfill-block-cidr.py + docker exec -e PYTHONPATH=/app -e APPLY=1 yttrx-welcomebot python /tmp/backfill-block-cidr.py + +Result of the 2026-09-13 run: 154 blocked rows, 65 backfilled to a real range, +79 that genuinely were single-address blocks (ipapi.is returned no asn.route +for them), 10 with no cache entry at all whose range can't be reconstructed +and stay as a /32. +""" +import os + +import app.main as m + +# Route-CIDR blocking landed 2026-07-15 (commit b39528a); the ipapi range +# cache landed 2026-07-13. Only signups blocked after the former had a network +# CIDR handed to register_ip_block — for anything earlier only the /32 was +# ever registered, so backfilling a wider range from the cache would publish a +# block that doesn't exist. Under-reporting is safe; over-reporting isn't. +CUTOFF = "2026-07-16" +APPLY = os.environ.get("APPLY") == "1" + +# NB: _db_lock is not reentrant — never call cached_ip_intel() inside a _db() +# block, it opens its own. +with m._db() as conn: + rows = conn.execute( + "SELECT account_id, ip, classified_at FROM signup_ip " + "WHERE ipblock_registered = 1 AND block_cidr IS NULL" + ).fetchall() + +updates, skipped_old, no_cache = [], 0, 0 +for account_id, ip, classified_at in rows: + intel = m.cached_ip_intel(ip or "") + cidr = (intel or {}).get("cidr") or "" + if not cidr or cidr.endswith(("/32", "/128")): + no_cache += 1 + continue + if (classified_at or "") < CUTOFF: + skipped_old += 1 + continue + updates.append((cidr, account_id)) + +print(f"rows missing block_cidr : {len(rows)}") +print(f"no wider cached CIDR : {no_cache} (stay as their /32 or /128)") +print(f"pre-{CUTOFF} rows : {skipped_old} (only a /32 was ever blocked)") +print(f"to backfill : {len(updates)}") +for cidr, account_id in updates[:5]: + print(f" e.g. account {account_id} -> {cidr}") + +if APPLY: + with m._db() as conn: + conn.executemany("UPDATE signup_ip SET block_cidr = ? WHERE account_id = ?", updates) + print(f"applied {len(updates)} updates") +else: + print("DRY RUN — set APPLY=1 to write")