diff --git a/.env.example b/.env.example index b368c32..4c05128 100644 --- a/.env.example +++ b/.env.example @@ -241,6 +241,10 @@ SUSPICIOUS_COMBINED_DRY_RUN=true # paths 404 as if they didn't exist. yttrx.com sets this to true. BLOCKS_API_ENABLED=false +# The list is kept honest by app/reconcile_blocks.py (cron, hourly), which +# drops entries whose block no longer exists in Mastodon. That needs +# admin:read:ip_blocks + admin:read:email_domain_blocks on ABUSE_BOT_TOKEN. + # Hard ceiling on entries per response, whatever ?limit asks for — the # endpoint is unauthenticated, so response size can't be left to the caller. BLOCKS_API_MAX_ITEMS=5000 diff --git a/CLAUDE.md b/CLAUDE.md index 4ba9c2f..a5cb5eb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -206,6 +206,17 @@ as an error; nothing else in the bot is affected). report-triggered suspend override) will 403 (logged as an error; the suspend itself still goes through since that only needs `admin:write:accounts`). +**The blocklist reconcile (`app/reconcile_blocks.py`) needs two more scopes, +`admin:read:ip_blocks` and `admin:read:email_domain_blocks`**, to `GET` the +live `/api/v1/admin/ip_blocks` and `/api/v1/admin/email_domain_blocks` lists +and check whether the blocks we registered are still there. Added to the live +token **in place** on 2026-09-13 (`Doorkeeper::AccessToken#update!(scopes:)` +on id 11334, matched by SHA-256 digest against admin's `.env`) rather than by +minting and rotating — the token value is unchanged, so no `.env` edit was +needed. Without these scopes the reconcile 403s and exits having changed +nothing, which is the deliberate failure mode: never read an outage as "every +block was removed". + **The suspicious-signup sweep (`app/suspicious_sweep.py`) needs another scope, `admin:read:accounts`**, to `GET /api/v1/admin/accounts/:id` (fetches current post/follow counts + role/suspended state to decide whether to suspend) — @@ -217,7 +228,7 @@ Mint a correctly-scoped token from the Rails console on mammut: ```bash ssh mammut 'docker exec $(docker ps -f name=live-web-1 -q) bin/rails runner " acct = Account.find_local(%q{bot}) -scopes = %q{read:statuses write:statuses admin:read:reports admin:write:accounts admin:write:ip_blocks admin:write:email_domain_blocks admin:read:accounts} +scopes = %q{read:statuses write:statuses admin:read:reports admin:write:accounts admin:write:ip_blocks admin:write:email_domain_blocks admin:read:accounts admin:read:ip_blocks admin:read:email_domain_blocks} app = Doorkeeper::Application.create!(name: %q{Abuse handler vN}, scopes: scopes, redirect_uri: %q{urn:ietf:wg:oauth:2.0:oob}) tok = Doorkeeper::AccessToken.create!(application_id: app.id, resource_owner_id: acct.user.id, scopes: scopes) puts tok.token diff --git a/README.md b/README.md index 464fd56..e973d40 100644 --- a/README.md +++ b/README.md @@ -301,6 +301,32 @@ the ipapi range cache by `bin/backfill-block-cidr.py`; ten rows with no cache entry left are shown as their `/32`, since a narrower entry is a safer error than claiming a range is blocked when it isn't. +### Staying accurate (`app/reconcile_blocks.py`) + +`ipblock_registered` / `domain_blocked` only ever meant "we posted this block +once" — they say nothing about whether it's still there. A moderator clearing +entries by hand, or an `expires_at` elapsing, would leave the endpoint +publishing blocks that no longer block anything. (This was not hypothetical: +when the API first went live it was publishing 24 `/32` ranges that had been +removed on 2026-07-21.) + +The reconcile pass fetches Mastodon's live `Admin::IpBlock` and +`Admin::EmailDomainBlock` lists and flips `block_active` off for rows whose +block is gone or expired, back on for any that reappear. The API serves only +active rows, so `?format=txt` hands an importer only blocks actually in force. + +```bash +docker exec yttrx-welcomebot python -m app.reconcile_blocks --dry-run +docker exec yttrx-welcomebot python -m app.reconcile_blocks +``` + +Runs hourly by cron on admin.yttrx.com, alongside the suspicious-signup sweep. +It needs `admin:read:ip_blocks` and `admin:read:email_domain_blocks` on +`ABUSE_BOT_TOKEN`. **A failed fetch changes nothing** — a 403 from a missing +scope, or Mastodon being down, must never be read as "every block was +removed", so rows keep their existing state and the list degrades to +over-reporting rather than emptying itself. + Scope: this is the **bot's own record**, not Mastodon's full blocklist — blocks a moderator added by hand in the admin UI are not in it. Entries appear only once the block actually landed on the Mastodon side (a failed @@ -312,6 +338,7 @@ only once the block actually landed on the Mastodon side (a failed |---|---| | `app/main.py` | The FastAPI app | | `app/suspicious_sweep.py` | Scheduled sweep: suspends flagged signups with no activity (run via cron, not the webserver) | +| `app/reconcile_blocks.py` | Scheduled pass: marks blocks that no longer exist in Mastodon inactive (cron) | | `Dockerfile`, `docker-compose.yml` | Container build + run | | `.env.example` | Config template (copy to `.env`, never commit) | | `nginx/hooks.yttrx.com.conf` | nginx site for admin.yttrx.com (webhook, healthz, blocklist API) | diff --git a/app/main.py b/app/main.py index b512a3b..8af7d1a 100644 --- a/app/main.py +++ b/app/main.py @@ -301,6 +301,8 @@ def _init_db() -> None: " flagged INTEGER," " ipblock_registered INTEGER DEFAULT 0," " block_cidr TEXT," + " block_active INTEGER DEFAULT 1," + " block_checked_at TEXT," " classified_at TEXT DEFAULT CURRENT_TIMESTAMP" ")" ) @@ -312,6 +314,18 @@ def _init_db() -> None: # ASN route CIDR only started 2026-07-15. if not _column_exists(conn, "signup_ip", "block_cidr"): conn.execute("ALTER TABLE signup_ip ADD COLUMN block_cidr TEXT") + # block_active tracks whether the block we registered is STILL present + # on the Mastodon side. ipblock_registered only ever meant "we posted + # it once"; a moderator clearing blocks by hand (as happened to 24 + # rows on 2026-07-21) or an expires_at elapsing would leave us + # publishing entries that no longer block anything. Maintained by + # app/reconcile_blocks.py; rows default to active until it says + # otherwise, so a reconcile that never runs degrades to the old + # behaviour rather than emptying the list. + if not _column_exists(conn, "signup_ip", "block_active"): + conn.execute("ALTER TABLE signup_ip ADD COLUMN block_active INTEGER DEFAULT 1") + if not _column_exists(conn, "signup_ip", "block_checked_at"): + conn.execute("ALTER TABLE signup_ip ADD COLUMN block_checked_at TEXT") conn.execute( "CREATE TABLE IF NOT EXISTS ipapi_range_cache (" " family INTEGER NOT NULL," # 4 or 6 @@ -337,9 +351,15 @@ def _init_db() -> None: " risk INTEGER," " flagged INTEGER," " domain_blocked INTEGER DEFAULT 0," + " block_active INTEGER DEFAULT 1," + " block_checked_at TEXT," " classified_at TEXT DEFAULT CURRENT_TIMESTAMP" ")" ) + if not _column_exists(conn, "signup_email", "block_active"): + conn.execute("ALTER TABLE signup_email ADD COLUMN block_active INTEGER DEFAULT 1") + if not _column_exists(conn, "signup_email", "block_checked_at"): + conn.execute("ALTER TABLE signup_email ADD COLUMN block_checked_at TEXT") conn.execute( "CREATE TABLE IF NOT EXISTS check_mail_cache (" " domain TEXT PRIMARY KEY," @@ -431,9 +451,12 @@ def get_signup_flag(account_id: str) -> bool: def mark_ipblock_registered(account_id: str) -> None: + # block_active is reset too: if this range was reconciled away earlier and + # we've just registered it again, it is live again. with _db() as conn: conn.execute( - "UPDATE signup_ip SET ipblock_registered = 1 WHERE account_id = ?", + "UPDATE signup_ip SET ipblock_registered = 1, block_active = 1 " + "WHERE account_id = ?", (account_id,), ) @@ -528,7 +551,8 @@ def get_signup_email(account_id: str) -> tuple[str, bool]: def mark_email_domain_blocked(account_id: str) -> None: with _db() as conn: conn.execute( - "UPDATE signup_email SET domain_blocked = 1 WHERE account_id = ?", + "UPDATE signup_email SET domain_blocked = 1, block_active = 1 " + "WHERE account_id = ?", (account_id,), ) @@ -579,8 +603,10 @@ def _fallback_cidr(ip: str) -> str: def blocked_ip_ranges(limit: int = 0) -> list[dict]: - """Every network this bot has registered an Admin::IpBlock for, newest - first, collapsed to one entry per distinct CIDR. + """Every network this bot has registered an Admin::IpBlock for that is + still present on the Mastodon side, newest first, collapsed to one entry + per distinct CIDR. Rows the reconcile pass found gone or expired are + excluded — see app/reconcile_blocks.py. Repeat signups out of the same range are common (that's the point of blocking the range rather than the address), so entries carry a `signups` @@ -596,7 +622,7 @@ def blocked_ip_ranges(limit: int = 0) -> list[dict]: with _db() as conn: rows = conn.execute( "SELECT block_cidr, ip, classified_at " - "FROM signup_ip WHERE ipblock_registered = 1 " + "FROM signup_ip WHERE ipblock_registered = 1 AND block_active = 1 " "ORDER BY classified_at DESC" ).fetchall() @@ -622,8 +648,9 @@ def blocked_ip_ranges(limit: int = 0) -> list[dict]: def blocked_email_domains(limit: int = 0) -> list[dict]: - """Every domain this bot has registered an Admin::EmailDomainBlock for, - newest first, one entry per distinct domain. + """Every domain this bot has registered an Admin::EmailDomainBlock for + that is still present on the Mastodon side, newest first, one entry per + distinct domain. See app/reconcile_blocks.py. Domains only — the bot never stores or forwards a full signup address (see classify_email_domain), so there is none to expose here. @@ -635,7 +662,7 @@ def blocked_email_domains(limit: int = 0) -> list[dict]: with _db() as conn: rows = conn.execute( "SELECT domain, classified_at " - "FROM signup_email WHERE domain_blocked = 1 " + "FROM signup_email WHERE domain_blocked = 1 AND block_active = 1 " "ORDER BY classified_at DESC" ).fetchall() diff --git a/app/reconcile_blocks.py b/app/reconcile_blocks.py new file mode 100644 index 0000000..d8dc308 --- /dev/null +++ b/app/reconcile_blocks.py @@ -0,0 +1,205 @@ +"""Blocklist reconcile pass. + +The bot's `signup_ip.ipblock_registered` / `signup_email.domain_blocked` flags +only ever meant "we successfully posted this block once". They say nothing +about whether the block is still there. A moderator clearing entries by hand +(24 bot-registered `/32` rows were removed on 2026-07-21), or an `expires_at` +elapsing, leaves the bot's record claiming blocks that no longer block +anything — and the public /api/blocks endpoint publishing them. + +This pass closes that gap: it fetches Mastodon's live `Admin::IpBlock` and +`Admin::EmailDomainBlock` lists, and flips `block_active` off for any of our +rows whose block is gone (or has expired), back on for any that reappeared. +The API then serves only entries that are actually in force. + +Run on a schedule (cron on admin.yttrx.com, see CLAUDE.md): + + docker exec yttrx-welcomebot python -m app.reconcile_blocks + +Requires ABUSE_BOT_TOKEN to carry `admin:read:ip_blocks` and +`admin:read:email_domain_blocks` in addition to its existing scopes. Without +them the fetch 403s and the pass exits without changing anything — rows keep +whatever state they had, so a missing scope degrades to the old (over- +reporting) behaviour rather than silently emptying the published list. + +`--dry-run` reports what would change without writing. +""" + +from __future__ import annotations + +import argparse +import ipaddress +import logging +import sys +from datetime import datetime, timezone + +import httpx + +from app.main import ( + MASTODON_BASE_URL, + _admin_headers, + _db, +) + +log = logging.getLogger("welcomebot.reconcile_blocks") + + +def _fetch_all(path: str) -> list[dict] | None: + """Page through an admin list endpoint. Returns None on failure, which the + caller treats as "don't touch anything" — an outage or a missing scope must + never be read as "every block was removed".""" + results: list[dict] = [] + url = f"{MASTODON_BASE_URL}{path}?limit=200" + try: + while url: + resp = httpx.get(url, headers=_admin_headers(), timeout=30.0) + if resp.status_code in (401, 403): + log.error("%s returned %d — ABUSE_BOT_TOKEN is missing the matching " + "admin read scope; leaving all rows untouched", + path, resp.status_code) + return None + resp.raise_for_status() + page = resp.json() + if not page: + break + results.extend(page) + url = resp.links.get("next", {}).get("url") + except (httpx.HTTPError, ValueError) as exc: + log.error("failed to fetch %s: %s — leaving all rows untouched", path, exc) + return None + return results + + +def _expired(entry: dict) -> bool: + raw = entry.get("expires_at") + if not raw: + return False + try: + when = datetime.fromisoformat(raw.replace("Z", "+00:00")) + except ValueError: + return False + if when.tzinfo is None: + when = when.replace(tzinfo=timezone.utc) + return when <= datetime.now(timezone.utc) + + +def live_ip_cidrs() -> set[str] | None: + """Normalised CIDR strings of every non-expired Admin::IpBlock.""" + entries = _fetch_all("/api/v1/admin/ip_blocks") + if entries is None: + return None + cidrs = set() + for entry in entries: + if _expired(entry): + continue + raw = entry.get("ip") or "" + try: + cidrs.add(str(ipaddress.ip_network(raw, strict=False))) + except ValueError: + log.warning("skipping unparseable live ip_block %r", raw) + return cidrs + + +def live_email_domains() -> set[str] | None: + entries = _fetch_all("/api/v1/admin/email_domain_blocks") + if entries is None: + return None + return {(e.get("domain") or "").strip().lower() for e in entries if e.get("domain")} + + +def _normalise(cidr: str) -> str: + try: + return str(ipaddress.ip_network(cidr, strict=False)) + except ValueError: + return cidr + + +def reconcile(dry_run: bool = False) -> int: + """Sync block_active against Mastodon. Returns the number of rows whose + state changed (or would change, under dry_run).""" + live_ips = live_ip_cidrs() + live_domains = live_email_domains() + if live_ips is None and live_domains is None: + log.error("both fetches failed — nothing reconciled") + return 0 + + changed = 0 + now = datetime.now(timezone.utc).replace(microsecond=0).isoformat() + + if live_ips is not None: + with _db() as conn: + rows = conn.execute( + "SELECT account_id, ip, block_cidr, block_active " + "FROM signup_ip WHERE ipblock_registered = 1" + ).fetchall() + + updates = [] + for account_id, ip, block_cidr, active in rows: + cidr = block_cidr or _single_cidr(ip or "") + present = bool(cidr) and _normalise(cidr) in live_ips + if bool(active) != present: + changed += 1 + log.info("ip_block %s: %s -> %s", cidr, + "active" if active else "inactive", + "active" if present else "inactive") + updates.append((int(present), now, account_id)) + + if not dry_run and updates: + with _db() as conn: + conn.executemany( + "UPDATE signup_ip SET block_active = ?, block_checked_at = ? " + "WHERE account_id = ?", updates) + log.info("ip_blocks: %d live, %d bot rows checked", len(live_ips), len(rows)) + + if live_domains is not None: + with _db() as conn: + rows = conn.execute( + "SELECT account_id, domain, block_active " + "FROM signup_email WHERE domain_blocked = 1" + ).fetchall() + + updates = [] + for account_id, domain, active in rows: + present = bool(domain) and domain.strip().lower() in live_domains + if bool(active) != present: + changed += 1 + log.info("email_domain_block %s: %s -> %s", domain, + "active" if active else "inactive", + "active" if present else "inactive") + updates.append((int(present), now, account_id)) + + if not dry_run and updates: + with _db() as conn: + conn.executemany( + "UPDATE signup_email SET block_active = ?, block_checked_at = ? " + "WHERE account_id = ?", updates) + log.info("email_domain_blocks: %d live, %d bot rows checked", + len(live_domains), len(rows)) + + log.info("%s%d row(s) changed state", "[DRY-RUN] " if dry_run else "", changed) + return changed + + +def _single_cidr(ip: str) -> str: + try: + addr = ipaddress.ip_address(ip) + except ValueError: + return "" + return f"{addr}/{32 if addr.version == 4 else 128}" + + +def main() -> int: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + ) + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--dry-run", action="store_true", + help="report what would change without writing") + args = ap.parse_args() + reconcile(dry_run=args.dry_run) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test_local.py b/test_local.py index ecb8eaa..433154a 100644 --- a/test_local.py +++ b/test_local.py @@ -893,6 +893,92 @@ def blocks_api_tests(): main.BLOCKS_API_MAX_ITEMS = 5000 +def reconcile_blocks_tests(): + """Drive app.reconcile_blocks.reconcile() against seeded rows with the + Mastodon admin list fetches stubbed out (no network). The important case + is the failure path: a 403 or an outage must never be read as "every + block was removed".""" + import app.reconcile_blocks as rec + + # Seed: two blocked ranges and two blocked domains, all currently active. + main.record_signup_ip("950", "recone", "198.18.5.1", "datacenter", "X", True, "198.18.5.0/24") + main.mark_ipblock_registered("950") + main.record_signup_ip("951", "rectwo", "198.18.6.1", "datacenter", "X", True, "198.18.6.0/24") + main.mark_ipblock_registered("951") + main.record_signup_email("960", "recthree", "rec-blocked.test", True, 90, True) + main.mark_email_domain_blocked("960") + main.record_signup_email("961", "recfour", "rec-removed.test", True, 90, True) + main.mark_email_domain_blocked("961") + + def active_ips(): + return {e["cidr"] for e in main.blocked_ip_ranges()} + + def active_domains(): + return {e["domain"] for e in main.blocked_email_domains()} + + assert "198.18.5.0/24" in active_ips() and "198.18.6.0/24" in active_ips() + assert "rec-removed.test" in active_domains() + + # A. Mastodon no longer has 198.18.6.0/24 or rec-removed.test -> both drop + # out of what the API publishes; the survivors stay. + rec.live_ip_cidrs = lambda: {"198.18.5.0/24"} + rec.live_email_domains = lambda: {"rec-blocked.test"} + changed = rec.reconcile() + assert changed >= 2, changed + ips, domains = active_ips(), active_domains() + assert "198.18.5.0/24" in ips, "a block still present must stay published" + assert "198.18.6.0/24" not in ips, "a block removed in Mastodon must stop being published" + assert "rec-blocked.test" in domains + assert "rec-removed.test" not in domains + + # B. It reappears (re-registered by hand, or the bot blocks it again) -> + # the row comes back rather than being stuck off. + rec.live_ip_cidrs = lambda: {"198.18.5.0/24", "198.18.6.0/24"} + rec.live_email_domains = lambda: {"rec-blocked.test", "rec-removed.test"} + rec.reconcile() + assert "198.18.6.0/24" in active_ips(), "a reappearing block must be republished" + assert "rec-removed.test" in active_domains() + + # C. THE IMPORTANT ONE: the fetch fails (403 on a missing scope, or + # Mastodon down). Nothing may be marked inactive — an outage must not + # empty the published list. + rec.live_ip_cidrs = lambda: None + rec.live_email_domains = lambda: None + assert rec.reconcile() == 0, "a failed fetch must change nothing" + assert "198.18.5.0/24" in active_ips() and "198.18.6.0/24" in active_ips() + assert len(active_domains()) >= 2 + + # D. Half-failure: IP fetch works, domain fetch doesn't. Only the side + # that answered is reconciled. + rec.live_ip_cidrs = lambda: {"198.18.5.0/24"} + rec.live_email_domains = lambda: None + rec.reconcile() + assert "198.18.6.0/24" not in active_ips(), "the side that answered is reconciled" + assert "rec-removed.test" in active_domains(), "the side that failed is left alone" + + # E. --dry-run reports without writing. + rec.live_ip_cidrs = lambda: set() + rec.live_email_domains = lambda: set() + assert rec.reconcile(dry_run=True) > 0 + assert "198.18.5.0/24" in active_ips(), "dry-run must not change stored state" + + # F. An expired block is treated as gone even though Mastodon still lists it. + future = (datetime.now(timezone.utc) + timedelta(days=1)).isoformat() + past = (datetime.now(timezone.utc) - timedelta(days=1)).isoformat() + assert rec._expired({"expires_at": past}) is True + assert rec._expired({"expires_at": future}) is False + assert rec._expired({"expires_at": None}) is False + assert rec._expired({}) is False + + # G. A re-registration reactivates a row that had been reconciled away. + rec.live_ip_cidrs = lambda: set() + rec.live_email_domains = lambda: set() + rec.reconcile() + assert "198.18.5.0/24" not in active_ips() + main.mark_ipblock_registered("950") + assert "198.18.5.0/24" in active_ips(), "re-registering must reactivate the row" + + if __name__ == "__main__": routing_tests() policy_tests() @@ -904,4 +990,5 @@ if __name__ == "__main__": suspicious_watch_tests() suspicious_sweep_tests() blocks_api_tests() + reconcile_blocks_tests() print("ALL TESTS PASSED")