Only publish blocks that are still in force

ipblock_registered and domain_blocked only ever meant "we posted this
block once" — they said nothing about whether it still exists. When the
API went live it was publishing 24 /32 ranges that had been removed by
hand on 2026-07-21, so ~17% of the IP list didn't block anything. That's
a bad thing to hand another admin an importer for.

app/reconcile_blocks.py fetches Mastodon's live Admin::IpBlock and
Admin::EmailDomainBlock lists and maintains a new block_active column on
both tables: off for rows whose block is gone or whose expires_at has
elapsed, back on for any that reappear. The API serves only active rows.
Re-registering a block reactivates its row, so a range the bot blocks
again after a manual cleanup comes back on its own.

A failed fetch changes nothing — not a 403 from a missing scope, not
Mastodon being down, not a half-failure where one list answers and the
other doesn't. An outage must never be read as "every block was
removed"; the list degrades to over-reporting rather than emptying
itself, and there's a test for each of those paths.

Needs admin:read:ip_blocks + admin:read:email_domain_blocks, added to
the live token in place (scopes updated on the existing Doorkeeper
token, matched by digest) rather than by minting and rotating, so the
token value is unchanged and .env needed no edit.

app/main.py

@@ -301,6 +301,8 @@ def _init_db() -> None:
             "  flagged    INTEGER,"
             "  ipblock_registered INTEGER DEFAULT 0,"
             "  block_cidr TEXT,"
+            "  block_active INTEGER DEFAULT 1,"
+            "  block_checked_at TEXT,"
             "  classified_at TEXT DEFAULT CURRENT_TIMESTAMP"
             ")"
         )
@@ -312,6 +314,18 @@ def _init_db() -> None:
         # ASN route CIDR only started 2026-07-15.
         if not _column_exists(conn, "signup_ip", "block_cidr"):
             conn.execute("ALTER TABLE signup_ip ADD COLUMN block_cidr TEXT")
+        # block_active tracks whether the block we registered is STILL present
+        # on the Mastodon side. ipblock_registered only ever meant "we posted
+        # it once"; a moderator clearing blocks by hand (as happened to 24
+        # rows on 2026-07-21) or an expires_at elapsing would leave us
+        # publishing entries that no longer block anything. Maintained by
+        # app/reconcile_blocks.py; rows default to active until it says
+        # otherwise, so a reconcile that never runs degrades to the old
+        # behaviour rather than emptying the list.
+        if not _column_exists(conn, "signup_ip", "block_active"):
+            conn.execute("ALTER TABLE signup_ip ADD COLUMN block_active INTEGER DEFAULT 1")
+        if not _column_exists(conn, "signup_ip", "block_checked_at"):
+            conn.execute("ALTER TABLE signup_ip ADD COLUMN block_checked_at TEXT")
         conn.execute(
             "CREATE TABLE IF NOT EXISTS ipapi_range_cache ("
             "  family        INTEGER NOT NULL,"  # 4 or 6
@@ -337,9 +351,15 @@ def _init_db() -> None:
             "  risk       INTEGER,"
             "  flagged    INTEGER,"
             "  domain_blocked INTEGER DEFAULT 0,"
+            "  block_active INTEGER DEFAULT 1,"
+            "  block_checked_at TEXT,"
             "  classified_at TEXT DEFAULT CURRENT_TIMESTAMP"
             ")"
         )
+        if not _column_exists(conn, "signup_email", "block_active"):
+            conn.execute("ALTER TABLE signup_email ADD COLUMN block_active INTEGER DEFAULT 1")
+        if not _column_exists(conn, "signup_email", "block_checked_at"):
+            conn.execute("ALTER TABLE signup_email ADD COLUMN block_checked_at TEXT")
         conn.execute(
             "CREATE TABLE IF NOT EXISTS check_mail_cache ("
             "  domain     TEXT PRIMARY KEY,"
@@ -431,9 +451,12 @@ def get_signup_flag(account_id: str) -> bool:
 
 
 def mark_ipblock_registered(account_id: str) -> None:
+    # block_active is reset too: if this range was reconciled away earlier and
+    # we've just registered it again, it is live again.
     with _db() as conn:
         conn.execute(
-            "UPDATE signup_ip SET ipblock_registered = 1 WHERE account_id = ?",
+            "UPDATE signup_ip SET ipblock_registered = 1, block_active = 1 "
+            "WHERE account_id = ?",
             (account_id,),
         )
 
@@ -528,7 +551,8 @@ def get_signup_email(account_id: str) -> tuple[str, bool]:
 def mark_email_domain_blocked(account_id: str) -> None:
     with _db() as conn:
         conn.execute(
-            "UPDATE signup_email SET domain_blocked = 1 WHERE account_id = ?",
+            "UPDATE signup_email SET domain_blocked = 1, block_active = 1 "
+            "WHERE account_id = ?",
             (account_id,),
         )
 
@@ -579,8 +603,10 @@ def _fallback_cidr(ip: str) -> str:
 
 
 def blocked_ip_ranges(limit: int = 0) -> list[dict]:
-    """Every network this bot has registered an Admin::IpBlock for, newest
-    first, collapsed to one entry per distinct CIDR.
+    """Every network this bot has registered an Admin::IpBlock for that is
+    still present on the Mastodon side, newest first, collapsed to one entry
+    per distinct CIDR. Rows the reconcile pass found gone or expired are
+    excluded — see app/reconcile_blocks.py.
 
     Repeat signups out of the same range are common (that's the point of
     blocking the range rather than the address), so entries carry a `signups`
@@ -596,7 +622,7 @@ def blocked_ip_ranges(limit: int = 0) -> list[dict]:
     with _db() as conn:
         rows = conn.execute(
             "SELECT block_cidr, ip, classified_at "
-            "FROM signup_ip WHERE ipblock_registered = 1 "
+            "FROM signup_ip WHERE ipblock_registered = 1 AND block_active = 1 "
             "ORDER BY classified_at DESC"
         ).fetchall()
 
@@ -622,8 +648,9 @@ def blocked_ip_ranges(limit: int = 0) -> list[dict]:
 
 
 def blocked_email_domains(limit: int = 0) -> list[dict]:
-    """Every domain this bot has registered an Admin::EmailDomainBlock for,
-    newest first, one entry per distinct domain.
+    """Every domain this bot has registered an Admin::EmailDomainBlock for
+    that is still present on the Mastodon side, newest first, one entry per
+    distinct domain. See app/reconcile_blocks.py.
 
     Domains only — the bot never stores or forwards a full signup address (see
     classify_email_domain), so there is none to expose here.
@@ -635,7 +662,7 @@ def blocked_email_domains(limit: int = 0) -> list[dict]:
     with _db() as conn:
         rows = conn.execute(
             "SELECT domain, classified_at "
-            "FROM signup_email WHERE domain_blocked = 1 "
+            "FROM signup_email WHERE domain_blocked = 1 AND block_active = 1 "
             "ORDER BY classified_at DESC"
         ).fetchall()
 

app/reconcile_blocks.py

@@ -0,0 +1,205 @@
+"""Blocklist reconcile pass.
+
+The bot's `signup_ip.ipblock_registered` / `signup_email.domain_blocked` flags
+only ever meant "we successfully posted this block once". They say nothing
+about whether the block is still there. A moderator clearing entries by hand
+(24 bot-registered `/32` rows were removed on 2026-07-21), or an `expires_at`
+elapsing, leaves the bot's record claiming blocks that no longer block
+anything — and the public /api/blocks endpoint publishing them.
+
+This pass closes that gap: it fetches Mastodon's live `Admin::IpBlock` and
+`Admin::EmailDomainBlock` lists, and flips `block_active` off for any of our
+rows whose block is gone (or has expired), back on for any that reappeared.
+The API then serves only entries that are actually in force.
+
+Run on a schedule (cron on admin.yttrx.com, see CLAUDE.md):
+
+    docker exec yttrx-welcomebot python -m app.reconcile_blocks
+
+Requires ABUSE_BOT_TOKEN to carry `admin:read:ip_blocks` and
+`admin:read:email_domain_blocks` in addition to its existing scopes. Without
+them the fetch 403s and the pass exits without changing anything — rows keep
+whatever state they had, so a missing scope degrades to the old (over-
+reporting) behaviour rather than silently emptying the published list.
+
+`--dry-run` reports what would change without writing.
+"""
+
+from __future__ import annotations
+
+import argparse
+import ipaddress
+import logging
+import sys
+from datetime import datetime, timezone
+
+import httpx
+
+from app.main import (
+    MASTODON_BASE_URL,
+    _admin_headers,
+    _db,
+)
+
+log = logging.getLogger("welcomebot.reconcile_blocks")
+
+
+def _fetch_all(path: str) -> list[dict] | None:
+    """Page through an admin list endpoint. Returns None on failure, which the
+    caller treats as "don't touch anything" — an outage or a missing scope must
+    never be read as "every block was removed"."""
+    results: list[dict] = []
+    url = f"{MASTODON_BASE_URL}{path}?limit=200"
+    try:
+        while url:
+            resp = httpx.get(url, headers=_admin_headers(), timeout=30.0)
+            if resp.status_code in (401, 403):
+                log.error("%s returned %d — ABUSE_BOT_TOKEN is missing the matching "
+                          "admin read scope; leaving all rows untouched",
+                          path, resp.status_code)
+                return None
+            resp.raise_for_status()
+            page = resp.json()
+            if not page:
+                break
+            results.extend(page)
+            url = resp.links.get("next", {}).get("url")
+    except (httpx.HTTPError, ValueError) as exc:
+        log.error("failed to fetch %s: %s — leaving all rows untouched", path, exc)
+        return None
+    return results
+
+
+def _expired(entry: dict) -> bool:
+    raw = entry.get("expires_at")
+    if not raw:
+        return False
+    try:
+        when = datetime.fromisoformat(raw.replace("Z", "+00:00"))
+    except ValueError:
+        return False
+    if when.tzinfo is None:
+        when = when.replace(tzinfo=timezone.utc)
+    return when <= datetime.now(timezone.utc)
+
+
+def live_ip_cidrs() -> set[str] | None:
+    """Normalised CIDR strings of every non-expired Admin::IpBlock."""
+    entries = _fetch_all("/api/v1/admin/ip_blocks")
+    if entries is None:
+        return None
+    cidrs = set()
+    for entry in entries:
+        if _expired(entry):
+            continue
+        raw = entry.get("ip") or ""
+        try:
+            cidrs.add(str(ipaddress.ip_network(raw, strict=False)))
+        except ValueError:
+            log.warning("skipping unparseable live ip_block %r", raw)
+    return cidrs
+
+
+def live_email_domains() -> set[str] | None:
+    entries = _fetch_all("/api/v1/admin/email_domain_blocks")
+    if entries is None:
+        return None
+    return {(e.get("domain") or "").strip().lower() for e in entries if e.get("domain")}
+
+
+def _normalise(cidr: str) -> str:
+    try:
+        return str(ipaddress.ip_network(cidr, strict=False))
+    except ValueError:
+        return cidr
+
+
+def reconcile(dry_run: bool = False) -> int:
+    """Sync block_active against Mastodon. Returns the number of rows whose
+    state changed (or would change, under dry_run)."""
+    live_ips = live_ip_cidrs()
+    live_domains = live_email_domains()
+    if live_ips is None and live_domains is None:
+        log.error("both fetches failed — nothing reconciled")
+        return 0
+
+    changed = 0
+    now = datetime.now(timezone.utc).replace(microsecond=0).isoformat()
+
+    if live_ips is not None:
+        with _db() as conn:
+            rows = conn.execute(
+                "SELECT account_id, ip, block_cidr, block_active "
+                "FROM signup_ip WHERE ipblock_registered = 1"
+            ).fetchall()
+
+        updates = []
+        for account_id, ip, block_cidr, active in rows:
+            cidr = block_cidr or _single_cidr(ip or "")
+            present = bool(cidr) and _normalise(cidr) in live_ips
+            if bool(active) != present:
+                changed += 1
+                log.info("ip_block %s: %s -> %s", cidr,
+                         "active" if active else "inactive",
+                         "active" if present else "inactive")
+            updates.append((int(present), now, account_id))
+
+        if not dry_run and updates:
+            with _db() as conn:
+                conn.executemany(
+                    "UPDATE signup_ip SET block_active = ?, block_checked_at = ? "
+                    "WHERE account_id = ?", updates)
+        log.info("ip_blocks: %d live, %d bot rows checked", len(live_ips), len(rows))
+
+    if live_domains is not None:
+        with _db() as conn:
+            rows = conn.execute(
+                "SELECT account_id, domain, block_active "
+                "FROM signup_email WHERE domain_blocked = 1"
+            ).fetchall()
+
+        updates = []
+        for account_id, domain, active in rows:
+            present = bool(domain) and domain.strip().lower() in live_domains
+            if bool(active) != present:
+                changed += 1
+                log.info("email_domain_block %s: %s -> %s", domain,
+                         "active" if active else "inactive",
+                         "active" if present else "inactive")
+            updates.append((int(present), now, account_id))
+
+        if not dry_run and updates:
+            with _db() as conn:
+                conn.executemany(
+                    "UPDATE signup_email SET block_active = ?, block_checked_at = ? "
+                    "WHERE account_id = ?", updates)
+        log.info("email_domain_blocks: %d live, %d bot rows checked",
+                 len(live_domains), len(rows))
+
+    log.info("%s%d row(s) changed state", "[DRY-RUN] " if dry_run else "", changed)
+    return changed
+
+
+def _single_cidr(ip: str) -> str:
+    try:
+        addr = ipaddress.ip_address(ip)
+    except ValueError:
+        return ""
+    return f"{addr}/{32 if addr.version == 4 else 128}"
+
+
+def main() -> int:
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s %(levelname)s %(name)s %(message)s",
+    )
+    ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
+    ap.add_argument("--dry-run", action="store_true",
+                    help="report what would change without writing")
+    args = ap.parse_args()
+    reconcile(dry_run=args.dry_run)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())