Expire auto-registered IP blocks after 24 hours

Blocks were registered permanently. An IP block is a claim about who is
currently behind an address, and that goes stale: VPS addresses are recycled
between customers constantly, and now that the /24 ceiling means most blocks
cover a single address, a permanent block increasingly punishes whoever gets
that address next rather than the actor we saw.

24h is long enough to break a signup run -- the abuse arrives in bursts, not
as a steady trickle -- and short enough that the blocklist stays a live
picture rather than something that only ever grows.

Mastodon expires these itself and reconcile_blocks already reads an elapsed
expires_at as "gone", so expired blocks leave /api/blocks unaided. The one
piece that needed adding is an expiry on our own dedup record: reconcile runs
hourly, so between a block lapsing and the next pass, reputation_block_
installed would otherwise claim a block Mastodon had already dropped and skip
re-installing it.

expires_in is omitted rather than sent as 0 when the expiry is switched off.
Email domain blocks have no expiry in Mastodon and are untouched.

.env.example

@@ -276,6 +276,22 @@ BLOCKS_API_CACHE_SECONDS=300
 #
 # Applied at registration time, not retroactively: blocks already in force
 # keep whatever width they were registered with.
+# How long a registered Admin::IpBlock lasts, in seconds. 0 = never expires.
+#
+# An IP block is a statement about who is *currently* behind an address, and
+# that goes stale: VPS addresses get recycled between customers constantly,
+# and with the /24 ceiling below most blocks now cover a single address, so a
+# permanent block increasingly punishes whoever gets that address next rather
+# than the actor we saw. 24h (86400) breaks a signup run -- which is what the
+# abuse actually looks like, bursts rather than a steady trickle -- and keeps
+# the blocklist a live picture instead of something that only accumulates.
+#
+# Mastodon expires these itself (Scheduler::IpCleanupScheduler) and
+# app/reconcile_blocks.py treats an elapsed expires_at as "gone", so an
+# expired block drops out of /api/blocks without anyone doing anything.
+# Email domain blocks have no expiry in Mastodon and are unaffected.
+IP_SCRUTINY_IPBLOCK_EXPIRES_IN=86400
+
 IP_SCRUTINY_MAX_BLOCK_PREFIX_V4=24
 
 # The v6 ceiling. /48 is the standard end-site allocation — the rough

app/main.py

@@ -194,6 +194,22 @@ IP_SCRUTINY_MAX_BLOCK_PREFIX_V4 = int(os.environ.get("IP_SCRUTINY_MAX_BLOCK_PREF
 # equivalent of "one customer's block" that a /24 represents in v4.
 IP_SCRUTINY_MAX_BLOCK_PREFIX_V6 = int(os.environ.get("IP_SCRUTINY_MAX_BLOCK_PREFIX_V6", "48"))
 
+# How long a registered Admin::IpBlock lasts, in seconds. 0 = never expires.
+#
+# An IP block is a statement about who is *currently* behind an address, and
+# that goes stale: VPS addresses are recycled between customers constantly,
+# and with the /24 ceiling above most blocks now cover a single address, so a
+# permanent block increasingly punishes whoever gets that address next rather
+# than the actor we saw. 24h is long enough to break a signup run (which is
+# what the abuse actually looks like — bursts, not a steady trickle) and short
+# enough that the blocklist stays a live picture instead of accumulating
+# forever.
+#
+# Mastodon expires these itself (Scheduler::IpCleanupScheduler), and
+# app/reconcile_blocks.py already treats an elapsed expires_at as "gone", so
+# an expired block drops out of /api/blocks on its own.
+IP_SCRUTINY_IPBLOCK_EXPIRES_IN = int(os.environ.get("IP_SCRUTINY_IPBLOCK_EXPIRES_IN", "86400"))
+
 IP_SCRUTINY_STRONG_SIGNALS = frozenset({"datacenter", "proxy", "tor", "abuser"})
 IP_SCRUTINY_STRONG_SUSPEND_ENABLED = os.environ.get(
     "IP_SCRUTINY_STRONG_SUSPEND_ENABLED", "true"
@@ -491,9 +507,15 @@ def _init_db() -> None:
             "  last_seen_at     TEXT DEFAULT CURRENT_TIMESTAMP,"
             "  block_active     INTEGER DEFAULT 1,"
             "  block_checked_at TEXT,"
+            "  expires_at       TEXT,"
             "  PRIMARY KEY (kind, target)"
             ")"
         )
+        # expires_at arrived with IP_SCRUTINY_IPBLOCK_EXPIRES_IN. Rows written
+        # before it keep NULL, which reads as "no expiry" — correct, since
+        # they were registered without one.
+        if not _column_exists(conn, "reputation_block", "expires_at"):
+            conn.execute("ALTER TABLE reputation_block ADD COLUMN expires_at TEXT")
         conn.execute(
             "CREATE TABLE IF NOT EXISTS suspicious_watch ("
             "  account_id TEXT PRIMARY KEY,"
@@ -768,34 +790,53 @@ def record_reputation_query(kind: str, value: str, classification: str, org: str
 
 
 def reputation_block_installed(kind: str, target: str) -> bool:
-    """True if the query API already installed this block and the reconcile
-    pass hasn't since found it gone. Lets a repeat query skip the Mastodon
-    round trip — this runs on the signup request path, so a saved POST is
-    saved latency."""
+    """True if the query API already installed this block, the reconcile pass
+    hasn't since found it gone, and it hasn't expired. Lets a repeat query
+    skip the Mastodon round trip — this runs on the signup request path, so a
+    saved POST is saved latency.
+
+    The expiry check matters independently of the reconcile pass: IP blocks
+    now last IP_SCRUTINY_IPBLOCK_EXPIRES_IN, and reconcile only runs hourly,
+    so between a block lapsing and the next reconcile this would otherwise
+    claim a block that Mastodon has already dropped — and skip re-installing
+    it. A NULL expires_at means no expiry (email domain blocks never expire,
+    and neither did IP blocks before this was introduced).
+    """
     with _db() as conn:
         row = conn.execute(
             "SELECT 1 FROM reputation_block "
-            "WHERE kind = ? AND target = ? AND block_active = 1",
-            (kind, target),
+            "WHERE kind = ? AND target = ? AND block_active = 1 "
+            "  AND (expires_at IS NULL OR expires_at > ?)",
+            (kind, target, _now_iso()),
         ).fetchone()
         return row is not None
 
 
-def record_reputation_block(kind: str, target: str, classification: str, org: str) -> None:
+def record_reputation_block(kind: str, target: str, classification: str, org: str,
+                            expires_in: int = 0) -> None:
     """Record that this block is installed, or that an installed one was hit
     again. Reactivates a row the reconcile pass had marked gone, same as
-    mark_ipblock_registered does for the signup-derived rows."""
+    mark_ipblock_registered does for the signup-derived rows.
+
+    expires_in mirrors what was sent to Mastodon, so our dedup record lapses
+    at the same moment the block itself does; 0 means no expiry.
+    """
+    expires_at = None
+    if expires_in > 0:
+        expires_at = (datetime.now(timezone.utc).replace(microsecond=0)
+                      + timedelta(seconds=expires_in)).isoformat()
     with _db() as conn:
         conn.execute(
             "INSERT INTO reputation_block "
-            "(kind, target, classification, org) VALUES (?, ?, ?, ?) "
+            "(kind, target, classification, org, expires_at) VALUES (?, ?, ?, ?, ?) "
             "ON CONFLICT(kind, target) DO UPDATE SET "
             "  queries = queries + 1,"
             "  last_seen_at = CURRENT_TIMESTAMP,"
             "  block_active = 1,"
+            "  expires_at = excluded.expires_at,"
             "  classification = COALESCE(excluded.classification, classification),"
             "  org = COALESCE(excluded.org, org)",
-            (kind, target, classification or None, org or None),
+            (kind, target, classification or None, org or None, expires_at),
         )
 
 
@@ -805,8 +846,10 @@ def reputation_blocks(kind: str) -> list[tuple[str, str, str]]:
     with _db() as conn:
         return conn.execute(
             "SELECT target, first_blocked_at, last_seen_at FROM reputation_block "
-            "WHERE kind = ? AND block_active = 1 ORDER BY last_seen_at DESC",
-            (kind,),
+            "WHERE kind = ? AND block_active = 1 "
+            "  AND (expires_at IS NULL OR expires_at > ?) "
+            "ORDER BY last_seen_at DESC",
+            (kind, _now_iso()),
         ).fetchall()
 
 
@@ -1406,6 +1449,16 @@ def classify_signup_ip(ip: str) -> tuple[str, str, bool, str]:
     return classification, intel["org"], bool(reasons), block_cidr or f"{ip}/32"
 
 
+def _ip_block_params(cidr: str, comment: str) -> dict:
+    """POST body for Admin::IpBlock. expires_in is omitted entirely when the
+    expiry is switched off, rather than sent as 0 — Mastodon validates it
+    against a fixed list of durations and would reject the 0."""
+    params = {"ip": cidr, "severity": IP_SCRUTINY_IPBLOCK_SEVERITY, "comment": comment}
+    if IP_SCRUTINY_IPBLOCK_EXPIRES_IN > 0:
+        params["expires_in"] = IP_SCRUTINY_IPBLOCK_EXPIRES_IN
+    return params
+
+
 def register_ip_block(ip: str, acct: str, org: str, cidr: str = "",
                       comment: str = "") -> bool:
     """Register a flagged signup's network in Mastodon's native Admin::IpBlock.
@@ -1437,7 +1490,7 @@ def register_ip_block(ip: str, acct: str, org: str, cidr: str = "",
         resp = httpx.post(
             f"{MASTODON_BASE_URL}/api/v1/admin/ip_blocks",
             headers=_admin_headers(),
-            data={"ip": cidr, "severity": IP_SCRUTINY_IPBLOCK_SEVERITY, "comment": comment},
+            data=_ip_block_params(cidr, comment),
             timeout=15.0,
         )
         if resp.status_code == 422:
@@ -1522,12 +1575,14 @@ def _reputation_ip(ip: str, source: str) -> dict | None:
         elif reputation_block_installed("ip", cidr):
             # Already ours and still in force; Mastodon's own lookup will find
             # it. Skip the round trip, just count the hit.
-            record_reputation_block("ip", cidr, classification, org)
+            record_reputation_block("ip", cidr, classification, org,
+                                    IP_SCRUTINY_IPBLOCK_EXPIRES_IN)
             blocked = True
         elif register_ip_block(ip, "", org, cidr, comment=(
                 f"welcomebot: pre-signup block, {classification} "
                 f"({org or 'unknown org'})")):
-            record_reputation_block("ip", cidr, classification, org)
+            record_reputation_block("ip", cidr, classification, org,
+                                    IP_SCRUTINY_IPBLOCK_EXPIRES_IN)
             blocked = True
             log.warning("reputation query: pre-blocked ip=%s cidr=%s classification=%s org=%s source=%s",
                         ip, cidr, classification, org, source or "-")

test_local.py

@@ -1224,6 +1224,44 @@ def reputation_api_tests():
     active = {e["cidr"] for e in client.get("/api/blocks/ip-ranges").json()["items"]}
     assert "198.51.100.0/24" in active, "re-installing republishes it"
 
+    # M. IP blocks are registered with an expiry, and our own dedup record
+    #    lapses with them. Without that, a query between a block expiring and
+    #    the next hourly reconcile would claim a block Mastodon has already
+    #    dropped -- and skip re-installing it.
+    main.REPUTATION_API_ENABLED = True
+    params = main._ip_block_params("198.51.100.0/24", "test")
+    assert params["expires_in"] == main.IP_SCRUTINY_IPBLOCK_EXPIRES_IN, params
+    assert main.IP_SCRUTINY_IPBLOCK_EXPIRES_IN == 86400, "default is 24 hours"
+
+    saved = main.IP_SCRUTINY_IPBLOCK_EXPIRES_IN
+    main.IP_SCRUTINY_IPBLOCK_EXPIRES_IN = 0
+    assert "expires_in" not in main._ip_block_params("198.51.100.0/24", "test"), \
+        "a 0 expiry omits the field rather than sending 0"
+    main.IP_SCRUTINY_IPBLOCK_EXPIRES_IN = saved
+
+    #    An expired record reads as not-installed, so the next query
+    #    re-installs rather than assuming it's still there.
+    main.record_reputation_block("ip", "203.0.113.0/24", "datacenter", "Expiring Inc", 86400)
+    assert main.reputation_block_installed("ip", "203.0.113.0/24")
+    with main._db() as conn:
+        conn.execute("UPDATE reputation_block SET expires_at = ? WHERE target = ?",
+                     ("2020-01-01T00:00:00+00:00", "203.0.113.0/24"))
+    assert not main.reputation_block_installed("ip", "203.0.113.0/24"), \
+        "an expired block must not count as installed"
+    active = {e["cidr"] for e in client.get("/api/blocks/ip-ranges").json()["items"]}
+    assert "203.0.113.0/24" not in active, "an expired block must not be published"
+
+    #    Email domain blocks don't expire in Mastodon, so they carry none here
+    #    either: recorded with no expiry, they stay installed indefinitely.
+    main.record_reputation_block("email", "rep-forever.test", "", "")
+    assert main.reputation_block_installed("email", "rep-forever.test"), \
+        "email domain blocks have no expiry and stay installed"
+    with main._db() as conn:
+        row = conn.execute(
+            "SELECT expires_at FROM reputation_block WHERE target = ?",
+            ("rep-forever.test",)).fetchone()
+    assert row[0] is None, row
+
     main.REPUTATION_API_ENABLED = False