diff --git a/.env.example b/.env.example index d4e2b1a..f8f59e5 100644 --- a/.env.example +++ b/.env.example @@ -276,6 +276,22 @@ BLOCKS_API_CACHE_SECONDS=300 # # Applied at registration time, not retroactively: blocks already in force # keep whatever width they were registered with. +# How long a registered Admin::IpBlock lasts, in seconds. 0 = never expires. +# +# An IP block is a statement about who is *currently* behind an address, and +# that goes stale: VPS addresses get recycled between customers constantly, +# and with the /24 ceiling below most blocks now cover a single address, so a +# permanent block increasingly punishes whoever gets that address next rather +# than the actor we saw. 24h (86400) breaks a signup run -- which is what the +# abuse actually looks like, bursts rather than a steady trickle -- and keeps +# the blocklist a live picture instead of something that only accumulates. +# +# Mastodon expires these itself (Scheduler::IpCleanupScheduler) and +# app/reconcile_blocks.py treats an elapsed expires_at as "gone", so an +# expired block drops out of /api/blocks without anyone doing anything. +# Email domain blocks have no expiry in Mastodon and are unaffected. +IP_SCRUTINY_IPBLOCK_EXPIRES_IN=86400 + IP_SCRUTINY_MAX_BLOCK_PREFIX_V4=24 # The v6 ceiling. /48 is the standard end-site allocation — the rough diff --git a/app/main.py b/app/main.py index 7b60e5e..62d62a7 100644 --- a/app/main.py +++ b/app/main.py @@ -194,6 +194,22 @@ IP_SCRUTINY_MAX_BLOCK_PREFIX_V4 = int(os.environ.get("IP_SCRUTINY_MAX_BLOCK_PREF # equivalent of "one customer's block" that a /24 represents in v4. IP_SCRUTINY_MAX_BLOCK_PREFIX_V6 = int(os.environ.get("IP_SCRUTINY_MAX_BLOCK_PREFIX_V6", "48")) +# How long a registered Admin::IpBlock lasts, in seconds. 0 = never expires. +# +# An IP block is a statement about who is *currently* behind an address, and +# that goes stale: VPS addresses are recycled between customers constantly, +# and with the /24 ceiling above most blocks now cover a single address, so a +# permanent block increasingly punishes whoever gets that address next rather +# than the actor we saw. 24h is long enough to break a signup run (which is +# what the abuse actually looks like — bursts, not a steady trickle) and short +# enough that the blocklist stays a live picture instead of accumulating +# forever. +# +# Mastodon expires these itself (Scheduler::IpCleanupScheduler), and +# app/reconcile_blocks.py already treats an elapsed expires_at as "gone", so +# an expired block drops out of /api/blocks on its own. +IP_SCRUTINY_IPBLOCK_EXPIRES_IN = int(os.environ.get("IP_SCRUTINY_IPBLOCK_EXPIRES_IN", "86400")) + IP_SCRUTINY_STRONG_SIGNALS = frozenset({"datacenter", "proxy", "tor", "abuser"}) IP_SCRUTINY_STRONG_SUSPEND_ENABLED = os.environ.get( "IP_SCRUTINY_STRONG_SUSPEND_ENABLED", "true" @@ -491,9 +507,15 @@ def _init_db() -> None: " last_seen_at TEXT DEFAULT CURRENT_TIMESTAMP," " block_active INTEGER DEFAULT 1," " block_checked_at TEXT," + " expires_at TEXT," " PRIMARY KEY (kind, target)" ")" ) + # expires_at arrived with IP_SCRUTINY_IPBLOCK_EXPIRES_IN. Rows written + # before it keep NULL, which reads as "no expiry" — correct, since + # they were registered without one. + if not _column_exists(conn, "reputation_block", "expires_at"): + conn.execute("ALTER TABLE reputation_block ADD COLUMN expires_at TEXT") conn.execute( "CREATE TABLE IF NOT EXISTS suspicious_watch (" " account_id TEXT PRIMARY KEY," @@ -768,34 +790,53 @@ def record_reputation_query(kind: str, value: str, classification: str, org: str def reputation_block_installed(kind: str, target: str) -> bool: - """True if the query API already installed this block and the reconcile - pass hasn't since found it gone. Lets a repeat query skip the Mastodon - round trip — this runs on the signup request path, so a saved POST is - saved latency.""" + """True if the query API already installed this block, the reconcile pass + hasn't since found it gone, and it hasn't expired. Lets a repeat query + skip the Mastodon round trip — this runs on the signup request path, so a + saved POST is saved latency. + + The expiry check matters independently of the reconcile pass: IP blocks + now last IP_SCRUTINY_IPBLOCK_EXPIRES_IN, and reconcile only runs hourly, + so between a block lapsing and the next reconcile this would otherwise + claim a block that Mastodon has already dropped — and skip re-installing + it. A NULL expires_at means no expiry (email domain blocks never expire, + and neither did IP blocks before this was introduced). + """ with _db() as conn: row = conn.execute( "SELECT 1 FROM reputation_block " - "WHERE kind = ? AND target = ? AND block_active = 1", - (kind, target), + "WHERE kind = ? AND target = ? AND block_active = 1 " + " AND (expires_at IS NULL OR expires_at > ?)", + (kind, target, _now_iso()), ).fetchone() return row is not None -def record_reputation_block(kind: str, target: str, classification: str, org: str) -> None: +def record_reputation_block(kind: str, target: str, classification: str, org: str, + expires_in: int = 0) -> None: """Record that this block is installed, or that an installed one was hit again. Reactivates a row the reconcile pass had marked gone, same as - mark_ipblock_registered does for the signup-derived rows.""" + mark_ipblock_registered does for the signup-derived rows. + + expires_in mirrors what was sent to Mastodon, so our dedup record lapses + at the same moment the block itself does; 0 means no expiry. + """ + expires_at = None + if expires_in > 0: + expires_at = (datetime.now(timezone.utc).replace(microsecond=0) + + timedelta(seconds=expires_in)).isoformat() with _db() as conn: conn.execute( "INSERT INTO reputation_block " - "(kind, target, classification, org) VALUES (?, ?, ?, ?) " + "(kind, target, classification, org, expires_at) VALUES (?, ?, ?, ?, ?) " "ON CONFLICT(kind, target) DO UPDATE SET " " queries = queries + 1," " last_seen_at = CURRENT_TIMESTAMP," " block_active = 1," + " expires_at = excluded.expires_at," " classification = COALESCE(excluded.classification, classification)," " org = COALESCE(excluded.org, org)", - (kind, target, classification or None, org or None), + (kind, target, classification or None, org or None, expires_at), ) @@ -805,8 +846,10 @@ def reputation_blocks(kind: str) -> list[tuple[str, str, str]]: with _db() as conn: return conn.execute( "SELECT target, first_blocked_at, last_seen_at FROM reputation_block " - "WHERE kind = ? AND block_active = 1 ORDER BY last_seen_at DESC", - (kind,), + "WHERE kind = ? AND block_active = 1 " + " AND (expires_at IS NULL OR expires_at > ?) " + "ORDER BY last_seen_at DESC", + (kind, _now_iso()), ).fetchall() @@ -1406,6 +1449,16 @@ def classify_signup_ip(ip: str) -> tuple[str, str, bool, str]: return classification, intel["org"], bool(reasons), block_cidr or f"{ip}/32" +def _ip_block_params(cidr: str, comment: str) -> dict: + """POST body for Admin::IpBlock. expires_in is omitted entirely when the + expiry is switched off, rather than sent as 0 — Mastodon validates it + against a fixed list of durations and would reject the 0.""" + params = {"ip": cidr, "severity": IP_SCRUTINY_IPBLOCK_SEVERITY, "comment": comment} + if IP_SCRUTINY_IPBLOCK_EXPIRES_IN > 0: + params["expires_in"] = IP_SCRUTINY_IPBLOCK_EXPIRES_IN + return params + + def register_ip_block(ip: str, acct: str, org: str, cidr: str = "", comment: str = "") -> bool: """Register a flagged signup's network in Mastodon's native Admin::IpBlock. @@ -1437,7 +1490,7 @@ def register_ip_block(ip: str, acct: str, org: str, cidr: str = "", resp = httpx.post( f"{MASTODON_BASE_URL}/api/v1/admin/ip_blocks", headers=_admin_headers(), - data={"ip": cidr, "severity": IP_SCRUTINY_IPBLOCK_SEVERITY, "comment": comment}, + data=_ip_block_params(cidr, comment), timeout=15.0, ) if resp.status_code == 422: @@ -1522,12 +1575,14 @@ def _reputation_ip(ip: str, source: str) -> dict | None: elif reputation_block_installed("ip", cidr): # Already ours and still in force; Mastodon's own lookup will find # it. Skip the round trip, just count the hit. - record_reputation_block("ip", cidr, classification, org) + record_reputation_block("ip", cidr, classification, org, + IP_SCRUTINY_IPBLOCK_EXPIRES_IN) blocked = True elif register_ip_block(ip, "", org, cidr, comment=( f"welcomebot: pre-signup block, {classification} " f"({org or 'unknown org'})")): - record_reputation_block("ip", cidr, classification, org) + record_reputation_block("ip", cidr, classification, org, + IP_SCRUTINY_IPBLOCK_EXPIRES_IN) blocked = True log.warning("reputation query: pre-blocked ip=%s cidr=%s classification=%s org=%s source=%s", ip, cidr, classification, org, source or "-") diff --git a/test_local.py b/test_local.py index 550e6a7..31f61bd 100644 --- a/test_local.py +++ b/test_local.py @@ -1224,6 +1224,44 @@ def reputation_api_tests(): active = {e["cidr"] for e in client.get("/api/blocks/ip-ranges").json()["items"]} assert "198.51.100.0/24" in active, "re-installing republishes it" + # M. IP blocks are registered with an expiry, and our own dedup record + # lapses with them. Without that, a query between a block expiring and + # the next hourly reconcile would claim a block Mastodon has already + # dropped -- and skip re-installing it. + main.REPUTATION_API_ENABLED = True + params = main._ip_block_params("198.51.100.0/24", "test") + assert params["expires_in"] == main.IP_SCRUTINY_IPBLOCK_EXPIRES_IN, params + assert main.IP_SCRUTINY_IPBLOCK_EXPIRES_IN == 86400, "default is 24 hours" + + saved = main.IP_SCRUTINY_IPBLOCK_EXPIRES_IN + main.IP_SCRUTINY_IPBLOCK_EXPIRES_IN = 0 + assert "expires_in" not in main._ip_block_params("198.51.100.0/24", "test"), \ + "a 0 expiry omits the field rather than sending 0" + main.IP_SCRUTINY_IPBLOCK_EXPIRES_IN = saved + + # An expired record reads as not-installed, so the next query + # re-installs rather than assuming it's still there. + main.record_reputation_block("ip", "203.0.113.0/24", "datacenter", "Expiring Inc", 86400) + assert main.reputation_block_installed("ip", "203.0.113.0/24") + with main._db() as conn: + conn.execute("UPDATE reputation_block SET expires_at = ? WHERE target = ?", + ("2020-01-01T00:00:00+00:00", "203.0.113.0/24")) + assert not main.reputation_block_installed("ip", "203.0.113.0/24"), \ + "an expired block must not count as installed" + active = {e["cidr"] for e in client.get("/api/blocks/ip-ranges").json()["items"]} + assert "203.0.113.0/24" not in active, "an expired block must not be published" + + # Email domain blocks don't expire in Mastodon, so they carry none here + # either: recorded with no expiry, they stay installed indefinitely. + main.record_reputation_block("email", "rep-forever.test", "", "") + assert main.reputation_block_installed("email", "rep-forever.test"), \ + "email domain blocks have no expiry and stay installed" + with main._db() as conn: + row = conn.execute( + "SELECT expires_at FROM reputation_block WHERE target = ?", + ("rep-forever.test",)).fetchone() + assert row[0] is None, row + main.REPUTATION_API_ENABLED = False