diff --git a/.env.example b/.env.example index 56b00a0..be79efb 100644 --- a/.env.example +++ b/.env.example @@ -105,8 +105,13 @@ IP_SCRUTINY_HOLD_WELCOME=true # signup IP was flagged. IP_SCRUTINY_ABUSE_THRESHOLD=1 -# Auto-register a flagged IP into Mastodon's native Admin::IpBlock. Requires -# the ABUSE_BOT_TOKEN to carry the admin:write:ip_blocks scope (see +# Auto-register a flagged signup's network into Mastodon's native +# Admin::IpBlock — ipapi.is's asn.route CIDR when it's a valid network that +# actually contains the signup IP, else just that one /32 or /128. Blocking +# the whole network matters here because a flagged signup is almost always +# datacenter/VPN/proxy space, where a repeat bad actor is far more likely to +# come back from a different address in the same block than the exact one. +# Requires the ABUSE_BOT_TOKEN to carry the admin:write:ip_blocks scope (see # CLAUDE.md's moderator-token-gotcha section) — without it this 403s and is # logged as an error, but nothing else in the bot is affected. IP_SCRUTINY_AUTO_IPBLOCK=true diff --git a/README.md b/README.md index 3f95c21..6da201a 100644 --- a/README.md +++ b/README.md @@ -72,10 +72,16 @@ Every `account.created` delivery already carries the signup IP for free on `account.created` and only sent when `account.approved` fires, i.e. once a human clears yttrx's existing approval-required registration gate. If the signup is rejected instead, no welcome is ever sent. - - **Auto-registered IP block** (`IP_SCRUTINY_AUTO_IPBLOCK`) — the IP is + - **Auto-registered IP block** (`IP_SCRUTINY_AUTO_IPBLOCK`) — the *network* + the signup IP belongs to (ipapi.is's `asn.route`, when it's a valid CIDR + that actually contains the IP; otherwise just that one `/32`/`/128`) is added to Mastodon's native `Admin::IpBlock` at `IP_SCRUTINY_IPBLOCK_SEVERITY` (default `sign_up_requires_approval`, - reversible from the admin UI). + reversible from the admin UI). Blocking the whole network rather than + the single address matters here specifically because a flagged signup is + almost always datacenter/VPN/proxy space — a repeat bad actor is far more + likely to come back from a different address in the same block than the + exact one, unlike a residential IP where that reasoning wouldn't hold. - **Lowered abuse-bot threshold** — if this account is later reported, the usual `ABUSE_SOURCES_*` distinct-reporter threshold is replaced by `IP_SCRUTINY_ABUSE_THRESHOLD` (whichever is lower), since a flagged @@ -211,7 +217,7 @@ Copy `.env.example` to `.env` and fill in: | `IP_SCRUTINY_DRY_RUN` | `true` — classify + DM only, no held welcome, no ip_block write | | `IP_SCRUTINY_HOLD_WELCOME` | `true` — hold the welcome for a flagged signup until `account.approved` | | `IP_SCRUTINY_ABUSE_THRESHOLD` | Distinct-reporter threshold used (if lower) for accounts with a flagged signup IP | -| `IP_SCRUTINY_AUTO_IPBLOCK` | Auto-register a flagged IP into Mastodon's `Admin::IpBlock` | +| `IP_SCRUTINY_AUTO_IPBLOCK` | Auto-register a flagged signup's network (ipapi.is route, or its own `/32`/`/128` if no route) into Mastodon's `Admin::IpBlock` | | `IP_SCRUTINY_IPBLOCK_SEVERITY` | `sign_up_requires_approval` (default), `sign_up_block`, or `no_access` | | `CHECK_MAIL_ENABLED` | Master switch for disposable/high-risk email signup scrutiny | | `CHECK_MAIL_API_KEY` | check-mail.org API key; blank disables the check | diff --git a/app/main.py b/app/main.py index 3b0c972..87f27df 100644 --- a/app/main.py +++ b/app/main.py @@ -136,7 +136,13 @@ IP_SCRUTINY_HOLD_WELCOME = os.environ.get("IP_SCRUTINY_HOLD_WELCOME", "true").lo # Distinct-reporter threshold used INSTEAD of the tier's usual threshold (via # min()) when the reported account's signup IP was flagged. IP_SCRUTINY_ABUSE_THRESHOLD = int(os.environ.get("IP_SCRUTINY_ABUSE_THRESHOLD", "1")) -# Auto-register flagged IPs into Mastodon's native Admin::IpBlock. +# Auto-register a flagged signup's network into Mastodon's native +# Admin::IpBlock. Blocks the ASN/route-level CIDR ipapi.is reports the IP +# belonging to (see classify_signup_ip's block_cidr), not just the single +# address — a flagged signup is almost always datacenter/VPN/proxy space, and +# a repeat bad actor is far more likely to come back from a different address +# in the same block than the exact same one. Falls back to a single-address +# (/32 or /128) block if no usable route was returned/cached. IP_SCRUTINY_AUTO_IPBLOCK = os.environ.get("IP_SCRUTINY_AUTO_IPBLOCK", "true").lower() in ("1", "true", "yes") # severity: sign_up_requires_approval | sign_up_block | no_access IP_SCRUTINY_IPBLOCK_SEVERITY = os.environ.get("IP_SCRUTINY_IPBLOCK_SEVERITY", "sign_up_requires_approval") @@ -403,7 +409,7 @@ def cached_ip_intel(ip: str) -> dict | None: family, packed, _, _ = bounds with _db() as conn: row = conn.execute( - "SELECT is_datacenter, is_vpn, is_proxy, is_tor, is_abuser, org " + "SELECT is_datacenter, is_vpn, is_proxy, is_tor, is_abuser, org, cidr " "FROM ipapi_range_cache " "WHERE family = ? AND range_start <= ? AND range_end >= ? LIMIT 1", (family, packed, packed), @@ -414,6 +420,7 @@ def cached_ip_intel(ip: str) -> dict | None: "is_datacenter": bool(row[0]), "is_vpn": bool(row[1]), "is_proxy": bool(row[2]), "is_tor": bool(row[3]), "is_abuser": bool(row[4]), "org": row[5] or "", + "cidr": row[6] or bounds[3], } @@ -627,14 +634,14 @@ def process_signup(account_id: str, acct: str, ip: str, email: str = "", email_flagged = False if IP_SCRUTINY_ENABLED and ip: - classification, org, ip_flagged = classify_signup_ip(ip) + classification, org, ip_flagged, block_cidr = classify_signup_ip(ip) record_signup_ip(account_id, acct, ip, classification, org, ip_flagged) if ip_flagged: log.warning("flagged signup acct=%s ip=%s classification=%s org=%s", acct, ip, classification, org) if IP_SCRUTINY_AUTO_IPBLOCK and not IP_SCRUTINY_DRY_RUN: - register_ip_block(ip, acct, org) + register_ip_block(ip, acct, org, block_cidr) mark_ipblock_registered(account_id) prefix = "[DRY-RUN] " if IP_SCRUTINY_DRY_RUN else "" reasons.append(f"{prefix}IP {ip} ({org or 'unknown org'}, {classification})") @@ -844,16 +851,21 @@ def classify_account(target_id: str) -> dict: } -def classify_signup_ip(ip: str) -> tuple[str, str, bool]: +def classify_signup_ip(ip: str) -> tuple[str, str, bool, str]: """Classify a signup IP via ipapi.is, cached indefinitely in sqlite (an IP's owning org/abuse posture doesn't change on the timescale that matters here). - Returns (classification, org, flagged). classification is a "+"-joined - list of every matched signal (datacenter/vpn/proxy/tor/abuser), or - "clean" if none matched. flagged is True if any signal matched. API - failure yields ("unknown", "", False) — scrutiny should never trigger on - our own lookup errors. + Returns (classification, org, flagged, block_cidr). classification is a + "+"-joined list of every matched signal (datacenter/vpn/proxy/tor/abuser), + or "clean" if none matched. flagged is True if any signal matched. + block_cidr is the network register_ip_block should block when flagged — + ipapi.is's asn.route when it's a valid network that actually contains ip, + else ip's own /32 or /128 — since a flagged signup is almost always + datacenter/VPN/proxy space, where a repeat bad actor is far more likely to + come back from a different address in the same block than the exact same + one. API failure yields ("unknown", "", False, "/32|128") — scrutiny + should never trigger on our own lookup errors. """ intel = cached_ip_intel(ip) if intel is None: @@ -866,7 +878,8 @@ def classify_signup_ip(ip: str) -> tuple[str, str, bool]: data = resp.json() except (httpx.HTTPError, ValueError) as exc: log.warning("ipapi.is lookup failed for ip=%s: %s", ip, exc) - return "unknown", "", False + bounds = _ip_range_bounds(ip) + return "unknown", "", False, bounds[3] if bounds else f"{ip}/32" intel = { "is_datacenter": bool(data.get("is_datacenter")), "is_vpn": bool(data.get("is_vpn")), @@ -877,6 +890,8 @@ def classify_signup_ip(ip: str) -> tuple[str, str, bool]: or (data.get("asn") or {}).get("org") or ""), } route = (data.get("asn") or {}).get("route") or "" + bounds = _ip_range_bounds(ip, route) + intel["cidr"] = bounds[3] if bounds else f"{ip}/32" cache_ip_intel(ip, intel, route) reasons = [name for name, key in ( @@ -887,17 +902,25 @@ def classify_signup_ip(ip: str) -> tuple[str, str, bool]: ("abuser", "is_abuser"), ) if intel[key]] classification = "+".join(reasons) if reasons else "clean" - return classification, intel["org"], bool(reasons) + return classification, intel["org"], bool(reasons), intel.get("cidr") or f"{ip}/32" -def register_ip_block(ip: str, acct: str, org: str) -> None: - """Register a flagged signup IP in Mastodon's native Admin::IpBlock.""" - try: - prefix_len = 32 if ipaddress.ip_address(ip).version == 4 else 128 - except ValueError: - log.warning("skipping ip_block registration for unparseable ip=%r (acct=%s)", ip, acct) - return - cidr = f"{ip}/{prefix_len}" +def register_ip_block(ip: str, acct: str, org: str, cidr: str = "") -> None: + """Register a flagged signup's network in Mastodon's native Admin::IpBlock. + + cidr should be classify_signup_ip's block_cidr — the ASN/route-level + network the signup IP belongs to, so a repeat bad actor from the same + datacenter/VPN/proxy block is blocked too, not just this one address. + Falls back to a single-address block if no cidr is given or ip itself + doesn't parse (defensive only; classify_signup_ip always supplies one). + """ + if not cidr: + try: + prefix_len = 32 if ipaddress.ip_address(ip).version == 4 else 128 + except ValueError: + log.warning("skipping ip_block registration for unparseable ip=%r (acct=%s)", ip, acct) + return + cidr = f"{ip}/{prefix_len}" comment = f"welcomebot: flagged signup @{acct} ({org or 'unknown org'})"[:200] try: resp = httpx.post( diff --git a/test_local.py b/test_local.py index 1c1209b..b07bfcf 100644 --- a/test_local.py +++ b/test_local.py @@ -224,7 +224,7 @@ def ip_scrutiny_tests(): ipblocks = [] main.send_welcome = lambda account_id, acct: sent.append((account_id, acct)) main.dm_moderator = lambda message: dms.append(message) - main.register_ip_block = lambda ip, acct, org: ipblocks.append((ip, acct, org)) + main.register_ip_block = lambda ip, acct, org, cidr: ipblocks.append((ip, acct, org, cidr)) # A flagged-but-not-held signup starts the suspicious watch inline, which # would otherwise hit the network for a baseline snapshot — stub it. main.fetch_account_counts = lambda account_id: (0, 0) @@ -233,9 +233,9 @@ def ip_scrutiny_tests(): return classifications[ip] classifications = { - "203.0.113.10": ("clean", "Example Residential ISP", False), - "198.51.100.20": ("datacenter", "Example Cloud Hosting Inc", True), - "198.51.100.21": ("datacenter", "Example Cloud Hosting Inc", True), + "203.0.113.10": ("clean", "Example Residential ISP", False, "203.0.113.10/32"), + "198.51.100.20": ("datacenter", "Example Cloud Hosting Inc", True, "198.51.100.0/24"), + "198.51.100.21": ("datacenter", "Example Cloud Hosting Inc", True, "198.51.100.0/24"), } main.classify_signup_ip = classify @@ -256,7 +256,7 @@ def ip_scrutiny_tests(): sent.clear(); dms.clear(); ipblocks.clear() main.process_signup("102", "dc1", "198.51.100.20") assert ("102", "dc1") not in sent, "welcome should be held for a flagged signup" - assert ipblocks == [("198.51.100.20", "dc1", "Example Cloud Hosting Inc")], ipblocks + assert ipblocks == [("198.51.100.20", "dc1", "Example Cloud Hosting Inc", "198.51.100.0/24")], ipblocks assert any("dc1" in d and "held" in d for d in dms), dms assert main.get_signup_flag("102") is True @@ -268,7 +268,7 @@ def ip_scrutiny_tests(): # fire, so process_signup must welcome + start the watch immediately # instead of waiting for an event that isn't coming. sent.clear(); dms.clear(); ipblocks.clear() - classifications["198.51.100.22"] = ("datacenter", "Example Cloud Hosting Inc", True) + classifications["198.51.100.22"] = ("datacenter", "Example Cloud Hosting Inc", True, "198.51.100.0/24") main.process_signup("104", "dc3", "198.51.100.22", "", True) assert ("104", "dc3") in sent, "already-approved flagged signup must be welcomed immediately" assert main.get_signup_flag("104") is True @@ -404,6 +404,45 @@ def email_scrutiny_tests(): assert domain_blocks == [], domain_blocks +def range_cache_tests(): + """Drive the ipapi range-cache helpers (_ip_range_bounds/cache_ip_intel/ + cached_ip_intel) directly — pure sqlite + ipaddress logic, no network. + This is what classify_signup_ip's block_cidr (the network register_ + ip_block blocks) is built on.""" + intel = { + "is_datacenter": True, "is_vpn": False, "is_proxy": False, + "is_tor": False, "is_abuser": False, "org": "Example Cloud Hosting Inc", + } + + # A. a valid route containing the ip caches (and returns) the wider CIDR, + # not just the single address. + main.cache_ip_intel("198.51.100.77", intel, route="198.51.100.0/24") + cached = main.cached_ip_intel("198.51.100.77") + assert cached["cidr"] == "198.51.100.0/24", cached + assert cached["is_datacenter"] is True + + # B. any other address in that same cached range hits the cache with the + # same wider CIDR — this is the point of range-based caching, and + # exactly what lets a repeat signup from elsewhere in the block also + # resolve to blocking the whole range. + cached2 = main.cached_ip_intel("198.51.100.200") + assert cached2["cidr"] == "198.51.100.0/24", cached2 + + # C. an address outside the cached range is a cache miss. + assert main.cached_ip_intel("198.51.101.1") is None + + # D. a route that doesn't actually contain the ip falls back to a /32 — + # never trust a route wide enough to not even cover the IP it came from. + main.cache_ip_intel("203.0.113.9", intel, route="10.0.0.0/8") + cached3 = main.cached_ip_intel("203.0.113.9") + assert cached3["cidr"] == "203.0.113.9/32", cached3 + + # E. no route at all -> /32. + main.cache_ip_intel("203.0.113.10", intel, route="") + cached4 = main.cached_ip_intel("203.0.113.10") + assert cached4["cidr"] == "203.0.113.10/32", cached4 + + def combined_signal_tests(): """Drive process_signup's combined IP+email immediate-suspend path (SUSPICIOUS_COMBINED_*), independent of the OR-based hourly sweep.""" @@ -415,7 +454,7 @@ def combined_signal_tests(): main.send_welcome = lambda account_id, acct: sent.append((account_id, acct)) main.dm_moderator = lambda message: dms.append(message) main.apply_action = lambda target_id, action, text: actions.append((target_id, action)) - main.register_ip_block = lambda ip, acct, org: ipblocks.append((ip, acct, org)) + main.register_ip_block = lambda ip, acct, org, cidr: ipblocks.append((ip, acct, org, cidr)) main.register_email_domain_block = lambda domain, acct: domain_blocks.append((domain, acct)) main.fetch_account_counts = lambda account_id: (0, 0) @@ -434,8 +473,8 @@ def combined_signal_tests(): main.ABUSE_ALLOWLIST = {"trustedstaff"} ip_classifications = { - "198.51.100.40": ("datacenter", "Example Cloud Hosting Inc", True), - "198.51.100.41": ("datacenter", "Example Cloud Hosting Inc", True), + "198.51.100.40": ("datacenter", "Example Cloud Hosting Inc", True, "198.51.100.32/28"), + "198.51.100.41": ("datacenter", "Example Cloud Hosting Inc", True, "198.51.100.32/28"), } main.classify_signup_ip = lambda ip: ip_classifications[ip] email_classifications = { @@ -456,7 +495,7 @@ def combined_signal_tests(): "SELECT 1 FROM suspicious_watch WHERE account_id = ?", ("501",), ).fetchone() assert row is None, "an immediately-suspended signup must not also enter the hourly watch" - assert ("198.51.100.40", "bothbad", "Example Cloud Hosting Inc") in ipblocks, ipblocks + assert ("198.51.100.40", "bothbad", "Example Cloud Hosting Inc", "198.51.100.32/28") in ipblocks, ipblocks assert ("temp-mail.org", "bothbad") in domain_blocks, domain_blocks # B. only IP flagged -> untouched by the combined path, falls through to @@ -643,6 +682,7 @@ if __name__ == "__main__": policy_tests() ip_scrutiny_tests() email_scrutiny_tests() + range_cache_tests() combined_signal_tests() suspicious_watch_tests() suspicious_sweep_tests()