From eb6d7cad46459495366f6f677549f8a39000bdde Mon Sep 17 00:00:00 2001 From: waffle2k Date: Sun, 13 Sep 2026 00:41:48 -0700 Subject: [PATCH] Add read-only blocklist API (off by default) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Publishes the IP ranges and email domains the bot has auto-blocked as JSON: GET /api/blocks, /api/blocks/ip-ranges, /api/blocks/email-domains, each taking an optional ?limit=N. The endpoints are unauthenticated, so BLOCKS_API_ENABLED gates them and defaults to false — with it unset the paths 404 as if they didn't exist. Publishing which ranges are already blocked also tells a spam signup which ones to avoid, so opting in is a per-deployment call; yttrx.com sets it true in the box's .env. Responses assert only that a block exists: the blocked range/domain, a signups count, and first/last seen. No account identifiers, and none of the ipapi.is or check-mail.org classification that drove the decision — republishing a paid reputation feed's verdicts on an open endpoint would be redistributing their data rather than ours. Supporting changes: - signup_ip gains a block_cidr column (migrated in place on open) so the API reports the network actually blocked rather than the signup IP. Rows predating it fall back to that IP's /32 or /128, which is exactly what was registered before range-blocking started 2026-07-15. - register_ip_block / register_email_domain_block now return whether the block landed, and the caller only marks it registered on success, so a failed POST is no longer published as a block that exists. --- .env.example | 27 +++++ CLAUDE.md | 16 +++ README.md | 59 ++++++++- app/main.py | 237 ++++++++++++++++++++++++++++++++++--- nginx/hooks.yttrx.com.conf | 12 ++ test_local.py | 93 ++++++++++++++- 6 files changed, 422 insertions(+), 22 deletions(-) diff --git a/.env.example b/.env.example index e0acdf2..22e79e9 100644 --- a/.env.example +++ b/.env.example @@ -222,3 +222,30 @@ SUSPICIOUS_COMBINED_ACTION=suspend # SUSPICIOUS_DRY_RUN's by-design "false" default above, since this is a # brand-new action path. Flip to "false" once the moderator DMs look right. SUSPICIOUS_COMBINED_DRY_RUN=true + +# --- Blocklist API (app/main.py, GET /api/blocks) --------------------------- +# Read-only JSON of the IP ranges and email domains this bot has auto-blocked: +# GET /api/blocks both lists in one document +# GET /api/blocks/ip-ranges registered Admin::IpBlock networks +# GET /api/blocks/email-domains registered Admin::EmailDomainBlock domains +# Optional ?limit=N on any of them. Responses assert only that a range/domain +# is blocked — no account handles, account ids or email addresses, and none of +# the ipapi.is / check-mail.org classification behind the decision (publishing +# those verdicts would be redistributing a paid reputation feed's data). Note +# this reports the *bot's own* record, not Mastodon's full blocklist: blocks a +# moderator added by hand in the admin UI aren't in it. +# +# SERVED WITHOUT AUTHENTICATION, so it defaults to off and every deployment +# must opt in: publishing which ranges and domains are already blocked also +# tells a would-be spam signup which ones to avoid. With this unset the three +# paths 404 as if they didn't exist. yttrx.com sets this to true. +BLOCKS_API_ENABLED=false + +# Hard ceiling on entries per response, whatever ?limit asks for — the +# endpoint is unauthenticated, so response size can't be left to the caller. +BLOCKS_API_MAX_ITEMS=5000 + +# Cache-Control max-age (seconds) on blocklist responses. The data only +# changes when a signup is flagged, so a few minutes of client/intermediate +# caching costs nothing. +BLOCKS_API_CACHE_SECONDS=300 diff --git a/CLAUDE.md b/CLAUDE.md index 98716d3..4ba9c2f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,6 +19,16 @@ A FastAPI webhook server. One Mastodon admin webhook delivers `account.created`, and auto-**silence** young/dormant accounts past the threshold — lowered for accounts with a flagged signup IP (see README). +Alongside the webhook it serves an optional read-only **blocklist API** — +`GET /api/blocks`, `/api/blocks/ip-ranges`, `/api/blocks/email-domains` — +publishing the IP ranges and email domains the bot has auto-blocked. It is +unauthenticated, so it 404s unless `BLOCKS_API_ENABLED=true`; yttrx.com sets +that in the box's `.env`, the shipped default is off. Responses assert only +*that* a block exists — no account identifiers, and deliberately none of the +ipapi.is / check-mail.org classification behind the decision, so we're not +republishing a paid reputation feed's verdicts. See README.md's "Blocklist +API" section. + A separate scheduled module, `app/suspicious_sweep.py` (run via `python -m app.suspicious_sweep`, not part of the webhook server), suspends any IP- or email-flagged signup that shows zero new posts and zero new @@ -46,6 +56,12 @@ user-memory note `project_welcomebot` has the current state. - The bot **silences** (reversible), it does not suspend by default (`ABUSE_ACTION=silence`). It applies the action **without** `report_id` so the report stays open for human review. +- **The blocklist API must stay classification-free.** `/api/blocks` is + public and unauthenticated; entries may say a range/domain is blocked and + when, never *why*. Don't add the ipapi.is verdict, the org name, or + check-mail.org's disposable/risk fields back into the response — that would + be redistributing a paid reputation feed's data. Account identifiers + (handles, ids, emails, raw signup IPs) are out for the usual reasons. - **`admin.yttrx.com`'s `/root/yttrx-welcomebot/` is not a git repo any more** (cleaned up 2026-07-06). Don't `git init`/clone it back into place — the registry image + `docker-compose.yml`/`.env` is the whole deploy surface now. diff --git a/README.md b/README.md index 09dc8ae..37fa5b0 100644 --- a/README.md +++ b/README.md @@ -191,6 +191,63 @@ section for the remint procedure. Inspect the watch list: `welcomebot-suspicious` / `welcomebot-suspicious --pending` (see `bin/welcomebot-suspicious`). +## Blocklist API (`/api/blocks`) + +An optional, **read-only** JSON view of what the bot has auto-blocked. Three +endpoints, all `GET`, all accepting an optional `?limit=N`: + +| Path | Returns | +|---|---| +| `/api/blocks` | both lists in one document | +| `/api/blocks/ip-ranges` | networks registered as `Admin::IpBlock` | +| `/api/blocks/email-domains` | domains registered as `Admin::EmailDomainBlock` | + +```json +{ + "generated_at": "2026-09-13T00:00:00+00:00", + "ip_ranges": { + "count": 1, + "items": [ + {"cidr": "198.51.100.0/24", "signups": 3, + "first_blocked_at": "2026-07-15 09:12:04", + "last_blocked_at": "2026-08-30 22:41:18"} + ] + }, + "email_domains": { + "count": 1, + "items": [ + {"domain": "temp-mail.org", "signups": 2, + "first_blocked_at": "2026-07-03 11:02:55", + "last_blocked_at": "2026-07-19 16:30:09"} + ] + } +} +``` + +**Off by default.** These paths are served *without authentication*, so they +404 unless `BLOCKS_API_ENABLED=true` is set — publishing which ranges and +domains are already blocked also tells a would-be spam signup which ones to +avoid, and that's a per-deployment call. yttrx.com opts in; the shipped +default doesn't. `BLOCKS_API_MAX_ITEMS` (default 5000) caps the response size +whatever `?limit` asks for, and `BLOCKS_API_CACHE_SECONDS` (default 300) sets +the `Cache-Control: public, max-age=…` on responses. + +What the response deliberately does **not** contain: + +- **No account identifiers** — no handles, account ids, emails or signup IPs. + Repeat signups out of one range collapse into a single entry with a + `signups` count. +- **No classification data.** The ipapi.is verdict (`datacenter`/`proxy`/…), + the org name, and check-mail.org's disposable flag and risk score all stay + out of the response. They're what *drove* each decision, but republishing a + paid reputation feed's verdicts on an open endpoint would be redistributing + their data rather than ours. An entry asserts only that the block exists. + +Scope: this is the **bot's own record**, not Mastodon's full blocklist — +blocks a moderator added by hand in the admin UI are not in it. Entries appear +only once the block actually landed on the Mastodon side (a failed +`POST /api/v1/admin/ip_blocks` is logged and *not* recorded as blocked). + ## Layout | Path | Purpose | @@ -199,7 +256,7 @@ Inspect the watch list: `welcomebot-suspicious` / `welcomebot-suspicious --pendi | `app/suspicious_sweep.py` | Scheduled sweep: suspends flagged signups with no activity (run via cron, not the webserver) | | `Dockerfile`, `docker-compose.yml` | Container build + run | | `.env.example` | Config template (copy to `.env`, never commit) | -| `nginx/hooks.yttrx.com.conf` | nginx site for admin.yttrx.com | +| `nginx/hooks.yttrx.com.conf` | nginx site for admin.yttrx.com (webhook, healthz, blocklist API) | | `bin/welcomebot-logs`, `bin/welcomebot-signups`, `bin/welcomebot-suspicious` | Ops CLIs installed to `/root/bin` on admin.yttrx.com | | `test_local.py` | Offline smoke test (no network) | diff --git a/app/main.py b/app/main.py index a73eb66..cde485a 100644 --- a/app/main.py +++ b/app/main.py @@ -29,6 +29,11 @@ Mastodon signs every delivery with: One Mastodon webhook subscribing to both events points at /webhook; both share the single WEBHOOK_SECRET. See README.md for the deploy + Mastodon-side configuration runbook. + +Alongside the webhook, an optional read-only blocklist API publishes what the +bot has auto-blocked — ``GET /api/blocks``, ``/api/blocks/ip-ranges``, +``/api/blocks/email-domains``. It is unauthenticated and therefore off unless +``BLOCKS_API_ENABLED`` is set; see that config entry. """ from __future__ import annotations @@ -45,6 +50,7 @@ from datetime import datetime, timedelta, timezone import httpx from fastapi import BackgroundTasks, FastAPI, Header, HTTPException, Request +from fastapi.responses import JSONResponse logging.basicConfig( level=logging.INFO, @@ -220,6 +226,23 @@ SUSPICIOUS_COMBINED_ACTION = os.environ.get("SUSPICIOUS_COMBINED_ACTION", "suspe # by-design "false" default above) since this is a brand-new action path. SUSPICIOUS_COMBINED_DRY_RUN = os.environ.get("SUSPICIOUS_COMBINED_DRY_RUN", "true").lower() in ("1", "true", "yes") +# --- Public blocklist API --------------------------------------------------- +# Read-only JSON of the blocks this bot has registered (GET /api/blocks and +# friends). Served WITHOUT authentication when enabled, so it defaults to +# "off" and must be opted into per-deployment: publishing which ranges and +# domains are already blocked also tells a would-be spam signup which ones to +# avoid, and that's a per-instance call, not a safe default. yttrx.com sets +# BLOCKS_API_ENABLED=true in its .env; every other deployment 404s. +BLOCKS_API_ENABLED = os.environ.get("BLOCKS_API_ENABLED", "false").lower() in ("1", "true", "yes") +# Hard ceiling on how many entries one response may return, whatever ?limit +# asks for — this endpoint is unauthenticated, so the response size can't be +# left to the caller. +BLOCKS_API_MAX_ITEMS = int(os.environ.get("BLOCKS_API_MAX_ITEMS", "5000")) +# Cache-Control max-age (seconds) on blocklist responses. The underlying data +# changes only when a signup is flagged, so letting clients and any +# intermediate cache hold it for a few minutes costs nothing. +BLOCKS_API_CACHE_SECONDS = int(os.environ.get("BLOCKS_API_CACHE_SECONDS", "300")) + if not WEBHOOK_SECRET: log.warning("WEBHOOK_SECRET is empty — signature verification will reject all requests.") if not BOT_ACCESS_TOKEN: @@ -238,6 +261,10 @@ app = FastAPI(title="yttrx welcome-bot + abuse-bot") _db_lock = threading.Lock() +def _column_exists(conn: sqlite3.Connection, table: str, column: str) -> bool: + return any(row[1] == column for row in conn.execute(f"PRAGMA table_info({table})")) + + def _init_db() -> None: os.makedirs(os.path.dirname(DB_PATH), exist_ok=True) with sqlite3.connect(DB_PATH) as conn: @@ -268,9 +295,18 @@ def _init_db() -> None: " org TEXT," " flagged INTEGER," " ipblock_registered INTEGER DEFAULT 0," + " block_cidr TEXT," " classified_at TEXT DEFAULT CURRENT_TIMESTAMP" ")" ) + # block_cidr was added after the table shipped (it records the network + # register_ip_block actually blocks, which is what /api/blocks + # publishes). Rows written before this column existed keep it NULL and + # fall back to the single-address /32 or /128 of their signup IP — + # which is exactly what was registered back then, since blocking the + # ASN route CIDR only started 2026-07-15. + if not _column_exists(conn, "signup_ip", "block_cidr"): + conn.execute("ALTER TABLE signup_ip ADD COLUMN block_cidr TEXT") conn.execute( "CREATE TABLE IF NOT EXISTS ipapi_range_cache (" " family INTEGER NOT NULL," # 4 or 6 @@ -368,15 +404,16 @@ def record_action(target_id: str, acct: str, action: str, tier: str, def record_signup_ip(account_id: str, acct: str, ip: str, classification: str, - org: str, flagged: bool) -> None: + org: str, flagged: bool, block_cidr: str = "") -> None: with _db() as conn: conn.execute( "INSERT OR REPLACE INTO signup_ip " "(account_id, acct, ip, classification, org, flagged, " - " ipblock_registered) " + " ipblock_registered, block_cidr) " "VALUES (?, ?, ?, ?, ?, ?, " - " COALESCE((SELECT ipblock_registered FROM signup_ip WHERE account_id = ?), 0))", - (account_id, acct, ip, classification, org, int(flagged), account_id), + " COALESCE((SELECT ipblock_registered FROM signup_ip WHERE account_id = ?), 0), ?)", + (account_id, acct, ip, classification, org, int(flagged), account_id, + block_cidr or None), ) @@ -526,6 +563,104 @@ def start_suspicious_watch(account_id: str, acct: str, reasons: str, ) +def _fallback_cidr(ip: str) -> str: + """The single-address CIDR for ip, used for signup_ip rows written before + block_cidr existed — that's the block that was registered for them.""" + try: + addr = ipaddress.ip_address(ip) + except ValueError: + return "" + return f"{addr}/{32 if addr.version == 4 else 128}" + + +def blocked_ip_ranges(limit: int = 0) -> list[dict]: + """Every network this bot has registered an Admin::IpBlock for, newest + first, collapsed to one entry per distinct CIDR. + + Repeat signups out of the same range are common (that's the point of + blocking the range rather than the address), so entries carry a `signups` + count and the first/last time the range was hit rather than appearing once + per account. + + Entries state only *that* a range is blocked. The ipapi.is classification + and org that drove the decision stay out of the response on purpose: this + is an unauthenticated endpoint, and republishing a paid IP-reputation + feed's verdicts would be redistributing their data, not ours. The same + goes for identifying anything about the accounts involved. + """ + with _db() as conn: + rows = conn.execute( + "SELECT block_cidr, ip, classified_at " + "FROM signup_ip WHERE ipblock_registered = 1 " + "ORDER BY classified_at DESC" + ).fetchall() + + grouped: dict[str, dict] = {} + for block_cidr, ip, classified_at in rows: + cidr = block_cidr or _fallback_cidr(ip or "") + if not cidr: + continue + entry = grouped.get(cidr) + if entry is None: + grouped[cidr] = { + "cidr": cidr, + "signups": 1, + "first_blocked_at": classified_at, + "last_blocked_at": classified_at, + } + else: + # rows are newest-first, so every later row is an older sighting + entry["signups"] += 1 + entry["first_blocked_at"] = classified_at + + return _capped(list(grouped.values()), limit) + + +def blocked_email_domains(limit: int = 0) -> list[dict]: + """Every domain this bot has registered an Admin::EmailDomainBlock for, + newest first, one entry per distinct domain. + + Domains only — the bot never stores or forwards a full signup address (see + classify_email_domain), so there is none to expose here. + + As with blocked_ip_ranges, entries state only *that* a domain is blocked: + check-mail.org's disposable flag and risk score stay out of the response + so this doesn't republish a third-party reputation feed's verdicts. + """ + with _db() as conn: + rows = conn.execute( + "SELECT domain, classified_at " + "FROM signup_email WHERE domain_blocked = 1 " + "ORDER BY classified_at DESC" + ).fetchall() + + grouped: dict[str, dict] = {} + for domain, classified_at in rows: + if not domain: + continue + entry = grouped.get(domain) + if entry is None: + grouped[domain] = { + "domain": domain, + "signups": 1, + "first_blocked_at": classified_at, + "last_blocked_at": classified_at, + } + else: + entry["signups"] += 1 + entry["first_blocked_at"] = classified_at + + return _capped(list(grouped.values()), limit) + + +def _capped(items: list[dict], limit: int) -> list[dict]: + """Apply a caller's ?limit on top of BLOCKS_API_MAX_ITEMS. limit <= 0 + means "as many as the ceiling allows".""" + ceiling = max(BLOCKS_API_MAX_ITEMS, 0) + n = ceiling if limit <= 0 else min(limit, ceiling) + return items[:n] + + _init_db() # --- Signature verification ------------------------------------------------ @@ -604,12 +739,16 @@ def classify_email_domain(domain: str) -> tuple[bool, int]: return is_disposable, risk -def register_email_domain_block(domain: str, acct: str) -> None: +def register_email_domain_block(domain: str, acct: str) -> bool: """Register a flagged signup's email domain in Mastodon's native Admin::EmailDomainBlock, blocking future signups from that domain. Requires the admin:write:email_domain_blocks OAuth scope on ABUSE_BOT_TOKEN — see CLAUDE.md's moderator-token-gotcha section. + + Returns True if the domain is blocked on the Mastodon side afterwards + (including the 422 "already blocked" case), so the caller only records it + as blocked when it really is — /api/blocks publishes those rows. """ try: resp = httpx.post( @@ -624,6 +763,8 @@ def register_email_domain_block(domain: str, acct: str) -> None: resp.raise_for_status() except httpx.HTTPError as exc: log.error("failed to register email_domain_block for %s (acct=%s): %s", domain, acct, exc) + return False + return True def process_signup(account_id: str, acct: str, ip: str, email: str = "", @@ -662,15 +803,15 @@ def process_signup(account_id: str, acct: str, ip: str, email: str = "", if IP_SCRUTINY_ENABLED and ip: classification, org, ip_flagged, block_cidr = classify_signup_ip(ip) - record_signup_ip(account_id, acct, ip, classification, org, ip_flagged) + record_signup_ip(account_id, acct, ip, classification, org, ip_flagged, block_cidr) if ip_flagged: ip_strong_flagged = bool(set(classification.split("+")) & IP_SCRUTINY_STRONG_SIGNALS) log.warning("flagged signup acct=%s ip=%s classification=%s org=%s", acct, ip, classification, org) if IP_SCRUTINY_AUTO_IPBLOCK and not IP_SCRUTINY_DRY_RUN: - register_ip_block(ip, acct, org, block_cidr) - mark_ipblock_registered(account_id) + if register_ip_block(ip, acct, org, block_cidr): + mark_ipblock_registered(account_id) prefix = "[DRY-RUN] " if IP_SCRUTINY_DRY_RUN else "" reasons.append(f"{prefix}IP {ip} ({org or 'unknown org'}, {classification})") hold = hold or (IP_SCRUTINY_HOLD_WELCOME and not IP_SCRUTINY_DRY_RUN) @@ -685,8 +826,8 @@ def process_signup(account_id: str, acct: str, ip: str, email: str = "", log.warning("flagged signup acct=%s email_domain=%s is_disposable=%s risk=%d", acct, domain, is_disposable, risk) if CHECK_MAIL_AUTO_DOMAIN_BLOCK and not CHECK_MAIL_DRY_RUN: - register_email_domain_block(domain, acct) - mark_email_domain_blocked(account_id) + if register_email_domain_block(domain, acct): + mark_email_domain_blocked(account_id) prefix = "[DRY-RUN] " if CHECK_MAIL_DRY_RUN else "" reasons.append(f"{prefix}email domain {domain} (disposable={is_disposable}, risk={risk})") hold = hold or (CHECK_MAIL_HOLD_WELCOME and not CHECK_MAIL_DRY_RUN) @@ -975,7 +1116,7 @@ def classify_signup_ip(ip: str) -> tuple[str, str, bool, str]: return classification, intel["org"], bool(reasons), intel.get("cidr") or f"{ip}/32" -def register_ip_block(ip: str, acct: str, org: str, cidr: str = "") -> None: +def register_ip_block(ip: str, acct: str, org: str, cidr: str = "") -> bool: """Register a flagged signup's network in Mastodon's native Admin::IpBlock. cidr should be classify_signup_ip's block_cidr — the ASN/route-level @@ -983,13 +1124,17 @@ def register_ip_block(ip: str, acct: str, org: str, cidr: str = "") -> None: datacenter/VPN/proxy block is blocked too, not just this one address. Falls back to a single-address block if no cidr is given or ip itself doesn't parse (defensive only; classify_signup_ip always supplies one). + + Returns True if the range is blocked on the Mastodon side afterwards + (including the 422 "already blocked" case), so the caller only records it + as blocked when it really is — /api/blocks publishes those rows. """ if not cidr: try: prefix_len = 32 if ipaddress.ip_address(ip).version == 4 else 128 except ValueError: log.warning("skipping ip_block registration for unparseable ip=%r (acct=%s)", ip, acct) - return + return False cidr = f"{ip}/{prefix_len}" comment = f"welcomebot: flagged signup @{acct} ({org or 'unknown org'})"[:200] try: @@ -1005,7 +1150,8 @@ def register_ip_block(ip: str, acct: str, org: str, cidr: str = "") -> None: resp.raise_for_status() except httpx.HTTPError as exc: log.error("failed to register ip_block for %s (acct=%s): %s", cidr, acct, exc) - return + return False + return True def count_distinct_reporters(target_id: str) -> int: @@ -1147,8 +1293,8 @@ def handle_report(report_id: str, target_id: str, acct: str, is_local: bool, if CHECK_MAIL_AUTO_DOMAIN_BLOCK: record_signup_email(target_id, acct, domain, is_disposable, risk, True) - register_email_domain_block(domain, acct) - mark_email_domain_blocked(target_id) + if register_email_domain_block(domain, acct): + mark_email_domain_blocked(target_id) return try: @@ -1215,6 +1361,67 @@ def healthz(): return {"ok": True} +# --- Blocklist API --- +# Read-only JSON view of what this bot has auto-blocked, served unauthenticated +# when BLOCKS_API_ENABLED is on (off by default — see the config block). It +# reports only the bot's own record: blocks a human added by hand in Mastodon's +# admin UI are not in here. Responses assert only that a range or domain is +# blocked — no account handles, account ids or email addresses, and none of +# the ipapi.is / check-mail.org classification data behind the decision (see +# blocked_ip_ranges). + + +def _blocks_guard(limit: int) -> int: + """404 unless the API is switched on for this deployment; a disabled + endpoint should be indistinguishable from one that doesn't exist. Returns + the sanitised limit.""" + if not BLOCKS_API_ENABLED: + raise HTTPException(status_code=404, detail="Not Found") + return max(limit, 0) + + +def _blocks_response(payload: dict) -> JSONResponse: + return JSONResponse( + payload, + headers={"Cache-Control": f"public, max-age={BLOCKS_API_CACHE_SECONDS}"}, + ) + + +def _now_iso() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat() + + +@app.get("/api/blocks") +def api_blocks(limit: int = 0): + """Both blocklists in one document.""" + limit = _blocks_guard(limit) + ip_ranges = blocked_ip_ranges(limit) + email_domains = blocked_email_domains(limit) + return _blocks_response({ + "generated_at": _now_iso(), + "ip_ranges": {"count": len(ip_ranges), "items": ip_ranges}, + "email_domains": {"count": len(email_domains), "items": email_domains}, + }) + + +@app.get("/api/blocks/ip-ranges") +def api_blocks_ip_ranges(limit: int = 0): + limit = _blocks_guard(limit) + items = blocked_ip_ranges(limit) + return _blocks_response({ + "generated_at": _now_iso(), "count": len(items), "items": items, + }) + + +@app.get("/api/blocks/email-domains") +def api_blocks_email_domains(limit: int = 0): + limit = _blocks_guard(limit) + items = blocked_email_domains(limit) + return _blocks_response({ + "generated_at": _now_iso(), "count": len(items), "items": items, + }) + + @app.post("/webhook") async def webhook( request: Request, diff --git a/nginx/hooks.yttrx.com.conf b/nginx/hooks.yttrx.com.conf index 0f4f5d4..43ea65c 100644 --- a/nginx/hooks.yttrx.com.conf +++ b/nginx/hooks.yttrx.com.conf @@ -31,6 +31,18 @@ server { proxy_pass http://127.0.0.1:8087; } + # Read-only blocklist API (GET /api/blocks, /api/blocks/ip-ranges, + # /api/blocks/email-domains). Unauthenticated by design, and gated in the + # app by BLOCKS_API_ENABLED — with that unset the app 404s these paths, + # so proxying them here exposes nothing on its own. + location ^~ /api/blocks { + proxy_pass http://127.0.0.1:8087; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + location / { return 404; } diff --git a/test_local.py b/test_local.py index cb4c2c9..90a56ba 100644 --- a/test_local.py +++ b/test_local.py @@ -224,7 +224,7 @@ def ip_scrutiny_tests(): ipblocks = [] main.send_welcome = lambda account_id, acct: sent.append((account_id, acct)) main.dm_moderator = lambda message: dms.append(message) - main.register_ip_block = lambda ip, acct, org, cidr: ipblocks.append((ip, acct, org, cidr)) + main.register_ip_block = lambda ip, acct, org, cidr: ipblocks.append((ip, acct, org, cidr)) or True # A flagged-but-not-held signup starts the suspicious watch inline, which # would otherwise hit the network for a baseline snapshot — stub it. main.fetch_account_counts = lambda account_id: (0, 0) @@ -315,7 +315,7 @@ def email_scrutiny_tests(): domain_blocks = [] main.send_welcome = lambda account_id, acct: sent.append((account_id, acct)) main.dm_moderator = lambda message: dms.append(message) - main.register_email_domain_block = lambda domain, acct: domain_blocks.append((domain, acct)) + main.register_email_domain_block = lambda domain, acct: domain_blocks.append((domain, acct)) or True # A flagged-but-not-held signup starts the suspicious watch inline, which # would otherwise hit the network for a baseline snapshot — stub it. main.fetch_account_counts = lambda account_id: (0, 0) @@ -454,8 +454,8 @@ def combined_signal_tests(): main.send_welcome = lambda account_id, acct: sent.append((account_id, acct)) main.dm_moderator = lambda message: dms.append(message) main.apply_action = lambda target_id, action, text: actions.append((target_id, action)) - main.register_ip_block = lambda ip, acct, org, cidr: ipblocks.append((ip, acct, org, cidr)) - main.register_email_domain_block = lambda domain, acct: domain_blocks.append((domain, acct)) + main.register_ip_block = lambda ip, acct, org, cidr: ipblocks.append((ip, acct, org, cidr)) or True + main.register_email_domain_block = lambda domain, acct: domain_blocks.append((domain, acct)) or True main.fetch_account_counts = lambda account_id: (0, 0) main.IP_SCRUTINY_ENABLED = True @@ -546,8 +546,8 @@ def strong_ip_suspend_tests(): main.send_welcome = lambda account_id, acct: sent.append((account_id, acct)) main.dm_moderator = lambda message: dms.append(message) main.apply_action = lambda target_id, action, text: actions.append((target_id, action)) - main.register_ip_block = lambda ip, acct, org, cidr: ipblocks.append((ip, acct, org, cidr)) - main.register_email_domain_block = lambda domain, acct: None + main.register_ip_block = lambda ip, acct, org, cidr: ipblocks.append((ip, acct, org, cidr)) or True + main.register_email_domain_block = lambda domain, acct: True main.fetch_account_counts = lambda account_id: (0, 0) main.IP_SCRUTINY_ENABLED = True @@ -780,6 +780,86 @@ def suspicious_sweep_tests(): assert any("[DRY-RUN]" in d and "dryrunbot" in d for d in dms), dms +def blocks_api_tests(): + """Drive the read-only blocklist API: the BLOCKS_API_ENABLED gate, the + response shape, the deliberate omission of provider classification data + and account identifiers, and the pre-block_cidr fallback.""" + # Seed deterministic rows rather than relying on what earlier tests left + # in the shared db; assertions below check for presence, not exact lists. + main.record_signup_ip("900", "blockedone", "198.18.0.90", "datacenter", + "Cloud Inc", True, "198.18.0.0/24") + main.mark_ipblock_registered("900") + # A second signup out of the same range collapses into one entry. + main.record_signup_ip("901", "blockedtwo", "198.18.0.91", "datacenter", + "Cloud Inc", True, "198.18.0.0/24") + main.mark_ipblock_registered("901") + # Classified but never blocked -> must not appear at all. + main.record_signup_ip("902", "notblockedacct", "203.0.113.199", "clean", "", False) + # A row predating the block_cidr column (NULL) -> falls back to the /32 + # that was in fact registered for it back then. + main.record_signup_ip("903", "legacyacct", "192.0.2.77", "datacenter", "Old Corp", True) + main.mark_ipblock_registered("903") + + main.record_signup_email("910", "spamone", "blocked-example.test", True, 95, True) + main.mark_email_domain_blocked("910") + main.record_signup_email("911", "spamtwo", "blocked-example.test", True, 95, True) + main.mark_email_domain_blocked("911") + main.record_signup_email("912", "cleanacct", "notblocked-example.test", False, 0, False) + + paths = ("/api/blocks", "/api/blocks/ip-ranges", "/api/blocks/email-domains") + + # A. Disabled (the shipped default) -> 404, indistinguishable from a path + # that doesn't exist. + main.BLOCKS_API_ENABLED = False + for path in paths: + assert client.get(path).status_code == 404, path + + # B. Enabled -> 200, with the ranges and domains actually blocked. + main.BLOCKS_API_ENABLED = True + r = client.get("/api/blocks") + assert r.status_code == 200, r.status_code + assert r.headers["cache-control"] == f"public, max-age={main.BLOCKS_API_CACHE_SECONDS}" + body = r.json() + ranges = {e["cidr"]: e for e in body["ip_ranges"]["items"]} + domains = {e["domain"]: e for e in body["email_domains"]["items"]} + assert body["ip_ranges"]["count"] == len(body["ip_ranges"]["items"]) + assert body["email_domains"]["count"] == len(body["email_domains"]["items"]) + + assert "198.18.0.0/24" in ranges, ranges + assert ranges["198.18.0.0/24"]["signups"] == 2, "repeat signups collapse into one entry" + assert "192.0.2.77/32" in ranges, "a NULL block_cidr row falls back to its /32" + assert "blocked-example.test" in domains, domains + assert domains["blocked-example.test"]["signups"] == 2 + assert "notblocked-example.test" not in domains, "an unblocked domain must not be listed" + + # C. Nothing that identifies an account, and nothing sourced from the + # ipapi.is / check-mail.org feeds, may appear in the response. + raw = r.text + for leak in ("blockedone", "legacyacct", "notblockedacct", "spamone", "cleanacct", + "203.0.113.199", "datacenter", "Cloud Inc", "Old Corp"): + assert leak not in raw, f"{leak!r} must not be exposed by the blocklist API" + for entry in body["ip_ranges"]["items"]: + assert set(entry) == {"cidr", "signups", "first_blocked_at", "last_blocked_at"}, entry + for entry in body["email_domains"]["items"]: + assert set(entry) == {"domain", "signups", "first_blocked_at", "last_blocked_at"}, entry + + # D. The per-list endpoints agree with the combined document. + ips_only = client.get("/api/blocks/ip-ranges").json() + domains_only = client.get("/api/blocks/email-domains").json() + assert ips_only["items"] == body["ip_ranges"]["items"] + assert domains_only["items"] == body["email_domains"]["items"] + + # E. ?limit truncates; a nonsensical limit is treated as "no limit". + assert len(client.get("/api/blocks/ip-ranges?limit=1").json()["items"]) == 1 + assert (client.get("/api/blocks/ip-ranges?limit=-5").json()["items"] + == ips_only["items"]), "a negative limit must not empty the list" + + # F. BLOCKS_API_MAX_ITEMS caps the caller's limit, not the other way round. + main.BLOCKS_API_MAX_ITEMS = 1 + assert len(client.get("/api/blocks/ip-ranges?limit=999").json()["items"]) == 1 + main.BLOCKS_API_MAX_ITEMS = 5000 + + if __name__ == "__main__": routing_tests() policy_tests() @@ -790,4 +870,5 @@ if __name__ == "__main__": strong_ip_suspend_tests() suspicious_watch_tests() suspicious_sweep_tests() + blocks_api_tests() print("ALL TESTS PASSED")