From 8966de2a1d21dd0cc5bff4796b34ac78089cc792 Mon Sep 17 00:00:00 2001 From: waffle2k Date: Sun, 13 Sep 2026 00:59:12 -0700 Subject: [PATCH] Add text format, CORS, and an importer so other instances can use the list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mastodon has no RBL/DNSBL support — nothing in it consults an outside list when deciding whether to accept a signup, confirmed against v4.7.1 (the only outbound lookup in that path is EmailMxValidator's MX resolution, which feeds the local email_domain_blocks table). So a shared blocklist has to be imported rather than queried live, the same way Garden Fence and other community lists propagate. These are the pieces that make ours importable. - ?format=txt on both per-list endpoints: one entry per line under a '#' comment header, the form blocklists are traded in and the form a cron job can consume without a JSON parser. The combined /api/blocks stays JSON-only — a text rendering would mix CIDRs with domains in one file, defeating the point. An unrecognised ?format is a 400 rather than a silent fallback to JSON, and the enabled-gate still runs first so a disabled deployment 404s either way. - Access-Control-Allow-Origin: * on blocklist responses, so browser-side admin tooling can read them. The data is already public. - bin/yttrx-blocklist-import: standard-library-only importer another admin can run against their own instance. Dry run unless --apply; 422 from Mastodon counts as "already blocked", not an error. The importer defaults IP entries to sign_up_requires_approval rather than the sign_up_block yttrx uses itself. The entries are whole ASN routes, which suits yttrx because a flagged signup here is nearly always datacenter/VPN/proxy space — that calibration isn't transferable, so an importer gets the reviewable severity by default and has to opt into the hard one. --- .env.example | 5 ++ README.md | 49 ++++++++++ app/main.py | 76 ++++++++++++++-- bin/yttrx-blocklist-import | 177 +++++++++++++++++++++++++++++++++++++ test_local.py | 33 +++++++ 5 files changed, 331 insertions(+), 9 deletions(-) create mode 100755 bin/yttrx-blocklist-import diff --git a/.env.example b/.env.example index 22e79e9..b368c32 100644 --- a/.env.example +++ b/.env.example @@ -245,6 +245,11 @@ BLOCKS_API_ENABLED=false # endpoint is unauthenticated, so response size can't be left to the caller. BLOCKS_API_MAX_ITEMS=5000 +# Public base URL this API is reached at. Used only to print a "source:" line +# in the text format's header comments, so a downloaded list says where it +# came from; it plays no part in routing. +BLOCKS_API_PUBLIC_URL=https://hooks.yttrx.com + # Cache-Control max-age (seconds) on blocklist responses. The data only # changes when a signup is flagged, so a few minutes of client/intermediate # caching costs nothing. diff --git a/README.md b/README.md index 13e2523..464fd56 100644 --- a/README.md +++ b/README.md @@ -224,6 +224,54 @@ endpoints, all `GET`, all accepting an optional `?limit=N`: } ``` +### Plain-text format + +Both per-list endpoints also take `?format=txt`, which renders one entry per +line under a `#` comment header — the form blocklists are normally traded in, +and the form another admin's cron job can consume without a JSON parser: + +```bash +curl -s 'https://hooks.yttrx.com/api/blocks/ip-ranges?format=txt' | grep -v '^#' +curl -s 'https://hooks.yttrx.com/api/blocks/email-domains?format=txt' | grep -v '^#' +``` + +The combined `/api/blocks` is JSON-only: a text rendering would put CIDRs and +domains in one file, and the point of the text form is that it pipes straight +into something expecting one kind of entry. Responses carry +`Access-Control-Allow-Origin: *` so browser-side tooling can read them. + +### Importing it elsewhere (`bin/yttrx-blocklist-import`) + +Mastodon has **no RBL/DNSBL support** — nothing in it consults an outside list +when deciding whether to accept a signup. (The only outbound lookup in that +path is `EmailMxValidator`'s MX resolution, and that feeds the *local* +`email_domain_blocks` table.) So another instance can't point at this feed +live; it has to import, the same way [Garden Fence](https://gardenfence.github.io) +and other community lists propagate. + +`bin/yttrx-blocklist-import` does that — standard library only, no install: + +```bash +# dry run (the default — nothing is written) +./bin/yttrx-blocklist-import --instance https://your.instance --token "$TOKEN" +# actually write the blocks +./bin/yttrx-blocklist-import --instance https://your.instance --token "$TOKEN" --apply +``` + +The token needs `admin:write:ip_blocks` and `admin:write:email_domain_blocks` +(the script's `--help` has a Rails-console snippet that mints one). Entries +already present come back `422` and are counted as "already blocked", not as +errors. `--min-signups N` skips one-off entries, `--ips-only`/`--domains-only` +take one list, `--limit` caps each. + +**On severity:** the IP entries are whole networks, because on yttrx a flagged +signup is almost always datacenter/VPN/proxy space where the next attempt comes +from a neighbouring address. That's calibrated to yttrx's traffic, not someone +else's, so the importer defaults to `sign_up_requires_approval` — a flagged +signup lands in the importer's moderation queue rather than being rejected — +even though yttrx runs these at `sign_up_block` itself. `--ip-severity +sign_up_block` opts into the harder behaviour. + **Off by default.** These paths are served *without authentication*, so they 404 unless `BLOCKS_API_ENABLED=true` is set — publishing which ranges and domains are already blocked also tells a would-be spam signup which ones to @@ -268,6 +316,7 @@ only once the block actually landed on the Mastodon side (a failed | `.env.example` | Config template (copy to `.env`, never commit) | | `nginx/hooks.yttrx.com.conf` | nginx site for admin.yttrx.com (webhook, healthz, blocklist API) | | `bin/welcomebot-logs`, `bin/welcomebot-signups`, `bin/welcomebot-suspicious` | Ops CLIs installed to `/root/bin` on admin.yttrx.com | +| `bin/yttrx-blocklist-import` | Importer other admins can run to pull this list into their own instance | | `bin/backfill-block-cidr.py` | One-shot: reconstruct `block_cidr` for rows predating it (run 2026-09-13) | | `test_local.py` | Offline smoke test (no network) | diff --git a/app/main.py b/app/main.py index cde485a..b512a3b 100644 --- a/app/main.py +++ b/app/main.py @@ -50,7 +50,7 @@ from datetime import datetime, timedelta, timezone import httpx from fastapi import BackgroundTasks, FastAPI, Header, HTTPException, Request -from fastapi.responses import JSONResponse +from fastapi.responses import JSONResponse, PlainTextResponse logging.basicConfig( level=logging.INFO, @@ -238,6 +238,11 @@ BLOCKS_API_ENABLED = os.environ.get("BLOCKS_API_ENABLED", "false").lower() in (" # asks for — this endpoint is unauthenticated, so the response size can't be # left to the caller. BLOCKS_API_MAX_ITEMS = int(os.environ.get("BLOCKS_API_MAX_ITEMS", "5000")) +# Public base URL this API is reached at, used only to print a "source:" line +# in the text format's header comments so a downloaded list says where it came +# from. Not used for routing. +BLOCKS_API_PUBLIC_URL = os.environ.get( + "BLOCKS_API_PUBLIC_URL", "https://hooks.yttrx.com").rstrip("/") # Cache-Control max-age (seconds) on blocklist responses. The underlying data # changes only when a signup is flagged, so letting clients and any # intermediate cache hold it for a few minutes costs nothing. @@ -1380,11 +1385,46 @@ def _blocks_guard(limit: int) -> int: return max(limit, 0) +def _blocks_headers() -> dict: + return { + "Cache-Control": f"public, max-age={BLOCKS_API_CACHE_SECONDS}", + # The list is public anyway, and other instances' admin tooling may + # want to read it from a browser context. + "Access-Control-Allow-Origin": "*", + } + + def _blocks_response(payload: dict) -> JSONResponse: - return JSONResponse( - payload, - headers={"Cache-Control": f"public, max-age={BLOCKS_API_CACHE_SECONDS}"}, - ) + return JSONResponse(payload, headers=_blocks_headers()) + + +def _blocks_text(items: list[dict], key: str, what: str, path: str) -> PlainTextResponse: + """Render one list as a plain-text blocklist: comment header, then one + entry per line. Blocklists are traded as text files, so this is the form + another admin's cron job can consume without a JSON parser — strip the + header with `grep -v '^#'`.""" + lines = [ + f"# yttrx.com — {what} automatically blocked at signup", + f"# source: {BLOCKS_API_PUBLIC_URL}{path}", + f"# generated: {_now_iso()}", + f"# entries: {len(items)}", + "#", + "# One entry per line; lines starting with '#' are comments. The JSON", + "# form of this URL adds first/last-seen timestamps and signup counts.", + ] + lines.extend(str(entry[key]) for entry in items) + return PlainTextResponse("\n".join(lines) + "\n", headers=_blocks_headers()) + + +def _blocks_format(value: str) -> str: + """Normalise the ?format= query param, rejecting anything unrecognised + rather than silently serving JSON.""" + fmt = (value or "json").strip().lower() + if fmt in ("txt", "text", "plain"): + return "txt" + if fmt == "json": + return "json" + raise HTTPException(status_code=400, detail="format must be 'json' or 'txt'") def _now_iso() -> str: @@ -1392,9 +1432,21 @@ def _now_iso() -> str: @app.get("/api/blocks") -def api_blocks(limit: int = 0): - """Both blocklists in one document.""" +def api_blocks(limit: int = 0, format: str = "json"): + """Both blocklists in one document. + + JSON only: a text rendering would put CIDRs and domains in one file, and + the whole point of the text format is that it pipes straight into a tool + that expects one kind of entry. Callers wanting text use the per-list + endpoints. + """ limit = _blocks_guard(limit) + if _blocks_format(format) == "txt": + raise HTTPException( + status_code=400, + detail="text format is per-list: use /api/blocks/ip-ranges?format=txt " + "or /api/blocks/email-domains?format=txt", + ) ip_ranges = blocked_ip_ranges(limit) email_domains = blocked_email_domains(limit) return _blocks_response({ @@ -1405,18 +1457,24 @@ def api_blocks(limit: int = 0): @app.get("/api/blocks/ip-ranges") -def api_blocks_ip_ranges(limit: int = 0): +def api_blocks_ip_ranges(limit: int = 0, format: str = "json"): limit = _blocks_guard(limit) + fmt = _blocks_format(format) items = blocked_ip_ranges(limit) + if fmt == "txt": + return _blocks_text(items, "cidr", "IP ranges", "/api/blocks/ip-ranges") return _blocks_response({ "generated_at": _now_iso(), "count": len(items), "items": items, }) @app.get("/api/blocks/email-domains") -def api_blocks_email_domains(limit: int = 0): +def api_blocks_email_domains(limit: int = 0, format: str = "json"): limit = _blocks_guard(limit) + fmt = _blocks_format(format) items = blocked_email_domains(limit) + if fmt == "txt": + return _blocks_text(items, "domain", "email domains", "/api/blocks/email-domains") return _blocks_response({ "generated_at": _now_iso(), "count": len(items), "items": items, }) diff --git a/bin/yttrx-blocklist-import b/bin/yttrx-blocklist-import new file mode 100755 index 0000000..153bd51 --- /dev/null +++ b/bin/yttrx-blocklist-import @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +"""Import yttrx.com's published signup blocklist into another Mastodon instance. + +Mastodon has no RBL/DNSBL support — nothing in it consults an outside list when +deciding whether to accept a signup (the only outbound lookup in that path is +EmailMxValidator's MX resolution, which feeds the *local* email_domain_blocks +table). So a shared blocklist has to be imported, not queried live. This script +does that: it fetches the published list and POSTs each entry to your own +instance's admin API. + +Standard library only — no pip install, no venv. + + # see what it would do (default) + ./yttrx-blocklist-import --instance https://your.instance --token "$TOKEN" + + # actually write the blocks + ./yttrx-blocklist-import --instance https://your.instance --token "$TOKEN" --apply + +Your token needs `admin:write:ip_blocks` and `admin:write:email_domain_blocks`. +Mint one from the Rails console on your instance: + + bin/rails runner ' + acct = Account.find_local("youradmin") + scopes = "admin:write:ip_blocks admin:write:email_domain_blocks" + app = Doorkeeper::Application.create!(name: "blocklist import", scopes: scopes, + redirect_uri: "urn:ietf:wg:oauth:2.0:oob") + puts Doorkeeper::AccessToken.create!(application_id: app.id, + resource_owner_id: acct.user.id, + scopes: scopes).token' + +A NOTE ON SEVERITY, worth reading before --apply: + +The IP entries are whole networks (the ASN route a flagged signup's address +belongs to), because on yttrx a flagged signup is almost always datacenter, +VPN or proxy space where the next attempt comes from a neighbouring address. +That is calibrated to yttrx's traffic, not yours. This script therefore +defaults to `sign_up_requires_approval` — a flagged signup lands in your +moderation queue rather than being rejected outright — even though yttrx runs +these at `sign_up_block` itself. Pass --ip-severity sign_up_block if you want +the harder behaviour; think about whether you have users behind a VPN first. + +Entries already present come back 422 from Mastodon and are counted as +"already blocked", not as errors. +""" + +from __future__ import annotations + +import argparse +import json +import sys +import urllib.error +import urllib.parse +import urllib.request + +DEFAULT_SOURCE = "https://hooks.yttrx.com/api/blocks" +IP_SEVERITIES = ("sign_up_requires_approval", "sign_up_block", "no_access") + + +def fetch_source(url: str, timeout: float) -> dict: + req = urllib.request.Request(url, headers={"Accept": "application/json"}) + with urllib.request.urlopen(req, timeout=timeout) as resp: + return json.loads(resp.read().decode("utf-8")) + + +def post(base: str, path: str, token: str, fields: dict, timeout: float) -> int: + """POST form-encoded fields; returns the HTTP status (422 included, which + Mastodon uses for 'this block already exists').""" + req = urllib.request.Request( + f"{base}{path}", + data=urllib.parse.urlencode(fields).encode(), + headers={"Authorization": f"Bearer {token}"}, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + return resp.status + except urllib.error.HTTPError as exc: + return exc.code + + +def import_entries(entries, label, base, path, token, field_fn, timeout, apply_changes): + created = existing = failed = 0 + for entry in entries: + fields = field_fn(entry) + shown = next(iter(fields.values())) + if not apply_changes: + print(f" would block {shown}") + created += 1 + continue + status = post(base, path, token, fields, timeout) + if status in (200, 201): + created += 1 + print(f" blocked {shown}") + elif status == 422: + existing += 1 + elif status in (401, 403): + print(f" ERROR {status} on {shown} — token is missing the required " + f"admin write scope; stopping.", file=sys.stderr) + failed += 1 + break + else: + failed += 1 + print(f" ERROR {status} on {shown}", file=sys.stderr) + verb = "would block" if not apply_changes else "blocked" + print(f"{label}: {created} {verb}, {existing} already blocked, {failed} failed") + return failed + + +def main() -> int: + ap = argparse.ArgumentParser( + description="Import yttrx.com's published signup blocklist into your Mastodon instance.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog="Runs as a dry run unless --apply is given.", + ) + ap.add_argument("--instance", required=True, + help="your instance's base URL, e.g. https://your.instance") + ap.add_argument("--token", required=True, + help="admin token with admin:write:ip_blocks + admin:write:email_domain_blocks") + ap.add_argument("--source", default=DEFAULT_SOURCE, + help=f"blocklist JSON to import (default: {DEFAULT_SOURCE})") + ap.add_argument("--ip-severity", default="sign_up_requires_approval", choices=IP_SEVERITIES, + help="severity for imported IP ranges (default: %(default)s — see the note " + "at the top of this script before choosing sign_up_block)") + ap.add_argument("--ips-only", action="store_true", help="import only the IP ranges") + ap.add_argument("--domains-only", action="store_true", help="import only the email domains") + ap.add_argument("--min-signups", type=int, default=1, + help="skip entries seen fewer than this many times (default: %(default)s)") + ap.add_argument("--limit", type=int, default=0, help="import at most N of each kind") + ap.add_argument("--timeout", type=float, default=30.0, help="HTTP timeout in seconds") + ap.add_argument("--apply", action="store_true", + help="actually write the blocks (without this, nothing is changed)") + args = ap.parse_args() + + if args.ips_only and args.domains_only: + ap.error("--ips-only and --domains-only are mutually exclusive") + + base = args.instance.rstrip("/") + try: + data = fetch_source(args.source, args.timeout) + except (urllib.error.URLError, ValueError) as exc: + print(f"could not fetch {args.source}: {exc}", file=sys.stderr) + return 1 + + def pick(section): + items = [e for e in data.get(section, {}).get("items", []) + if e.get("signups", 1) >= args.min_signups] + return items[:args.limit] if args.limit > 0 else items + + ip_ranges = [] if args.domains_only else pick("ip_ranges") + domains = [] if args.ips_only else pick("email_domains") + + print(f"source : {args.source}") + print(f"generated : {data.get('generated_at', 'unknown')}") + print(f"target : {base}") + print(f"mode : {'APPLY — writing blocks' if args.apply else 'dry run (pass --apply to write)'}") + print(f"ip ranges : {len(ip_ranges)} (severity {args.ip_severity})") + print(f"domains : {len(domains)}") + print() + + failed = 0 + if ip_ranges: + failed += import_entries( + ip_ranges, "IP ranges", base, "/api/v1/admin/ip_blocks", args.token, + lambda e: {"ip": e["cidr"], "severity": args.ip_severity, + "comment": f"imported from {args.source}"}, + args.timeout, args.apply) + if domains: + failed += import_entries( + domains, "email domains", base, "/api/v1/admin/email_domain_blocks", args.token, + lambda e: {"domain": e["domain"]}, + args.timeout, args.apply) + + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test_local.py b/test_local.py index 90a56ba..ecb8eaa 100644 --- a/test_local.py +++ b/test_local.py @@ -859,6 +859,39 @@ def blocks_api_tests(): assert len(client.get("/api/blocks/ip-ranges?limit=999").json()["items"]) == 1 main.BLOCKS_API_MAX_ITEMS = 5000 + # G. CORS is open — other instances' tooling may fetch this from a browser. + assert r.headers["access-control-allow-origin"] == "*" + + # H. ?format=txt renders one entry per line under a comment header, so it + # pipes into another admin's tooling without a JSON parser. + t = client.get("/api/blocks/ip-ranges?format=txt") + assert t.status_code == 200, t.status_code + assert t.headers["content-type"].startswith("text/plain"), t.headers["content-type"] + assert t.headers["access-control-allow-origin"] == "*" + lines = t.text.strip().split("\n") + header = [ln for ln in lines if ln.startswith("#")] + payload = [ln for ln in lines if not ln.startswith("#")] + assert any("source:" in ln for ln in header), header + assert f"entries: {len(ips_only['items'])}" in "\n".join(header) + assert payload == [e["cidr"] for e in ips_only["items"]], payload[:3] + + td = client.get("/api/blocks/email-domains?format=txt") + assert [ln for ln in td.text.strip().split("\n") if not ln.startswith("#")] \ + == [e["domain"] for e in domains_only["items"]] + + # I. An unknown format is rejected rather than quietly serving JSON, and + # the combined endpoint refuses text (it would mix CIDRs with domains). + assert client.get("/api/blocks/ip-ranges?format=yaml").status_code == 400 + assert client.get("/api/blocks?format=txt").status_code == 400 + assert client.get("/api/blocks?format=json").status_code == 200 + + # J. The format gate never leaks existence when the API is switched off. + main.BLOCKS_API_ENABLED = False + assert client.get("/api/blocks/ip-ranges?format=txt").status_code == 404 + assert client.get("/api/blocks?format=txt").status_code == 404 + main.BLOCKS_API_ENABLED = True + main.BLOCKS_API_MAX_ITEMS = 5000 + if __name__ == "__main__": routing_tests()