From 0356fe7997f27231c05d384479413f5458569913 Mon Sep 17 00:00:00 2001 From: waffle2k Date: Mon, 6 Jul 2026 08:57:55 -0700 Subject: [PATCH] Replace RDAP/ipwhois signup-IP classifier with ipapi.is Flags datacenter, vpn, proxy, tor, and independently-scored abuser signals instead of regex-matching RDAP org names; anonymous tier covers 1000 req/day, well above signup volume. --- .env.example | 20 +++---- CLAUDE.md | 7 ++- README.md | 19 ++++--- app/main.py | 132 ++++++++++++++++++++++++++------------------ bin/welcomebot-logs | 4 +- requirements.txt | 1 - test_local.py | 6 +- 7 files changed, 106 insertions(+), 83 deletions(-) diff --git a/.env.example b/.env.example index cf8b037..dcac911 100644 --- a/.env.example +++ b/.env.example @@ -80,11 +80,16 @@ ABUSE_USER_DM=Hi @{acct} — your yttrx account has been temporarily limited whi # --- IP-based signup scrutiny ----------------------------------------------- # Classifies each new signup's IP (Admin::Account.ip, delivered free on the -# account.created webhook) via RDAP org lookup, and treats non-residential/ -# non-mobile ("datacenter"/hosting/VPN) signups with more scrutiny. Master -# switch: +# account.created webhook) via ipapi.is, and flags it if ipapi.is reports it +# as any of datacenter, vpn, proxy, tor, or an independently-scored abuser. +# Master switch: IP_SCRUTINY_ENABLED=true +# Optional ipapi.is API key for higher rate limits. Anonymous (blank) is +# capped at 1,000 req/day, which comfortably covers yttrx's signup volume; +# get a key at https://ipapi.is/ only if you expect to exceed that. +IP_SCRUTINY_IPAPI_KEY= + # Rollout safety: when true, classify + DM a moderator but take NO action # (no held welcome, no ip_blocks write). Recommended for the first days in # production; flip to false once you've reviewed the false-positive rate. @@ -97,7 +102,7 @@ IP_SCRUTINY_HOLD_WELCOME=true # Distinct-reporter threshold used instead of the tier's usual # ABUSE_SOURCES_* threshold (whichever is lower) when the reported account's -# signup IP was flagged as datacenter/hosting. +# signup IP was flagged. IP_SCRUTINY_ABUSE_THRESHOLD=1 # Auto-register a flagged IP into Mastodon's native Admin::IpBlock. Requires @@ -111,13 +116,6 @@ IP_SCRUTINY_AUTO_IPBLOCK=true # or no_access (blocks all access, not just signups). IP_SCRUTINY_IPBLOCK_SEVERITY=sign_up_requires_approval -# Case-insensitive regexes matched against the RDAP org/ASN description. -# Anything matching neither is treated as "residential" (never flagged). -# Only IP_SCRUTINY_HOSTING_RE matches are flagged; IP_SCRUTINY_MOBILE_RE is -# informational only (mobile carriers are never scrutinized). -IP_SCRUTINY_HOSTING_RE=amazon|aws|google|microsoft|azure|digitalocean|ovh|hetzner|linode|vultr|contabo|choopa|m247|scaleway|leaseweb|hostinger|namecheap|godaddy|cloudflare|oracle|alibaba|tencent|akamai|fastly|packet|vpn|proxy|hosting|datacenter|data center|colo(?:cation)?|server -IP_SCRUTINY_MOBILE_RE=mobile|wireless|cellular|verizon|t-mobile|tmobile|at&t|att mobility|vodafone|telstra|sprint|three\b|orange mobile|deutsche telekom|o2\b - # check-mail.org disposable/high-risk email domain scrutiny (roadmap item B, # anti-abuse.md). Domain-only query (never the full email) against # POST https://api.check-mail.org/v2/, Authorization: Bearer . diff --git a/CLAUDE.md b/CLAUDE.md index 5d780f3..9291df2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,9 +8,10 @@ Guidance for Claude Code when working on **yttrx-welcomebot** (the welcome bot A FastAPI webhook server. One Mastodon admin webhook delivers `account.created`, `account.approved`, and `report.created` to `POST /webhook`; the app dispatches: -- **account.created** → classify the signup IP (RDAP org lookup); flagged - (datacenter/hosting) signups get more scrutiny (held welcome, ip_blocks - registration, DM to moderator) before falling through to a normal welcome. +- **account.created** → classify the signup IP via ipapi.is; flagged + (datacenter/vpn/proxy/tor/abuser) signups get more scrutiny (held welcome, + ip_blocks registration, DM to moderator) before falling through to a normal + welcome. - **account.approved** → always DM the welcome (dedup-safe; this is what releases a held welcome once a human clears the approval queue). Also where a held, flagged signup's suspicious-watch baseline gets captured. diff --git a/README.md b/README.md index c7cad57..3f95c21 100644 --- a/README.md +++ b/README.md @@ -61,11 +61,12 @@ without touching any account — keep it on until you've watched it for a while. Every `account.created` delivery already carries the signup IP for free (`Admin::Account.ip`). On each new local signup, the bot: -1. **Classifies the IP** via RDAP org lookup (`ipwhois`, cached in sqlite) as - `datacenter` (hosting/VPN-keyword match), `mobile` (carrier-keyword match, - informational only), or `residential` (everything else — never flagged). - RDAP failures classify as `unknown` and are never flagged. -2. If **`datacenter`**, the signup is treated with more scrutiny: +1. **Classifies the IP** via [ipapi.is](https://ipapi.is/) (cached in sqlite; + free, keyless tier is 1,000 req/day — `IP_SCRUTINY_IPAPI_KEY` raises the + limit if needed) as any combination of `datacenter`, `vpn`, `proxy`, `tor`, + and `abuser` (ipapi.is's own abuse score), joined with `+`, or `clean` if + none matched. API failures classify as `unknown` and are never flagged. +2. If **any signal matched**, the signup is treated with more scrutiny: - **Moderator DM** with the IP, org, and classification. - **Held welcome** (`IP_SCRUTINY_HOLD_WELCOME`) — the welcome DM is skipped on `account.created` and only sent when `account.approved` fires, i.e. @@ -130,8 +131,8 @@ A scheduled companion to the two signup-scrutiny signals above (`app/suspicious_ run via `docker exec yttrx-welcomebot python -m app.suspicious_sweep`, e.g. hourly cron on `admin.yttrx.com`): -1. Any account flagged by **either** IP-scrutiny (datacenter/hosting) or - email-domain scrutiny (disposable/high-risk) has a grace-period clock +1. Any account flagged by **either** IP-scrutiny (datacenter/vpn/proxy/tor/ + abuser) or email-domain scrutiny (disposable/high-risk) has a grace-period clock started the moment it goes live — `account.created` if signups are open (or auto-approved), `account.approved` if this instance requires moderator approval. Same dual-event handling the welcome flow already @@ -140,7 +141,7 @@ run via `docker exec yttrx-welcomebot python -m app.suspicious_sweep`, e.g. hour that moment (`app.main.maybe_start_suspicious_watch`, table `suspicious_watch`). 2. The sweep looks for watches past `SUSPICIOUS_GRACE_HOURS` (default 1; - lowered from the original 24 on 2026-07-06 — a datacenter/hosting-IP + lowered from the original 24 on 2026-07-06 — a flagged-IP signup is already a strong enough spam signal that a full day of grace was mostly just delaying an inevitable suspend) and compares current counts to the baseline: @@ -206,12 +207,12 @@ Copy `.env.example` to `.env` and fill in: | `ABUSE_HELP_URL` | Appeals/help page the silenced user is linked to | | `ABUSE_USER_DM` | Template DMed to the silenced user (`{acct}`, `{help_url}`); blank disables | | `IP_SCRUTINY_ENABLED` | Master switch for IP-based signup scrutiny | +| `IP_SCRUTINY_IPAPI_KEY` | Optional ipapi.is API key for higher rate limits; blank uses the anonymous 1,000 req/day tier | | `IP_SCRUTINY_DRY_RUN` | `true` — classify + DM only, no held welcome, no ip_block write | | `IP_SCRUTINY_HOLD_WELCOME` | `true` — hold the welcome for a flagged signup until `account.approved` | | `IP_SCRUTINY_ABUSE_THRESHOLD` | Distinct-reporter threshold used (if lower) for accounts with a flagged signup IP | | `IP_SCRUTINY_AUTO_IPBLOCK` | Auto-register a flagged IP into Mastodon's `Admin::IpBlock` | | `IP_SCRUTINY_IPBLOCK_SEVERITY` | `sign_up_requires_approval` (default), `sign_up_block`, or `no_access` | -| `IP_SCRUTINY_HOSTING_RE` / `IP_SCRUTINY_MOBILE_RE` | Keyword regexes matched against the RDAP org/ASN description | | `CHECK_MAIL_ENABLED` | Master switch for disposable/high-risk email signup scrutiny | | `CHECK_MAIL_API_KEY` | check-mail.org API key; blank disables the check | | `CHECK_MAIL_DRY_RUN` | `true` — classify + DM only, no held welcome, no email_domain_block write, no report-triggered suspend | diff --git a/app/main.py b/app/main.py index 2273e78..8107f1e 100644 --- a/app/main.py +++ b/app/main.py @@ -7,8 +7,9 @@ and dispatches by event: account a welcome toot via the welcome bot's token (scope ``write:statuses``). Both events are handled so the welcome works whether or not registration approval is enabled; the dedup store welcomes each account exactly once. - ``account.created`` also classifies the signup's IP (datacenter/hosting via - RDAP) and email domain (disposable/high-risk via check-mail.org); a flagged + ``account.created`` also classifies the signup's IP (datacenter/vpn/proxy/ + tor/abuser via ipapi.is) and email domain (disposable/high-risk via + check-mail.org); a flagged signup gets more scrutiny (held welcome, auto ip_block/email_domain_block, moderator DM) before falling through to a normal welcome. * ``report.created`` — evaluates the reported account and, when enough @@ -35,7 +36,6 @@ import hmac import ipaddress import logging import os -import re import sqlite3 import threading from contextlib import contextmanager @@ -43,7 +43,6 @@ from datetime import datetime, timedelta, timezone import httpx from fastapi import BackgroundTasks, FastAPI, Header, HTTPException, Request -from ipwhois import IPWhois logging.basicConfig( level=logging.INFO, @@ -119,9 +118,14 @@ ABUSE_USER_DM = os.environ.get( # --- IP-based signup scrutiny ----------------------------------------------- # Classifies each new signup's IP (already delivered free on the -# account.created payload as Admin::Account.ip) via RDAP org lookup, and -# treats non-residential/non-mobile ("datacenter") signups with more scrutiny. +# account.created payload as Admin::Account.ip) via ipapi.is (free, keyless, +# up to 1000 req/day), and flags it if ipapi.is reports it as datacenter, +# vpn, proxy, tor, or an independently-scored abuser. IP_SCRUTINY_ENABLED = os.environ.get("IP_SCRUTINY_ENABLED", "true").lower() in ("1", "true", "yes") +# Optional ipapi.is API key for higher rate limits (anonymous is capped at +# 1000 req/day, shared across all callers of that IP from this host). Empty +# uses the anonymous tier. +IP_SCRUTINY_IPAPI_KEY = os.environ.get("IP_SCRUTINY_IPAPI_KEY", "") # Log/DM only — no held welcome, no ip_blocks write. Rollout safety, same role # as ABUSE_DRY_RUN. IP_SCRUTINY_DRY_RUN = os.environ.get("IP_SCRUTINY_DRY_RUN", "true").lower() in ("1", "true", "yes") @@ -136,20 +140,6 @@ IP_SCRUTINY_ABUSE_THRESHOLD = int(os.environ.get("IP_SCRUTINY_ABUSE_THRESHOLD", IP_SCRUTINY_AUTO_IPBLOCK = os.environ.get("IP_SCRUTINY_AUTO_IPBLOCK", "true").lower() in ("1", "true", "yes") # severity: sign_up_requires_approval | sign_up_block | no_access IP_SCRUTINY_IPBLOCK_SEVERITY = os.environ.get("IP_SCRUTINY_IPBLOCK_SEVERITY", "sign_up_requires_approval") -# Org-name keyword regexes (case-insensitive) used to classify the RDAP -# asn_description/network name. Anything matching neither is "residential". -IP_SCRUTINY_HOSTING_RE = re.compile(os.environ.get( - "IP_SCRUTINY_HOSTING_RE", - r"amazon|aws|google|microsoft|azure|digitalocean|ovh|hetzner|linode|vultr|" - r"contabo|choopa|m247|scaleway|leaseweb|hostinger|namecheap|godaddy|" - r"cloudflare|oracle|alibaba|tencent|akamai|fastly|packet|vpn|proxy|hosting|" - r"datacenter|data center|colo(?:cation)?|server", -), re.I) -IP_SCRUTINY_MOBILE_RE = re.compile(os.environ.get( - "IP_SCRUTINY_MOBILE_RE", - r"mobile|wireless|cellular|verizon|t-mobile|tmobile|at&t|att mobility|" - r"vodafone|telstra|sprint|three\b|orange mobile|deutsche telekom|o2\b", -), re.I) # --- Disposable/high-risk email signup scrutiny (check-mail.org) ----------- # Every account.created delivery already carries the signup email for free @@ -245,10 +235,15 @@ def _init_db() -> None: ")" ) conn.execute( - "CREATE TABLE IF NOT EXISTS whois_cache (" - " ip TEXT PRIMARY KEY," - " org TEXT," - " cached_at TEXT DEFAULT CURRENT_TIMESTAMP" + "CREATE TABLE IF NOT EXISTS ipapi_cache (" + " ip TEXT PRIMARY KEY," + " is_datacenter INTEGER," + " is_vpn INTEGER," + " is_proxy INTEGER," + " is_tor INTEGER," + " is_abuser INTEGER," + " org TEXT," + " cached_at TEXT DEFAULT CURRENT_TIMESTAMP" ")" ) conn.execute( @@ -360,19 +355,30 @@ def mark_ipblock_registered(account_id: str) -> None: ) -def cached_whois_org(ip: str) -> str | None: +def cached_ip_intel(ip: str) -> dict | None: with _db() as conn: row = conn.execute( - "SELECT org FROM whois_cache WHERE ip = ?", (ip,) + "SELECT is_datacenter, is_vpn, is_proxy, is_tor, is_abuser, org " + "FROM ipapi_cache WHERE ip = ?", (ip,) ).fetchone() - return row[0] if row else None + if row is None: + return None + return { + "is_datacenter": bool(row[0]), "is_vpn": bool(row[1]), + "is_proxy": bool(row[2]), "is_tor": bool(row[3]), + "is_abuser": bool(row[4]), "org": row[5] or "", + } -def cache_whois_org(ip: str, org: str) -> None: +def cache_ip_intel(ip: str, intel: dict) -> None: with _db() as conn: conn.execute( - "INSERT OR IGNORE INTO whois_cache (ip, org) VALUES (?, ?)", - (ip, org), + "INSERT OR IGNORE INTO ipapi_cache " + "(ip, is_datacenter, is_vpn, is_proxy, is_tor, is_abuser, org) " + "VALUES (?, ?, ?, ?, ?, ?, ?)", + (ip, int(intel["is_datacenter"]), int(intel["is_vpn"]), + int(intel["is_proxy"]), int(intel["is_tor"]), + int(intel["is_abuser"]), intel["org"]), ) @@ -494,7 +500,7 @@ def classify_email_domain(domain: str) -> tuple[bool, int]: Cached indefinitely in sqlite (a domain's disposable/risk classification doesn't meaningfully change hour to hour, and the free tier is capped at 1,000 req/month). API failure yields (False, 0) — never flag on our own - lookup errors, same philosophy as classify_signup_ip's RDAP failure path. + lookup errors, same philosophy as classify_signup_ip's ipapi.is failure path. """ cached = cached_check_mail(domain) if cached is not None: @@ -555,8 +561,7 @@ def process_signup(account_id: str, acct: str, ip: str, email: str = "") -> None hold = False if IP_SCRUTINY_ENABLED and ip: - classification, org = classify_signup_ip(ip) - ip_flagged = classification == "datacenter" + classification, org, ip_flagged = classify_signup_ip(ip) record_signup_ip(account_id, acct, ip, classification, org, ip_flagged) if ip_flagged: @@ -566,7 +571,7 @@ def process_signup(account_id: str, acct: str, ip: str, email: str = "") -> None register_ip_block(ip, acct, org) mark_ipblock_registered(account_id) prefix = "[DRY-RUN] " if IP_SCRUTINY_DRY_RUN else "" - reasons.append(f"{prefix}IP {ip} ({org or 'unknown org'}, datacenter/hosting)") + reasons.append(f"{prefix}IP {ip} ({org or 'unknown org'}, {classification})") hold = hold or (IP_SCRUTINY_HOLD_WELCOME and not IP_SCRUTINY_DRY_RUN) if CHECK_MAIL_ENABLED and CHECK_MAIL_API_KEY and email and "@" in email: @@ -727,30 +732,49 @@ def classify_account(target_id: str) -> dict: } -def classify_signup_ip(ip: str) -> tuple[str, str]: - """Classify a signup IP as 'datacenter' | 'mobile' | 'residential' | 'unknown'. +def classify_signup_ip(ip: str) -> tuple[str, str, bool]: + """Classify a signup IP via ipapi.is, cached indefinitely in sqlite (an + IP's owning org/abuse posture doesn't change on the timescale that + matters here). - Looks up the owning org via RDAP (cached indefinitely in sqlite — an IP's - *owning org* doesn't change on the timescale that matters here) and - keyword-matches it against IP_SCRUTINY_HOSTING_RE / IP_SCRUTINY_MOBILE_RE. - RDAP failure yields 'unknown', which is deliberately never flagged — - scrutiny should never trigger on our own lookup errors. + Returns (classification, org, flagged). classification is a "+"-joined + list of every matched signal (datacenter/vpn/proxy/tor/abuser), or + "clean" if none matched. flagged is True if any signal matched. API + failure yields ("unknown", "", False) — scrutiny should never trigger on + our own lookup errors. """ - org = cached_whois_org(ip) - if org is None: + intel = cached_ip_intel(ip) + if intel is None: + params = {"q": ip} + if IP_SCRUTINY_IPAPI_KEY: + params["key"] = IP_SCRUTINY_IPAPI_KEY try: - result = IPWhois(ip).lookup_rdap(depth=1) - org = result.get("asn_description") or (result.get("network") or {}).get("name") or "" - except Exception as exc: # noqa: BLE001 - any RDAP failure -> unknown, never raise - log.warning("RDAP lookup failed for ip=%s: %s", ip, exc) - return "unknown", "" - cache_whois_org(ip, org) + resp = httpx.get("https://api.ipapi.is", params=params, timeout=10.0) + resp.raise_for_status() + data = resp.json() + except (httpx.HTTPError, ValueError) as exc: + log.warning("ipapi.is lookup failed for ip=%s: %s", ip, exc) + return "unknown", "", False + intel = { + "is_datacenter": bool(data.get("is_datacenter")), + "is_vpn": bool(data.get("is_vpn")), + "is_proxy": bool(data.get("is_proxy")), + "is_tor": bool(data.get("is_tor")), + "is_abuser": bool(data.get("is_abuser")), + "org": ((data.get("company") or {}).get("name") + or (data.get("asn") or {}).get("org") or ""), + } + cache_ip_intel(ip, intel) - if IP_SCRUTINY_HOSTING_RE.search(org): - return "datacenter", org - if IP_SCRUTINY_MOBILE_RE.search(org): - return "mobile", org - return "residential", org + reasons = [name for name, key in ( + ("datacenter", "is_datacenter"), + ("vpn", "is_vpn"), + ("proxy", "is_proxy"), + ("tor", "is_tor"), + ("abuser", "is_abuser"), + ) if intel[key]] + classification = "+".join(reasons) if reasons else "clean" + return classification, intel["org"], bool(reasons) def register_ip_block(ip: str, acct: str, org: str) -> None: diff --git a/bin/welcomebot-logs b/bin/welcomebot-logs index b54ee6d..a298732 100644 --- a/bin/welcomebot-logs +++ b/bin/welcomebot-logs @@ -24,7 +24,7 @@ Options: --since WHEN Only logs newer than WHEN (e.g. 1h, 30m, 2026-06-17, 2026-06-17T20:00). --abuse Only abuse-handler lines (reports, silences, dry-run, tiers). --welcome Only welcome lines. - --ip Only IP-scrutiny lines (classifications, ip_block writes, RDAP errors). + --ip Only IP-scrutiny lines (classifications, ip_block writes, ipapi.is errors). -g, --grep PATTERN Only lines matching PATTERN (extended regex). -h, --help Show this help. @@ -44,7 +44,7 @@ while [ $# -gt 0 ]; do --since) shift; since="${1:?--since needs a value}" ;; --abuse) pattern='welcomebot (report |auto-|\[DRY-RUN\]|tier=|silenc|holds a staff|allowlisted)' ;; --welcome) pattern='welcomebot (welcomed|already welcomed|skipping non-local|queued)' ;; - --ip) pattern='welcomebot (flagged signup|RDAP lookup failed|ip_block|unparseable ip)' ;; + --ip) pattern='welcomebot (flagged signup|ipapi\.is lookup failed|ip_block|unparseable ip)' ;; -g|--grep) shift; pattern="${1:?--grep needs a value}" ;; -h|--help) usage; exit 0 ;; *) echo "welcomebot-logs: unknown option: $1" >&2; usage >&2; exit 2 ;; diff --git a/requirements.txt b/requirements.txt index 584941f..457a82e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ fastapi==0.115.6 uvicorn[standard]==0.34.0 httpx==0.28.1 -ipwhois==1.3.0 diff --git a/test_local.py b/test_local.py index 7e89358..45e3caf 100644 --- a/test_local.py +++ b/test_local.py @@ -233,9 +233,9 @@ def ip_scrutiny_tests(): return classifications[ip] classifications = { - "203.0.113.10": ("residential", "Example Residential ISP"), - "198.51.100.20": ("datacenter", "Example Cloud Hosting Inc"), - "198.51.100.21": ("datacenter", "Example Cloud Hosting Inc"), + "203.0.113.10": ("clean", "Example Residential ISP", False), + "198.51.100.20": ("datacenter", "Example Cloud Hosting Inc", True), + "198.51.100.21": ("datacenter", "Example Cloud Hosting Inc", True), } main.classify_signup_ip = classify