Replace RDAP/ipwhois signup-IP classifier with ipapi.is
Flags datacenter, vpn, proxy, tor, and independently-scored abuser signals instead of regex-matching RDAP org names; anonymous tier covers 1000 req/day, well above signup volume.
This commit is contained in:
+78
-54
@@ -7,8 +7,9 @@ and dispatches by event:
|
||||
account a welcome toot via the welcome bot's token (scope ``write:statuses``).
|
||||
Both events are handled so the welcome works whether or not registration
|
||||
approval is enabled; the dedup store welcomes each account exactly once.
|
||||
``account.created`` also classifies the signup's IP (datacenter/hosting via
|
||||
RDAP) and email domain (disposable/high-risk via check-mail.org); a flagged
|
||||
``account.created`` also classifies the signup's IP (datacenter/vpn/proxy/
|
||||
tor/abuser via ipapi.is) and email domain (disposable/high-risk via
|
||||
check-mail.org); a flagged
|
||||
signup gets more scrutiny (held welcome, auto ip_block/email_domain_block,
|
||||
moderator DM) before falling through to a normal welcome.
|
||||
* ``report.created`` — evaluates the reported account and, when enough
|
||||
@@ -35,7 +36,6 @@ import hmac
|
||||
import ipaddress
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import threading
|
||||
from contextlib import contextmanager
|
||||
@@ -43,7 +43,6 @@ from datetime import datetime, timedelta, timezone
|
||||
|
||||
import httpx
|
||||
from fastapi import BackgroundTasks, FastAPI, Header, HTTPException, Request
|
||||
from ipwhois import IPWhois
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
@@ -119,9 +118,14 @@ ABUSE_USER_DM = os.environ.get(
|
||||
|
||||
# --- IP-based signup scrutiny -----------------------------------------------
|
||||
# Classifies each new signup's IP (already delivered free on the
|
||||
# account.created payload as Admin::Account.ip) via RDAP org lookup, and
|
||||
# treats non-residential/non-mobile ("datacenter") signups with more scrutiny.
|
||||
# account.created payload as Admin::Account.ip) via ipapi.is (free, keyless,
|
||||
# up to 1000 req/day), and flags it if ipapi.is reports it as datacenter,
|
||||
# vpn, proxy, tor, or an independently-scored abuser.
|
||||
IP_SCRUTINY_ENABLED = os.environ.get("IP_SCRUTINY_ENABLED", "true").lower() in ("1", "true", "yes")
|
||||
# Optional ipapi.is API key for higher rate limits (anonymous is capped at
|
||||
# 1000 req/day, shared across all callers of that IP from this host). Empty
|
||||
# uses the anonymous tier.
|
||||
IP_SCRUTINY_IPAPI_KEY = os.environ.get("IP_SCRUTINY_IPAPI_KEY", "")
|
||||
# Log/DM only — no held welcome, no ip_blocks write. Rollout safety, same role
|
||||
# as ABUSE_DRY_RUN.
|
||||
IP_SCRUTINY_DRY_RUN = os.environ.get("IP_SCRUTINY_DRY_RUN", "true").lower() in ("1", "true", "yes")
|
||||
@@ -136,20 +140,6 @@ IP_SCRUTINY_ABUSE_THRESHOLD = int(os.environ.get("IP_SCRUTINY_ABUSE_THRESHOLD",
|
||||
IP_SCRUTINY_AUTO_IPBLOCK = os.environ.get("IP_SCRUTINY_AUTO_IPBLOCK", "true").lower() in ("1", "true", "yes")
|
||||
# severity: sign_up_requires_approval | sign_up_block | no_access
|
||||
IP_SCRUTINY_IPBLOCK_SEVERITY = os.environ.get("IP_SCRUTINY_IPBLOCK_SEVERITY", "sign_up_requires_approval")
|
||||
# Org-name keyword regexes (case-insensitive) used to classify the RDAP
|
||||
# asn_description/network name. Anything matching neither is "residential".
|
||||
IP_SCRUTINY_HOSTING_RE = re.compile(os.environ.get(
|
||||
"IP_SCRUTINY_HOSTING_RE",
|
||||
r"amazon|aws|google|microsoft|azure|digitalocean|ovh|hetzner|linode|vultr|"
|
||||
r"contabo|choopa|m247|scaleway|leaseweb|hostinger|namecheap|godaddy|"
|
||||
r"cloudflare|oracle|alibaba|tencent|akamai|fastly|packet|vpn|proxy|hosting|"
|
||||
r"datacenter|data center|colo(?:cation)?|server",
|
||||
), re.I)
|
||||
IP_SCRUTINY_MOBILE_RE = re.compile(os.environ.get(
|
||||
"IP_SCRUTINY_MOBILE_RE",
|
||||
r"mobile|wireless|cellular|verizon|t-mobile|tmobile|at&t|att mobility|"
|
||||
r"vodafone|telstra|sprint|three\b|orange mobile|deutsche telekom|o2\b",
|
||||
), re.I)
|
||||
|
||||
# --- Disposable/high-risk email signup scrutiny (check-mail.org) -----------
|
||||
# Every account.created delivery already carries the signup email for free
|
||||
@@ -245,10 +235,15 @@ def _init_db() -> None:
|
||||
")"
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS whois_cache ("
|
||||
" ip TEXT PRIMARY KEY,"
|
||||
" org TEXT,"
|
||||
" cached_at TEXT DEFAULT CURRENT_TIMESTAMP"
|
||||
"CREATE TABLE IF NOT EXISTS ipapi_cache ("
|
||||
" ip TEXT PRIMARY KEY,"
|
||||
" is_datacenter INTEGER,"
|
||||
" is_vpn INTEGER,"
|
||||
" is_proxy INTEGER,"
|
||||
" is_tor INTEGER,"
|
||||
" is_abuser INTEGER,"
|
||||
" org TEXT,"
|
||||
" cached_at TEXT DEFAULT CURRENT_TIMESTAMP"
|
||||
")"
|
||||
)
|
||||
conn.execute(
|
||||
@@ -360,19 +355,30 @@ def mark_ipblock_registered(account_id: str) -> None:
|
||||
)
|
||||
|
||||
|
||||
def cached_whois_org(ip: str) -> str | None:
|
||||
def cached_ip_intel(ip: str) -> dict | None:
|
||||
with _db() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT org FROM whois_cache WHERE ip = ?", (ip,)
|
||||
"SELECT is_datacenter, is_vpn, is_proxy, is_tor, is_abuser, org "
|
||||
"FROM ipapi_cache WHERE ip = ?", (ip,)
|
||||
).fetchone()
|
||||
return row[0] if row else None
|
||||
if row is None:
|
||||
return None
|
||||
return {
|
||||
"is_datacenter": bool(row[0]), "is_vpn": bool(row[1]),
|
||||
"is_proxy": bool(row[2]), "is_tor": bool(row[3]),
|
||||
"is_abuser": bool(row[4]), "org": row[5] or "",
|
||||
}
|
||||
|
||||
|
||||
def cache_whois_org(ip: str, org: str) -> None:
|
||||
def cache_ip_intel(ip: str, intel: dict) -> None:
|
||||
with _db() as conn:
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO whois_cache (ip, org) VALUES (?, ?)",
|
||||
(ip, org),
|
||||
"INSERT OR IGNORE INTO ipapi_cache "
|
||||
"(ip, is_datacenter, is_vpn, is_proxy, is_tor, is_abuser, org) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?)",
|
||||
(ip, int(intel["is_datacenter"]), int(intel["is_vpn"]),
|
||||
int(intel["is_proxy"]), int(intel["is_tor"]),
|
||||
int(intel["is_abuser"]), intel["org"]),
|
||||
)
|
||||
|
||||
|
||||
@@ -494,7 +500,7 @@ def classify_email_domain(domain: str) -> tuple[bool, int]:
|
||||
Cached indefinitely in sqlite (a domain's disposable/risk classification
|
||||
doesn't meaningfully change hour to hour, and the free tier is capped at
|
||||
1,000 req/month). API failure yields (False, 0) — never flag on our own
|
||||
lookup errors, same philosophy as classify_signup_ip's RDAP failure path.
|
||||
lookup errors, same philosophy as classify_signup_ip's ipapi.is failure path.
|
||||
"""
|
||||
cached = cached_check_mail(domain)
|
||||
if cached is not None:
|
||||
@@ -555,8 +561,7 @@ def process_signup(account_id: str, acct: str, ip: str, email: str = "") -> None
|
||||
hold = False
|
||||
|
||||
if IP_SCRUTINY_ENABLED and ip:
|
||||
classification, org = classify_signup_ip(ip)
|
||||
ip_flagged = classification == "datacenter"
|
||||
classification, org, ip_flagged = classify_signup_ip(ip)
|
||||
record_signup_ip(account_id, acct, ip, classification, org, ip_flagged)
|
||||
|
||||
if ip_flagged:
|
||||
@@ -566,7 +571,7 @@ def process_signup(account_id: str, acct: str, ip: str, email: str = "") -> None
|
||||
register_ip_block(ip, acct, org)
|
||||
mark_ipblock_registered(account_id)
|
||||
prefix = "[DRY-RUN] " if IP_SCRUTINY_DRY_RUN else ""
|
||||
reasons.append(f"{prefix}IP {ip} ({org or 'unknown org'}, datacenter/hosting)")
|
||||
reasons.append(f"{prefix}IP {ip} ({org or 'unknown org'}, {classification})")
|
||||
hold = hold or (IP_SCRUTINY_HOLD_WELCOME and not IP_SCRUTINY_DRY_RUN)
|
||||
|
||||
if CHECK_MAIL_ENABLED and CHECK_MAIL_API_KEY and email and "@" in email:
|
||||
@@ -727,30 +732,49 @@ def classify_account(target_id: str) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def classify_signup_ip(ip: str) -> tuple[str, str]:
|
||||
"""Classify a signup IP as 'datacenter' | 'mobile' | 'residential' | 'unknown'.
|
||||
def classify_signup_ip(ip: str) -> tuple[str, str, bool]:
|
||||
"""Classify a signup IP via ipapi.is, cached indefinitely in sqlite (an
|
||||
IP's owning org/abuse posture doesn't change on the timescale that
|
||||
matters here).
|
||||
|
||||
Looks up the owning org via RDAP (cached indefinitely in sqlite — an IP's
|
||||
*owning org* doesn't change on the timescale that matters here) and
|
||||
keyword-matches it against IP_SCRUTINY_HOSTING_RE / IP_SCRUTINY_MOBILE_RE.
|
||||
RDAP failure yields 'unknown', which is deliberately never flagged —
|
||||
scrutiny should never trigger on our own lookup errors.
|
||||
Returns (classification, org, flagged). classification is a "+"-joined
|
||||
list of every matched signal (datacenter/vpn/proxy/tor/abuser), or
|
||||
"clean" if none matched. flagged is True if any signal matched. API
|
||||
failure yields ("unknown", "", False) — scrutiny should never trigger on
|
||||
our own lookup errors.
|
||||
"""
|
||||
org = cached_whois_org(ip)
|
||||
if org is None:
|
||||
intel = cached_ip_intel(ip)
|
||||
if intel is None:
|
||||
params = {"q": ip}
|
||||
if IP_SCRUTINY_IPAPI_KEY:
|
||||
params["key"] = IP_SCRUTINY_IPAPI_KEY
|
||||
try:
|
||||
result = IPWhois(ip).lookup_rdap(depth=1)
|
||||
org = result.get("asn_description") or (result.get("network") or {}).get("name") or ""
|
||||
except Exception as exc: # noqa: BLE001 - any RDAP failure -> unknown, never raise
|
||||
log.warning("RDAP lookup failed for ip=%s: %s", ip, exc)
|
||||
return "unknown", ""
|
||||
cache_whois_org(ip, org)
|
||||
resp = httpx.get("https://api.ipapi.is", params=params, timeout=10.0)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
except (httpx.HTTPError, ValueError) as exc:
|
||||
log.warning("ipapi.is lookup failed for ip=%s: %s", ip, exc)
|
||||
return "unknown", "", False
|
||||
intel = {
|
||||
"is_datacenter": bool(data.get("is_datacenter")),
|
||||
"is_vpn": bool(data.get("is_vpn")),
|
||||
"is_proxy": bool(data.get("is_proxy")),
|
||||
"is_tor": bool(data.get("is_tor")),
|
||||
"is_abuser": bool(data.get("is_abuser")),
|
||||
"org": ((data.get("company") or {}).get("name")
|
||||
or (data.get("asn") or {}).get("org") or ""),
|
||||
}
|
||||
cache_ip_intel(ip, intel)
|
||||
|
||||
if IP_SCRUTINY_HOSTING_RE.search(org):
|
||||
return "datacenter", org
|
||||
if IP_SCRUTINY_MOBILE_RE.search(org):
|
||||
return "mobile", org
|
||||
return "residential", org
|
||||
reasons = [name for name, key in (
|
||||
("datacenter", "is_datacenter"),
|
||||
("vpn", "is_vpn"),
|
||||
("proxy", "is_proxy"),
|
||||
("tor", "is_tor"),
|
||||
("abuser", "is_abuser"),
|
||||
) if intel[key]]
|
||||
classification = "+".join(reasons) if reasons else "clean"
|
||||
return classification, intel["org"], bool(reasons)
|
||||
|
||||
|
||||
def register_ip_block(ip: str, acct: str, org: str) -> None:
|
||||
|
||||
Reference in New Issue
Block a user