From a7208a84d1eaee08398299ec7a3ae7a84dd08614 Mon Sep 17 00:00:00 2001 From: waffle2k Date: Mon, 13 Jul 2026 08:35:55 -0700 Subject: [PATCH] Cache ipapi.is results by CIDR range instead of exact IP Uses asn.route from ipapi.is to bucket the classification cache by network block (IPv4 and IPv6), so signups from the same datacenter/ASN block no longer each burn a separate lookup against the 1000/day quota. Falls back to a single-address range when route is missing or doesn't actually contain the queried IP. --- app/main.py | 62 ++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 52 insertions(+), 10 deletions(-) diff --git a/app/main.py b/app/main.py index c4da1db..d5596b6 100644 --- a/app/main.py +++ b/app/main.py @@ -235,15 +235,19 @@ def _init_db() -> None: ")" ) conn.execute( - "CREATE TABLE IF NOT EXISTS ipapi_cache (" - " ip TEXT PRIMARY KEY," + "CREATE TABLE IF NOT EXISTS ipapi_range_cache (" + " family INTEGER NOT NULL," # 4 or 6 + " range_start BLOB NOT NULL," # packed address bytes, fixed width per family + " range_end BLOB NOT NULL," + " cidr TEXT," # human-readable, for inspection only " is_datacenter INTEGER," " is_vpn INTEGER," " is_proxy INTEGER," " is_tor INTEGER," " is_abuser INTEGER," " org TEXT," - " cached_at TEXT DEFAULT CURRENT_TIMESTAMP" + " cached_at TEXT DEFAULT CURRENT_TIMESTAMP," + " PRIMARY KEY (family, range_start, range_end)" ")" ) conn.execute( @@ -355,11 +359,42 @@ def mark_ipblock_registered(account_id: str) -> None: ) +def _ip_range_bounds(ip: str, route: str = "") -> tuple[int, bytes, bytes, str] | None: + """Compute the (family, range_start, range_end, cidr) to cache ip's + classification under: route (ipapi.is's asn.route CIDR) when it's a + valid network that actually contains ip, else a single-address range. + bytes are fixed-width packed addresses (4 bytes for v4, 16 for v6), so + lexicographic BLOB comparison in sqlite matches numeric address order. + Returns None if ip itself doesn't parse. + """ + try: + addr = ipaddress.ip_address(ip) + except ValueError: + log.warning("skipping ipapi range cache lookup for unparseable ip=%r", ip) + return None + if route: + try: + net = ipaddress.ip_network(route, strict=False) + except ValueError: + net = None + if net is not None and net.version == addr.version and addr in net: + return (net.version, net.network_address.packed, net.broadcast_address.packed, + f"{net.network_address}/{net.prefixlen}") + prefix_len = 32 if addr.version == 4 else 128 + return addr.version, addr.packed, addr.packed, f"{addr}/{prefix_len}" + + def cached_ip_intel(ip: str) -> dict | None: + bounds = _ip_range_bounds(ip) + if bounds is None: + return None + family, packed, _, _ = bounds with _db() as conn: row = conn.execute( "SELECT is_datacenter, is_vpn, is_proxy, is_tor, is_abuser, org " - "FROM ipapi_cache WHERE ip = ?", (ip,) + "FROM ipapi_range_cache " + "WHERE family = ? AND range_start <= ? AND range_end >= ? LIMIT 1", + (family, packed, packed), ).fetchone() if row is None: return None @@ -370,13 +405,19 @@ def cached_ip_intel(ip: str) -> dict | None: } -def cache_ip_intel(ip: str, intel: dict) -> None: +def cache_ip_intel(ip: str, intel: dict, route: str = "") -> None: + bounds = _ip_range_bounds(ip, route) + if bounds is None: + return + family, range_start, range_end, cidr = bounds with _db() as conn: conn.execute( - "INSERT OR IGNORE INTO ipapi_cache " - "(ip, is_datacenter, is_vpn, is_proxy, is_tor, is_abuser, org) " - "VALUES (?, ?, ?, ?, ?, ?, ?)", - (ip, int(intel["is_datacenter"]), int(intel["is_vpn"]), + "INSERT OR IGNORE INTO ipapi_range_cache " + "(family, range_start, range_end, cidr, is_datacenter, is_vpn, " + " is_proxy, is_tor, is_abuser, org) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + (family, range_start, range_end, cidr, + int(intel["is_datacenter"]), int(intel["is_vpn"]), int(intel["is_proxy"]), int(intel["is_tor"]), int(intel["is_abuser"]), intel["org"]), ) @@ -783,7 +824,8 @@ def classify_signup_ip(ip: str) -> tuple[str, str, bool]: "org": ((data.get("company") or {}).get("name") or (data.get("asn") or {}).get("org") or ""), } - cache_ip_intel(ip, intel) + route = (data.get("asn") or {}).get("route") or "" + cache_ip_intel(ip, intel, route) reasons = [name for name, key in ( ("datacenter", "is_datacenter"),