Cache ipapi.is results by CIDR range instead of exact IP
docker-build-push / build-push (push) Successful in 5s
docker-build-push / build-push (push) Successful in 5s
Uses asn.route from ipapi.is to bucket the classification cache by network block (IPv4 and IPv6), so signups from the same datacenter/ASN block no longer each burn a separate lookup against the 1000/day quota. Falls back to a single-address range when route is missing or doesn't actually contain the queried IP.
This commit is contained in:
+52
-10
@@ -235,15 +235,19 @@ def _init_db() -> None:
|
||||
")"
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS ipapi_cache ("
|
||||
" ip TEXT PRIMARY KEY,"
|
||||
"CREATE TABLE IF NOT EXISTS ipapi_range_cache ("
|
||||
" family INTEGER NOT NULL," # 4 or 6
|
||||
" range_start BLOB NOT NULL," # packed address bytes, fixed width per family
|
||||
" range_end BLOB NOT NULL,"
|
||||
" cidr TEXT," # human-readable, for inspection only
|
||||
" is_datacenter INTEGER,"
|
||||
" is_vpn INTEGER,"
|
||||
" is_proxy INTEGER,"
|
||||
" is_tor INTEGER,"
|
||||
" is_abuser INTEGER,"
|
||||
" org TEXT,"
|
||||
" cached_at TEXT DEFAULT CURRENT_TIMESTAMP"
|
||||
" cached_at TEXT DEFAULT CURRENT_TIMESTAMP,"
|
||||
" PRIMARY KEY (family, range_start, range_end)"
|
||||
")"
|
||||
)
|
||||
conn.execute(
|
||||
@@ -355,11 +359,42 @@ def mark_ipblock_registered(account_id: str) -> None:
|
||||
)
|
||||
|
||||
|
||||
def _ip_range_bounds(ip: str, route: str = "") -> tuple[int, bytes, bytes, str] | None:
|
||||
"""Compute the (family, range_start, range_end, cidr) to cache ip's
|
||||
classification under: route (ipapi.is's asn.route CIDR) when it's a
|
||||
valid network that actually contains ip, else a single-address range.
|
||||
bytes are fixed-width packed addresses (4 bytes for v4, 16 for v6), so
|
||||
lexicographic BLOB comparison in sqlite matches numeric address order.
|
||||
Returns None if ip itself doesn't parse.
|
||||
"""
|
||||
try:
|
||||
addr = ipaddress.ip_address(ip)
|
||||
except ValueError:
|
||||
log.warning("skipping ipapi range cache lookup for unparseable ip=%r", ip)
|
||||
return None
|
||||
if route:
|
||||
try:
|
||||
net = ipaddress.ip_network(route, strict=False)
|
||||
except ValueError:
|
||||
net = None
|
||||
if net is not None and net.version == addr.version and addr in net:
|
||||
return (net.version, net.network_address.packed, net.broadcast_address.packed,
|
||||
f"{net.network_address}/{net.prefixlen}")
|
||||
prefix_len = 32 if addr.version == 4 else 128
|
||||
return addr.version, addr.packed, addr.packed, f"{addr}/{prefix_len}"
|
||||
|
||||
|
||||
def cached_ip_intel(ip: str) -> dict | None:
|
||||
bounds = _ip_range_bounds(ip)
|
||||
if bounds is None:
|
||||
return None
|
||||
family, packed, _, _ = bounds
|
||||
with _db() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT is_datacenter, is_vpn, is_proxy, is_tor, is_abuser, org "
|
||||
"FROM ipapi_cache WHERE ip = ?", (ip,)
|
||||
"FROM ipapi_range_cache "
|
||||
"WHERE family = ? AND range_start <= ? AND range_end >= ? LIMIT 1",
|
||||
(family, packed, packed),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
@@ -370,13 +405,19 @@ def cached_ip_intel(ip: str) -> dict | None:
|
||||
}
|
||||
|
||||
|
||||
def cache_ip_intel(ip: str, intel: dict) -> None:
|
||||
def cache_ip_intel(ip: str, intel: dict, route: str = "") -> None:
|
||||
bounds = _ip_range_bounds(ip, route)
|
||||
if bounds is None:
|
||||
return
|
||||
family, range_start, range_end, cidr = bounds
|
||||
with _db() as conn:
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO ipapi_cache "
|
||||
"(ip, is_datacenter, is_vpn, is_proxy, is_tor, is_abuser, org) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?)",
|
||||
(ip, int(intel["is_datacenter"]), int(intel["is_vpn"]),
|
||||
"INSERT OR IGNORE INTO ipapi_range_cache "
|
||||
"(family, range_start, range_end, cidr, is_datacenter, is_vpn, "
|
||||
" is_proxy, is_tor, is_abuser, org) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(family, range_start, range_end, cidr,
|
||||
int(intel["is_datacenter"]), int(intel["is_vpn"]),
|
||||
int(intel["is_proxy"]), int(intel["is_tor"]),
|
||||
int(intel["is_abuser"]), intel["org"]),
|
||||
)
|
||||
@@ -783,7 +824,8 @@ def classify_signup_ip(ip: str) -> tuple[str, str, bool]:
|
||||
"org": ((data.get("company") or {}).get("name")
|
||||
or (data.get("asn") or {}).get("org") or ""),
|
||||
}
|
||||
cache_ip_intel(ip, intel)
|
||||
route = (data.get("asn") or {}).get("route") or ""
|
||||
cache_ip_intel(ip, intel, route)
|
||||
|
||||
reasons = [name for name, key in (
|
||||
("datacenter", "is_datacenter"),
|
||||
|
||||
Reference in New Issue
Block a user