Cache ipapi.is results by CIDR range instead of exact IP
docker-build-push / build-push (push) Successful in 5s
docker-build-push / build-push (push) Successful in 5s
Uses asn.route from ipapi.is to bucket the classification cache by network block (IPv4 and IPv6), so signups from the same datacenter/ASN block no longer each burn a separate lookup against the 1000/day quota. Falls back to a single-address range when route is missing or doesn't actually contain the queried IP.
This commit is contained in:
+52
-10
@@ -235,15 +235,19 @@ def _init_db() -> None:
|
|||||||
")"
|
")"
|
||||||
)
|
)
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"CREATE TABLE IF NOT EXISTS ipapi_cache ("
|
"CREATE TABLE IF NOT EXISTS ipapi_range_cache ("
|
||||||
" ip TEXT PRIMARY KEY,"
|
" family INTEGER NOT NULL," # 4 or 6
|
||||||
|
" range_start BLOB NOT NULL," # packed address bytes, fixed width per family
|
||||||
|
" range_end BLOB NOT NULL,"
|
||||||
|
" cidr TEXT," # human-readable, for inspection only
|
||||||
" is_datacenter INTEGER,"
|
" is_datacenter INTEGER,"
|
||||||
" is_vpn INTEGER,"
|
" is_vpn INTEGER,"
|
||||||
" is_proxy INTEGER,"
|
" is_proxy INTEGER,"
|
||||||
" is_tor INTEGER,"
|
" is_tor INTEGER,"
|
||||||
" is_abuser INTEGER,"
|
" is_abuser INTEGER,"
|
||||||
" org TEXT,"
|
" org TEXT,"
|
||||||
" cached_at TEXT DEFAULT CURRENT_TIMESTAMP"
|
" cached_at TEXT DEFAULT CURRENT_TIMESTAMP,"
|
||||||
|
" PRIMARY KEY (family, range_start, range_end)"
|
||||||
")"
|
")"
|
||||||
)
|
)
|
||||||
conn.execute(
|
conn.execute(
|
||||||
@@ -355,11 +359,42 @@ def mark_ipblock_registered(account_id: str) -> None:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _ip_range_bounds(ip: str, route: str = "") -> tuple[int, bytes, bytes, str] | None:
|
||||||
|
"""Compute the (family, range_start, range_end, cidr) to cache ip's
|
||||||
|
classification under: route (ipapi.is's asn.route CIDR) when it's a
|
||||||
|
valid network that actually contains ip, else a single-address range.
|
||||||
|
bytes are fixed-width packed addresses (4 bytes for v4, 16 for v6), so
|
||||||
|
lexicographic BLOB comparison in sqlite matches numeric address order.
|
||||||
|
Returns None if ip itself doesn't parse.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
addr = ipaddress.ip_address(ip)
|
||||||
|
except ValueError:
|
||||||
|
log.warning("skipping ipapi range cache lookup for unparseable ip=%r", ip)
|
||||||
|
return None
|
||||||
|
if route:
|
||||||
|
try:
|
||||||
|
net = ipaddress.ip_network(route, strict=False)
|
||||||
|
except ValueError:
|
||||||
|
net = None
|
||||||
|
if net is not None and net.version == addr.version and addr in net:
|
||||||
|
return (net.version, net.network_address.packed, net.broadcast_address.packed,
|
||||||
|
f"{net.network_address}/{net.prefixlen}")
|
||||||
|
prefix_len = 32 if addr.version == 4 else 128
|
||||||
|
return addr.version, addr.packed, addr.packed, f"{addr}/{prefix_len}"
|
||||||
|
|
||||||
|
|
||||||
def cached_ip_intel(ip: str) -> dict | None:
|
def cached_ip_intel(ip: str) -> dict | None:
|
||||||
|
bounds = _ip_range_bounds(ip)
|
||||||
|
if bounds is None:
|
||||||
|
return None
|
||||||
|
family, packed, _, _ = bounds
|
||||||
with _db() as conn:
|
with _db() as conn:
|
||||||
row = conn.execute(
|
row = conn.execute(
|
||||||
"SELECT is_datacenter, is_vpn, is_proxy, is_tor, is_abuser, org "
|
"SELECT is_datacenter, is_vpn, is_proxy, is_tor, is_abuser, org "
|
||||||
"FROM ipapi_cache WHERE ip = ?", (ip,)
|
"FROM ipapi_range_cache "
|
||||||
|
"WHERE family = ? AND range_start <= ? AND range_end >= ? LIMIT 1",
|
||||||
|
(family, packed, packed),
|
||||||
).fetchone()
|
).fetchone()
|
||||||
if row is None:
|
if row is None:
|
||||||
return None
|
return None
|
||||||
@@ -370,13 +405,19 @@ def cached_ip_intel(ip: str) -> dict | None:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def cache_ip_intel(ip: str, intel: dict) -> None:
|
def cache_ip_intel(ip: str, intel: dict, route: str = "") -> None:
|
||||||
|
bounds = _ip_range_bounds(ip, route)
|
||||||
|
if bounds is None:
|
||||||
|
return
|
||||||
|
family, range_start, range_end, cidr = bounds
|
||||||
with _db() as conn:
|
with _db() as conn:
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"INSERT OR IGNORE INTO ipapi_cache "
|
"INSERT OR IGNORE INTO ipapi_range_cache "
|
||||||
"(ip, is_datacenter, is_vpn, is_proxy, is_tor, is_abuser, org) "
|
"(family, range_start, range_end, cidr, is_datacenter, is_vpn, "
|
||||||
"VALUES (?, ?, ?, ?, ?, ?, ?)",
|
" is_proxy, is_tor, is_abuser, org) "
|
||||||
(ip, int(intel["is_datacenter"]), int(intel["is_vpn"]),
|
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||||
|
(family, range_start, range_end, cidr,
|
||||||
|
int(intel["is_datacenter"]), int(intel["is_vpn"]),
|
||||||
int(intel["is_proxy"]), int(intel["is_tor"]),
|
int(intel["is_proxy"]), int(intel["is_tor"]),
|
||||||
int(intel["is_abuser"]), intel["org"]),
|
int(intel["is_abuser"]), intel["org"]),
|
||||||
)
|
)
|
||||||
@@ -783,7 +824,8 @@ def classify_signup_ip(ip: str) -> tuple[str, str, bool]:
|
|||||||
"org": ((data.get("company") or {}).get("name")
|
"org": ((data.get("company") or {}).get("name")
|
||||||
or (data.get("asn") or {}).get("org") or ""),
|
or (data.get("asn") or {}).get("org") or ""),
|
||||||
}
|
}
|
||||||
cache_ip_intel(ip, intel)
|
route = (data.get("asn") or {}).get("route") or ""
|
||||||
|
cache_ip_intel(ip, intel, route)
|
||||||
|
|
||||||
reasons = [name for name, key in (
|
reasons = [name for name, key in (
|
||||||
("datacenter", "is_datacenter"),
|
("datacenter", "is_datacenter"),
|
||||||
|
|||||||
Reference in New Issue
Block a user