add conservative domain intelligence
This commit is contained in:
Marco0300
@@ -0,0 +1,174 @@
|
||||
"""Conservative, dependency-free domain intelligence helpers.
|
||||
|
||||
This module deliberately reports uncertainty rather than inferring DNS or registrar facts.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import re
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
import unicodedata
|
||||
from datetime import datetime, timezone
|
||||
from difflib import SequenceMatcher
|
||||
from urllib.parse import urlparse
|
||||
|
||||
# Small explicitly maintained PSL subset. Unknown suffixes are never guessed.
|
||||
PUBLIC_SUFFIXES = frozenset({
|
||||
"com", "org", "net", "za", "co.za", "org.za", "net.za", "ac.za", "gov.za", "edu.za",
|
||||
})
|
||||
DEFAULT_TTL_SECONDS = 300
|
||||
MAX_DOMAIN_LENGTH = 253
|
||||
MAX_CANDIDATES = 20
|
||||
|
||||
|
||||
def _host(value: str | None) -> str:
|
||||
raw = str(value or "").strip().lower()
|
||||
if not raw:
|
||||
return ""
|
||||
parsed = urlparse(raw if "://" in raw else "//" + raw)
|
||||
host = (parsed.hostname or "").rstrip(".").lower()
|
||||
if host.startswith("www."):
|
||||
host = host[4:]
|
||||
return host
|
||||
|
||||
|
||||
def _valid_hostname(host: str) -> bool:
|
||||
if not host or len(host) > MAX_DOMAIN_LENGTH or "." not in host:
|
||||
return False
|
||||
try:
|
||||
ipaddress.ip_address(host)
|
||||
return False
|
||||
except ValueError:
|
||||
pass
|
||||
if any(len(label) > 63 or not re.fullmatch(r"[a-z0-9](?:[a-z0-9-]*[a-z0-9])?", label) for label in host.split(".")):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def normalize_registrable_domain(value: str | None) -> str:
|
||||
"""Return the registrable domain, or ``unknown`` for unsupported/unsafe input."""
|
||||
host = _host(value)
|
||||
if not _valid_hostname(host):
|
||||
return "unknown"
|
||||
labels = host.split(".")
|
||||
suffix = next((".".join(labels[-n:]) for n in (3, 2, 1) if ".".join(labels[-n:]) in PUBLIC_SUFFIXES), None)
|
||||
if not suffix or len(labels) <= suffix.count(".") + 1:
|
||||
return "unknown"
|
||||
return ".".join(labels[-(suffix.count(".") + 2):])
|
||||
|
||||
|
||||
def domain_info(value: str | None) -> dict:
|
||||
host = _host(value)
|
||||
registered = normalize_registrable_domain(value)
|
||||
return {"input": str(value or ""), "hostname": host, "registrable_domain": registered,
|
||||
"status": "ok" if registered != "unknown" else "unknown"}
|
||||
|
||||
|
||||
def _metadata(ttl_seconds: int = DEFAULT_TTL_SECONDS) -> dict:
|
||||
now = time.time()
|
||||
checked = datetime.fromtimestamp(now, timezone.utc).replace(microsecond=0).isoformat()
|
||||
return {"checked_at": checked, "ttl_seconds": ttl_seconds,
|
||||
"cache_expires_at": datetime.fromtimestamp(now + ttl_seconds, timezone.utc).replace(microsecond=0).isoformat()}
|
||||
|
||||
|
||||
def resolve_domain(domain: str, *, timeout: float = 3.0, resolver=None, ttl_seconds: int = DEFAULT_TTL_SECONDS) -> dict:
|
||||
"""Resolve only a validated hostname using stdlib socket; never fetches a URL."""
|
||||
normalized = normalize_registrable_domain(domain)
|
||||
host = _host(domain)
|
||||
result = {"domain": normalized, "hostname": host, "status": "unknown", "addresses": [], **_metadata(ttl_seconds)}
|
||||
if normalized == "unknown" or not _valid_hostname(host):
|
||||
return result
|
||||
resolver = resolver or socket.getaddrinfo
|
||||
try:
|
||||
# getaddrinfo has no per-call timeout. Run it in a daemon worker so a
|
||||
# resolver stall cannot block an HTTP handler indefinitely.
|
||||
outcome = {}
|
||||
def lookup():
|
||||
try: outcome["records"] = resolver(host, None, 0, socket.SOCK_STREAM)
|
||||
except BaseException as exc: outcome["exception"] = exc
|
||||
worker = threading.Thread(target=lookup, daemon=True)
|
||||
worker.start(); worker.join(max(0.01, float(timeout)))
|
||||
if worker.is_alive():
|
||||
result["status"] = "timeout"
|
||||
return result
|
||||
if "exception" in outcome: raise outcome["exception"]
|
||||
records = outcome.get("records", [])
|
||||
addresses = sorted({str(item[4][0]) for item in records if item and len(item) > 4 and item[0] in (socket.AF_INET, socket.AF_INET6)})
|
||||
result["addresses"] = addresses
|
||||
result["status"] = "ok" if addresses else "unknown"
|
||||
except socket.gaierror as exc:
|
||||
code = getattr(exc, "errno", None)
|
||||
if code is None and exc.args: code = exc.args[0]
|
||||
result["status"] = "nxdomain" if code in (socket.EAI_NONAME, socket.EAI_NODATA, -2, -5) else "error"
|
||||
result["error_code"] = code
|
||||
except (TimeoutError, socket.timeout):
|
||||
result["status"] = "timeout"
|
||||
except OSError as exc:
|
||||
result["status"] = "error"; result["error_code"] = exc.__class__.__name__
|
||||
return result
|
||||
|
||||
|
||||
def capability_hook(domain: str, capability: str, *, resolver=None, ttl_seconds: int = DEFAULT_TTL_SECONDS) -> dict:
|
||||
"""MX/NS/TXT hook. No optional resolver means not_configured, not empty."""
|
||||
normalized = normalize_registrable_domain(domain)
|
||||
base = {"domain": normalized, "capability": capability, "status": "unknown", "records": [], **_metadata(ttl_seconds)}
|
||||
if normalized == "unknown": return base
|
||||
if capability not in {"mx", "ns", "txt"}: base.update(status="error", error_code="unsupported_capability"); return base
|
||||
if resolver is None:
|
||||
base.update(status="not_configured", reason="resolver_not_configured")
|
||||
return base
|
||||
try:
|
||||
records = resolver(normalized, capability)
|
||||
base.update(status="ok" if records else "unknown", records=list(records or []))
|
||||
except TimeoutError: base["status"] = "timeout"
|
||||
except socket.gaierror: base["status"] = "nxdomain"
|
||||
except Exception as exc: base.update(status="error", error_code=exc.__class__.__name__)
|
||||
return base
|
||||
|
||||
|
||||
def resolve_mx(domain: str, *, resolver=None, ttl_seconds: int = DEFAULT_TTL_SECONDS) -> dict:
|
||||
return capability_hook(domain, "mx", resolver=resolver, ttl_seconds=ttl_seconds)
|
||||
|
||||
|
||||
def resolve_ns(domain: str, *, resolver=None, ttl_seconds: int = DEFAULT_TTL_SECONDS) -> dict:
|
||||
return capability_hook(domain, "ns", resolver=resolver, ttl_seconds=ttl_seconds)
|
||||
|
||||
|
||||
def resolve_txt(domain: str, *, resolver=None, ttl_seconds: int = DEFAULT_TTL_SECONDS) -> dict:
|
||||
return capability_hook(domain, "txt", resolver=resolver, ttl_seconds=ttl_seconds)
|
||||
|
||||
|
||||
registrable_domain = normalize_registrable_domain
|
||||
|
||||
|
||||
def association_confidence(candidate_domain: str, observed_domain: str, *, business_name: str = "") -> dict:
|
||||
candidate = normalize_registrable_domain(candidate_domain); observed = normalize_registrable_domain(observed_domain)
|
||||
if candidate == "unknown" or observed == "unknown": return {"level":"unknown", "score":0.0, "reasons":["unsupported_domain"]}
|
||||
if candidate == observed: return {"level":"high", "score":1.0, "reasons":["same_registrable_domain"]}
|
||||
token = re.sub(r"[^a-z0-9]", "", unicodedata.normalize("NFKD", str(business_name).lower()))
|
||||
score = SequenceMatcher(None, token, candidate.split(".")[0]).ratio() if token else 0.0
|
||||
return {"level":"medium" if score >= .8 else "low", "score":round(score, 4), "reasons":["name_domain_similarity"] if score >= .8 else ["different_registrable_domain"]}
|
||||
|
||||
|
||||
def _slug(value: object) -> str:
|
||||
text = unicodedata.normalize("NFKD", str(value or "")).encode("ascii", "ignore").decode().lower()
|
||||
words = re.findall(r"[a-z0-9]+", text)
|
||||
return "-".join(words)[:40].strip("-")
|
||||
|
||||
|
||||
def generate_candidate_domains(business_name: str, service: str = "", location: str = "", *, suffixes=("co.za", "com"), limit: int = MAX_CANDIDATES) -> list[str]:
|
||||
limit = max(0, min(int(limit), MAX_CANDIDATES)); name, svc, loc = _slug(business_name), _slug(service), _slug(location)
|
||||
if not name: return []
|
||||
parts = [name, f"{name}-{svc}" if svc else "", f"{name}-{loc}" if loc else "", f"{svc}-{loc}" if svc and loc else ""]
|
||||
if svc: parts += [f"{name}{svc}"]
|
||||
out=[]
|
||||
for base in parts:
|
||||
if not base or len(base) > 63: continue
|
||||
for suffix in suffixes:
|
||||
if suffix not in PUBLIC_SUFFIXES: continue
|
||||
domain = f"{base}.{suffix}"
|
||||
if normalize_registrable_domain(domain) != "unknown" and domain not in out: out.append(domain)
|
||||
if len(out) >= limit: return out
|
||||
return out
|
||||
+68
-1
@@ -9,15 +9,17 @@ if __package__ in (None, ""):
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from app.domain import deduplication_key, deduplicate_businesses, is_suppressed, normalize_business, score_business, normalize_domain, normalize_phone, match_businesses
|
||||
from app.sources import adapter_for, contains_secret
|
||||
from app.domain_intelligence import normalize_registrable_domain, resolve_domain, generate_candidate_domains
|
||||
else:
|
||||
from .domain import deduplication_key, deduplicate_businesses, is_suppressed, normalize_business, score_business, normalize_domain, normalize_phone, match_businesses
|
||||
from .sources import adapter_for, contains_secret
|
||||
from .domain_intelligence import normalize_registrable_domain, resolve_domain, generate_candidate_domains
|
||||
ORGANIZATION_ID = "demo-tenant"
|
||||
SCHEMA = Path(__file__).resolve().parents[1] / "schema.sql"
|
||||
SESSION_DAYS = 7
|
||||
PBKDF2_ITERATIONS = 300_000
|
||||
MUTATING_ROLES = {"owner", "admin", "researcher"}
|
||||
JOB_TYPES = {"noop", "prospect_recalculate", "source_discovery"}
|
||||
JOB_TYPES = {"noop", "prospect_recalculate", "source_discovery", "domain_check"}
|
||||
JOB_PAGE_SIZE = 100
|
||||
SECRET_KEYS = {"password", "passwd", "secret", "token", "api_key", "apikey", "authorization", "credential", "private_key"}
|
||||
CHILD_TABLES = {"contacts": ("name", "email", "phone", "title", "do_not_contact"), "domains": ("domain", "kind"), "websites": ("url", "website_class"), "evidence": ("kind", "url", "claim"), "notes": ("body",)}
|
||||
@@ -117,16 +119,79 @@ class ApiHandler(BaseHTTPRequestHandler):
|
||||
if path=="/api/v1/discovery-queries": return self.list_queries(db,org)
|
||||
if path=="/api/v1/source-records": return self.list_source_records(db,org,parse_qs(parsed.query))
|
||||
if path=="/api/v1/jobs": return self.list_jobs(db,org,parse_qs(parsed.query))
|
||||
if path=="/api/v1/domain-checks": return self.list_domain_checks(db,org,parse_qs(parsed.query))
|
||||
if path.startswith("/api/v1/jobs/"): return self.get_job_route(db,org,path,parse_qs(parsed.query))
|
||||
if path.startswith("/api/v1/businesses/"):
|
||||
bits=path.split("/"); ident=bits[4] if len(bits)>4 else ""
|
||||
if not ident.isdigit(): return self.send_json(404,{"error":"not_found"})
|
||||
row=self.business(db,int(ident),org)
|
||||
if not row:return self.send_json(404,{"error":"not_found"})
|
||||
if len(bits)==7 and bits[5:]==["domains","check"]: return self.get_domain_check(int(ident),db,user,parse_qs(parsed.query))
|
||||
if len(bits)==6 and bits[5]=="domain-candidates": return self.list_domain_candidates(int(ident),db,org)
|
||||
if len(bits)==6 and bits[5]=="matches": return self.matches(int(ident),db,org)
|
||||
payload=row_json(row); payload.update(self.nested(db,int(ident),org)); return self.send_json(200,payload)
|
||||
return self.send_json(404,{"error":"not_found"})
|
||||
finally: db.close()
|
||||
def _domain_result(self, row, cache_hit=False):
|
||||
try: result=json.loads(row["result_json"] or "{}")
|
||||
except (TypeError,ValueError): result={}
|
||||
result.update({"id":row["id"],"business_id":row["business_id"],"domain":row["domain"],"status":row["status"],"cache_hit":cache_hit,"checked_at":row["checked_at"],"cache_expires_at":row["cache_expires_at"]})
|
||||
return result
|
||||
|
||||
def _check_domain(self, bid, domain, db, user):
|
||||
normalized=normalize_registrable_domain(domain)
|
||||
if normalized == "unknown": return self.send_json(400,{"error":"unsupported_domain","status":"unknown"})
|
||||
now=datetime.now(timezone.utc).replace(microsecond=0).isoformat()
|
||||
cache_key=normalized+":a_aaaa:v1"
|
||||
cached=db.execute("SELECT * FROM domain_checks WHERE organization_id=? AND business_id=? AND cache_key=? AND cache_expires_at>? ORDER BY id DESC LIMIT 1",(user["organization_id"],bid,cache_key,now)).fetchone()
|
||||
if cached:return self.send_json(200,self._domain_result(cached,True))
|
||||
result=resolve_domain(normalized)
|
||||
try: cur=db.execute("INSERT INTO domain_checks(organization_id,business_id,domain,status,result_json,cache_key,checked_at,cache_expires_at) VALUES(?,?,?,?,?,?,?,?)",(user["organization_id"],bid,normalized,result["status"],json.dumps(result,sort_keys=True),cache_key,result["checked_at"],result["cache_expires_at"]))
|
||||
except sqlite3.IntegrityError:
|
||||
existing=db.execute("SELECT id FROM domain_checks WHERE organization_id=? AND business_id=? AND cache_key=?",(user["organization_id"],bid,cache_key)).fetchone()
|
||||
if not existing: return self.send_json(409,{"error":"domain_check_conflict"})
|
||||
db.execute("UPDATE domain_checks SET domain=?,status=?,result_json=?,checked_at=?,cache_expires_at=? WHERE id=?",(normalized,result["status"],json.dumps(result,sort_keys=True),result["checked_at"],result["cache_expires_at"],existing["id"]))
|
||||
self.audit(db,user,"domain.checked",f"{bid}:{normalized}:{result['status']}"); db.commit()
|
||||
return self.send_json(200,self._domain_result(db.execute("SELECT * FROM domain_checks WHERE id=?",(existing["id"],)).fetchone()))
|
||||
self.audit(db,user,"domain.checked",f"{bid}:{normalized}:{result['status']}"); db.commit()
|
||||
return self.send_json(200,self._domain_result(db.execute("SELECT * FROM domain_checks WHERE id=?",(cur.lastrowid,)).fetchone()))
|
||||
|
||||
def post_domain_check(self,bid,payload,db,user):
|
||||
if not self.business(db,bid,user["organization_id"]): return self.send_json(404,{"error":"not_found"})
|
||||
domain=payload.get("domain") or self.business(db,bid,user["organization_id"])["website_domain"]
|
||||
if not str(domain).strip(): return self.send_json(400,{"error":"domain_required"})
|
||||
return self._check_domain(bid,str(domain),db,user)
|
||||
|
||||
def get_domain_check(self,bid,db,user,query):
|
||||
domain=(query.get("domain") or [""])[0]
|
||||
if not domain:return self.send_json(400,{"error":"domain_required"})
|
||||
return self._check_domain(bid,domain,db,user)
|
||||
|
||||
def list_domain_checks(self,db,org,query):
|
||||
try: limit=max(1,min(int((query.get("page_size") or [50])[0]),100)); offset=max(0,int((query.get("offset") or [0])[0]))
|
||||
except (ValueError,TypeError): return self.send_json(400,{"error":"invalid_pagination"})
|
||||
rows=db.execute("SELECT * FROM domain_checks WHERE organization_id=? ORDER BY id DESC LIMIT ? OFFSET ?",(org,limit+1,offset)).fetchall()
|
||||
return self.send_json(200,{"organization_id":org,"items":[self._domain_result(r) for r in rows[:limit]],"limit":limit,"offset":offset,"has_more":len(rows)>limit})
|
||||
|
||||
def list_domain_candidates(self,bid,db,org):
|
||||
business=self.business(db,bid,org)
|
||||
if not business:return self.send_json(404,{"error":"not_found"})
|
||||
rows=db.execute("SELECT * FROM domain_candidates WHERE organization_id=? AND business_id=? ORDER BY rank,id",(org,bid)).fetchall()
|
||||
if not rows:
|
||||
generated=generate_candidate_domains(business["name"],business["description"],business["city"])
|
||||
for rank,domain in enumerate(generated,1):
|
||||
db.execute("INSERT OR IGNORE INTO domain_candidates(organization_id,business_id,domain,rank) VALUES(?,?,?,?)",(org,bid,domain,rank))
|
||||
db.commit(); rows=db.execute("SELECT * FROM domain_candidates WHERE organization_id=? AND business_id=? ORDER BY rank,id",(org,bid)).fetchall()
|
||||
return self.send_json(200,{"business_id":bid,"items":[row_json(r) for r in rows]})
|
||||
|
||||
def check_availability(self,bid,payload,db,user):
|
||||
if not self.business(db,bid,user["organization_id"]):return self.send_json(404,{"error":"not_found"})
|
||||
candidates=db.execute("SELECT domain FROM domain_candidates WHERE organization_id=? AND business_id=? ORDER BY rank,id",(user["organization_id"],bid)).fetchall()
|
||||
domains=[r["domain"] for r in candidates]
|
||||
if isinstance(payload.get("domains"),list): domains=[normalize_registrable_domain(x) for x in payload["domains"] if normalize_registrable_domain(x)!="unknown"][:20]
|
||||
self.audit(db,user,"domain.availability.checked",str(bid)); db.commit()
|
||||
return self.send_json(200,{"business_id":bid,"status":"unknown","reason":"not_configured","provider_configured":False,"items":[{"domain":d,"status":"unknown","reason":"not_configured"} for d in domains]})
|
||||
|
||||
def list_jobs(self, db, org, query):
|
||||
try: limit=max(1,min(int(query.get("page_size",[50])[0]),JOB_PAGE_SIZE)); offset=max(0,int(query.get("offset",[0])[0]))
|
||||
except (ValueError, TypeError): return self.send_json(400,{"error":"invalid_pagination"})
|
||||
@@ -221,6 +286,8 @@ class ApiHandler(BaseHTTPRequestHandler):
|
||||
if path=="/api/v1/discovery-queries":return self.create_query(payload,db,user)
|
||||
if path=="/api/v1/suppressions":return self.create_suppression(payload,db,user)
|
||||
if path=="/api/v1/imports/preview":return self.preview_import(payload,db,org)
|
||||
if len(path.split("/"))==7 and path.split("/")[3:6]==["businesses",path.split("/")[4],"domains"] and path.split("/")[6]=="check": return self.post_domain_check(int(path.split("/")[4]) if path.split("/")[4].isdigit() else -1,payload,db,user)
|
||||
if len(path.split("/"))==7 and path.split("/")[3:6]==["businesses",path.split("/")[4],"domain-candidates"] and path.split("/")[6]=="check-availability": return self.check_availability(int(path.split("/")[4]) if path.split("/")[4].isdigit() else -1,payload,db,user)
|
||||
if path.startswith("/api/v1/merge-history/") and path.endswith("/reverse"):
|
||||
ident=path.split("/")[4]
|
||||
return self.reverse_merge(int(ident) if ident.isdigit() else -1,db,user)
|
||||
|
||||
Reference in New Issue
Block a user