diff --git a/api/intelligence.py b/api/intelligence.py index b0fd526..f31de78 100644 --- a/api/intelligence.py +++ b/api/intelligence.py @@ -8,6 +8,7 @@ from __future__ import annotations import html import ipaddress +import os import re import socket import ssl @@ -23,7 +24,11 @@ EMAIL_RE = re.compile(r"[A-Z0-9._%+\-]+@[A-Z0-9.\-]+\.[A-Z]{2,63}", re.I) MAX_BODY_BYTES = 700_000 MAX_REDIRECTS = 2 MAX_PAGES = 2 -TIMEOUT_SECONDS = 8 +TIMEOUT_SECONDS = max(2, min(30, int(os.environ.get("WEBSITE_SCAN_TIMEOUT", "8")))) +# getaddrinfo has no per-call timeout parameter on all supported platforms. +# Set a bounded process default so a broken DNS server cannot hold a discovery +# worker forever during website analysis. +socket.setdefaulttimeout(TIMEOUT_SECONDS) class _PageParser(HTMLParser): diff --git a/api/server.py b/api/server.py index 2bd201b..2fd6dcc 100644 --- a/api/server.py +++ b/api/server.py @@ -802,13 +802,21 @@ def insert_catalogue(run_id: int, query: str, raw_results: list[dict[str, Any]], suppressed_emails = {str(row[0]).lower() for row in conn.execute("SELECT value FROM suppressions WHERE kind='email'")} total = len(mapped) for index, item in enumerate(mapped, 1): - item = enrich_place(item) + # A hostile or slow public site must not invalidate an otherwise + # successful Maps run. Keep the raw listing and mark enrichment as + # unknown; it can be rescanned later from the explorer. + try: + item = enrich_place(item) + except Exception as exc: + item.update({"website_status": "unknown", "website_checked_at": now(), "website_error": "analysis_failed:" + str(exc)[:240], "email_evidence": item.get("email_evidence", "[]"), "score_reasons": "[]"}) + item["opportunity_score"] = score_opportunity(item)[0] if progress: progress(index, total) emails = set(normalize_public_emails(item.get("emails"))) if item["domain"].lower() in suppressed_domains or emails.intersection(suppressed_emails): continue - cur = conn.execute("""INSERT OR IGNORE INTO prospects( + try: + cur = conn.execute("""INSERT OR IGNORE INTO prospects( name,website,domain,description,query,discovered_at,source,source_id, category,address,phone,emails,rating,review_count,maps_url,latitude, longitude,listing_status,website_status,website_http_status,website_final_url, @@ -825,7 +833,11 @@ def insert_catalogue(run_id: int, query: str, raw_results: list[dict[str, Any]], item.get("website_tls_expires_at", ""), item.get("website_checked_at", ""), item.get("website_error", ""), item.get("email_evidence", "[]"), item.get("score_reasons", "[]"), None, "", "", "", "", "", "", item["opportunity_score"], item["raw_json"], - )) + )) + except (sqlite3.Error, TypeError, ValueError): + # Keep processing the remaining listings if one malformed + # provider row cannot be written. + continue if cur.rowcount: added += 1 conn.commit()