deploy Maps v13 update

api/intelligence.py

@@ -8,6 +8,7 @@ from __future__ import annotations
 
 import html
 import ipaddress
+import os
 import re
 import socket
 import ssl
@@ -23,7 +24,11 @@ EMAIL_RE = re.compile(r"[A-Z0-9._%+\-]+@[A-Z0-9.\-]+\.[A-Z]{2,63}", re.I)
 MAX_BODY_BYTES = 700_000
 MAX_REDIRECTS = 2
 MAX_PAGES = 2
-TIMEOUT_SECONDS = 8
+TIMEOUT_SECONDS = max(2, min(30, int(os.environ.get("WEBSITE_SCAN_TIMEOUT", "8"))))
+# getaddrinfo has no per-call timeout parameter on all supported platforms.
+# Set a bounded process default so a broken DNS server cannot hold a discovery
+# worker forever during website analysis.
+socket.setdefaulttimeout(TIMEOUT_SECONDS)
 
 
 class _PageParser(HTMLParser):

api/server.py

@@ -802,13 +802,21 @@ def insert_catalogue(run_id: int, query: str, raw_results: list[dict[str, Any]],
         suppressed_emails = {str(row[0]).lower() for row in conn.execute("SELECT value FROM suppressions WHERE kind='email'")}
         total = len(mapped)
         for index, item in enumerate(mapped, 1):
-            item = enrich_place(item)
+            # A hostile or slow public site must not invalidate an otherwise
+            # successful Maps run. Keep the raw listing and mark enrichment as
+            # unknown; it can be rescanned later from the explorer.
+            try:
+                item = enrich_place(item)
+            except Exception as exc:
+                item.update({"website_status": "unknown", "website_checked_at": now(), "website_error": "analysis_failed:" + str(exc)[:240], "email_evidence": item.get("email_evidence", "[]"), "score_reasons": "[]"})
+                item["opportunity_score"] = score_opportunity(item)[0]
             if progress:
                 progress(index, total)
             emails = set(normalize_public_emails(item.get("emails")))
             if item["domain"].lower() in suppressed_domains or emails.intersection(suppressed_emails):
                 continue
-            cur = conn.execute("""INSERT OR IGNORE INTO prospects(
+            try:
+                cur = conn.execute("""INSERT OR IGNORE INTO prospects(
                 name,website,domain,description,query,discovered_at,source,source_id,
                 category,address,phone,emails,rating,review_count,maps_url,latitude,
                 longitude,listing_status,website_status,website_http_status,website_final_url,
@@ -825,7 +833,11 @@ def insert_catalogue(run_id: int, query: str, raw_results: list[dict[str, Any]],
                 item.get("website_tls_expires_at", ""), item.get("website_checked_at", ""), item.get("website_error", ""),
                 item.get("email_evidence", "[]"), item.get("score_reasons", "[]"), None, "", "", "", "", "", "",
                 item["opportunity_score"], item["raw_json"],
-            ))
+                ))
+            except (sqlite3.Error, TypeError, ValueError):
+                # Keep processing the remaining listings if one malformed
+                # provider row cannot be written.
+                continue
             if cur.rowcount:
                 added += 1
         conn.commit()