deploy DOM search v1 update
This commit is contained in:
Marco0300
+122
-20
@@ -27,10 +27,10 @@ from urllib.parse import parse_qs, quote_plus, unquote, urlparse
|
||||
|
||||
try:
|
||||
from intelligence import domain_candidates, scan_website
|
||||
from browser_investigator import investigate_websites
|
||||
from browser_investigator import direct_search_candidates, investigate_websites
|
||||
except ImportError: # tests import the module as api.server
|
||||
from api.intelligence import domain_candidates, scan_website
|
||||
from api.browser_investigator import investigate_websites
|
||||
from api.browser_investigator import direct_search_candidates, investigate_websites
|
||||
|
||||
DB_PATH = os.environ.get("PROSPECT_DB", "/data/prospects.db")
|
||||
MAX_RESULTS = 50
|
||||
@@ -417,6 +417,95 @@ def recover_stale_runs() -> None:
|
||||
conn.close()
|
||||
|
||||
|
||||
def persist_website_candidates(prospect_id: int, candidates: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""Upsert candidate domains immediately, preserving accepted decisions.
|
||||
|
||||
Search-page extraction and Browser Use are both allowed to return partial
|
||||
results. Persisting in one small transaction before any optional AI work
|
||||
means a timeout, model error, or disconnected UI cannot discard links
|
||||
that were already visible to the browser.
|
||||
"""
|
||||
conn = db()
|
||||
try:
|
||||
for candidate in candidates:
|
||||
url = str(candidate.get("url") or "").strip()
|
||||
domain = str(candidate.get("domain") or clean_domain(url)).lower().strip()
|
||||
if not url or not domain:
|
||||
continue
|
||||
evidence = candidate.get("evidence")
|
||||
if not isinstance(evidence, list):
|
||||
evidence = []
|
||||
raw = dict(candidate)
|
||||
source = str(candidate.get("source") or "browser_use").strip()[:80] or "browser_use"
|
||||
conn.execute(
|
||||
"""INSERT INTO website_candidates(prospect_id,url,domain,confidence,evidence,source,status,discovered_at,raw_json)
|
||||
VALUES(?,?,?,?,?,?,?,?,?)
|
||||
ON CONFLICT(prospect_id,domain) DO UPDATE SET
|
||||
url=excluded.url, confidence=MAX(website_candidates.confidence, excluded.confidence),
|
||||
evidence=excluded.evidence, source=excluded.source, discovered_at=excluded.discovered_at,
|
||||
raw_json=excluded.raw_json,
|
||||
status=CASE WHEN website_candidates.status='accepted' THEN 'accepted' ELSE 'candidate' END""",
|
||||
(prospect_id, url[:1000], domain[:255], max(0, min(100, as_int(candidate.get("confidence")))),
|
||||
json.dumps(evidence[:12], ensure_ascii=False), source, "candidate", now(),
|
||||
json.dumps(raw, ensure_ascii=False, default=str)[:100_000]),
|
||||
)
|
||||
conn.commit()
|
||||
rows = conn.execute("SELECT * FROM website_candidates WHERE prospect_id=? ORDER BY confidence DESC, id DESC", (prospect_id,)).fetchall()
|
||||
return [dict(item) for item in rows]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def verify_website_candidates(prospect: dict[str, Any], candidates: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""Attach a small, evidence-grounded reachability/identity check.
|
||||
|
||||
Search results are useful leads, not proof of ownership. A bounded
|
||||
website scan confirms whether the domain responds and whether its title or
|
||||
final URL contains business-name tokens. Failed checks leave the result
|
||||
as a reviewable candidate instead of deleting it.
|
||||
"""
|
||||
name_tokens = [token for token in re.findall(r"[a-z0-9]+", str(prospect.get("name") or "").lower())
|
||||
if len(token) >= 3 and token not in {"and", "the", "services", "service", "company", "pty", "ltd", "electrical"}]
|
||||
output: list[dict[str, Any]] = []
|
||||
for original in candidates:
|
||||
candidate = dict(original)
|
||||
url = str(candidate.get("url") or "").strip()
|
||||
if not url:
|
||||
output.append(candidate)
|
||||
continue
|
||||
try:
|
||||
checked = scan_website(url, timeout_seconds=website_scan_timeout_seconds())
|
||||
status = str(checked.get("website_status") or "unknown")
|
||||
title = str(checked.get("website_title") or "").strip()
|
||||
final_url = str(checked.get("website_final_url") or url).strip()
|
||||
haystack = (title + " " + final_url).lower()
|
||||
evidence = [str(item).strip() for item in (candidate.get("evidence") or []) if str(item).strip()]
|
||||
matched = [token for token in name_tokens if token in haystack]
|
||||
if matched:
|
||||
evidence.append("website title or URL matches: " + ", ".join(matched[:4]))
|
||||
candidate["confidence"] = min(100, max(int(candidate.get("confidence") or 0), int(candidate.get("confidence") or 0) + 8))
|
||||
if checked.get("website_http_status"):
|
||||
evidence.append(f"website responded HTTP {int(checked.get('website_http_status'))}")
|
||||
if status in {"healthy", "redirect", "http_only", "slow"}:
|
||||
evidence.append("domain is reachable for review")
|
||||
elif checked.get("website_error"):
|
||||
evidence.append("website check: " + str(checked.get("website_error"))[:160])
|
||||
candidate["evidence"] = list(dict.fromkeys(evidence))[:12]
|
||||
candidate["verification"] = {
|
||||
"status": status,
|
||||
"http_status": checked.get("website_http_status"),
|
||||
"final_url": final_url[:1000],
|
||||
"title": title[:300],
|
||||
"identity_match": bool(matched),
|
||||
"emails_found": len(checked.get("website_emails") or []),
|
||||
}
|
||||
except Exception as exc:
|
||||
candidate.setdefault("evidence", []).append("website verification failed: " + str(exc)[:160])
|
||||
candidate["verification"] = {"status": "unknown", "identity_match": False, "error": str(exc)[:200]}
|
||||
output.append(candidate)
|
||||
return output
|
||||
|
||||
|
||||
def json_bytes(value: Any) -> bytes:
|
||||
return json.dumps(value, ensure_ascii=False).encode("utf-8")
|
||||
|
||||
@@ -1711,6 +1800,34 @@ class Handler(BaseHTTPRequestHandler):
|
||||
def browser_log(kind: str, event: str, details: dict[str, Any]) -> None:
|
||||
record_log(kind, event, prospect_id=prospect_id, details=details)
|
||||
|
||||
# First use deterministic DOM extraction. This does not need
|
||||
# an AI key and is deliberately run before Browser Use so a
|
||||
# model timeout or malformed final response cannot erase a
|
||||
# link that was already visible on Bing.
|
||||
direct_candidates = direct_search_candidates(
|
||||
dict(row), user_agent=OUTBOUND_USER_AGENT, log_callback=browser_log,
|
||||
requested=5, timeout_seconds=min(180, int(browser_runtime["timeout_seconds"])),
|
||||
)
|
||||
if direct_candidates:
|
||||
# Save the raw search links first. Verification is a
|
||||
# separate bounded step and must never erase a useful
|
||||
# result if a public site is slow or unavailable.
|
||||
persist_website_candidates(prospect_id, direct_candidates)
|
||||
verified_candidates = verify_website_candidates(dict(row), direct_candidates)
|
||||
saved_rows = persist_website_candidates(prospect_id, verified_candidates)
|
||||
for candidate in verified_candidates:
|
||||
record_log("search", "website_candidate_verified", prospect_id=prospect_id,
|
||||
details={"domain": candidate.get("domain"), "confidence": candidate.get("confidence"),
|
||||
"verification": candidate.get("verification", {})})
|
||||
record_log("browser", "website_candidates_persisted", prospect_id=prospect_id,
|
||||
details={"source": "browser_dom", "candidate_count": len(direct_candidates),
|
||||
"domains": [item.get("domain") for item in direct_candidates]})
|
||||
return self.send_json(200, {"items": saved_rows, "count": len(saved_rows), "source": "browser_dom"})
|
||||
|
||||
# Keep Browser Use as a bounded fallback for cases where the
|
||||
# public result pages yielded no parseable links.
|
||||
record_log("browser", "browser_use_fallback_started", prospect_id=prospect_id,
|
||||
details={"reason": "direct_search_no_candidates", "model": runtime["model"]})
|
||||
candidates = investigate_websites(
|
||||
dict(row), base_url=runtime["base_url"], api_key=runtime["api_key"],
|
||||
model=runtime["model"], user_agent=OUTBOUND_USER_AGENT, log_callback=browser_log,
|
||||
@@ -1718,24 +1835,9 @@ class Handler(BaseHTTPRequestHandler):
|
||||
timeout_seconds=browser_runtime["timeout_seconds"], llm_timeout_seconds=browser_runtime["llm_timeout_seconds"],
|
||||
)
|
||||
record_log("browser", "website_investigation_completed", prospect_id=prospect_id,
|
||||
details={"candidate_count": len(candidates), "model": runtime["model"]})
|
||||
conn = db()
|
||||
for candidate in candidates:
|
||||
encoded_evidence = json.dumps(candidate.get("evidence") or [], ensure_ascii=False)
|
||||
encoded_raw = json.dumps(candidate, ensure_ascii=False)
|
||||
conn.execute(
|
||||
"""INSERT INTO website_candidates(prospect_id,url,domain,confidence,evidence,source,status,discovered_at,raw_json)
|
||||
VALUES(?,?,?,?,?,?,?,?,?)
|
||||
ON CONFLICT(prospect_id,domain) DO UPDATE SET
|
||||
url=excluded.url, confidence=excluded.confidence, evidence=excluded.evidence,
|
||||
source=excluded.source, discovered_at=excluded.discovered_at, raw_json=excluded.raw_json,
|
||||
status=CASE WHEN website_candidates.status='accepted' THEN 'accepted' ELSE 'candidate' END""",
|
||||
(prospect_id, candidate["url"], candidate["domain"], candidate["confidence"], encoded_evidence,
|
||||
"browser_use", "candidate", now(), encoded_raw),
|
||||
)
|
||||
conn.commit()
|
||||
rows = conn.execute("SELECT * FROM website_candidates WHERE prospect_id=? ORDER BY confidence DESC, id DESC", (prospect_id,)).fetchall()
|
||||
return self.send_json(200, {"items": [dict(item) for item in rows], "count": len(rows)})
|
||||
details={"candidate_count": len(candidates), "model": runtime["model"], "source": "browser_use"})
|
||||
saved_rows = persist_website_candidates(prospect_id, candidates)
|
||||
return self.send_json(200, {"items": saved_rows, "count": len(saved_rows), "source": "browser_use"})
|
||||
except RuntimeError as exc:
|
||||
record_log("browser", "website_investigation_failed", level="error", prospect_id=prospect_id, error=str(exc))
|
||||
return self.send_json(503, {"error": str(exc)})
|
||||
|
||||
Reference in New Issue
Block a user