2042 lines
111 KiB
Python
2042 lines
111 KiB
Python
#!/usr/bin/env python3
|
|
"""Personal ProspectOS service backed by Google Maps Scraper.
|
|
|
|
The app owns the small local catalogue and review workflow. Google Maps
|
|
Scraper runs as a separate Playwright service and is called through its
|
|
asynchronous REST API. A browser-search implementation remains available as
|
|
an explicit fallback (DISCOVERY_BACKEND=browser) for local testing.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import csv
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import re
|
|
import sqlite3
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from datetime import datetime, timezone
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
from pathlib import Path
|
|
from typing import Any, Callable
|
|
from urllib.parse import parse_qs, quote_plus, unquote, urlparse
|
|
|
|
try:
|
|
from intelligence import domain_candidates, scan_website
|
|
from browser_investigator import direct_search_candidates, investigate_websites
|
|
except ImportError: # tests import the module as api.server
|
|
from api.intelligence import domain_candidates, scan_website
|
|
from api.browser_investigator import direct_search_candidates, investigate_websites
|
|
|
|
DB_PATH = os.environ.get("PROSPECT_DB", "/data/prospects.db")
|
|
MAX_RESULTS = 50
|
|
GMAPS_POLL_SECONDS = float(os.environ.get("GMAPS_POLL_SECONDS", "2"))
|
|
# Maps searches can legitimately take several minutes, especially when email
|
|
# extraction is enabled. Fifteen minutes is the safe default; it remains
|
|
# configurable for smaller or larger personal runs.
|
|
GMAPS_REQUEST_TIMEOUT = int(os.environ.get("GMAPS_REQUEST_TIMEOUT", "900"))
|
|
GMAPS_URL = os.environ.get("GMAPS_URL", "").strip().rstrip("/")
|
|
# The bundled gosom/google-maps-scraper image exposes a local, unauthenticated
|
|
# REST API. The optional SaaS edition uses a different API and requires an
|
|
# X-API-Key header; keep support for it only when explicitly selected.
|
|
GMAPS_API_MODE = os.environ.get("GMAPS_API_MODE", "standard").strip().lower()
|
|
GMAPS_API_KEY = os.environ.get("GMAPS_API_KEY", "").strip()
|
|
AI_BASE_URL = os.environ.get("AI_BASE_URL", "").strip().rstrip("/")
|
|
AI_API_KEY = os.environ.get("AI_API_KEY", "").strip()
|
|
AI_MODEL = os.environ.get("AI_MODEL", "").strip()
|
|
AI_PROVIDER = os.environ.get("AI_PROVIDER", "custom").strip().lower()
|
|
# Cloudflare Browser Integrity checks can reject Python's default
|
|
# ``Python-urllib/<version>`` fingerprint before an OpenAI-compatible request
|
|
# reaches the provider. Identify this application consistently on provider
|
|
# requests instead of relying on that default.
|
|
OUTBOUND_USER_AGENT = os.environ.get("OUTBOUND_USER_AGENT", "ProspectOS/2.0 (+personal-use)").strip() or "ProspectOS/2.0 (+personal-use)"
|
|
BROWSER_USE_ENABLED = os.environ.get("BROWSER_USE_ENABLED", "1").strip().lower() not in {"0", "false", "no", "off"}
|
|
NOUS_BASE_URL = "https://inference-api.nousresearch.com/v1"
|
|
NOUS_FREE_MODELS = [
|
|
{"id": "meituan/longcat-2.0:free", "label": "Meituan · LongCat 2.0", "free": True},
|
|
{"id": "upstage/solar-pro-4:free", "label": "Upstage · Solar Pro 4", "free": True},
|
|
{"id": "stepfun/step-3.7-flash:free", "label": "StepFun · Step 3.7 Flash", "free": True},
|
|
{"id": "poolside/laguna-s-2.1:free", "label": "Poolside · Laguna S 2.1", "free": True},
|
|
{"id": "poolside/laguna-xs-2.1:free", "label": "Poolside · Laguna XS 2.1", "free": True},
|
|
{"id": "inclusionai/ling-3.0-flash-fin:free", "label": "inclusionAI · Ling 3.0 Flash Fin", "free": True},
|
|
{"id": "inclusionai/ling-3.0-flash-sante:free", "label": "inclusionAI · Ling 3.0 Flash Sante", "free": True},
|
|
]
|
|
_nous_models_cache: list[dict[str, Any]] = NOUS_FREE_MODELS.copy()
|
|
_nous_models_cached_at = 0.0
|
|
ALLOWED_REVIEW = {"pending", "approved", "rejected", "contacted"}
|
|
_discover_lock = threading.Lock()
|
|
_schema_lock = threading.Lock()
|
|
_schema_path: str | None = None
|
|
|
|
|
|
def _saved_settings() -> dict[str, str]:
|
|
"""Read persisted runtime settings, falling back cleanly during startup."""
|
|
try:
|
|
conn = db()
|
|
try:
|
|
return {str(row["key"]): str(row["value"]) for row in conn.execute("SELECT key,value FROM settings")}
|
|
finally:
|
|
conn.close()
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
def _setting_int(saved: dict[str, str], key: str, env_key: str, default: int, minimum: int, maximum: int) -> int:
|
|
try:
|
|
value = int(saved.get(key, os.environ.get(env_key, str(default))))
|
|
except (TypeError, ValueError):
|
|
value = default
|
|
return max(minimum, min(maximum, value))
|
|
|
|
|
|
def _setting_float(saved: dict[str, str], key: str, env_key: str, default: float, minimum: float, maximum: float) -> float:
|
|
try:
|
|
value = float(saved.get(key, os.environ.get(env_key, str(default))))
|
|
except (TypeError, ValueError):
|
|
value = default
|
|
return max(minimum, min(maximum, value))
|
|
|
|
|
|
def _setting_bool(saved: dict[str, str], key: str, env_key: str, default: bool) -> bool:
|
|
raw = str(saved.get(key, os.environ.get(env_key, "true" if default else "false"))).strip().lower()
|
|
return raw not in {"0", "false", "no", "off", "disabled"}
|
|
|
|
|
|
def browser_use_settings() -> dict[str, Any]:
|
|
"""Return the live Browser Use controls from Settings or .env defaults."""
|
|
saved = _saved_settings()
|
|
return {
|
|
"enabled": _setting_bool(saved, "BROWSER_USE_ENABLED", "BROWSER_USE_ENABLED", BROWSER_USE_ENABLED),
|
|
"max_steps": _setting_int(saved, "BROWSER_USE_MAX_STEPS", "BROWSER_USE_MAX_STEPS", 10, 3, 50),
|
|
"timeout_seconds": _setting_int(saved, "BROWSER_USE_TIMEOUT_SECONDS", "BROWSER_USE_TIMEOUT_SECONDS", 600, 30, 1800),
|
|
"llm_timeout_seconds": _setting_int(saved, "BROWSER_USE_LLM_TIMEOUT_SECONDS", "BROWSER_USE_LLM_TIMEOUT_SECONDS", 90, 10, 300),
|
|
"headless": True,
|
|
"public_only": True,
|
|
}
|
|
|
|
|
|
def browser_use_limits() -> tuple[int, int]:
|
|
"""Return bounded Browser Use steps/timeout for compatibility with callers."""
|
|
runtime = browser_use_settings()
|
|
return int(runtime["max_steps"]), int(runtime["timeout_seconds"])
|
|
|
|
|
|
def gmaps_runtime_settings() -> dict[str, Any]:
|
|
"""Return live Maps worker timing controls from Settings or .env defaults."""
|
|
saved = _saved_settings()
|
|
return {
|
|
"request_timeout_seconds": _setting_int(saved, "GMAPS_REQUEST_TIMEOUT", "GMAPS_REQUEST_TIMEOUT", GMAPS_REQUEST_TIMEOUT, 60, 7200),
|
|
"poll_seconds": _setting_float(saved, "GMAPS_POLL_SECONDS", "GMAPS_POLL_SECONDS", GMAPS_POLL_SECONDS, 0.5, 60.0),
|
|
"max_depth": _setting_int(saved, "GMAPS_MAX_DEPTH", "GMAPS_MAX_DEPTH", 1, 1, 10),
|
|
}
|
|
|
|
|
|
def website_scan_timeout_seconds() -> int:
|
|
saved = _saved_settings()
|
|
return _setting_int(saved, "WEBSITE_SCAN_TIMEOUT", "WEBSITE_SCAN_TIMEOUT", 8, 2, 60)
|
|
|
|
|
|
def now() -> str:
|
|
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
|
|
|
|
|
|
try:
|
|
LOG_RETENTION = max(2_000, min(50_000, int(os.environ.get("LOG_RETENTION", "20000"))))
|
|
except (TypeError, ValueError):
|
|
LOG_RETENTION = 20_000
|
|
_LOG_SECRET_KEYS = {"api_key", "authorization", "password", "secret", "token", "access_token", "refresh_token"}
|
|
|
|
|
|
def _redact_log_value(value: Any, depth: int = 0) -> Any:
|
|
"""Keep logs useful without persisting provider credentials."""
|
|
if depth > 5:
|
|
return "[truncated]"
|
|
if isinstance(value, dict):
|
|
output: dict[str, Any] = {}
|
|
for key, item in list(value.items())[:80]:
|
|
key_text = str(key)
|
|
if key_text.lower() in _LOG_SECRET_KEYS or key_text.lower().endswith("_api_key"):
|
|
output[key_text] = "[redacted]"
|
|
else:
|
|
output[key_text] = _redact_log_value(item, depth + 1)
|
|
return output
|
|
if isinstance(value, (list, tuple, set)):
|
|
return [_redact_log_value(item, depth + 1) for item in list(value)[:100]]
|
|
if isinstance(value, str):
|
|
return value[:4_000] + ("…" if len(value) > 4_000 else "")
|
|
if isinstance(value, (int, float, bool)) or value is None:
|
|
return value
|
|
return str(value)[:4_000]
|
|
|
|
|
|
def record_log(kind: str, event: str, *, level: str = "info", method: str = "", path: str = "",
|
|
status: int | None = None, duration_ms: int | None = None, details: Any = None,
|
|
error: str = "", run_id: int | None = None, prospect_id: int | None = None) -> None:
|
|
"""Persist one bounded operational event; logging must never break a request."""
|
|
try:
|
|
safe_kind = str(kind or "system").strip().lower()[:30] or "system"
|
|
safe_level = str(level or "info").strip().lower()[:12] or "info"
|
|
safe_details = _redact_log_value(details if details is not None else {})
|
|
encoded = json.dumps(safe_details, ensure_ascii=False, default=str)[:20_000]
|
|
conn = db()
|
|
try:
|
|
cur = conn.execute(
|
|
"""INSERT INTO logs(created_at,kind,level,event,method,path,status,duration_ms,run_id,prospect_id,details,error)
|
|
VALUES(?,?,?,?,?,?,?,?,?,?,?,?)""",
|
|
(now(), safe_kind, safe_level, str(event or "event")[:240], str(method or "")[:12],
|
|
str(path or "")[:500], status, duration_ms, run_id, prospect_id, encoded, str(error or "")[:2_000]),
|
|
)
|
|
# Retain enough history for investigation without allowing a busy
|
|
# browser session to grow the SQLite volume indefinitely.
|
|
if int(cur.lastrowid or 0) % 100 == 0:
|
|
conn.execute("DELETE FROM logs WHERE id NOT IN (SELECT id FROM logs ORDER BY id DESC LIMIT ?)", (LOG_RETENTION,))
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
except Exception:
|
|
return
|
|
|
|
|
|
def backend_mode() -> str:
|
|
configured = os.environ.get("DISCOVERY_BACKEND", "").strip().lower()
|
|
if configured in {"gmaps", "browser"}:
|
|
return configured
|
|
return "gmaps" if os.environ.get("GMAPS_URL", "").strip() else "browser"
|
|
|
|
|
|
def db() -> sqlite3.Connection:
|
|
global _schema_path
|
|
os.makedirs(os.path.dirname(DB_PATH) or ".", exist_ok=True)
|
|
conn = sqlite3.connect(DB_PATH, timeout=30)
|
|
conn.row_factory = sqlite3.Row
|
|
conn.execute("PRAGMA busy_timeout=30000")
|
|
if _schema_path == DB_PATH:
|
|
return conn
|
|
_schema_lock.acquire()
|
|
if _schema_path == DB_PATH:
|
|
_schema_lock.release()
|
|
return conn
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS prospects (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
name TEXT NOT NULL,
|
|
website TEXT NOT NULL DEFAULT '',
|
|
domain TEXT NOT NULL UNIQUE,
|
|
description TEXT NOT NULL DEFAULT '',
|
|
query TEXT NOT NULL DEFAULT '',
|
|
discovered_at TEXT NOT NULL,
|
|
review_status TEXT NOT NULL DEFAULT 'pending',
|
|
review_notes TEXT NOT NULL DEFAULT '',
|
|
source TEXT NOT NULL DEFAULT 'google_maps',
|
|
source_id TEXT NOT NULL DEFAULT '',
|
|
category TEXT NOT NULL DEFAULT '',
|
|
search_category TEXT NOT NULL DEFAULT '',
|
|
address TEXT NOT NULL DEFAULT '',
|
|
phone TEXT NOT NULL DEFAULT '',
|
|
emails TEXT NOT NULL DEFAULT '',
|
|
rating REAL,
|
|
review_count INTEGER NOT NULL DEFAULT 0,
|
|
maps_url TEXT NOT NULL DEFAULT '',
|
|
latitude REAL,
|
|
longitude REAL,
|
|
listing_status TEXT NOT NULL DEFAULT '',
|
|
website_status TEXT NOT NULL DEFAULT 'unknown',
|
|
website_http_status INTEGER,
|
|
website_final_url TEXT NOT NULL DEFAULT '',
|
|
website_latency_ms INTEGER NOT NULL DEFAULT 0,
|
|
website_https INTEGER NOT NULL DEFAULT 0,
|
|
website_title TEXT NOT NULL DEFAULT '',
|
|
website_technology TEXT NOT NULL DEFAULT '',
|
|
website_has_viewport INTEGER NOT NULL DEFAULT 0,
|
|
website_tls_expires_at TEXT NOT NULL DEFAULT '',
|
|
website_checked_at TEXT NOT NULL DEFAULT '',
|
|
website_error TEXT NOT NULL DEFAULT '',
|
|
email_evidence TEXT NOT NULL DEFAULT '[]',
|
|
score_reasons TEXT NOT NULL DEFAULT '[]',
|
|
ai_probability REAL,
|
|
ai_recommendation TEXT NOT NULL DEFAULT '',
|
|
ai_reasoning TEXT NOT NULL DEFAULT '',
|
|
ai_checked_at TEXT NOT NULL DEFAULT '',
|
|
ai_subject TEXT NOT NULL DEFAULT '',
|
|
ai_message TEXT NOT NULL DEFAULT '',
|
|
ai_offer TEXT NOT NULL DEFAULT '',
|
|
ai_unknowns TEXT NOT NULL DEFAULT '[]',
|
|
ai_evidence_ids TEXT NOT NULL DEFAULT '[]',
|
|
ai_follow_up TEXT NOT NULL DEFAULT '',
|
|
opportunity_score INTEGER NOT NULL DEFAULT 0,
|
|
raw_json TEXT NOT NULL DEFAULT '{}'
|
|
)""")
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS runs (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
query TEXT NOT NULL,
|
|
location TEXT NOT NULL,
|
|
requested INTEGER NOT NULL,
|
|
found INTEGER NOT NULL DEFAULT 0,
|
|
added INTEGER NOT NULL DEFAULT 0,
|
|
status TEXT NOT NULL,
|
|
error TEXT NOT NULL DEFAULT '',
|
|
created_at TEXT NOT NULL,
|
|
source TEXT NOT NULL DEFAULT 'google_maps',
|
|
external_job_id TEXT NOT NULL DEFAULT '',
|
|
stage TEXT NOT NULL DEFAULT 'queued',
|
|
progress INTEGER NOT NULL DEFAULT 0,
|
|
completed_at TEXT NOT NULL DEFAULT '',
|
|
request_json TEXT NOT NULL DEFAULT '{}'
|
|
)""")
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS suppressions (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
kind TEXT NOT NULL,
|
|
value TEXT NOT NULL,
|
|
reason TEXT NOT NULL DEFAULT '',
|
|
created_at TEXT NOT NULL,
|
|
UNIQUE(kind, value)
|
|
)""")
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS schedules (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT, category TEXT NOT NULL, keywords TEXT NOT NULL,
|
|
location TEXT NOT NULL DEFAULT 'South Africa', max_results INTEGER NOT NULL DEFAULT 25,
|
|
interval_hours INTEGER NOT NULL DEFAULT 24, enabled INTEGER NOT NULL DEFAULT 1,
|
|
next_run_at TEXT NOT NULL, last_run_id INTEGER, created_at TEXT NOT NULL
|
|
)""")
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS settings (
|
|
key TEXT PRIMARY KEY, value TEXT NOT NULL DEFAULT '', updated_at TEXT NOT NULL
|
|
)""")
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS website_candidates (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
prospect_id INTEGER NOT NULL,
|
|
url TEXT NOT NULL,
|
|
domain TEXT NOT NULL,
|
|
confidence INTEGER NOT NULL DEFAULT 0,
|
|
evidence TEXT NOT NULL DEFAULT '[]',
|
|
source TEXT NOT NULL DEFAULT 'browser_use',
|
|
status TEXT NOT NULL DEFAULT 'candidate',
|
|
discovered_at TEXT NOT NULL,
|
|
raw_json TEXT NOT NULL DEFAULT '{}',
|
|
UNIQUE(prospect_id, domain)
|
|
)""")
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS logs (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
created_at TEXT NOT NULL,
|
|
kind TEXT NOT NULL DEFAULT 'system',
|
|
level TEXT NOT NULL DEFAULT 'info',
|
|
event TEXT NOT NULL,
|
|
method TEXT NOT NULL DEFAULT '',
|
|
path TEXT NOT NULL DEFAULT '',
|
|
status INTEGER,
|
|
duration_ms INTEGER,
|
|
run_id INTEGER,
|
|
prospect_id INTEGER,
|
|
details TEXT NOT NULL DEFAULT '{}',
|
|
error TEXT NOT NULL DEFAULT ''
|
|
)""")
|
|
|
|
# Migrate databases created by v1-v5 without destroying the user's
|
|
# catalogue. SQLite permits adding nullable/defaulted columns in place.
|
|
migrations = {
|
|
"source": "TEXT NOT NULL DEFAULT 'google_maps'",
|
|
"source_id": "TEXT NOT NULL DEFAULT ''",
|
|
"category": "TEXT NOT NULL DEFAULT ''",
|
|
"search_category": "TEXT NOT NULL DEFAULT ''",
|
|
"address": "TEXT NOT NULL DEFAULT ''",
|
|
"phone": "TEXT NOT NULL DEFAULT ''",
|
|
"emails": "TEXT NOT NULL DEFAULT ''",
|
|
"rating": "REAL",
|
|
"review_count": "INTEGER NOT NULL DEFAULT 0",
|
|
"maps_url": "TEXT NOT NULL DEFAULT ''",
|
|
"latitude": "REAL",
|
|
"longitude": "REAL",
|
|
"listing_status": "TEXT NOT NULL DEFAULT ''",
|
|
"website_status": "TEXT NOT NULL DEFAULT 'unknown'",
|
|
"website_http_status": "INTEGER",
|
|
"website_final_url": "TEXT NOT NULL DEFAULT ''",
|
|
"website_latency_ms": "INTEGER NOT NULL DEFAULT 0",
|
|
"website_https": "INTEGER NOT NULL DEFAULT 0",
|
|
"website_title": "TEXT NOT NULL DEFAULT ''",
|
|
"website_technology": "TEXT NOT NULL DEFAULT ''",
|
|
"website_has_viewport": "INTEGER NOT NULL DEFAULT 0",
|
|
"website_tls_expires_at": "TEXT NOT NULL DEFAULT ''",
|
|
"website_checked_at": "TEXT NOT NULL DEFAULT ''",
|
|
"website_error": "TEXT NOT NULL DEFAULT ''",
|
|
"email_evidence": "TEXT NOT NULL DEFAULT '[]'",
|
|
"score_reasons": "TEXT NOT NULL DEFAULT '[]'",
|
|
"ai_probability": "REAL",
|
|
"ai_recommendation": "TEXT NOT NULL DEFAULT ''",
|
|
"ai_reasoning": "TEXT NOT NULL DEFAULT ''",
|
|
"ai_checked_at": "TEXT NOT NULL DEFAULT ''",
|
|
"ai_subject": "TEXT NOT NULL DEFAULT ''",
|
|
"ai_message": "TEXT NOT NULL DEFAULT ''",
|
|
"ai_offer": "TEXT NOT NULL DEFAULT ''",
|
|
"ai_unknowns": "TEXT NOT NULL DEFAULT '[]'",
|
|
"ai_evidence_ids": "TEXT NOT NULL DEFAULT '[]'",
|
|
"ai_follow_up": "TEXT NOT NULL DEFAULT ''",
|
|
"opportunity_score": "INTEGER NOT NULL DEFAULT 0",
|
|
"raw_json": "TEXT NOT NULL DEFAULT '{}'",
|
|
}
|
|
existing = {row[1] for row in conn.execute("PRAGMA table_info(prospects)")}
|
|
for name, ddl in migrations.items():
|
|
if name not in existing:
|
|
conn.execute(f"ALTER TABLE prospects ADD COLUMN {name} {ddl}")
|
|
run_migrations = {
|
|
"source": "TEXT NOT NULL DEFAULT 'google_maps'",
|
|
"external_job_id": "TEXT NOT NULL DEFAULT ''",
|
|
"stage": "TEXT NOT NULL DEFAULT 'queued'",
|
|
"progress": "INTEGER NOT NULL DEFAULT 0",
|
|
"completed_at": "TEXT NOT NULL DEFAULT ''",
|
|
"request_json": "TEXT NOT NULL DEFAULT '{}'",
|
|
}
|
|
existing_runs = {row[1] for row in conn.execute("PRAGMA table_info(runs)")}
|
|
for name, ddl in run_migrations.items():
|
|
if name not in existing_runs:
|
|
conn.execute(f"ALTER TABLE runs ADD COLUMN {name} {ddl}")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_prospects_review ON prospects(review_status)")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_prospects_score ON prospects(opportunity_score DESC)")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_runs_created ON runs(id DESC)")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_website_candidates_prospect ON website_candidates(prospect_id, confidence DESC)")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_logs_created ON logs(id DESC)")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_logs_kind ON logs(kind, id DESC)")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_logs_level ON logs(level, id DESC)")
|
|
conn.commit()
|
|
_schema_path = DB_PATH
|
|
_schema_lock.release()
|
|
return conn
|
|
|
|
|
|
def recover_stale_runs() -> None:
|
|
"""Recover abandoned runs once at process startup, not on every request."""
|
|
conn = db()
|
|
try:
|
|
conn.execute("""UPDATE runs SET status='failed', stage='failed',
|
|
error=CASE WHEN error='' THEN 'stale_run_recovered' ELSE error END,
|
|
completed_at=CASE WHEN completed_at='' THEN ? ELSE completed_at END
|
|
WHERE status IN ('queued','running')
|
|
AND julianday(created_at) < julianday('now','-30 minutes')""", (now(),))
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def persist_website_candidates(prospect_id: int, candidates: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
"""Upsert candidate domains immediately, preserving accepted decisions.
|
|
|
|
Search-page extraction and Browser Use are both allowed to return partial
|
|
results. Persisting in one small transaction before any optional AI work
|
|
means a timeout, model error, or disconnected UI cannot discard links
|
|
that were already visible to the browser.
|
|
"""
|
|
conn = db()
|
|
try:
|
|
for candidate in candidates:
|
|
url = str(candidate.get("url") or "").strip()
|
|
domain = str(candidate.get("domain") or clean_domain(url)).lower().strip()
|
|
if not url or not domain:
|
|
continue
|
|
evidence = candidate.get("evidence")
|
|
if not isinstance(evidence, list):
|
|
evidence = []
|
|
raw = dict(candidate)
|
|
source = str(candidate.get("source") or "browser_use").strip()[:80] or "browser_use"
|
|
conn.execute(
|
|
"""INSERT INTO website_candidates(prospect_id,url,domain,confidence,evidence,source,status,discovered_at,raw_json)
|
|
VALUES(?,?,?,?,?,?,?,?,?)
|
|
ON CONFLICT(prospect_id,domain) DO UPDATE SET
|
|
url=excluded.url, confidence=MAX(website_candidates.confidence, excluded.confidence),
|
|
evidence=excluded.evidence, source=excluded.source, discovered_at=excluded.discovered_at,
|
|
raw_json=excluded.raw_json,
|
|
status=CASE WHEN website_candidates.status='accepted' THEN 'accepted' ELSE 'candidate' END""",
|
|
(prospect_id, url[:1000], domain[:255], max(0, min(100, as_int(candidate.get("confidence")))),
|
|
json.dumps(evidence[:12], ensure_ascii=False), source, "candidate", now(),
|
|
json.dumps(raw, ensure_ascii=False, default=str)[:100_000]),
|
|
)
|
|
conn.commit()
|
|
rows = conn.execute("SELECT * FROM website_candidates WHERE prospect_id=? ORDER BY confidence DESC, id DESC", (prospect_id,)).fetchall()
|
|
return [dict(item) for item in rows]
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def verify_website_candidates(prospect: dict[str, Any], candidates: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
"""Attach a small, evidence-grounded reachability/identity check.
|
|
|
|
Search results are useful leads, not proof of ownership. A bounded
|
|
website scan confirms whether the domain responds and whether its title or
|
|
final URL contains business-name tokens. Failed checks leave the result
|
|
as a reviewable candidate instead of deleting it.
|
|
"""
|
|
name_tokens = [token for token in re.findall(r"[a-z0-9]+", str(prospect.get("name") or "").lower())
|
|
if len(token) >= 3 and token not in {"and", "the", "services", "service", "company", "pty", "ltd", "electrical"}]
|
|
output: list[dict[str, Any]] = []
|
|
for original in candidates:
|
|
candidate = dict(original)
|
|
url = str(candidate.get("url") or "").strip()
|
|
if not url:
|
|
output.append(candidate)
|
|
continue
|
|
try:
|
|
checked = scan_website(url, timeout_seconds=website_scan_timeout_seconds())
|
|
status = str(checked.get("website_status") or "unknown")
|
|
title = str(checked.get("website_title") or "").strip()
|
|
final_url = str(checked.get("website_final_url") or url).strip()
|
|
haystack = (title + " " + final_url).lower()
|
|
evidence = [str(item).strip() for item in (candidate.get("evidence") or []) if str(item).strip()]
|
|
matched = [token for token in name_tokens if token in haystack]
|
|
if matched:
|
|
evidence.append("website title or URL matches: " + ", ".join(matched[:4]))
|
|
candidate["confidence"] = min(100, max(int(candidate.get("confidence") or 0), int(candidate.get("confidence") or 0) + 8))
|
|
if checked.get("website_http_status"):
|
|
evidence.append(f"website responded HTTP {int(checked.get('website_http_status'))}")
|
|
if status in {"healthy", "redirect", "http_only", "slow"}:
|
|
evidence.append("domain is reachable for review")
|
|
elif checked.get("website_error"):
|
|
evidence.append("website check: " + str(checked.get("website_error"))[:160])
|
|
candidate["evidence"] = list(dict.fromkeys(evidence))[:12]
|
|
candidate["verification"] = {
|
|
"status": status,
|
|
"http_status": checked.get("website_http_status"),
|
|
"final_url": final_url[:1000],
|
|
"title": title[:300],
|
|
"identity_match": bool(matched),
|
|
"emails_found": len(checked.get("website_emails") or []),
|
|
}
|
|
except Exception as exc:
|
|
candidate.setdefault("evidence", []).append("website verification failed: " + str(exc)[:160])
|
|
candidate["verification"] = {"status": "unknown", "identity_match": False, "error": str(exc)[:200]}
|
|
output.append(candidate)
|
|
return output
|
|
|
|
|
|
def json_bytes(value: Any) -> bytes:
|
|
return json.dumps(value, ensure_ascii=False).encode("utf-8")
|
|
|
|
|
|
def clean_domain(url: str) -> str:
|
|
parsed = urlparse(url if "://" in url else "https://" + url)
|
|
host = (parsed.hostname or "").lower().strip().rstrip(".")
|
|
if host.startswith("www."):
|
|
host = host[4:]
|
|
return host
|
|
|
|
|
|
def normalize_url(value: Any) -> str:
|
|
text = str(value or "").strip()
|
|
if not text:
|
|
return ""
|
|
if text.startswith("//"):
|
|
text = "https:" + text
|
|
if not text.startswith(("http://", "https://")):
|
|
return ""
|
|
return text
|
|
|
|
|
|
def as_text(value: Any) -> str:
|
|
if value is None:
|
|
return ""
|
|
if isinstance(value, (list, tuple, set)):
|
|
return ", ".join(as_text(item) for item in value if as_text(item))
|
|
if isinstance(value, dict):
|
|
return ", ".join(as_text(item) for item in value.values() if as_text(item))
|
|
return " ".join(str(value).split()).strip()
|
|
|
|
|
|
def field_value(value: Any, *names: str) -> Any:
|
|
"""Read JSON fields from both tagged and default Go JSON casing."""
|
|
if not isinstance(value, dict):
|
|
return None
|
|
for name in names:
|
|
if name in value:
|
|
return value[name]
|
|
wanted = {name.lower() for name in names}
|
|
for key, item in value.items():
|
|
if str(key).lower() in wanted:
|
|
return item
|
|
return None
|
|
|
|
|
|
def as_float(value: Any) -> float | None:
|
|
try:
|
|
if value in (None, ""):
|
|
return None
|
|
return float(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def as_int(value: Any) -> int:
|
|
try:
|
|
if value in (None, ""):
|
|
return 0
|
|
return max(0, int(float(str(value).replace(",", ""))))
|
|
except (TypeError, ValueError):
|
|
return 0
|
|
|
|
|
|
EMAIL_RE = re.compile(r"[A-Z0-9._%+\-]+@[A-Z0-9.\-]+\.[A-Z]{2,63}", re.I)
|
|
FREE_EMAIL_DOMAINS = {"gmail.com", "googlemail.com", "outlook.com", "hotmail.com", "yahoo.com", "icloud.com"}
|
|
|
|
|
|
def normalize_public_emails(value: Any) -> list[str]:
|
|
found: list[str] = []
|
|
text = as_text(value)
|
|
for address in EMAIL_RE.findall(text):
|
|
address = address.strip(".,;:()[]{}<>").lower()
|
|
if address not in found:
|
|
found.append(address)
|
|
return found[:30]
|
|
|
|
|
|
def score_opportunity(item: dict[str, Any]) -> tuple[int, list[str]]:
|
|
"""Transparent, explainable triage score; it is not an AI verdict."""
|
|
score = 20
|
|
reasons: list[str] = []
|
|
website = str(item.get("website") or "")
|
|
website_status = str(item.get("website_status") or "unknown")
|
|
emails = normalize_public_emails(item.get("emails"))
|
|
phone = str(item.get("phone") or "")
|
|
rating = as_float(item.get("rating"))
|
|
review_count = as_int(item.get("review_count"))
|
|
if not website or website_status in {"no_detected_website", "missing"}:
|
|
score += 35
|
|
reasons.append("No detected website")
|
|
elif website_status in {"broken", "unreachable", "error"}:
|
|
score += 30
|
|
reasons.append("Website appears unavailable or broken")
|
|
elif website_status == "slow":
|
|
score += 15
|
|
reasons.append("Website response is slow")
|
|
elif website_status == "http_only":
|
|
score += 10
|
|
reasons.append("Website has no HTTPS")
|
|
elif website_status == "redirect":
|
|
score += 5
|
|
reasons.append("Website relies on redirects")
|
|
if emails:
|
|
score += 12
|
|
reasons.append("Public email found")
|
|
if any(address.split("@", 1)[-1] in FREE_EMAIL_DOMAINS for address in emails):
|
|
score += 8
|
|
reasons.append("Free-mail address used")
|
|
if phone:
|
|
score += 10
|
|
reasons.append("Phone listed")
|
|
if rating is not None and rating >= 4:
|
|
score += 10
|
|
reasons.append("Strong public rating")
|
|
if review_count >= 10:
|
|
score += 10
|
|
reasons.append("Established review activity")
|
|
if as_text(item.get("category") or item.get("type")):
|
|
score += 5
|
|
return min(100, score), reasons
|
|
|
|
|
|
def score_place(item: dict[str, Any], website: str, phone: str, rating: float | None, review_count: int) -> int:
|
|
item = {**item, "website": website, "phone": phone, "rating": rating, "review_count": review_count,
|
|
"website_status": "no_detected_website" if not website else "unknown"}
|
|
return score_opportunity(item)[0]
|
|
|
|
|
|
def map_gmaps_result(item: dict[str, Any], query: str, search_category: str = "") -> dict[str, Any]:
|
|
"""Normalize gosom/google-maps-scraper's place object to our catalogue."""
|
|
name = as_text(item.get("title") or item.get("name") or item.get("business_name")) or "Unnamed place"
|
|
website = normalize_url(item.get("website") or item.get("website_url"))
|
|
maps_url = normalize_url(item.get("link") or item.get("maps_url") or item.get("google_maps_url"))
|
|
address = as_text(item.get("complete_address") or item.get("address"))
|
|
phone = as_text(item.get("phone") or item.get("phone_number"))
|
|
category = as_text(item.get("category") or item.get("type") or item.get("categories"))
|
|
description = as_text(item.get("descriptions") or item.get("description") or category)
|
|
email_values = normalize_public_emails(item.get("emails") or item.get("email"))
|
|
emails = ", ".join(email_values)
|
|
rating = as_float(item.get("review_rating") if item.get("review_rating") is not None else item.get("rating"))
|
|
review_count = as_int(item.get("review_count") if item.get("review_count") is not None else item.get("reviews"))
|
|
source_id = as_text(item.get("place_id") or item.get("data_id") or item.get("cid") or maps_url)
|
|
if not source_id:
|
|
source_id = hashlib.sha1((name + "|" + address).encode("utf-8")).hexdigest()[:20]
|
|
domain = clean_domain(website) if website else "maps:" + source_id
|
|
latitude = as_float(item.get("latitude"))
|
|
longitude = as_float(item.get("longitude"))
|
|
return {
|
|
"name": name[:200],
|
|
"website": website,
|
|
"domain": domain[:255],
|
|
"description": description[:1000],
|
|
"query": query,
|
|
"source": "google_maps",
|
|
"source_id": source_id[:255],
|
|
"category": category[:255],
|
|
"search_category": as_text(search_category)[:255],
|
|
"address": address[:1000],
|
|
"phone": phone[:120],
|
|
"emails": emails[:500],
|
|
"rating": rating,
|
|
"review_count": review_count,
|
|
"maps_url": maps_url,
|
|
"latitude": latitude,
|
|
"longitude": longitude,
|
|
"listing_status": as_text(item.get("status"))[:120],
|
|
"website_status": "unknown" if website else "no_detected_website",
|
|
"email_evidence": json.dumps([{"email": value, "source_url": website or maps_url, "found_at": now(), "method": "maps_worker", "confidence": "medium"} for value in email_values], ensure_ascii=False),
|
|
"score_reasons": "[]",
|
|
"opportunity_score": score_place(item, website, phone, rating, review_count),
|
|
"raw_json": json.dumps(item, ensure_ascii=False)[:100_000],
|
|
}
|
|
|
|
|
|
def _looks_like_place(value: dict[str, Any]) -> bool:
|
|
keys = {str(key).lower() for key in value}
|
|
return bool(keys.intersection({"title", "name", "business_name"})) and bool(
|
|
keys.intersection({"address", "complete_address", "website", "phone", "place_id", "link"})
|
|
)
|
|
|
|
|
|
def extract_gmaps_results(raw: Any) -> list[dict[str, Any]]:
|
|
"""Handle both current API payloads and older nested result formats."""
|
|
output: list[dict[str, Any]] = []
|
|
seen: set[str] = set()
|
|
|
|
def visit(value: Any) -> None:
|
|
if isinstance(value, list):
|
|
for item in value:
|
|
visit(item)
|
|
elif isinstance(value, dict):
|
|
if _looks_like_place(value):
|
|
marker = as_text(value.get("place_id") or value.get("data_id") or value.get("cid") or value.get("link") or value.get("title") or value.get("name"))
|
|
if marker and marker not in seen:
|
|
seen.add(marker)
|
|
output.append(value)
|
|
return
|
|
for key in ("results", "data", "items", "places", "result"):
|
|
if key in value:
|
|
visit(value[key])
|
|
if not any(key in value for key in ("results", "data", "items", "places", "result")):
|
|
for child in value.values():
|
|
if isinstance(child, (list, dict)):
|
|
visit(child)
|
|
|
|
visit(raw)
|
|
return output
|
|
|
|
|
|
def _upstream_error(prefix: str, code: int, detail: str) -> str:
|
|
"""Return an actionable, provider-specific upstream error string."""
|
|
lowered = detail.lower()
|
|
if code == 403 and ("error 1010" in lowered or "error_code\":1010" in lowered or "error code: 1010" in lowered):
|
|
return f"{prefix}_cloudflare_1010:{detail}"
|
|
return f"{prefix}_http_{code}:{detail}"
|
|
|
|
|
|
def _log_http_kind(error_prefix: str) -> str:
|
|
return "ai" if error_prefix == "ai" else "search" if error_prefix == "gmaps" else "api"
|
|
|
|
|
|
def _safe_log_url(url: str) -> str:
|
|
parsed = urlparse(url)
|
|
return f"{parsed.scheme}://{parsed.netloc}{parsed.path}"[:700]
|
|
|
|
|
|
def http_json(method: str, url: str, payload: Any = None, headers: dict[str, str] | None = None, timeout: float = 20, error_prefix: str = "upstream") -> Any:
|
|
started = time.monotonic()
|
|
log_kind = _log_http_kind(error_prefix)
|
|
log_details = {"url": _safe_log_url(url), "timeout_seconds": timeout, "request": payload}
|
|
data = None if payload is None else json.dumps(payload).encode("utf-8")
|
|
request_headers = {"Accept": "application/json", "User-Agent": OUTBOUND_USER_AGENT, **(headers or {})}
|
|
if data is not None:
|
|
request_headers["Content-Type"] = "application/json"
|
|
request = urllib.request.Request(url, data=data, headers=request_headers, method=method.upper())
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
body = response.read()
|
|
response_status = int(getattr(response, "status", 200) or 200)
|
|
except urllib.error.HTTPError as exc:
|
|
detail = exc.read().decode("utf-8", "replace")[:300]
|
|
record_log(log_kind, "upstream_http_error", level="error", method=method, path=_safe_log_url(url), status=int(exc.code),
|
|
duration_ms=int((time.monotonic() - started) * 1000), details={**log_details, "response": detail}, error=_upstream_error(error_prefix, int(exc.code), detail))
|
|
raise RuntimeError(_upstream_error(error_prefix, int(exc.code), detail)) from exc
|
|
except urllib.error.URLError as exc:
|
|
record_log(log_kind, "upstream_unreachable", level="error", method=method, path=_safe_log_url(url),
|
|
duration_ms=int((time.monotonic() - started) * 1000), details=log_details, error=str(exc.reason)[:500])
|
|
raise RuntimeError(f"{error_prefix}_unreachable:{exc.reason}") from exc
|
|
try:
|
|
parsed = json.loads(body or b"{}")
|
|
record_log(log_kind, "upstream_request", method=method, path=_safe_log_url(url), status=response_status,
|
|
duration_ms=int((time.monotonic() - started) * 1000), details={**log_details, "response": parsed})
|
|
return parsed
|
|
except json.JSONDecodeError as exc:
|
|
record_log(log_kind, "upstream_invalid_json", level="error", method=method, path=_safe_log_url(url), status=response_status,
|
|
duration_ms=int((time.monotonic() - started) * 1000), details={**log_details, "response": body[:2_000].decode("utf-8", "replace")}, error=str(exc))
|
|
raise RuntimeError(f"{error_prefix}_invalid_json") from exc
|
|
|
|
|
|
def http_bytes(method: str, url: str, payload: Any = None, headers: dict[str, str] | None = None, timeout: float = 20, error_prefix: str = "upstream") -> bytes:
|
|
"""Make a small HTTP request where the response is not JSON (CSV download)."""
|
|
started = time.monotonic()
|
|
log_kind = _log_http_kind(error_prefix)
|
|
log_details = {"url": _safe_log_url(url), "timeout_seconds": timeout, "request": payload}
|
|
data = None if payload is None else json.dumps(payload).encode("utf-8")
|
|
request_headers = {"Accept": "*/*", "User-Agent": OUTBOUND_USER_AGENT, **(headers or {})}
|
|
if data is not None:
|
|
request_headers["Content-Type"] = "application/json"
|
|
request = urllib.request.Request(url, data=data, headers=request_headers, method=method.upper())
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
body = response.read()
|
|
response_status = int(getattr(response, "status", 200) or 200)
|
|
record_log(log_kind, "upstream_request", method=method, path=_safe_log_url(url), status=response_status,
|
|
duration_ms=int((time.monotonic() - started) * 1000), details={**log_details, "response_bytes": len(body)})
|
|
return body
|
|
except urllib.error.HTTPError as exc:
|
|
detail = exc.read().decode("utf-8", "replace")[:300]
|
|
record_log(log_kind, "upstream_http_error", level="error", method=method, path=_safe_log_url(url), status=int(exc.code),
|
|
duration_ms=int((time.monotonic() - started) * 1000), details={**log_details, "response": detail}, error=_upstream_error(error_prefix, int(exc.code), detail))
|
|
raise RuntimeError(_upstream_error(error_prefix, int(exc.code), detail)) from exc
|
|
except urllib.error.URLError as exc:
|
|
record_log(log_kind, "upstream_unreachable", level="error", method=method, path=_safe_log_url(url),
|
|
duration_ms=int((time.monotonic() - started) * 1000), details=log_details, error=str(exc.reason)[:500])
|
|
raise RuntimeError(f"{error_prefix}_unreachable:{exc.reason}") from exc
|
|
|
|
|
|
def ai_runtime_settings() -> dict[str, str]:
|
|
saved: dict[str, str] = {}
|
|
try:
|
|
conn = db()
|
|
saved = {str(r["key"]): str(r["value"]) for r in conn.execute("SELECT key,value FROM settings")}
|
|
conn.close()
|
|
except Exception:
|
|
pass
|
|
provider = saved.get("AI_PROVIDER", AI_PROVIDER).strip().lower() or "custom"
|
|
base = saved.get("AI_BASE_URL", AI_BASE_URL).strip().rstrip("/")
|
|
if provider == "nous":
|
|
base = NOUS_BASE_URL
|
|
return {"provider": provider, "base_url": base, "api_key": saved.get("AI_API_KEY", AI_API_KEY).strip(), "model": saved.get("AI_MODEL", AI_MODEL).strip()}
|
|
|
|
|
|
def nous_models() -> list[dict[str, Any]]:
|
|
"""Return the live Nous catalog when reachable, with stable free fallbacks."""
|
|
global _nous_models_cache, _nous_models_cached_at
|
|
# Avoid making every Settings refresh wait on an external catalog request.
|
|
# A successful catalog is reused for ten minutes; fallback data is also
|
|
# cached after a failed request and can be replaced on the next refresh.
|
|
if time.monotonic() - _nous_models_cached_at < 600:
|
|
return _nous_models_cache
|
|
try:
|
|
response = http_json("GET", NOUS_BASE_URL + "/models", timeout=10, error_prefix="ai")
|
|
raw = response.get("data", response) if isinstance(response, dict) else response
|
|
models: list[dict[str, Any]] = []
|
|
if isinstance(raw, list):
|
|
for item in raw:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
model_id = str(item.get("id") or "").strip()
|
|
if not model_id:
|
|
continue
|
|
pricing = item.get("pricing") if isinstance(item.get("pricing"), dict) else {}
|
|
prices = [str(pricing[k]).strip() for k in ("prompt", "completion", "input", "output") if k in pricing]
|
|
free = bool(prices) and all(price in {"0", "0.0", "0.00"} for price in prices)
|
|
models.append({"id": model_id, "label": model_id, "free": free, "context_length": item.get("context_length")})
|
|
free = [item for item in models if item.get("free")]
|
|
if free:
|
|
_nous_models_cache = free
|
|
_nous_models_cached_at = time.monotonic()
|
|
return free
|
|
except Exception:
|
|
pass
|
|
_nous_models_cached_at = time.monotonic()
|
|
return _nous_models_cache
|
|
|
|
|
|
def gmaps_headers() -> dict[str, str]:
|
|
if GMAPS_API_MODE == "saas":
|
|
if not GMAPS_API_KEY:
|
|
raise RuntimeError("gmaps_api_key_missing_for_saas_mode")
|
|
return {"X-API-Key": GMAPS_API_KEY}
|
|
return {}
|
|
|
|
|
|
def download_standard_results(job_id: str) -> list[dict[str, Any]]:
|
|
"""Download and parse the CSV produced by the standard web worker."""
|
|
body = http_bytes(
|
|
"GET",
|
|
GMAPS_URL + "/api/v1/jobs/" + quote_plus(str(job_id)) + "/download",
|
|
headers=gmaps_headers(),
|
|
timeout=30,
|
|
error_prefix="gmaps",
|
|
)
|
|
try:
|
|
text = body.decode("utf-8-sig")
|
|
return [dict(row) for row in csv.DictReader(io.StringIO(text))]
|
|
except (UnicodeDecodeError, csv.Error) as exc:
|
|
raise RuntimeError("gmaps_csv_invalid") from exc
|
|
|
|
|
|
def submit_gmaps_job(query: str, include_email: bool = False, extra_reviews: bool = False) -> str:
|
|
if not GMAPS_URL:
|
|
raise RuntimeError("gmaps_not_configured")
|
|
include_email = True
|
|
lang = os.environ.get("GMAPS_LANG", "en")
|
|
maps_runtime = gmaps_runtime_settings()
|
|
depth = int(maps_runtime["max_depth"])
|
|
request_timeout = int(maps_runtime["request_timeout_seconds"])
|
|
if GMAPS_API_MODE == "saas":
|
|
endpoint = "/api/v1/scrape"
|
|
payload = {
|
|
"keyword": query,
|
|
"lang": lang,
|
|
"max_depth": depth,
|
|
"email": bool(include_email),
|
|
"extra_reviews": bool(extra_reviews),
|
|
"timeout": max(1, min(7200, request_timeout)),
|
|
}
|
|
else:
|
|
# Standard gosom web mode uses the local job API. It has no API-key
|
|
# authentication and returns a UUID in {"id": ...} with HTTP 201.
|
|
endpoint = "/api/v1/jobs"
|
|
payload = {
|
|
"name": query[:180],
|
|
"keywords": [query],
|
|
"lang": lang,
|
|
"zoom": 15,
|
|
"depth": depth,
|
|
"email": bool(include_email),
|
|
"extra_reviews": bool(extra_reviews),
|
|
"max_time": max(60, min(7200, request_timeout)),
|
|
}
|
|
response = http_json("POST", GMAPS_URL + endpoint, payload, gmaps_headers(), timeout=30, error_prefix="gmaps")
|
|
job_id = field_value(response, "job_id", "id", "JobID", "ID")
|
|
if not job_id:
|
|
raise RuntimeError("gmaps_job_id_missing")
|
|
return str(job_id)
|
|
|
|
|
|
def poll_gmaps_job(job_id: str, progress: Callable[[str, int, int], None] | None = None) -> list[dict[str, Any]]:
|
|
if not GMAPS_URL:
|
|
raise RuntimeError("gmaps_not_configured")
|
|
maps_runtime = gmaps_runtime_settings()
|
|
deadline = time.monotonic() + max(60, int(maps_runtime["request_timeout_seconds"]) + 60)
|
|
poll_seconds = float(maps_runtime["poll_seconds"])
|
|
if GMAPS_API_MODE == "saas":
|
|
terminal_ok = {"completed", "succeeded", "success"}
|
|
else:
|
|
terminal_ok = {"ok", "completed", "succeeded", "success"}
|
|
terminal_bad = {"failed", "cancelled", "canceled", "discarded"}
|
|
while time.monotonic() < deadline:
|
|
response = http_json("GET", GMAPS_URL + "/api/v1/jobs/" + quote_plus(str(job_id)), headers=gmaps_headers(), timeout=30, error_prefix="gmaps")
|
|
status = str(field_value(response, "status", "Status") or "pending").lower()
|
|
result_count = as_int(field_value(response, "result_count", "ResultCount"))
|
|
if progress:
|
|
progress(status, result_count, 0)
|
|
if status in terminal_ok:
|
|
if "results" in response:
|
|
return extract_gmaps_results(response.get("results"))
|
|
if GMAPS_API_MODE == "standard":
|
|
return download_standard_results(job_id)
|
|
return extract_gmaps_results(response)
|
|
if status in terminal_bad:
|
|
error = field_value(response, "error", "Error")
|
|
raise RuntimeError("gmaps_job_" + status + (":" + as_text(error) if error else ""))
|
|
time.sleep(poll_seconds)
|
|
# The worker can finish writing its CSV just after its status endpoint
|
|
# exceeds our polling window. Recover a completed/partial result when it
|
|
# is available rather than discarding the run as a false failure.
|
|
if GMAPS_API_MODE == "standard":
|
|
try:
|
|
recovered = download_standard_results(job_id)
|
|
if recovered:
|
|
return recovered
|
|
except Exception:
|
|
pass
|
|
raise RuntimeError("gmaps_job_timeout")
|
|
|
|
|
|
# The browser fallback is bounded and does not bypass challenge pages. It is
|
|
# retained for local testing or when the user deliberately sets backend=browser.
|
|
SEARCH_DEADLINE_SECONDS = 25
|
|
ENGINE_TIMEOUT_MS = 7000
|
|
|
|
|
|
def blocked(text: str) -> bool:
|
|
lowered = text.lower()
|
|
return any(marker in lowered for marker in (
|
|
"unusual traffic", "recaptcha", "captcha", "automated queries",
|
|
"access denied", "sorry...", "detected unusual traffic",
|
|
))
|
|
|
|
|
|
def normalize_result_href(raw_href: str) -> str:
|
|
href = (raw_href or "").strip()
|
|
if href.startswith("//"):
|
|
href = "https:" + href
|
|
if href.startswith("/"):
|
|
return ""
|
|
parsed = urlparse(href)
|
|
if parsed.hostname and parsed.hostname.endswith("duckduckgo.com"):
|
|
params = parse_qs(parsed.query)
|
|
for key in ("uddg", "url", "u"):
|
|
target = params.get(key, [""])[0]
|
|
if target.startswith(("http://", "https://")):
|
|
href = unquote(target)
|
|
break
|
|
return href
|
|
|
|
|
|
def extract_search_results(page: Any, selector: str, title_selector: str | None,
|
|
excluded_hosts: set[str], requested: int) -> list[dict[str, Any]]:
|
|
output: list[dict[str, Any]] = []
|
|
seen_domains: set[str] = set()
|
|
|
|
def add_anchor(anchor: Any) -> None:
|
|
if len(output) >= requested:
|
|
return
|
|
try:
|
|
title_node = anchor.locator(title_selector) if title_selector else anchor
|
|
title = " ".join((title_node.inner_text() or "").split())[:200]
|
|
if not title:
|
|
title = " ".join(((anchor.get_attribute("aria-label") or anchor.get_attribute("title") or "").split()))[:200]
|
|
href = normalize_result_href(anchor.get_attribute("href") or "")
|
|
except Exception:
|
|
return
|
|
domain = clean_domain(href)
|
|
if not title or len(title) < 3 or not href.startswith(("http://", "https://")):
|
|
return
|
|
if not domain or domain in excluded_hosts or domain in seen_domains:
|
|
return
|
|
seen_domains.add(domain)
|
|
output.append({"title": title, "website": href, "name": title})
|
|
|
|
try:
|
|
anchors = page.locator(selector)
|
|
for i in range(min(anchors.count(), requested * 4)):
|
|
add_anchor(anchors.nth(i))
|
|
except Exception:
|
|
pass
|
|
if not output:
|
|
try:
|
|
anchors = page.locator("a")
|
|
for i in range(min(anchors.count(), 250)):
|
|
add_anchor(anchors.nth(i))
|
|
if len(output) >= requested:
|
|
break
|
|
except Exception:
|
|
pass
|
|
return output
|
|
|
|
|
|
def google_search(query: str, requested: int, run_id: int | None = None) -> list[dict[str, Any]]:
|
|
try:
|
|
from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeout
|
|
except ImportError as exc:
|
|
raise RuntimeError("playwright_not_installed") from exc
|
|
executable = os.environ.get("PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH", "").strip()
|
|
options: dict[str, Any] = {"headless": True}
|
|
if executable:
|
|
options["executable_path"] = executable
|
|
engines = [
|
|
("brave", "https://search.brave.com/search?q=" + quote_plus(query), "a.heading-serpresult,a.result-header", None, {"search.brave.com"}),
|
|
("startpage", "https://www.startpage.com/sp/search?query=" + quote_plus(query), "a.w-gl__result-title", None, {"startpage.com"}),
|
|
("duckduckgo", "https://html.duckduckgo.com/html/?q=" + quote_plus(query), "a.result__a,a[data-testid='result-title-a']", None, {"duckduckgo.com"}),
|
|
("bing", "https://www.bing.com/search?q=" + quote_plus(query) + "&count=" + str(requested), "li.b_algo h2 a", "h2", {"bing.com", "bingj.com"}),
|
|
("google", "https://www.google.com/search?q=" + quote_plus(query) + "&num=" + str(requested), "a:has(h3)", "h3", {"google.com", "google.co.za"}),
|
|
]
|
|
errors: list[str] = []
|
|
deadline = time.monotonic() + SEARCH_DEADLINE_SECONDS
|
|
with sync_playwright() as p:
|
|
browser = p.chromium.launch(**options)
|
|
try:
|
|
for engine, url, selector, title_selector, excluded_hosts in engines:
|
|
remaining = deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
errors.append("search_deadline_exceeded")
|
|
break
|
|
context = None
|
|
try:
|
|
record_log("search", "search_engine_started", run_id=run_id, details={"engine": engine, "query": query, "requested": requested})
|
|
context = browser.new_context(locale="en-ZA", java_script_enabled=True, service_workers="block", viewport={"width": 1280, "height": 900})
|
|
page = context.new_page()
|
|
page.set_default_timeout(ENGINE_TIMEOUT_MS)
|
|
navigation_timed_out = False
|
|
try:
|
|
page.goto(url, wait_until="domcontentloaded", timeout=min(ENGINE_TIMEOUT_MS, max(1000, int(remaining * 1000))))
|
|
except PlaywrightTimeout:
|
|
navigation_timed_out = True
|
|
page.wait_for_timeout(min(500, max(0, int((deadline - time.monotonic()) * 1000))))
|
|
body = page.locator("body").inner_text()
|
|
html = page.content()
|
|
if blocked(body) or blocked(html):
|
|
errors.append(engine + "_blocked")
|
|
record_log("search", "search_engine_blocked", level="warn", run_id=run_id, details={"engine": engine, "query": query})
|
|
continue
|
|
output = extract_search_results(page, selector, title_selector, excluded_hosts, requested)
|
|
if output:
|
|
record_log("search", "search_engine_results", run_id=run_id, details={"engine": engine, "query": query, "count": len(output), "results": output})
|
|
return output
|
|
errors.append(engine + ("_timeout" if navigation_timed_out else "_empty"))
|
|
record_log("search", "search_engine_empty", level="warn", run_id=run_id, details={"engine": engine, "query": query, "navigation_timed_out": navigation_timed_out})
|
|
except PlaywrightTimeout:
|
|
errors.append(engine + "_timeout")
|
|
record_log("search", "search_engine_timeout", level="warn", run_id=run_id, details={"engine": engine, "query": query})
|
|
except Exception as exc:
|
|
errors.append(engine + "_error")
|
|
record_log("search", "search_engine_error", level="error", run_id=run_id, details={"engine": engine, "query": query}, error=str(exc)[:500])
|
|
finally:
|
|
if context is not None:
|
|
try:
|
|
context.close()
|
|
except Exception:
|
|
pass
|
|
finally:
|
|
browser.close()
|
|
message = "search_engines_unavailable:" + ",".join(errors)
|
|
record_log("search", "search_failed", level="error", run_id=run_id, details={"query": query, "errors": errors}, error=message)
|
|
raise RuntimeError(message)
|
|
|
|
|
|
def validate_payload(payload: dict[str, Any]) -> tuple[str, str, str, int, bool, bool]:
|
|
category = str(payload.get("category", "")).strip()
|
|
keywords = str(payload.get("keywords", "")).strip()
|
|
location = str(payload.get("location", "South Africa")).strip() or "South Africa"
|
|
if not category or not keywords:
|
|
raise ValueError("category_and_keywords_required")
|
|
try:
|
|
requested = max(1, min(MAX_RESULTS, int(payload.get("max_results", 10))))
|
|
except (TypeError, ValueError) as exc:
|
|
raise ValueError("invalid_max_results") from exc
|
|
query = f"{keywords} {category} {location}".strip()
|
|
# Email discovery is part of this app's core use case. The server forces
|
|
# it on so a client/UI change cannot accidentally create a run without
|
|
# attempting public email extraction.
|
|
include_email = True
|
|
extra_reviews = bool(payload.get("extra_reviews", False))
|
|
return category, query, location, requested, include_email, extra_reviews
|
|
|
|
|
|
def update_run(run_id: int, **fields: Any) -> None:
|
|
if not fields:
|
|
return
|
|
allowed = {"found", "added", "status", "error", "source", "external_job_id", "stage", "progress", "completed_at"}
|
|
safe = {key: value for key, value in fields.items() if key in allowed}
|
|
if not safe:
|
|
return
|
|
assignments = ", ".join(f"{key}=?" for key in safe)
|
|
conn = db()
|
|
try:
|
|
conn.execute(f"UPDATE runs SET {assignments} WHERE id=?", (*safe.values(), run_id))
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
if any(key in safe for key in ("stage", "status", "error")):
|
|
record_log("system", "run_state_updated", level="error" if safe.get("status") == "failed" else "info",
|
|
run_id=run_id, details={key: value for key, value in safe.items() if key in {"stage", "status", "progress", "found", "added", "error", "external_job_id"}},
|
|
error=str(safe.get("error") or ""))
|
|
|
|
|
|
def _json_list(value: Any) -> list[Any]:
|
|
try:
|
|
parsed = json.loads(str(value or "[]"))
|
|
return parsed if isinstance(parsed, list) else []
|
|
except (TypeError, ValueError, json.JSONDecodeError):
|
|
return []
|
|
|
|
|
|
def enrich_place(item: dict[str, Any]) -> dict[str, Any]:
|
|
"""Add website health, public email evidence, and explainable scoring."""
|
|
prior_evidence = _json_list(item.get("email_evidence"))
|
|
if item.get("website"):
|
|
scan_started = time.monotonic()
|
|
record_log("search", "website_scan_started", prospect_id=as_int(item.get("id")) or None,
|
|
details={"business": item.get("name"), "url": _safe_log_url(str(item.get("website"))), "timeout_seconds": website_scan_timeout_seconds()})
|
|
try:
|
|
item.update(scan_website(str(item["website"]), timeout_seconds=website_scan_timeout_seconds()))
|
|
record_log("search", "website_scan_completed", prospect_id=as_int(item.get("id")) or None,
|
|
duration_ms=int((time.monotonic() - scan_started) * 1000),
|
|
details={"business": item.get("name"), "url": _safe_log_url(str(item.get("website"))), "status": item.get("website_status"), "emails_found": len(item.get("website_emails") or [])})
|
|
except Exception as exc: # a single hostile site must not abort a run
|
|
item.update({"website_status": "unknown", "website_checked_at": now(), "website_error": str(exc)[:300], "website_emails": [], "email_evidence": []})
|
|
record_log("search", "website_scan_failed", level="error", prospect_id=as_int(item.get("id")) or None,
|
|
duration_ms=int((time.monotonic() - scan_started) * 1000),
|
|
details={"business": item.get("name"), "url": _safe_log_url(str(item.get("website")))}, error=str(exc)[:500])
|
|
else:
|
|
item.setdefault("website_status", "no_detected_website")
|
|
item.setdefault("website_emails", [])
|
|
item.setdefault("email_evidence", [])
|
|
existing_emails = normalize_public_emails(item.get("emails"))
|
|
new_emails = normalize_public_emails(item.get("website_emails"))
|
|
all_emails = list(dict.fromkeys(existing_emails + new_emails))[:30]
|
|
item["emails"] = ", ".join(all_emails)
|
|
evidence = prior_evidence
|
|
scanned_evidence = item.get("email_evidence")
|
|
if isinstance(scanned_evidence, list):
|
|
evidence.extend(scanned_evidence)
|
|
# The Maps worker evidence was serialized before the website scan. Keep it
|
|
# only once, then append the scanner's page-level evidence.
|
|
seen_evidence: set[tuple[str, str]] = set()
|
|
compact: list[dict[str, Any]] = []
|
|
for record in evidence:
|
|
if not isinstance(record, dict):
|
|
continue
|
|
marker = (str(record.get("email", "")), str(record.get("source_url", "")))
|
|
if marker[0] and marker not in seen_evidence:
|
|
seen_evidence.add(marker)
|
|
compact.append(record)
|
|
item["email_evidence"] = json.dumps(compact[:50], ensure_ascii=False)
|
|
score, reasons = score_opportunity(item)
|
|
item["opportunity_score"] = score
|
|
item["score_reasons"] = json.dumps(reasons, ensure_ascii=False)
|
|
return item
|
|
|
|
|
|
def insert_catalogue(run_id: int, query: str, raw_results: list[dict[str, Any]], requested: int,
|
|
progress: Callable[[int, int], None] | None = None,
|
|
search_category: str = "") -> tuple[list[dict[str, Any]], int]:
|
|
mapped = [map_gmaps_result(item, query, search_category) for item in raw_results[:requested]]
|
|
conn = db()
|
|
added = 0
|
|
try:
|
|
suppressed_domains = {str(row[0]).lower() for row in conn.execute("SELECT value FROM suppressions WHERE kind='domain'")}
|
|
suppressed_emails = {str(row[0]).lower() for row in conn.execute("SELECT value FROM suppressions WHERE kind='email'")}
|
|
total = len(mapped)
|
|
for index, item in enumerate(mapped, 1):
|
|
# A hostile or slow public site must not invalidate an otherwise
|
|
# successful Maps run. Keep the raw listing and mark enrichment as
|
|
# unknown; it can be rescanned later from the explorer.
|
|
try:
|
|
item = enrich_place(item)
|
|
except Exception as exc:
|
|
item.update({"website_status": "unknown", "website_checked_at": now(), "website_error": "analysis_failed:" + str(exc)[:240], "email_evidence": item.get("email_evidence", "[]"), "score_reasons": "[]"})
|
|
item["opportunity_score"] = score_opportunity(item)[0]
|
|
# Release SQLite's read/write transaction before the progress
|
|
# callback opens its own connection to update the run row. Without
|
|
# this commit, a multi-result enrichment can deadlock forever in
|
|
# the website_analysis stage while waiting on the catalogue read.
|
|
conn.commit()
|
|
if progress:
|
|
progress(index, total)
|
|
emails = set(normalize_public_emails(item.get("emails")))
|
|
if item["domain"].lower() in suppressed_domains or emails.intersection(suppressed_emails):
|
|
record_log("system", "prospect_suppressed", run_id=run_id, details={"name": item.get("name"), "domain": item.get("domain"), "email_match": bool(emails.intersection(suppressed_emails))})
|
|
continue
|
|
try:
|
|
cur = conn.execute("""INSERT OR IGNORE INTO prospects(
|
|
name,website,domain,description,query,discovered_at,source,source_id,
|
|
category,search_category,address,phone,emails,rating,review_count,maps_url,latitude,
|
|
longitude,listing_status,website_status,website_http_status,website_final_url,
|
|
website_latency_ms,website_https,website_title,website_technology,website_has_viewport,
|
|
website_tls_expires_at,website_checked_at,website_error,email_evidence,score_reasons,
|
|
ai_probability,ai_recommendation,ai_reasoning,ai_checked_at,ai_subject,ai_message,ai_offer,opportunity_score,raw_json
|
|
) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""", (
|
|
item["name"], item["website"], item["domain"], item["description"], item["query"], now(),
|
|
item["source"], item["source_id"], item["category"], item.get("search_category", ""), item["address"], item["phone"], item["emails"],
|
|
item["rating"], item["review_count"], item["maps_url"], item["latitude"], item["longitude"],
|
|
item.get("listing_status", ""), item.get("website_status", "unknown"), item.get("website_http_status"),
|
|
item.get("website_final_url", ""), item.get("website_latency_ms", 0), item.get("website_https", 0),
|
|
item.get("website_title", ""), item.get("website_technology", ""), item.get("website_has_viewport", 0),
|
|
item.get("website_tls_expires_at", ""), item.get("website_checked_at", ""), item.get("website_error", ""),
|
|
item.get("email_evidence", "[]"), item.get("score_reasons", "[]"), None, "", "", "", "", "", "",
|
|
item["opportunity_score"], item["raw_json"],
|
|
))
|
|
except (sqlite3.Error, TypeError, ValueError):
|
|
# Keep processing the remaining listings if one malformed
|
|
# provider row cannot be written.
|
|
continue
|
|
if cur.rowcount:
|
|
added += 1
|
|
conn.commit()
|
|
record_log("system", "prospect_added", run_id=run_id, details={"name": item.get("name"), "domain": item.get("domain"), "score": item.get("opportunity_score")})
|
|
else:
|
|
conn.commit()
|
|
record_log("system", "prospect_duplicate", run_id=run_id, details={"name": item.get("name"), "domain": item.get("domain")})
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
return mapped, added
|
|
|
|
|
|
def run_discovery(run_id: int, payload: dict[str, Any]) -> None:
|
|
category, query, location, requested, include_email, extra_reviews = validate_payload(payload)
|
|
# Serialize local runs so two personal browser jobs do not fight over the
|
|
# same machine's Chromium/RAM. The Maps worker itself remains async.
|
|
with _discover_lock:
|
|
record_log("search", "discovery_started", run_id=run_id, details={"query": query, "location": location, "requested": requested, "backend": backend_mode(), "email_collection": include_email})
|
|
update_run(run_id, status="running", stage="connecting", progress=5, source=backend_mode())
|
|
try:
|
|
if backend_mode() == "gmaps":
|
|
external_job_id = submit_gmaps_job(query, include_email, extra_reviews)
|
|
update_run(run_id, external_job_id=external_job_id, stage="scraping", progress=10)
|
|
|
|
def on_progress(status: str, found: int, _unused: int) -> None:
|
|
progress = 15 if status == "pending" else 45 if status == "running" else 75
|
|
update_run(run_id, stage="scraping", progress=progress, found=found)
|
|
|
|
raw_results = poll_gmaps_job(external_job_id, on_progress)
|
|
else:
|
|
update_run(run_id, stage="browser_search", progress=20)
|
|
raw_results = google_search(query, requested, run_id=run_id)
|
|
update_run(run_id, stage="website_analysis", progress=76, found=len(raw_results))
|
|
|
|
def enrichment_progress(index: int, total: int) -> None:
|
|
if total:
|
|
update_run(run_id, stage="website_analysis", progress=76 + int(index * 19 / total), found=index)
|
|
|
|
results, added = insert_catalogue(run_id, query, raw_results, requested, enrichment_progress, search_category=category)
|
|
update_run(run_id, found=len(results), added=added, status="succeeded", stage="complete", progress=100, completed_at=now(), error="")
|
|
record_log("search", "discovery_completed", run_id=run_id, details={"found": len(results), "added": added, "backend": backend_mode()})
|
|
except Exception as exc:
|
|
message = str(exc).strip()[:500] or "discovery_failed"
|
|
update_run(run_id, status="failed", stage="failed", progress=100, error=message, completed_at=now())
|
|
record_log("search", "discovery_failed", level="error", run_id=run_id, details={"backend": backend_mode()}, error=message)
|
|
|
|
|
|
def create_discovery(payload: dict[str, Any]) -> dict[str, Any]:
|
|
category, query, location, requested, include_email, extra_reviews = validate_payload(payload)
|
|
request_json = json.dumps({
|
|
"category": category,
|
|
"keywords": str(payload.get("keywords", "")).strip(),
|
|
"location": location,
|
|
"max_results": requested,
|
|
"include_email": include_email,
|
|
"extra_reviews": extra_reviews,
|
|
}, ensure_ascii=False)
|
|
conn = db()
|
|
try:
|
|
cur = conn.execute("""INSERT INTO runs(query,location,requested,status,created_at,source,stage,request_json)
|
|
VALUES(?,?,?,?,?,?,?,?)""", (query, location, requested, "queued", now(), backend_mode(), "queued", request_json))
|
|
run_id = int(cur.lastrowid)
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
worker = threading.Thread(target=run_discovery, args=(run_id, payload), name=f"prospect-run-{run_id}", daemon=True)
|
|
worker.start()
|
|
return {"id": run_id, "status": "queued", "source": backend_mode(), "query": query, "requested": requested}
|
|
|
|
|
|
def scheduler_loop() -> None:
|
|
while True:
|
|
try:
|
|
conn = db(); due = conn.execute("SELECT * FROM schedules WHERE enabled=1 AND next_run_at<=?", (now(),)).fetchall()
|
|
for row in due:
|
|
payload = {"category": row["category"], "keywords": row["keywords"], "location": row["location"], "max_results": row["max_results"], "extra_reviews": False}
|
|
run = create_discovery(payload)
|
|
conn.execute("UPDATE schedules SET next_run_at=?,last_run_id=? WHERE id=?", (datetime.now(timezone.utc).replace(microsecond=0).timestamp() and datetime.fromtimestamp(datetime.now(timezone.utc).timestamp()+int(row["interval_hours"])*3600,timezone.utc).isoformat(), run["id"], row["id"]))
|
|
conn.commit(); conn.close()
|
|
except Exception:
|
|
try: conn.close()
|
|
except Exception: pass
|
|
time.sleep(30)
|
|
|
|
|
|
def ai_generate_message(prospect: dict[str, Any], offer: str) -> dict[str, Any]:
|
|
"""Generate an evidence-grounded, human-editable outreach draft."""
|
|
runtime = ai_runtime_settings()
|
|
ai_base_url = runtime["base_url"]
|
|
ai_api_key = runtime["api_key"]
|
|
ai_model = runtime["model"]
|
|
if runtime["provider"] == "nous" and not ai_api_key:
|
|
raise RuntimeError("nous_api_key_required_for_standalone_app")
|
|
if not ai_base_url or not ai_model:
|
|
raise RuntimeError("ai_not_configured_set_AI_BASE_URL_and_AI_MODEL")
|
|
evidence_ids = ["prospect_record"]
|
|
if prospect.get("maps_url"):
|
|
evidence_ids.append("public_maps_listing")
|
|
if prospect.get("website"):
|
|
evidence_ids.append("website_scan")
|
|
if prospect.get("emails"):
|
|
evidence_ids.append("public_email_listing")
|
|
if prospect.get("phone"):
|
|
evidence_ids.append("public_phone_listing")
|
|
evidence = {
|
|
"evidence_ids": evidence_ids,
|
|
"business_name": prospect.get("name"),
|
|
"search_category": prospect.get("search_category") or prospect.get("category"),
|
|
"address_or_location": prospect.get("address"),
|
|
"public_listing": {
|
|
"maps_url": prospect.get("maps_url"), "phone": prospect.get("phone"),
|
|
"rating": prospect.get("rating"), "review_count": prospect.get("review_count"),
|
|
},
|
|
"website": {
|
|
"url": prospect.get("website"), "status": prospect.get("website_status"),
|
|
"http_status": prospect.get("website_http_status"), "title": prospect.get("website_title"),
|
|
"technology": prospect.get("website_technology"), "https": bool(prospect.get("website_https")),
|
|
"latency_ms": prospect.get("website_latency_ms"),
|
|
},
|
|
"public_emails": prospect.get("emails"),
|
|
"opportunity_score": prospect.get("opportunity_score"),
|
|
"score_reasons": _json_list(prospect.get("score_reasons")),
|
|
}
|
|
system = (
|
|
"You are a senior B2B copywriter for a South African web-hosting and website-design business. "
|
|
"Create a short, respectful, genuinely personalised first-contact email from the supplied evidence. "
|
|
"Use only stored facts and the user's offer; never invent a person's name, company activity, price, "
|
|
"service, problem, result, testimonial, urgency, or link. Do not mention scraping, scoring, hidden "
|
|
"analysis, or imply certainty where the evidence is uncertain. Do not insult the prospect or call a "
|
|
"site bad; use neutral wording such as 'may be having trouble loading' when appropriate. "
|
|
"If there is no website, do not say you reviewed their website. If there is no mockup URL in the offer, "
|
|
"do not invent one. Keep the email around 90-150 words, with a natural greeting using the business name, "
|
|
"one specific evidence-based observation, the exact offer/instructions supplied by the user, one simple "
|
|
"call to action, and a polite opt-out sentence. Avoid spammy hype, excessive emojis, all-caps, and fake "
|
|
"personal familiarity. Return ONLY strict JSON with exactly these keys: subject, message, rationale, "
|
|
"recommended_offer, unknowns, evidence_used, follow_up. subject is 5-10 natural words. message is plain "
|
|
"text. rationale is concise. recommended_offer is a short summary of the user's offer. unknowns is an "
|
|
"array of facts that should be confirmed. evidence_used is an array containing only the supplied evidence "
|
|
"IDs. follow_up is a short human review note, not part of the email."
|
|
)
|
|
prompt = json.dumps({"evidence": evidence, "user_offer_or_instruction": offer[:2000]}, ensure_ascii=False)
|
|
headers = {"Authorization": "Bearer " + ai_api_key} if ai_api_key else {}
|
|
request_payload: dict[str, Any] = {
|
|
"model": ai_model,
|
|
"temperature": 0.4,
|
|
"max_tokens": 900,
|
|
"messages": [{"role": "system", "content": system}, {"role": "user", "content": prompt}],
|
|
}
|
|
# Nous exposes an OpenAI-compatible chat endpoint, but not every free model
|
|
# accepts OpenAI's optional response_format parameter. The prompt still
|
|
# requires strict JSON and the parser below validates the response.
|
|
if runtime["provider"] != "nous":
|
|
request_payload["response_format"] = {"type": "json_object"}
|
|
result = http_json("POST", ai_base_url + "/chat/completions", request_payload, headers=headers, timeout=90, error_prefix="ai")
|
|
content = (((result.get("choices") or [{}])[0].get("message") or {}).get("content") or "") if isinstance(result, dict) else ""
|
|
parsed = _parse_ai_object(content)
|
|
if not isinstance(parsed, dict) or not str(parsed.get("subject") or "").strip() or not str(parsed.get("message") or "").strip():
|
|
raise RuntimeError("ai_incomplete_message")
|
|
def text_list(value: Any, limit: int = 6, size: int = 400) -> list[str]:
|
|
if not isinstance(value, list):
|
|
return []
|
|
return [str(item).strip()[:size] for item in value if str(item).strip()][:limit]
|
|
used = [item for item in text_list(parsed.get("evidence_used"), limit=10, size=80) if item in evidence_ids]
|
|
return {
|
|
"subject": str(parsed["subject"]).strip()[:200],
|
|
"message": str(parsed["message"]).strip()[:8000],
|
|
"rationale": str(parsed.get("rationale") or "").strip()[:2000],
|
|
"recommended_offer": str(parsed.get("recommended_offer") or offer).strip()[:2000],
|
|
"unknowns": text_list(parsed.get("unknowns")),
|
|
"evidence_used": used,
|
|
"follow_up": str(parsed.get("follow_up") or "Review the draft and confirm the offer before sending.").strip()[:1000],
|
|
}
|
|
|
|
|
|
def _parse_ai_object(content: Any) -> dict[str, Any]:
|
|
"""Parse a JSON object from models that occasionally add markdown fences."""
|
|
text = str(content or "").strip()
|
|
if text.startswith("```"):
|
|
text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text, flags=re.I | re.S).strip()
|
|
try:
|
|
value = json.loads(text)
|
|
except (TypeError, ValueError, json.JSONDecodeError):
|
|
start, end = text.find("{"), text.rfind("}")
|
|
if start < 0 or end <= start:
|
|
raise RuntimeError("ai_invalid_json_response")
|
|
try:
|
|
value = json.loads(text[start:end + 1])
|
|
except (TypeError, ValueError, json.JSONDecodeError) as exc:
|
|
raise RuntimeError("ai_invalid_json_response") from exc
|
|
if not isinstance(value, dict):
|
|
raise RuntimeError("ai_invalid_json_response")
|
|
return value
|
|
|
|
|
|
def ai_assess_prospect(prospect: dict[str, Any]) -> dict[str, Any]:
|
|
"""Assess whether a prospect is worth human review/contact."""
|
|
runtime = ai_runtime_settings()
|
|
if runtime["provider"] == "nous" and not runtime["api_key"]:
|
|
raise RuntimeError("nous_api_key_required_for_standalone_app")
|
|
if not runtime["base_url"] or not runtime["model"]:
|
|
raise RuntimeError("ai_not_configured_set_AI_BASE_URL_and_AI_MODEL")
|
|
evidence = {
|
|
"business": prospect.get("name"), "category": prospect.get("category"),
|
|
"address": prospect.get("address"), "phone": prospect.get("phone"),
|
|
"website": prospect.get("website"), "website_status": prospect.get("website_status"),
|
|
"website_http_status": prospect.get("website_http_status"),
|
|
"website_title": prospect.get("website_title"), "technology": prospect.get("website_technology"),
|
|
"emails": prospect.get("emails"), "rating": prospect.get("rating"),
|
|
"review_count": prospect.get("review_count"), "opportunity_score": prospect.get("opportunity_score"),
|
|
"score_reasons": _json_list(prospect.get("score_reasons")),
|
|
}
|
|
system = (
|
|
"You qualify prospects for a South African web hosting and web design business. "
|
|
"Assess only the supplied public evidence. A high probability means the business appears active "
|
|
"and has a plausible hosting, domain, website repair, or redesign need. Do not invent facts, "
|
|
"intent, budget, decision-maker names, or contact details. Return strict JSON with exactly these "
|
|
"keys: probability (integer 0-100), recommendation (one of contact, review, do_not_contact), "
|
|
"reasoning (one concise paragraph)."
|
|
)
|
|
prompt = json.dumps({"prospect": evidence}, ensure_ascii=False)
|
|
headers = {"Authorization": "Bearer " + runtime["api_key"]} if runtime["api_key"] else {}
|
|
request_payload: dict[str, Any] = {
|
|
"model": runtime["model"], "temperature": 0.1, "max_tokens": 450,
|
|
"messages": [{"role": "system", "content": system}, {"role": "user", "content": prompt}],
|
|
}
|
|
if runtime["provider"] != "nous":
|
|
request_payload["response_format"] = {"type": "json_object"}
|
|
result = http_json("POST", runtime["base_url"] + "/chat/completions", request_payload, headers=headers, timeout=90, error_prefix="ai")
|
|
content = (((result.get("choices") or [{}])[0].get("message") or {}).get("content") or "") if isinstance(result, dict) else ""
|
|
parsed = _parse_ai_object(content)
|
|
try:
|
|
probability = max(0, min(100, int(round(float(parsed.get("probability", 0))))))
|
|
except (TypeError, ValueError):
|
|
raise RuntimeError("ai_invalid_probability")
|
|
recommendation = str(parsed.get("recommendation") or "").strip().lower().replace(" ", "_")
|
|
aliases = {
|
|
"yes": "contact", "contact_now": "contact", "high": "contact",
|
|
"maybe": "review", "needs_review": "review", "medium": "review",
|
|
"no": "do_not_contact", "do_not_contact_now": "do_not_contact", "low": "do_not_contact",
|
|
}
|
|
recommendation = aliases.get(recommendation, recommendation)
|
|
if recommendation not in {"contact", "review", "do_not_contact"}:
|
|
recommendation = "contact" if probability >= 70 else "review" if probability >= 45 else "do_not_contact"
|
|
return {
|
|
"ai_probability": probability,
|
|
"ai_recommendation": recommendation,
|
|
"ai_reasoning": str(parsed.get("reasoning") or "").strip()[:3000],
|
|
}
|
|
|
|
|
|
class Handler(BaseHTTPRequestHandler):
|
|
server_version = "PersonalProspectOS/2.0"
|
|
|
|
def handle_one_request(self) -> None:
|
|
self._request_started_at = time.monotonic()
|
|
super().handle_one_request()
|
|
|
|
def send_json(self, status: int, value: Any) -> None:
|
|
body = json_bytes(value)
|
|
self.send_response(status)
|
|
self.send_header("Content-Type", "application/json; charset=utf-8")
|
|
self.send_header("Cache-Control", "no-store")
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
if str(getattr(self, "path", "")).startswith("/api/"):
|
|
error_value = value.get("error") if isinstance(value, dict) else ""
|
|
record_log(
|
|
"api", f"{getattr(self, 'command', 'HTTP')} {urlparse(self.path).path}",
|
|
level="error" if status >= 400 else "info", method=getattr(self, "command", "HTTP"),
|
|
path=urlparse(self.path).path, status=status,
|
|
duration_ms=int((time.monotonic() - getattr(self, "_request_started_at", time.monotonic())) * 1000),
|
|
details={"response_status": status}, error=str(error_value or "")[:500],
|
|
)
|
|
|
|
def read_json(self) -> dict[str, Any]:
|
|
length = min(int(self.headers.get("Content-Length", "0") or 0), 100_000)
|
|
value = json.loads(self.rfile.read(length) or b"{}")
|
|
if not isinstance(value, dict):
|
|
raise ValueError("json_object_required")
|
|
return value
|
|
|
|
def do_GET(self) -> None:
|
|
parsed = urlparse(self.path)
|
|
if not parsed.path.startswith("/api/"):
|
|
self.serve_static(parsed.path)
|
|
return
|
|
conn = db()
|
|
try:
|
|
if parsed.path == "/api/health":
|
|
configured = bool(GMAPS_URL) if backend_mode() == "gmaps" else True
|
|
if GMAPS_API_MODE == "saas":
|
|
configured = configured and bool(GMAPS_API_KEY)
|
|
return self.send_json(200, {"status": "ok", "source": backend_mode(), "configured": configured, "maps_service": bool(GMAPS_URL), "maps_api_mode": GMAPS_API_MODE, "maps_api_key": bool(GMAPS_API_KEY)})
|
|
if parsed.path == "/api/config":
|
|
auth_required = GMAPS_API_MODE == "saas"
|
|
ready = backend_mode() == "browser" or (bool(GMAPS_URL) and (not auth_required or bool(GMAPS_API_KEY)))
|
|
ai_runtime = ai_runtime_settings()
|
|
browser_runtime = browser_use_settings()
|
|
return self.send_json(200, {
|
|
"backend": backend_mode(),
|
|
"gmaps_url": bool(GMAPS_URL),
|
|
"gmaps_api_mode": GMAPS_API_MODE,
|
|
"gmaps_auth_required": auth_required,
|
|
"gmaps_api_key": bool(GMAPS_API_KEY),
|
|
"ready": ready,
|
|
"email_extraction_default": True,
|
|
"email_extraction_always": True,
|
|
"browser_use_enabled": browser_runtime["enabled"],
|
|
"ai_provider": ai_runtime["provider"],
|
|
"ai_configured": bool(ai_runtime["base_url"] and ai_runtime["model"] and (ai_runtime["provider"] != "nous" or ai_runtime["api_key"])),
|
|
})
|
|
if parsed.path == "/api/ai/models":
|
|
provider = (parse_qs(parsed.query).get("provider", [AI_PROVIDER])[0] or AI_PROVIDER).strip().lower()
|
|
if provider == "nous":
|
|
return self.send_json(200, {"provider": "nous", "base_url": NOUS_BASE_URL, "models": nous_models()})
|
|
return self.send_json(200, {"provider": provider or "custom", "models": []})
|
|
if parsed.path == "/api/logs":
|
|
params = parse_qs(parsed.query)
|
|
kind = (params.get("kind", [""])[0] or "").strip().lower()
|
|
level = (params.get("level", [""])[0] or "").strip().lower()
|
|
query = (params.get("q", [""])[0] or "").strip().lower()
|
|
try:
|
|
limit = max(1, min(500, int(params.get("limit", ["200"])[0])))
|
|
except (TypeError, ValueError):
|
|
limit = 200
|
|
clauses: list[str] = []
|
|
values: list[Any] = []
|
|
if kind and kind != "all":
|
|
clauses.append("kind=?"); values.append(kind)
|
|
if level and level != "all":
|
|
clauses.append("level=?"); values.append(level)
|
|
if query:
|
|
clauses.append("(lower(event) LIKE ? OR lower(path) LIKE ? OR lower(error) LIKE ? OR lower(details) LIKE ?)")
|
|
needle = "%" + query[:120] + "%"; values.extend([needle, needle, needle, needle])
|
|
where = " WHERE " + " AND ".join(clauses) if clauses else ""
|
|
rows = conn.execute("SELECT * FROM logs" + where + " ORDER BY id DESC LIMIT ?", (*values, limit)).fetchall()
|
|
items: list[dict[str, Any]] = []
|
|
for row in rows:
|
|
item = dict(row)
|
|
try:
|
|
item["details"] = json.loads(item.get("details") or "{}")
|
|
except (TypeError, ValueError, json.JSONDecodeError):
|
|
item["details"] = {"raw": str(item.get("details") or "")[:20_000]}
|
|
items.append(item)
|
|
total = int(conn.execute("SELECT COUNT(*) FROM logs" + where, tuple(values)).fetchone()[0])
|
|
counts = {str(row[0]): int(row[1]) for row in conn.execute("SELECT kind,COUNT(*) FROM logs GROUP BY kind")}
|
|
error_count = int(conn.execute("SELECT COUNT(*) FROM logs WHERE level='error'").fetchone()[0])
|
|
counts["errors"] = error_count
|
|
return self.send_json(200, {"items": items, "total": total, "counts": counts})
|
|
if parsed.path == "/api/settings":
|
|
ai_runtime = ai_runtime_settings()
|
|
models = nous_models() if ai_runtime["provider"] == "nous" else []
|
|
browser_runtime = browser_use_settings()
|
|
maps_runtime = gmaps_runtime_settings()
|
|
scan_timeout = website_scan_timeout_seconds()
|
|
return self.send_json(200, {
|
|
"maps": {
|
|
"backend": backend_mode(), "worker_url": GMAPS_URL,
|
|
"ready": backend_mode() == "browser" or bool(GMAPS_URL),
|
|
**maps_runtime,
|
|
},
|
|
"ai": {"provider": ai_runtime["provider"], "base_url": ai_runtime["base_url"], "model": ai_runtime["model"], "configured": bool(ai_runtime["base_url"] and ai_runtime["model"] and (ai_runtime["provider"] != "nous" or ai_runtime["api_key"])), "models": models},
|
|
"browser_use": browser_runtime,
|
|
"email_collection": {"always_on": True, "public_only": True},
|
|
"limits": {"max_results": MAX_RESULTS, "website_scan_timeout_seconds": scan_timeout},
|
|
})
|
|
if parsed.path == "/api/prospects":
|
|
params = parse_qs(parsed.query)
|
|
q = (params.get("q", [""])[0]).strip().lower()
|
|
status = params.get("status", [""])[0].strip().lower()
|
|
rows = conn.execute("SELECT * FROM prospects ORDER BY opportunity_score DESC, id DESC").fetchall()
|
|
items = []
|
|
for row in rows:
|
|
haystack = " ".join(str(row[key] or "") for key in ("name", "domain", "description", "address", "category", "search_category", "phone", "emails")).lower()
|
|
if q and q not in haystack:
|
|
continue
|
|
if status and row["review_status"] != status:
|
|
continue
|
|
items.append(dict(row))
|
|
return self.send_json(200, {"items": items, "total": len(items)})
|
|
if parsed.path == "/api/export.csv":
|
|
params = parse_qs(parsed.query); q = params.get("q", [""])[0].strip().lower()
|
|
rows = [dict(r) for r in conn.execute("SELECT * FROM prospects ORDER BY opportunity_score DESC, id DESC")]
|
|
if q: rows = [r for r in rows if q in " ".join(str(r.get(k) or "") for k in ("name","domain","address","category","search_category","emails")).lower()]
|
|
out = io.StringIO(); fields = ["id","name","search_category","category","address","phone","emails","domain","website","website_status","opportunity_score","score_reasons","review_status","last_seen"]
|
|
writer = csv.DictWriter(out, fieldnames=fields); writer.writeheader(); writer.writerows({k:r.get(k) for k in fields} for r in rows)
|
|
body = out.getvalue().encode(); self.send_response(200); self.send_header("Content-Type","text/csv; charset=utf-8"); self.send_header("Content-Disposition","attachment; filename=prospects.csv"); self.send_header("Content-Length",str(len(body))); self.end_headers(); self.wfile.write(body); return
|
|
if parsed.path == "/api/suppressions":
|
|
return self.send_json(200, {"items": [dict(r) for r in conn.execute("SELECT * FROM suppressions ORDER BY id DESC")]})
|
|
if parsed.path == "/api/schedules":
|
|
return self.send_json(200, {"items": [dict(r) for r in conn.execute("SELECT * FROM schedules ORDER BY id DESC")]})
|
|
match = re.fullmatch(r"/api/prospects/(\d+)/website-candidates", parsed.path)
|
|
if match:
|
|
prospect_id = int(match.group(1))
|
|
if not conn.execute("SELECT 1 FROM prospects WHERE id=?", (prospect_id,)).fetchone():
|
|
return self.send_json(404, {"error": "not_found"})
|
|
rows = conn.execute("SELECT * FROM website_candidates WHERE prospect_id=? ORDER BY confidence DESC, id DESC", (prospect_id,)).fetchall()
|
|
return self.send_json(200, {"items": [dict(row) for row in rows]})
|
|
match = re.fullmatch(r"/api/prospects/(\d+)/domains", parsed.path)
|
|
if match:
|
|
row = conn.execute("SELECT * FROM prospects WHERE id=?", (int(match.group(1)),)).fetchone()
|
|
if not row: return self.send_json(404, {"error":"not_found"})
|
|
return self.send_json(200, {"items": domain_candidates(row["name"], row["address"], row["category"])})
|
|
match = re.fullmatch(r"/api/prospects/(\d+)", parsed.path)
|
|
if match:
|
|
row = conn.execute("SELECT * FROM prospects WHERE id=?", (int(match.group(1)),)).fetchone()
|
|
if not row: return self.send_json(404, {"error":"not_found"})
|
|
return self.send_json(200, dict(row))
|
|
if parsed.path == "/api/runs":
|
|
rows = [dict(row) for row in conn.execute("SELECT * FROM runs ORDER BY id DESC LIMIT 50")]
|
|
return self.send_json(200, {"items": rows})
|
|
match = re.fullmatch(r"/api/runs/(\d+)", parsed.path)
|
|
if match:
|
|
row = conn.execute("SELECT * FROM runs WHERE id=?", (int(match.group(1)),)).fetchone()
|
|
if not row:
|
|
return self.send_json(404, {"error": "not_found"})
|
|
return self.send_json(200, dict(row))
|
|
if parsed.path == "/api/stats":
|
|
total = conn.execute("SELECT COUNT(*) FROM prospects").fetchone()[0]
|
|
pending = conn.execute("SELECT COUNT(*) FROM prospects WHERE review_status='pending'").fetchone()[0]
|
|
approved = conn.execute("SELECT COUNT(*) FROM prospects WHERE review_status='approved'").fetchone()[0]
|
|
contacted = conn.execute("SELECT COUNT(*) FROM prospects WHERE review_status='contacted'").fetchone()[0]
|
|
running = conn.execute("SELECT COUNT(*) FROM runs WHERE status IN ('queued','running')").fetchone()[0]
|
|
return self.send_json(200, {"total": total, "pending": pending, "approved": approved, "contacted": contacted, "running": running})
|
|
return self.send_json(404, {"error": "not_found"})
|
|
finally:
|
|
conn.close()
|
|
|
|
def serve_static(self, path: str) -> None:
|
|
relative = "index.html" if path in {"", "/"} else path.lstrip("/")
|
|
root = (Path(__file__).parent / "web").resolve()
|
|
if not root.is_dir():
|
|
root = (Path(__file__).parent.parent / "web").resolve()
|
|
candidate = (root / relative).resolve()
|
|
if root not in candidate.parents and candidate != root:
|
|
return self.send_json(400, {"error": "invalid_path"})
|
|
if not candidate.is_file():
|
|
return self.send_json(404, {"error": "not_found"})
|
|
content = candidate.read_bytes()
|
|
content_type = "text/plain; charset=utf-8"
|
|
if candidate.suffix == ".html": content_type = "text/html; charset=utf-8"
|
|
elif candidate.suffix == ".js": content_type = "text/javascript; charset=utf-8"
|
|
elif candidate.suffix == ".css": content_type = "text/css; charset=utf-8"
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", content_type)
|
|
self.send_header("Cache-Control", "no-cache")
|
|
self.send_header("Content-Length", str(len(content)))
|
|
self.end_headers()
|
|
self.wfile.write(content)
|
|
|
|
def do_POST(self) -> None:
|
|
if urlparse(self.path).path == "/api/logs":
|
|
try:
|
|
payload = self.read_json()
|
|
action = str(payload.get("action") or "").strip()
|
|
if not action:
|
|
return self.send_json(400, {"error": "action_required"})
|
|
level = str(payload.get("level") or "info").strip().lower()
|
|
if level not in {"info", "warn", "error"}:
|
|
level = "info"
|
|
details = payload.get("details") if isinstance(payload.get("details"), (dict, list, str, int, float, bool)) else {}
|
|
record_log("action", action, level=level, method="CLIENT", path=str(payload.get("path") or "")[:500], details=details,
|
|
error=str(payload.get("error") or "")[:500])
|
|
return self.send_json(202, {"logged": True})
|
|
except (ValueError, TypeError, json.JSONDecodeError) as exc:
|
|
return self.send_json(400, {"error": str(exc)})
|
|
if self.path == "/api/settings":
|
|
try:
|
|
payload = self.read_json()
|
|
ai = payload.get("ai") if isinstance(payload.get("ai"), dict) else {}
|
|
browser = payload.get("browser_use") if isinstance(payload.get("browser_use"), dict) else {}
|
|
maps = payload.get("maps") if isinstance(payload.get("maps"), dict) else {}
|
|
limits = payload.get("limits") if isinstance(payload.get("limits"), dict) else {}
|
|
conn = db()
|
|
saved_keys: list[str] = []
|
|
|
|
def save_setting(key: str, value: Any) -> None:
|
|
text_value = str(value).strip()[:500]
|
|
conn.execute("INSERT INTO settings(key,value,updated_at) VALUES(?,?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value,updated_at=excluded.updated_at", (key, text_value, now()))
|
|
saved_keys.append(key)
|
|
|
|
for key in ("AI_PROVIDER", "AI_BASE_URL", "AI_API_KEY", "AI_MODEL"):
|
|
if key in ai:
|
|
# An empty password means “keep the existing credential”
|
|
# so a normal Settings save cannot erase a working key.
|
|
if key == "AI_API_KEY" and not str(ai.get(key) or "").strip():
|
|
continue
|
|
save_setting(key, ai.get(key) or "")
|
|
|
|
if "enabled" in browser:
|
|
enabled_value = str(browser.get("enabled") or "").strip().lower() not in {"0", "false", "no", "off", "disabled"}
|
|
save_setting("BROWSER_USE_ENABLED", "true" if enabled_value else "false")
|
|
if "max_steps" in browser:
|
|
save_setting("BROWSER_USE_MAX_STEPS", max(3, min(50, int(browser.get("max_steps")))))
|
|
if "timeout_seconds" in browser:
|
|
save_setting("BROWSER_USE_TIMEOUT_SECONDS", max(30, min(1800, int(browser.get("timeout_seconds")))))
|
|
if "llm_timeout_seconds" in browser:
|
|
save_setting("BROWSER_USE_LLM_TIMEOUT_SECONDS", max(10, min(300, int(browser.get("llm_timeout_seconds")))))
|
|
|
|
if "request_timeout_seconds" in maps:
|
|
save_setting("GMAPS_REQUEST_TIMEOUT", max(60, min(7200, int(maps.get("request_timeout_seconds")))))
|
|
if "poll_seconds" in maps:
|
|
save_setting("GMAPS_POLL_SECONDS", max(0.5, min(60.0, float(maps.get("poll_seconds")))))
|
|
if "max_depth" in maps:
|
|
save_setting("GMAPS_MAX_DEPTH", max(1, min(10, int(maps.get("max_depth")))))
|
|
if "website_scan_timeout_seconds" in limits:
|
|
save_setting("WEBSITE_SCAN_TIMEOUT", max(2, min(60, int(limits.get("website_scan_timeout_seconds")))))
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
record_log("action", "settings_updated", details={"keys": saved_keys})
|
|
ai_runtime = ai_runtime_settings()
|
|
return self.send_json(200, {
|
|
"saved": True,
|
|
"saved_keys": saved_keys,
|
|
"ai": {"provider": ai_runtime["provider"], "base_url": ai_runtime["base_url"], "model": ai_runtime["model"], "configured": bool(ai_runtime["base_url"] and ai_runtime["model"] and (ai_runtime["provider"] != "nous" or ai_runtime["api_key"]))},
|
|
"browser_use": browser_use_settings(),
|
|
"maps": gmaps_runtime_settings(),
|
|
"limits": {"website_scan_timeout_seconds": website_scan_timeout_seconds()},
|
|
})
|
|
except Exception as exc:
|
|
return self.send_json(400, {"error": str(exc)})
|
|
if urlparse(self.path).path == "/api/ai/test":
|
|
try:
|
|
runtime = ai_runtime_settings()
|
|
if not runtime["base_url"] or not runtime["model"]:
|
|
return self.send_json(400, {"error": "ai_not_configured_set_provider_and_model"})
|
|
if runtime["provider"] == "nous" and not runtime["api_key"]:
|
|
return self.send_json(400, {"error": "nous_api_key_required_for_standalone_app"})
|
|
headers = {"Authorization": "Bearer " + runtime["api_key"]} if runtime["api_key"] else {}
|
|
result = http_json("POST", runtime["base_url"] + "/chat/completions", {"model": runtime["model"], "temperature": 0, "max_tokens": 8, "messages": [{"role": "user", "content": "Reply with exactly OK."}]}, headers=headers, timeout=30, error_prefix="ai")
|
|
content = (((result.get("choices") or [{}])[0].get("message") or {}).get("content") or "") if isinstance(result, dict) else ""
|
|
if not str(content).strip():
|
|
return self.send_json(502, {"error": "ai_empty_response"})
|
|
return self.send_json(200, {"ok": True, "provider": runtime["provider"], "model": runtime["model"], "reply": str(content).strip()[:200]})
|
|
except RuntimeError as exc:
|
|
return self.send_json(502, {"error": str(exc)})
|
|
except Exception as exc:
|
|
return self.send_json(502, {"error": "ai_test_failed:" + str(exc)[:200]})
|
|
candidates_match = re.fullmatch(r"/api/prospects/(\d+)/website-candidates", urlparse(self.path).path)
|
|
if candidates_match:
|
|
prospect_id = int(candidates_match.group(1))
|
|
conn = db()
|
|
try:
|
|
row = conn.execute("SELECT * FROM prospects WHERE id=?", (prospect_id,)).fetchone()
|
|
finally:
|
|
conn.close()
|
|
if not row:
|
|
return self.send_json(404, {"error": "not_found"})
|
|
browser_runtime = browser_use_settings()
|
|
if not browser_runtime["enabled"]:
|
|
return self.send_json(503, {"error": "browser_use_disabled"})
|
|
try:
|
|
runtime = ai_runtime_settings()
|
|
record_log("browser", "website_investigation_started", prospect_id=prospect_id,
|
|
details={"business": row["name"], "model": runtime["model"], "public_only": True,
|
|
"max_steps": browser_runtime["max_steps"], "timeout_seconds": browser_runtime["timeout_seconds"],
|
|
"llm_timeout_seconds": browser_runtime["llm_timeout_seconds"]})
|
|
|
|
def browser_log(kind: str, event: str, details: dict[str, Any]) -> None:
|
|
record_log(kind, event, prospect_id=prospect_id, details=details)
|
|
|
|
# First use deterministic DOM extraction. This does not need
|
|
# an AI key and is deliberately run before Browser Use so a
|
|
# model timeout or malformed final response cannot erase a
|
|
# link that was already visible on Bing.
|
|
direct_candidates = direct_search_candidates(
|
|
dict(row), user_agent=OUTBOUND_USER_AGENT, log_callback=browser_log,
|
|
requested=5, timeout_seconds=min(180, int(browser_runtime["timeout_seconds"])),
|
|
)
|
|
if direct_candidates:
|
|
# Save the raw search links first. Verification is a
|
|
# separate bounded step and must never erase a useful
|
|
# result if a public site is slow or unavailable.
|
|
persist_website_candidates(prospect_id, direct_candidates)
|
|
verified_candidates = verify_website_candidates(dict(row), direct_candidates)
|
|
saved_rows = persist_website_candidates(prospect_id, verified_candidates)
|
|
for candidate in verified_candidates:
|
|
record_log("search", "website_candidate_verified", prospect_id=prospect_id,
|
|
details={"domain": candidate.get("domain"), "confidence": candidate.get("confidence"),
|
|
"verification": candidate.get("verification", {})})
|
|
record_log("browser", "website_candidates_persisted", prospect_id=prospect_id,
|
|
details={"source": "browser_dom", "candidate_count": len(direct_candidates),
|
|
"domains": [item.get("domain") for item in direct_candidates]})
|
|
return self.send_json(200, {"items": saved_rows, "count": len(saved_rows), "source": "browser_dom"})
|
|
|
|
# Keep Browser Use as a bounded fallback for cases where the
|
|
# public result pages yielded no parseable links.
|
|
record_log("browser", "browser_use_fallback_started", prospect_id=prospect_id,
|
|
details={"reason": "direct_search_no_candidates", "model": runtime["model"]})
|
|
candidates = investigate_websites(
|
|
dict(row), base_url=runtime["base_url"], api_key=runtime["api_key"],
|
|
model=runtime["model"], user_agent=OUTBOUND_USER_AGENT, log_callback=browser_log,
|
|
enabled=browser_runtime["enabled"], max_steps=browser_runtime["max_steps"],
|
|
timeout_seconds=browser_runtime["timeout_seconds"], llm_timeout_seconds=browser_runtime["llm_timeout_seconds"],
|
|
)
|
|
record_log("browser", "website_investigation_completed", prospect_id=prospect_id,
|
|
details={"candidate_count": len(candidates), "model": runtime["model"], "source": "browser_use"})
|
|
saved_rows = persist_website_candidates(prospect_id, candidates)
|
|
return self.send_json(200, {"items": saved_rows, "count": len(saved_rows), "source": "browser_use"})
|
|
except RuntimeError as exc:
|
|
record_log("browser", "website_investigation_failed", level="error", prospect_id=prospect_id, error=str(exc))
|
|
return self.send_json(503, {"error": str(exc)})
|
|
except Exception as exc:
|
|
record_log("browser", "website_investigation_failed", level="error", prospect_id=prospect_id, error=str(exc)[:500])
|
|
return self.send_json(502, {"error": "browser_use_failed:" + str(exc)[:240]})
|
|
finally:
|
|
try:
|
|
conn.close()
|
|
except Exception:
|
|
pass
|
|
assess_match = re.fullmatch(r"/api/prospects/(\d+)/assess", urlparse(self.path).path)
|
|
if assess_match:
|
|
conn = None
|
|
try:
|
|
conn = db(); prospect_id = int(assess_match.group(1))
|
|
row = conn.execute("SELECT * FROM prospects WHERE id=?", (prospect_id,)).fetchone()
|
|
if not row:
|
|
return self.send_json(404, {"error": "not_found"})
|
|
assessment = ai_assess_prospect(dict(row))
|
|
stamp = now()
|
|
conn.execute(
|
|
"UPDATE prospects SET ai_probability=?,ai_recommendation=?,ai_reasoning=?,ai_checked_at=? WHERE id=?",
|
|
(assessment["ai_probability"], assessment["ai_recommendation"], assessment["ai_reasoning"], stamp, prospect_id),
|
|
)
|
|
conn.commit()
|
|
return self.send_json(200, {**assessment, "ai_checked_at": stamp, "prospect_id": prospect_id})
|
|
except RuntimeError as exc:
|
|
return self.send_json(503, {"error": str(exc)})
|
|
except (ValueError, TypeError, json.JSONDecodeError) as exc:
|
|
return self.send_json(400, {"error": str(exc)})
|
|
finally:
|
|
if conn is not None:
|
|
conn.close()
|
|
message_match = re.fullmatch(r"/api/prospects/(\d+)/message", urlparse(self.path).path)
|
|
if message_match:
|
|
conn = None
|
|
try:
|
|
conn = db(); row = conn.execute("SELECT * FROM prospects WHERE id=?", (int(message_match.group(1)),)).fetchone()
|
|
if not row: return self.send_json(404, {"error":"not_found"})
|
|
payload = self.read_json(); offer = str(payload.get("offer") or "").strip()
|
|
if not offer: return self.send_json(400, {"error":"offer_required"})
|
|
draft = ai_generate_message(dict(row), offer)
|
|
stamp = now(); conn.execute("UPDATE prospects SET ai_subject=?,ai_message=?,ai_offer=?,ai_reasoning=?,ai_unknowns=?,ai_evidence_ids=?,ai_follow_up=?,ai_checked_at=? WHERE id=?", (draft["subject"],draft["message"],offer,draft["rationale"],json.dumps(draft.get("unknowns") or [], ensure_ascii=False),json.dumps(draft.get("evidence_used") or [], ensure_ascii=False),draft.get("follow_up") or "",stamp,int(message_match.group(1)))); conn.commit()
|
|
return self.send_json(200, {**draft,"ai_checked_at":stamp,"prospect_id":int(message_match.group(1))})
|
|
except RuntimeError as exc: return self.send_json(503, {"error":str(exc)})
|
|
except (ValueError, TypeError, json.JSONDecodeError) as exc: return self.send_json(400, {"error":str(exc)})
|
|
finally:
|
|
if conn is not None: conn.close()
|
|
scan_match = re.fullmatch(r"/api/prospects/(\d+)/scan", urlparse(self.path).path)
|
|
if scan_match:
|
|
conn = None
|
|
try:
|
|
conn=db(); row=conn.execute("SELECT * FROM prospects WHERE id=?",(int(scan_match.group(1)),)).fetchone()
|
|
if not row: return self.send_json(404,{"error":"not_found"})
|
|
updated=enrich_place(dict(row)); cols=["website_status","website_http_status","website_final_url","website_latency_ms","website_https","website_title","website_technology","website_has_viewport","website_tls_expires_at","website_checked_at","website_error","emails","email_evidence","score_reasons","opportunity_score"]
|
|
conn.execute("UPDATE prospects SET "+",".join(c+"=?" for c in cols)+" WHERE id=?",tuple(updated.get(c) for c in cols)+(int(scan_match.group(1)),)); conn.commit(); return self.send_json(200,dict(conn.execute("SELECT * FROM prospects WHERE id=?",(int(scan_match.group(1)),)).fetchone()))
|
|
except Exception as exc: return self.send_json(500,{"error":str(exc)})
|
|
finally:
|
|
if conn is not None: conn.close()
|
|
if self.path == "/api/suppressions":
|
|
try:
|
|
p=self.read_json(); kind=str(p.get("kind","domain")).lower(); value=str(p.get("value","")).strip().lower()
|
|
if kind not in {"domain","email"} or not value: return self.send_json(400,{"error":"kind_and_value_required"})
|
|
conn=db(); conn.execute("INSERT OR IGNORE INTO suppressions(kind,value,reason,created_at) VALUES(?,?,?,?)",(kind,value,str(p.get("reason","manual"))[:200],now())); conn.commit(); row=conn.execute("SELECT * FROM suppressions WHERE kind=? AND value=?",(kind,value)).fetchone(); conn.close(); return self.send_json(201,dict(row))
|
|
except Exception as exc: return self.send_json(400,{"error":str(exc)})
|
|
if self.path == "/api/schedules":
|
|
try:
|
|
p=self.read_json(); category=str(p.get("category") or "").strip(); keywords=str(p.get("keywords") or "").strip(); location=str(p.get("location") or "South Africa").strip(); interval=max(1,min(168,int(p.get("interval_hours",24)))); max_results=max(1,min(MAX_RESULTS,int(p.get("max_results",25))))
|
|
if not category or not keywords: return self.send_json(400,{"error":"category_and_keywords_required"})
|
|
next_at=datetime.fromtimestamp(datetime.now(timezone.utc).timestamp()+interval*3600,timezone.utc).isoformat(); conn=db(); cur=conn.execute("INSERT INTO schedules(category,keywords,location,max_results,interval_hours,enabled,next_run_at,created_at) VALUES(?,?,?,?,?,?,?,?)",(category,keywords,location,max_results,interval,1,next_at,now())); conn.commit(); row=conn.execute("SELECT * FROM schedules WHERE id=?",(cur.lastrowid,)).fetchone(); conn.close(); return self.send_json(201,dict(row))
|
|
except Exception as exc: return self.send_json(400,{"error":str(exc)})
|
|
if self.path != "/api/discover":
|
|
return self.send_json(404, {"error": "not_found"})
|
|
try:
|
|
result = create_discovery(self.read_json())
|
|
return self.send_json(202, result)
|
|
except ValueError as exc:
|
|
return self.send_json(400, {"error": str(exc)})
|
|
except (TypeError, json.JSONDecodeError) as exc:
|
|
return self.send_json(400, {"error": str(exc)})
|
|
|
|
def do_PATCH(self) -> None:
|
|
candidate_match = re.fullmatch(r"/api/prospects/(\d+)/website-candidates/(\d+)", urlparse(self.path).path)
|
|
if candidate_match:
|
|
prospect_id, candidate_id = (int(candidate_match.group(1)), int(candidate_match.group(2)))
|
|
conn: sqlite3.Connection | None = None
|
|
try:
|
|
payload = self.read_json()
|
|
status = str(payload.get("status", "")).strip().lower()
|
|
if status not in {"candidate", "accepted", "rejected"}:
|
|
return self.send_json(400, {"error": "invalid_website_candidate_status"})
|
|
conn = db()
|
|
candidate = conn.execute(
|
|
"SELECT * FROM website_candidates WHERE id=? AND prospect_id=?",
|
|
(candidate_id, prospect_id),
|
|
).fetchone()
|
|
prospect = conn.execute("SELECT * FROM prospects WHERE id=?", (prospect_id,)).fetchone()
|
|
if not candidate or not prospect:
|
|
return self.send_json(404, {"error": "not_found"})
|
|
if status != "accepted":
|
|
conn.execute("UPDATE website_candidates SET status=? WHERE id=?", (status, candidate_id))
|
|
conn.commit()
|
|
updated_candidate = conn.execute("SELECT * FROM website_candidates WHERE id=?", (candidate_id,)).fetchone()
|
|
return self.send_json(200, {"candidate": dict(updated_candidate)})
|
|
|
|
# The candidate is now a real catalogue domain. Check the
|
|
# catalogue before scanning so SQLite's unique domain index
|
|
# produces an actionable response instead of a generic 500.
|
|
duplicate = conn.execute(
|
|
"SELECT id FROM prospects WHERE domain=? AND id<>?",
|
|
(str(candidate["domain"]).lower(), prospect_id),
|
|
).fetchone()
|
|
if duplicate:
|
|
return self.send_json(409, {"error": "domain_already_catalogued", "prospect_id": int(duplicate["id"])})
|
|
candidate_url = normalize_url(candidate["url"])
|
|
if not candidate_url:
|
|
return self.send_json(400, {"error": "invalid_candidate_url"})
|
|
|
|
# Website probing is network-bound. Release the SQLite handle
|
|
# while it runs, then persist the selected URL and evidence in
|
|
# one short transaction.
|
|
base_item = {**dict(prospect), "website": candidate_url, "domain": str(candidate["domain"]).lower()}
|
|
conn.close()
|
|
conn = None
|
|
updated = enrich_place(base_item)
|
|
cols = [
|
|
"website", "domain", "website_status", "website_http_status", "website_final_url",
|
|
"website_latency_ms", "website_https", "website_title", "website_technology",
|
|
"website_has_viewport", "website_tls_expires_at", "website_checked_at", "website_error",
|
|
"emails", "email_evidence", "score_reasons", "opportunity_score",
|
|
]
|
|
conn = db()
|
|
conn.execute(
|
|
"UPDATE website_candidates SET status='candidate' WHERE prospect_id=? AND id<>? AND status='accepted'",
|
|
(prospect_id, candidate_id),
|
|
)
|
|
conn.execute("UPDATE website_candidates SET status='accepted' WHERE id=? AND prospect_id=?", (candidate_id, prospect_id))
|
|
conn.execute(
|
|
"UPDATE prospects SET " + ",".join(f"{column}=?" for column in cols) + " WHERE id=?",
|
|
tuple(updated.get(column) for column in cols) + (prospect_id,),
|
|
)
|
|
conn.commit()
|
|
updated_candidate = conn.execute("SELECT * FROM website_candidates WHERE id=?", (candidate_id,)).fetchone()
|
|
updated_prospect = conn.execute("SELECT * FROM prospects WHERE id=?", (prospect_id,)).fetchone()
|
|
return self.send_json(200, {"candidate": dict(updated_candidate), "prospect": dict(updated_prospect)})
|
|
except sqlite3.IntegrityError:
|
|
if conn is not None:
|
|
conn.rollback()
|
|
return self.send_json(409, {"error": "domain_already_catalogued"})
|
|
except (ValueError, TypeError, json.JSONDecodeError) as exc:
|
|
return self.send_json(400, {"error": str(exc)})
|
|
except Exception as exc:
|
|
return self.send_json(502, {"error": "website_candidate_accept_failed:" + str(exc)[:240]})
|
|
finally:
|
|
if conn is not None:
|
|
conn.close()
|
|
match = re.fullmatch(r"/api/prospects/(\d+)/review", urlparse(self.path).path)
|
|
if not match:
|
|
return self.send_json(404, {"error": "not_found"})
|
|
conn: sqlite3.Connection | None = None
|
|
try:
|
|
payload = self.read_json()
|
|
status = str(payload.get("status", "")).strip().lower()
|
|
if status not in ALLOWED_REVIEW:
|
|
return self.send_json(400, {"error": "invalid_review_status"})
|
|
conn = db()
|
|
cur = conn.execute("UPDATE prospects SET review_status=?,review_notes=? WHERE id=?", (status, str(payload.get("notes", ""))[:1000], int(match.group(1))))
|
|
conn.commit()
|
|
if not cur.rowcount:
|
|
return self.send_json(404, {"error": "not_found"})
|
|
row = conn.execute("SELECT * FROM prospects WHERE id=?", (int(match.group(1)),)).fetchone()
|
|
return self.send_json(200, dict(row))
|
|
except (ValueError, TypeError, json.JSONDecodeError) as exc:
|
|
return self.send_json(400, {"error": str(exc)})
|
|
finally:
|
|
if conn is not None:
|
|
conn.close()
|
|
|
|
def do_DELETE(self) -> None:
|
|
match = re.fullmatch(r"/api/suppressions/(\d+)", urlparse(self.path).path)
|
|
if not match: return self.send_json(404, {"error":"not_found"})
|
|
conn=db()
|
|
try:
|
|
cur=conn.execute("DELETE FROM suppressions WHERE id=?",(int(match.group(1)),)); conn.commit()
|
|
return self.send_json(200,{"deleted":bool(cur.rowcount)})
|
|
finally: conn.close()
|
|
|
|
|
|
def serve() -> None:
|
|
host = os.environ.get("HOST", "0.0.0.0")
|
|
port = int(os.environ.get("PORT", "8000"))
|
|
db()
|
|
recover_stale_runs()
|
|
threading.Thread(target=scheduler_loop, name="prospect-scheduler", daemon=True).start()
|
|
print(f"Personal ProspectOS listening on http://{host}:{port} (backend={backend_mode()})", flush=True)
|
|
ThreadingHTTPServer((host, port), Handler).serve_forever()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
serve()
|