3470 lines
213 KiB
Python
3470 lines
213 KiB
Python
#!/usr/bin/env python3
|
|
"""Personal ProspectOS service backed by Google Maps Scraper.
|
|
|
|
The app owns the small local catalogue and review workflow. Google Maps
|
|
Scraper runs as a separate Playwright service and is called through its
|
|
asynchronous REST API. A browser-search implementation remains available as
|
|
an explicit fallback (DISCOVERY_BACKEND=browser) for local testing.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import csv
|
|
import base64
|
|
import hashlib
|
|
import hmac
|
|
import io
|
|
import json
|
|
import os
|
|
import re
|
|
import secrets
|
|
import sqlite3
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from datetime import datetime, timezone
|
|
from html.parser import HTMLParser
|
|
from html import escape as html_escape
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
from http.cookies import SimpleCookie
|
|
from pathlib import Path
|
|
from typing import Any, Callable
|
|
from urllib.parse import parse_qs, quote, quote_plus, unquote, urlparse
|
|
|
|
try:
|
|
from intelligence import domain_candidates, scan_website, _safe_url
|
|
from browser_investigator import direct_search_candidates, investigate_websites
|
|
except ImportError: # tests import the module as api.server
|
|
from api.intelligence import domain_candidates, scan_website, _safe_url
|
|
from api.browser_investigator import direct_search_candidates, investigate_websites
|
|
|
|
DB_PATH = os.environ.get("PROSPECT_DB", "/data/prospects.db")
|
|
MAX_RESULTS = 50
|
|
GMAPS_POLL_SECONDS = float(os.environ.get("GMAPS_POLL_SECONDS", "2"))
|
|
# Maps searches can legitimately take several minutes, especially when email
|
|
# extraction is enabled. Fifteen minutes is the safe default; it remains
|
|
# configurable for smaller or larger personal runs.
|
|
GMAPS_REQUEST_TIMEOUT = int(os.environ.get("GMAPS_REQUEST_TIMEOUT", "900"))
|
|
GMAPS_URL = os.environ.get("GMAPS_URL", "").strip().rstrip("/")
|
|
# The bundled gosom/google-maps-scraper image exposes a local, unauthenticated
|
|
# REST API. The optional SaaS edition uses a different API and requires an
|
|
# X-API-Key header; keep support for it only when explicitly selected.
|
|
GMAPS_API_MODE = os.environ.get("GMAPS_API_MODE", "standard").strip().lower()
|
|
GMAPS_API_KEY = os.environ.get("GMAPS_API_KEY", "").strip()
|
|
AI_BASE_URL = os.environ.get("AI_BASE_URL", "").strip().rstrip("/")
|
|
AI_API_KEY = os.environ.get("AI_API_KEY", "").strip()
|
|
AI_MODEL = os.environ.get("AI_MODEL", "").strip()
|
|
AI_PROVIDER = os.environ.get("AI_PROVIDER", "custom").strip().lower()
|
|
# Cloudflare Browser Integrity checks can reject Python's default
|
|
# ``Python-urllib/<version>`` fingerprint before an OpenAI-compatible request
|
|
# reaches the provider. Identify this application consistently on provider
|
|
# requests instead of relying on that default.
|
|
OUTBOUND_USER_AGENT = os.environ.get("OUTBOUND_USER_AGENT", "ProspectOS/2.0 (+personal-use)").strip() or "ProspectOS/2.0 (+personal-use)"
|
|
BROWSER_USE_ENABLED = os.environ.get("BROWSER_USE_ENABLED", "1").strip().lower() not in {"0", "false", "no", "off"}
|
|
AUTH_REQUIRED = os.environ.get("AUTH_REQUIRED", "true").strip().lower() not in {"0", "false", "no", "off"}
|
|
AUTH_COOKIE_NAME = "prospect_session"
|
|
AUTH_SESSION_TTL_SECONDS = max(3600, min(60 * 60 * 24 * 30, int(os.environ.get("AUTH_SESSION_TTL_SECONDS", str(60 * 60 * 24 * 7)))))
|
|
AUTH_COOKIE_SECURE = os.environ.get("AUTH_COOKIE_SECURE", "false").strip().lower() not in {"0", "false", "no", "off"}
|
|
WORKSPACE_NAME = os.environ.get("WORKSPACE_NAME", "ProspectOS workspace").strip() or "ProspectOS workspace"
|
|
PUBLIC_BASE_URL = os.environ.get("PUBLIC_BASE_URL", "").strip().rstrip("/")
|
|
BACKUP_DIR = os.environ.get("BACKUP_DIR", "").strip()
|
|
try:
|
|
BACKUP_RETENTION = max(1, min(100, int(os.environ.get("BACKUP_RETENTION", "14"))))
|
|
except (TypeError, ValueError):
|
|
BACKUP_RETENTION = 14
|
|
try:
|
|
BACKUP_INTERVAL_HOURS = max(0, min(168, int(os.environ.get("BACKUP_INTERVAL_HOURS", "24"))))
|
|
except (TypeError, ValueError):
|
|
BACKUP_INTERVAL_HOURS = 24
|
|
NOUS_BASE_URL = "https://inference-api.nousresearch.com/v1"
|
|
NOUS_FREE_MODELS = [
|
|
{"id": "meituan/longcat-2.0:free", "label": "Meituan · LongCat 2.0", "free": True},
|
|
{"id": "upstage/solar-pro-4:free", "label": "Upstage · Solar Pro 4", "free": True},
|
|
{"id": "stepfun/step-3.7-flash:free", "label": "StepFun · Step 3.7 Flash", "free": True},
|
|
{"id": "poolside/laguna-s-2.1:free", "label": "Poolside · Laguna S 2.1", "free": True},
|
|
{"id": "poolside/laguna-xs-2.1:free", "label": "Poolside · Laguna XS 2.1", "free": True},
|
|
{"id": "inclusionai/ling-3.0-flash-fin:free", "label": "inclusionAI · Ling 3.0 Flash Fin", "free": True},
|
|
{"id": "inclusionai/ling-3.0-flash-sante:free", "label": "inclusionAI · Ling 3.0 Flash Sante", "free": True},
|
|
]
|
|
_nous_models_cache: list[dict[str, Any]] = NOUS_FREE_MODELS.copy()
|
|
_nous_models_cached_at = 0.0
|
|
ALLOWED_REVIEW = {"pending", "approved", "rejected", "contacted"}
|
|
_discover_lock = threading.Lock()
|
|
_schema_lock = threading.Lock()
|
|
_schema_path: str | None = None
|
|
|
|
|
|
def password_hash(password: str) -> str:
|
|
"""Hash a password with the standard-library scrypt implementation."""
|
|
value = str(password or "")
|
|
if len(value) < 8:
|
|
raise ValueError("password_must_be_at_least_8_characters")
|
|
salt = secrets.token_bytes(16)
|
|
digest = hashlib.scrypt(value.encode("utf-8"), salt=salt, n=16_384, r=8, p=1, dklen=32)
|
|
return "scrypt$16384$8$1$" + base64.urlsafe_b64encode(salt).decode("ascii") + "$" + base64.urlsafe_b64encode(digest).decode("ascii")
|
|
|
|
|
|
def password_matches(password: str, stored: str) -> bool:
|
|
try:
|
|
scheme, n, r, p, salt_text, digest_text = str(stored or "").split("$", 5)
|
|
if scheme != "scrypt":
|
|
return False
|
|
salt = base64.urlsafe_b64decode(salt_text.encode("ascii"))
|
|
expected = base64.urlsafe_b64decode(digest_text.encode("ascii"))
|
|
actual = hashlib.scrypt(str(password or "").encode("utf-8"), salt=salt, n=int(n), r=int(r), p=int(p), dklen=len(expected))
|
|
return hmac.compare_digest(actual, expected)
|
|
except (TypeError, ValueError, base64.binascii.Error):
|
|
return False
|
|
|
|
|
|
def _session_hash(token: str) -> str:
|
|
return hashlib.sha256(str(token).encode("utf-8")).hexdigest()
|
|
|
|
|
|
def public_auth_path(path: str) -> bool:
|
|
return path in {"/api/health", "/api/config", "/api/auth/status", "/api/auth/login", "/api/auth/setup"}
|
|
|
|
|
|
def user_view(row: sqlite3.Row | dict[str, Any]) -> dict[str, Any]:
|
|
value = dict(row)
|
|
for secret in ("password_hash", "failed_login_count", "locked_until"):
|
|
value.pop(secret, None)
|
|
value["active"] = bool(value.get("active", True))
|
|
return value
|
|
|
|
|
|
def workspace_view(row: sqlite3.Row | dict[str, Any] | None) -> dict[str, Any] | None:
|
|
return dict(row) if row else None
|
|
|
|
|
|
def _saved_settings() -> dict[str, str]:
|
|
"""Read persisted runtime settings, falling back cleanly during startup."""
|
|
try:
|
|
conn = db()
|
|
try:
|
|
return {str(row["key"]): str(row["value"]) for row in conn.execute("SELECT key,value FROM settings")}
|
|
finally:
|
|
conn.close()
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
COMPANY_PROFILE_KEYS = {
|
|
"website_url": "COMPANY_WEBSITE_URL",
|
|
"name": "COMPANY_NAME",
|
|
"description": "COMPANY_DESCRIPTION",
|
|
"services": "COMPANY_SERVICES",
|
|
"packages": "COMPANY_PACKAGES",
|
|
"tone": "COMPANY_TONE",
|
|
"notes": "COMPANY_NOTES",
|
|
"website_content": "COMPANY_WEBSITE_CONTENT",
|
|
"website_title": "COMPANY_WEBSITE_TITLE",
|
|
"website_fetched_at": "COMPANY_WEBSITE_FETCHED_AT",
|
|
}
|
|
|
|
|
|
def company_profile_settings(saved: dict[str, str] | None = None) -> dict[str, Any]:
|
|
"""Return the owner's reusable company knowledge for AI prompts/UI."""
|
|
values = saved if saved is not None else _saved_settings()
|
|
profile = {field: str(values.get(key, ""))[:20_000] for field, key in COMPANY_PROFILE_KEYS.items()}
|
|
profile["configured"] = bool(profile.get("website_url") or profile.get("description") or profile.get("services") or profile.get("packages"))
|
|
return profile
|
|
|
|
|
|
class _CompanyPageParser(HTMLParser):
|
|
"""Small bounded text extractor used by Teach AI; no scripts are retained."""
|
|
|
|
def __init__(self) -> None:
|
|
super().__init__(convert_charrefs=True)
|
|
self.title: list[str] = []
|
|
self.parts: list[str] = []
|
|
self._tag_stack: list[str] = []
|
|
|
|
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
|
tag = tag.lower()
|
|
self._tag_stack.append(tag)
|
|
|
|
def handle_endtag(self, tag: str) -> None:
|
|
tag = tag.lower()
|
|
for index in range(len(self._tag_stack) - 1, -1, -1):
|
|
if self._tag_stack[index] == tag:
|
|
del self._tag_stack[index:]
|
|
break
|
|
|
|
def handle_data(self, data: str) -> None:
|
|
if any(tag in {"script", "style", "noscript", "svg", "template"} for tag in self._tag_stack):
|
|
return
|
|
clean = re.sub(r"\s+", " ", data).strip()
|
|
if not clean:
|
|
return
|
|
if "title" in self._tag_stack:
|
|
self.title.append(clean)
|
|
self.parts.append(clean)
|
|
|
|
|
|
def fetch_company_page(url: str) -> dict[str, str]:
|
|
"""Fetch a public company page and extract a safe, bounded knowledge snapshot."""
|
|
normalized = normalize_url(url)
|
|
if not normalized or urlparse(normalized).scheme not in {"http", "https"} or not _safe_url(normalized):
|
|
raise ValueError("public_http_website_url_required")
|
|
request = urllib.request.Request(normalized, headers={"User-Agent": OUTBOUND_USER_AGENT, "Accept": "text/html,application/xhtml+xml"}, method="GET")
|
|
started = time.monotonic()
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=20) as response:
|
|
content_type = str(response.headers.get("Content-Type") or "").lower()
|
|
body = response.read(1_000_000)
|
|
final_url = str(response.geturl() or normalized)
|
|
except urllib.error.HTTPError as exc:
|
|
raise RuntimeError(f"company_website_http_{exc.code}") from exc
|
|
except urllib.error.URLError as exc:
|
|
raise RuntimeError(f"company_website_unreachable:{exc.reason}") from exc
|
|
if content_type and "html" not in content_type and "xhtml" not in content_type:
|
|
raise RuntimeError("company_website_not_html")
|
|
parser = _CompanyPageParser()
|
|
parser.feed(body.decode("utf-8", "replace"))
|
|
text = " ".join(parser.parts)
|
|
# Keep enough context for packages/services while preventing a huge prompt.
|
|
text = re.sub(r"\s+", " ", text).strip()[:20_000]
|
|
title = " ".join(parser.title).strip()[:300]
|
|
record_log("ai", "company_website_learned", details={"url": _safe_log_url(final_url), "title": title, "characters": len(text), "duration_ms": int((time.monotonic() - started) * 1000)})
|
|
return {"website_url": final_url, "website_title": title, "website_content": text, "website_fetched_at": now()}
|
|
|
|
|
|
def _setting_int(saved: dict[str, str], key: str, env_key: str, default: int, minimum: int, maximum: int) -> int:
|
|
try:
|
|
value = int(saved.get(key, os.environ.get(env_key, str(default))))
|
|
except (TypeError, ValueError):
|
|
value = default
|
|
return max(minimum, min(maximum, value))
|
|
|
|
|
|
def _setting_float(saved: dict[str, str], key: str, env_key: str, default: float, minimum: float, maximum: float) -> float:
|
|
try:
|
|
value = float(saved.get(key, os.environ.get(env_key, str(default))))
|
|
except (TypeError, ValueError):
|
|
value = default
|
|
return max(minimum, min(maximum, value))
|
|
|
|
|
|
def _setting_bool(saved: dict[str, str], key: str, env_key: str, default: bool) -> bool:
|
|
raw = str(saved.get(key, os.environ.get(env_key, "true" if default else "false"))).strip().lower()
|
|
return raw not in {"0", "false", "no", "off", "disabled"}
|
|
|
|
|
|
def browser_use_settings() -> dict[str, Any]:
|
|
"""Return the live Browser Use controls from Settings or .env defaults."""
|
|
saved = _saved_settings()
|
|
return {
|
|
"enabled": _setting_bool(saved, "BROWSER_USE_ENABLED", "BROWSER_USE_ENABLED", BROWSER_USE_ENABLED),
|
|
"max_steps": _setting_int(saved, "BROWSER_USE_MAX_STEPS", "BROWSER_USE_MAX_STEPS", 10, 3, 50),
|
|
"timeout_seconds": _setting_int(saved, "BROWSER_USE_TIMEOUT_SECONDS", "BROWSER_USE_TIMEOUT_SECONDS", 600, 30, 1800),
|
|
"llm_timeout_seconds": _setting_int(saved, "BROWSER_USE_LLM_TIMEOUT_SECONDS", "BROWSER_USE_LLM_TIMEOUT_SECONDS", 90, 10, 300),
|
|
"headless": True,
|
|
"public_only": True,
|
|
}
|
|
|
|
|
|
def browser_use_limits() -> tuple[int, int]:
|
|
"""Return bounded Browser Use steps/timeout for compatibility with callers."""
|
|
runtime = browser_use_settings()
|
|
return int(runtime["max_steps"]), int(runtime["timeout_seconds"])
|
|
|
|
|
|
def gmaps_runtime_settings() -> dict[str, Any]:
|
|
"""Return live Maps worker timing controls from Settings or .env defaults."""
|
|
saved = _saved_settings()
|
|
return {
|
|
"request_timeout_seconds": _setting_int(saved, "GMAPS_REQUEST_TIMEOUT", "GMAPS_REQUEST_TIMEOUT", GMAPS_REQUEST_TIMEOUT, 60, 7200),
|
|
"poll_seconds": _setting_float(saved, "GMAPS_POLL_SECONDS", "GMAPS_POLL_SECONDS", GMAPS_POLL_SECONDS, 0.5, 60.0),
|
|
"max_depth": _setting_int(saved, "GMAPS_MAX_DEPTH", "GMAPS_MAX_DEPTH", 1, 1, 10),
|
|
}
|
|
|
|
|
|
def website_scan_timeout_seconds() -> int:
|
|
saved = _saved_settings()
|
|
return _setting_int(saved, "WEBSITE_SCAN_TIMEOUT", "WEBSITE_SCAN_TIMEOUT", 8, 2, 60)
|
|
|
|
|
|
def now() -> str:
|
|
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
|
|
|
|
|
|
try:
|
|
LOG_RETENTION = max(2_000, min(50_000, int(os.environ.get("LOG_RETENTION", "20000"))))
|
|
except (TypeError, ValueError):
|
|
LOG_RETENTION = 20_000
|
|
_LOG_SECRET_KEYS = {"api_key", "authorization", "password", "secret", "token", "access_token", "refresh_token"}
|
|
|
|
|
|
def _redact_log_value(value: Any, depth: int = 0) -> Any:
|
|
"""Keep logs useful without persisting provider credentials."""
|
|
if depth > 5:
|
|
return "[truncated]"
|
|
if isinstance(value, dict):
|
|
output: dict[str, Any] = {}
|
|
for key, item in list(value.items())[:80]:
|
|
key_text = str(key)
|
|
if key_text.lower() in _LOG_SECRET_KEYS or key_text.lower().endswith("_api_key"):
|
|
output[key_text] = "[redacted]"
|
|
else:
|
|
output[key_text] = _redact_log_value(item, depth + 1)
|
|
return output
|
|
if isinstance(value, (list, tuple, set)):
|
|
return [_redact_log_value(item, depth + 1) for item in list(value)[:100]]
|
|
if isinstance(value, str):
|
|
return value[:4_000] + ("…" if len(value) > 4_000 else "")
|
|
if isinstance(value, (int, float, bool)) or value is None:
|
|
return value
|
|
return str(value)[:4_000]
|
|
|
|
|
|
def record_log(kind: str, event: str, *, level: str = "info", method: str = "", path: str = "",
|
|
status: int | None = None, duration_ms: int | None = None, details: Any = None,
|
|
error: str = "", run_id: int | None = None, prospect_id: int | None = None,
|
|
organization_id: int | None = None, user_id: int | None = None) -> None:
|
|
"""Persist one bounded operational event; logging must never break a request."""
|
|
try:
|
|
safe_kind = str(kind or "system").strip().lower()[:30] or "system"
|
|
safe_level = str(level or "info").strip().lower()[:12] or "info"
|
|
safe_details = _redact_log_value(details if details is not None else {})
|
|
encoded = json.dumps(safe_details, ensure_ascii=False, default=str)[:20_000]
|
|
conn = db()
|
|
try:
|
|
resolved_org = organization_id
|
|
if resolved_org is None and prospect_id is not None:
|
|
lookup = conn.execute("SELECT organization_id FROM prospects WHERE id=?", (prospect_id,)).fetchone()
|
|
resolved_org = lookup[0] if lookup else None
|
|
if resolved_org is None and run_id is not None:
|
|
lookup = conn.execute("SELECT organization_id FROM runs WHERE id=?", (run_id,)).fetchone()
|
|
resolved_org = lookup[0] if lookup else None
|
|
cur = conn.execute(
|
|
"""INSERT INTO logs(created_at,kind,level,event,method,path,status,duration_ms,run_id,prospect_id,details,error,organization_id,user_id)
|
|
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
|
|
(now(), safe_kind, safe_level, str(event or "event")[:240], str(method or "")[:12],
|
|
str(path or "")[:500], status, duration_ms, run_id, prospect_id, encoded, str(error or "")[:2_000], int(resolved_org or 1), user_id),
|
|
)
|
|
# Retain enough history for investigation without allowing a busy
|
|
# browser session to grow the SQLite volume indefinitely.
|
|
if int(cur.lastrowid or 0) % 100 == 0:
|
|
conn.execute("DELETE FROM logs WHERE id NOT IN (SELECT id FROM logs ORDER BY id DESC LIMIT ?)", (LOG_RETENTION,))
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
except Exception:
|
|
return
|
|
|
|
|
|
def backend_mode() -> str:
|
|
configured = os.environ.get("DISCOVERY_BACKEND", "").strip().lower()
|
|
if configured in {"gmaps", "browser"}:
|
|
return configured
|
|
return "gmaps" if os.environ.get("GMAPS_URL", "").strip() else "browser"
|
|
|
|
|
|
def db() -> sqlite3.Connection:
|
|
global _schema_path
|
|
os.makedirs(os.path.dirname(DB_PATH) or ".", exist_ok=True)
|
|
conn = sqlite3.connect(DB_PATH, timeout=30)
|
|
conn.row_factory = sqlite3.Row
|
|
conn.execute("PRAGMA busy_timeout=30000")
|
|
if _schema_path == DB_PATH:
|
|
return conn
|
|
_schema_lock.acquire()
|
|
if _schema_path == DB_PATH:
|
|
_schema_lock.release()
|
|
return conn
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS organizations (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
name TEXT NOT NULL,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL
|
|
)""")
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS users (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
organization_id INTEGER NOT NULL,
|
|
email TEXT NOT NULL,
|
|
name TEXT NOT NULL DEFAULT '',
|
|
password_hash TEXT NOT NULL,
|
|
role TEXT NOT NULL DEFAULT 'reviewer',
|
|
active INTEGER NOT NULL DEFAULT 1,
|
|
failed_login_count INTEGER NOT NULL DEFAULT 0,
|
|
locked_until TEXT NOT NULL DEFAULT '',
|
|
last_login_at TEXT NOT NULL DEFAULT '',
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL,
|
|
UNIQUE(organization_id, email)
|
|
)""")
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS sessions (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
user_id INTEGER NOT NULL,
|
|
token_hash TEXT NOT NULL UNIQUE,
|
|
csrf_token TEXT NOT NULL,
|
|
created_at TEXT NOT NULL,
|
|
last_seen_at TEXT NOT NULL,
|
|
expires_at TEXT NOT NULL,
|
|
ip_address TEXT NOT NULL DEFAULT '',
|
|
user_agent TEXT NOT NULL DEFAULT ''
|
|
)""")
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS password_reset_tokens (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
user_id INTEGER NOT NULL,
|
|
token_hash TEXT NOT NULL UNIQUE,
|
|
created_at TEXT NOT NULL,
|
|
expires_at TEXT NOT NULL,
|
|
used_at TEXT NOT NULL DEFAULT ''
|
|
)""")
|
|
conn.execute("INSERT OR IGNORE INTO organizations(id,name,created_at,updated_at) VALUES(1,?,?,?)", (WORKSPACE_NAME, now(), now()))
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS prospects (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
organization_id INTEGER NOT NULL DEFAULT 1,
|
|
created_by INTEGER,
|
|
assigned_user_id INTEGER,
|
|
updated_at TEXT NOT NULL DEFAULT '',
|
|
name TEXT NOT NULL,
|
|
website TEXT NOT NULL DEFAULT '',
|
|
domain TEXT NOT NULL UNIQUE,
|
|
description TEXT NOT NULL DEFAULT '',
|
|
query TEXT NOT NULL DEFAULT '',
|
|
discovered_at TEXT NOT NULL,
|
|
review_status TEXT NOT NULL DEFAULT 'pending',
|
|
review_notes TEXT NOT NULL DEFAULT '',
|
|
source TEXT NOT NULL DEFAULT 'google_maps',
|
|
source_id TEXT NOT NULL DEFAULT '',
|
|
category TEXT NOT NULL DEFAULT '',
|
|
search_category TEXT NOT NULL DEFAULT '',
|
|
address TEXT NOT NULL DEFAULT '',
|
|
phone TEXT NOT NULL DEFAULT '',
|
|
emails TEXT NOT NULL DEFAULT '',
|
|
rating REAL,
|
|
review_count INTEGER NOT NULL DEFAULT 0,
|
|
maps_url TEXT NOT NULL DEFAULT '',
|
|
latitude REAL,
|
|
longitude REAL,
|
|
listing_status TEXT NOT NULL DEFAULT '',
|
|
website_status TEXT NOT NULL DEFAULT 'unknown',
|
|
website_http_status INTEGER,
|
|
website_final_url TEXT NOT NULL DEFAULT '',
|
|
website_latency_ms INTEGER NOT NULL DEFAULT 0,
|
|
website_https INTEGER NOT NULL DEFAULT 0,
|
|
website_title TEXT NOT NULL DEFAULT '',
|
|
website_technology TEXT NOT NULL DEFAULT '',
|
|
website_has_viewport INTEGER NOT NULL DEFAULT 0,
|
|
website_tls_expires_at TEXT NOT NULL DEFAULT '',
|
|
website_checked_at TEXT NOT NULL DEFAULT '',
|
|
website_error TEXT NOT NULL DEFAULT '',
|
|
email_evidence TEXT NOT NULL DEFAULT '[]',
|
|
score_reasons TEXT NOT NULL DEFAULT '[]',
|
|
ai_probability REAL,
|
|
ai_recommendation TEXT NOT NULL DEFAULT '',
|
|
ai_reasoning TEXT NOT NULL DEFAULT '',
|
|
ai_checked_at TEXT NOT NULL DEFAULT '',
|
|
ai_subject TEXT NOT NULL DEFAULT '',
|
|
ai_message TEXT NOT NULL DEFAULT '',
|
|
ai_offer TEXT NOT NULL DEFAULT '',
|
|
ai_unknowns TEXT NOT NULL DEFAULT '[]',
|
|
ai_evidence_ids TEXT NOT NULL DEFAULT '[]',
|
|
ai_follow_up TEXT NOT NULL DEFAULT '',
|
|
ai_mockup_id INTEGER,
|
|
opportunity_score INTEGER NOT NULL DEFAULT 0,
|
|
raw_json TEXT NOT NULL DEFAULT '{}'
|
|
)""")
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS runs (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
organization_id INTEGER NOT NULL DEFAULT 1,
|
|
created_by INTEGER,
|
|
query TEXT NOT NULL,
|
|
location TEXT NOT NULL,
|
|
requested INTEGER NOT NULL,
|
|
found INTEGER NOT NULL DEFAULT 0,
|
|
added INTEGER NOT NULL DEFAULT 0,
|
|
status TEXT NOT NULL,
|
|
error TEXT NOT NULL DEFAULT '',
|
|
created_at TEXT NOT NULL,
|
|
source TEXT NOT NULL DEFAULT 'google_maps',
|
|
external_job_id TEXT NOT NULL DEFAULT '',
|
|
stage TEXT NOT NULL DEFAULT 'queued',
|
|
progress INTEGER NOT NULL DEFAULT 0,
|
|
completed_at TEXT NOT NULL DEFAULT '',
|
|
request_json TEXT NOT NULL DEFAULT '{}',
|
|
attempts INTEGER NOT NULL DEFAULT 1,
|
|
last_heartbeat_at TEXT NOT NULL DEFAULT '',
|
|
cancel_requested INTEGER NOT NULL DEFAULT 0,
|
|
retry_of INTEGER,
|
|
failure_count INTEGER NOT NULL DEFAULT 0,
|
|
recovery_state TEXT NOT NULL DEFAULT ''
|
|
)""")
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS suppressions (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
organization_id INTEGER NOT NULL DEFAULT 1,
|
|
created_by INTEGER,
|
|
kind TEXT NOT NULL,
|
|
value TEXT NOT NULL,
|
|
reason TEXT NOT NULL DEFAULT '',
|
|
created_at TEXT NOT NULL,
|
|
UNIQUE(kind, value)
|
|
)""")
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS schedules (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT, organization_id INTEGER NOT NULL DEFAULT 1, created_by INTEGER, category TEXT NOT NULL, keywords TEXT NOT NULL,
|
|
location TEXT NOT NULL DEFAULT 'South Africa', max_results INTEGER NOT NULL DEFAULT 25,
|
|
interval_hours INTEGER NOT NULL DEFAULT 24, enabled INTEGER NOT NULL DEFAULT 1,
|
|
next_run_at TEXT NOT NULL, last_run_id INTEGER, created_at TEXT NOT NULL
|
|
)""")
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS settings (
|
|
key TEXT PRIMARY KEY, value TEXT NOT NULL DEFAULT '', updated_at TEXT NOT NULL
|
|
)""")
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS website_candidates (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
organization_id INTEGER NOT NULL DEFAULT 1,
|
|
prospect_id INTEGER NOT NULL,
|
|
url TEXT NOT NULL,
|
|
domain TEXT NOT NULL,
|
|
confidence INTEGER NOT NULL DEFAULT 0,
|
|
evidence TEXT NOT NULL DEFAULT '[]',
|
|
source TEXT NOT NULL DEFAULT 'browser_use',
|
|
status TEXT NOT NULL DEFAULT 'candidate',
|
|
discovered_at TEXT NOT NULL,
|
|
raw_json TEXT NOT NULL DEFAULT '{}',
|
|
UNIQUE(prospect_id, domain)
|
|
)""")
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS logs (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
organization_id INTEGER NOT NULL DEFAULT 1,
|
|
user_id INTEGER,
|
|
created_at TEXT NOT NULL,
|
|
kind TEXT NOT NULL DEFAULT 'system',
|
|
level TEXT NOT NULL DEFAULT 'info',
|
|
event TEXT NOT NULL,
|
|
method TEXT NOT NULL DEFAULT '',
|
|
path TEXT NOT NULL DEFAULT '',
|
|
status INTEGER,
|
|
duration_ms INTEGER,
|
|
run_id INTEGER,
|
|
prospect_id INTEGER,
|
|
details TEXT NOT NULL DEFAULT '{}',
|
|
error TEXT NOT NULL DEFAULT ''
|
|
)""")
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS promotions (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
organization_id INTEGER NOT NULL DEFAULT 1,
|
|
created_by INTEGER,
|
|
name TEXT NOT NULL,
|
|
description TEXT NOT NULL DEFAULT '',
|
|
terms TEXT NOT NULL DEFAULT '',
|
|
starts_at TEXT NOT NULL DEFAULT '',
|
|
ends_at TEXT NOT NULL DEFAULT '',
|
|
active INTEGER NOT NULL DEFAULT 1,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL
|
|
)""")
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS prospect_notes (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
organization_id INTEGER NOT NULL DEFAULT 1,
|
|
prospect_id INTEGER NOT NULL,
|
|
user_id INTEGER,
|
|
body TEXT NOT NULL,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL
|
|
)""")
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS prospect_tasks (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
organization_id INTEGER NOT NULL DEFAULT 1,
|
|
prospect_id INTEGER NOT NULL,
|
|
created_by INTEGER,
|
|
assigned_user_id INTEGER,
|
|
title TEXT NOT NULL,
|
|
description TEXT NOT NULL DEFAULT '',
|
|
due_at TEXT NOT NULL DEFAULT '',
|
|
status TEXT NOT NULL DEFAULT 'open',
|
|
completed_at TEXT NOT NULL DEFAULT '',
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL
|
|
)""")
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS score_history (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
organization_id INTEGER NOT NULL DEFAULT 1,
|
|
prospect_id INTEGER NOT NULL,
|
|
user_id INTEGER,
|
|
score_type TEXT NOT NULL DEFAULT 'opportunity',
|
|
score INTEGER NOT NULL DEFAULT 0,
|
|
reasons TEXT NOT NULL DEFAULT '[]',
|
|
model TEXT NOT NULL DEFAULT '',
|
|
created_at TEXT NOT NULL
|
|
)""")
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS ai_assessment_history (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
organization_id INTEGER NOT NULL DEFAULT 1,
|
|
prospect_id INTEGER NOT NULL,
|
|
user_id INTEGER,
|
|
probability INTEGER NOT NULL DEFAULT 0,
|
|
recommendation TEXT NOT NULL DEFAULT '',
|
|
reasoning TEXT NOT NULL DEFAULT '',
|
|
model TEXT NOT NULL DEFAULT '',
|
|
created_at TEXT NOT NULL
|
|
)""")
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS mockups (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
organization_id INTEGER NOT NULL DEFAULT 1,
|
|
prospect_id INTEGER NOT NULL,
|
|
created_by INTEGER,
|
|
token TEXT NOT NULL UNIQUE,
|
|
title TEXT NOT NULL DEFAULT '',
|
|
source_url TEXT NOT NULL DEFAULT '',
|
|
html TEXT NOT NULL,
|
|
ai_model TEXT NOT NULL DEFAULT '',
|
|
generation_source TEXT NOT NULL DEFAULT 'fallback',
|
|
instructions TEXT NOT NULL DEFAULT '',
|
|
status TEXT NOT NULL DEFAULT 'published',
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL
|
|
)""")
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS audit_events (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
organization_id INTEGER NOT NULL DEFAULT 1,
|
|
user_id INTEGER,
|
|
action TEXT NOT NULL,
|
|
entity_type TEXT NOT NULL DEFAULT '',
|
|
entity_id INTEGER,
|
|
details TEXT NOT NULL DEFAULT '{}',
|
|
ip_address TEXT NOT NULL DEFAULT '',
|
|
created_at TEXT NOT NULL
|
|
)""")
|
|
conn.execute("""CREATE TABLE IF NOT EXISTS run_events (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
organization_id INTEGER NOT NULL DEFAULT 1,
|
|
run_id INTEGER NOT NULL,
|
|
user_id INTEGER,
|
|
stage TEXT NOT NULL DEFAULT '',
|
|
status TEXT NOT NULL DEFAULT '',
|
|
progress INTEGER NOT NULL DEFAULT 0,
|
|
message TEXT NOT NULL DEFAULT '',
|
|
details TEXT NOT NULL DEFAULT '{}',
|
|
created_at TEXT NOT NULL
|
|
)""")
|
|
|
|
# Migrate databases created by v1-v5 without destroying the user's
|
|
# catalogue. SQLite permits adding nullable/defaulted columns in place.
|
|
migrations = {
|
|
"organization_id": "INTEGER NOT NULL DEFAULT 1",
|
|
"created_by": "INTEGER",
|
|
"assigned_user_id": "INTEGER",
|
|
"updated_at": "TEXT NOT NULL DEFAULT ''",
|
|
"source": "TEXT NOT NULL DEFAULT 'google_maps'",
|
|
"source_id": "TEXT NOT NULL DEFAULT ''",
|
|
"category": "TEXT NOT NULL DEFAULT ''",
|
|
"search_category": "TEXT NOT NULL DEFAULT ''",
|
|
"address": "TEXT NOT NULL DEFAULT ''",
|
|
"phone": "TEXT NOT NULL DEFAULT ''",
|
|
"emails": "TEXT NOT NULL DEFAULT ''",
|
|
"rating": "REAL",
|
|
"review_count": "INTEGER NOT NULL DEFAULT 0",
|
|
"maps_url": "TEXT NOT NULL DEFAULT ''",
|
|
"latitude": "REAL",
|
|
"longitude": "REAL",
|
|
"listing_status": "TEXT NOT NULL DEFAULT ''",
|
|
"website_status": "TEXT NOT NULL DEFAULT 'unknown'",
|
|
"website_http_status": "INTEGER",
|
|
"website_final_url": "TEXT NOT NULL DEFAULT ''",
|
|
"website_latency_ms": "INTEGER NOT NULL DEFAULT 0",
|
|
"website_https": "INTEGER NOT NULL DEFAULT 0",
|
|
"website_title": "TEXT NOT NULL DEFAULT ''",
|
|
"website_technology": "TEXT NOT NULL DEFAULT ''",
|
|
"website_has_viewport": "INTEGER NOT NULL DEFAULT 0",
|
|
"website_tls_expires_at": "TEXT NOT NULL DEFAULT ''",
|
|
"website_checked_at": "TEXT NOT NULL DEFAULT ''",
|
|
"website_error": "TEXT NOT NULL DEFAULT ''",
|
|
"email_evidence": "TEXT NOT NULL DEFAULT '[]'",
|
|
"score_reasons": "TEXT NOT NULL DEFAULT '[]'",
|
|
"ai_probability": "REAL",
|
|
"ai_recommendation": "TEXT NOT NULL DEFAULT ''",
|
|
"ai_reasoning": "TEXT NOT NULL DEFAULT ''",
|
|
"ai_checked_at": "TEXT NOT NULL DEFAULT ''",
|
|
"ai_subject": "TEXT NOT NULL DEFAULT ''",
|
|
"ai_message": "TEXT NOT NULL DEFAULT ''",
|
|
"ai_offer": "TEXT NOT NULL DEFAULT ''",
|
|
"ai_unknowns": "TEXT NOT NULL DEFAULT '[]'",
|
|
"ai_evidence_ids": "TEXT NOT NULL DEFAULT '[]'",
|
|
"ai_follow_up": "TEXT NOT NULL DEFAULT ''",
|
|
"ai_promotion_id": "INTEGER",
|
|
"ai_mockup_id": "INTEGER",
|
|
"opportunity_score": "INTEGER NOT NULL DEFAULT 0",
|
|
"raw_json": "TEXT NOT NULL DEFAULT '{}'",
|
|
}
|
|
existing = {row[1] for row in conn.execute("PRAGMA table_info(prospects)")}
|
|
for name, ddl in migrations.items():
|
|
if name not in existing:
|
|
conn.execute(f"ALTER TABLE prospects ADD COLUMN {name} {ddl}")
|
|
run_migrations = {
|
|
"organization_id": "INTEGER NOT NULL DEFAULT 1",
|
|
"created_by": "INTEGER",
|
|
"source": "TEXT NOT NULL DEFAULT 'google_maps'",
|
|
"external_job_id": "TEXT NOT NULL DEFAULT ''",
|
|
"stage": "TEXT NOT NULL DEFAULT 'queued'",
|
|
"progress": "INTEGER NOT NULL DEFAULT 0",
|
|
"completed_at": "TEXT NOT NULL DEFAULT ''",
|
|
"request_json": "TEXT NOT NULL DEFAULT '{}'",
|
|
"attempts": "INTEGER NOT NULL DEFAULT 1",
|
|
"last_heartbeat_at": "TEXT NOT NULL DEFAULT ''",
|
|
"cancel_requested": "INTEGER NOT NULL DEFAULT 0",
|
|
"retry_of": "INTEGER",
|
|
"failure_count": "INTEGER NOT NULL DEFAULT 0",
|
|
"recovery_state": "TEXT NOT NULL DEFAULT ''",
|
|
}
|
|
existing_runs = {row[1] for row in conn.execute("PRAGMA table_info(runs)")}
|
|
for name, ddl in run_migrations.items():
|
|
if name not in existing_runs:
|
|
conn.execute(f"ALTER TABLE runs ADD COLUMN {name} {ddl}")
|
|
for table, fields in {
|
|
"suppressions": {"organization_id": "INTEGER NOT NULL DEFAULT 1", "created_by": "INTEGER"},
|
|
"schedules": {"organization_id": "INTEGER NOT NULL DEFAULT 1", "created_by": "INTEGER"},
|
|
"website_candidates": {"organization_id": "INTEGER NOT NULL DEFAULT 1"},
|
|
"logs": {"organization_id": "INTEGER NOT NULL DEFAULT 1", "user_id": "INTEGER"},
|
|
"promotions": {"organization_id": "INTEGER NOT NULL DEFAULT 1", "created_by": "INTEGER"},
|
|
"prospect_notes": {"organization_id": "INTEGER NOT NULL DEFAULT 1"},
|
|
"prospect_tasks": {"organization_id": "INTEGER NOT NULL DEFAULT 1"},
|
|
"score_history": {"organization_id": "INTEGER NOT NULL DEFAULT 1"},
|
|
"ai_assessment_history": {"organization_id": "INTEGER NOT NULL DEFAULT 1"},
|
|
"audit_events": {"organization_id": "INTEGER NOT NULL DEFAULT 1"},
|
|
"run_events": {"organization_id": "INTEGER NOT NULL DEFAULT 1"},
|
|
"mockups": {"organization_id": "INTEGER NOT NULL DEFAULT 1"},
|
|
}.items():
|
|
existing_fields = {row[1] for row in conn.execute(f"PRAGMA table_info({table})")}
|
|
for name, ddl in fields.items():
|
|
if name not in existing_fields:
|
|
conn.execute(f"ALTER TABLE {table} ADD COLUMN {name} {ddl}")
|
|
for table in ("prospects", "runs", "suppressions", "schedules", "website_candidates", "logs", "promotions", "prospect_notes", "prospect_tasks", "score_history", "ai_assessment_history", "audit_events", "run_events", "mockups"):
|
|
conn.execute(f"UPDATE {table} SET organization_id=1 WHERE organization_id IS NULL OR organization_id=0")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_prospects_review ON prospects(review_status)")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_prospects_score ON prospects(opportunity_score DESC)")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_runs_created ON runs(id DESC)")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_website_candidates_prospect ON website_candidates(prospect_id, confidence DESC)")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_logs_created ON logs(id DESC)")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_logs_kind ON logs(kind, id DESC)")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_logs_level ON logs(level, id DESC)")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_users_org ON users(organization_id, active)")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_sessions_token ON sessions(token_hash)")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_prospects_org_score ON prospects(organization_id, opportunity_score DESC)")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_prospects_assigned ON prospects(organization_id, assigned_user_id)")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_notes_prospect ON prospect_notes(organization_id, prospect_id, id DESC)")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_tasks_prospect ON prospect_tasks(organization_id, prospect_id, status, due_at)")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_score_history_prospect ON score_history(organization_id, prospect_id, id DESC)")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_ai_history_prospect ON ai_assessment_history(organization_id, prospect_id, id DESC)")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_audit_org_created ON audit_events(organization_id, id DESC)")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_run_events_run ON run_events(organization_id, run_id, id DESC)")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_mockups_prospect ON mockups(organization_id, prospect_id, id DESC)")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_mockups_token ON mockups(token)")
|
|
conn.commit()
|
|
_schema_path = DB_PATH
|
|
_schema_lock.release()
|
|
return conn
|
|
|
|
|
|
ROLE_ORDER = {"viewer": 10, "reviewer": 20, "researcher": 30, "admin": 40, "owner": 50}
|
|
ALLOWED_ROLES = set(ROLE_ORDER)
|
|
|
|
|
|
def _future_iso(seconds: int) -> str:
|
|
return datetime.fromtimestamp(time.time() + int(seconds), timezone.utc).replace(microsecond=0).isoformat()
|
|
|
|
|
|
def _expired(timestamp: str) -> bool:
|
|
try:
|
|
value = datetime.fromisoformat(str(timestamp).replace("Z", "+00:00"))
|
|
if value.tzinfo is None:
|
|
value = value.replace(tzinfo=timezone.utc)
|
|
return value <= datetime.now(timezone.utc)
|
|
except (TypeError, ValueError):
|
|
return True
|
|
|
|
|
|
def default_workspace_id() -> int:
|
|
conn = db()
|
|
try:
|
|
row = conn.execute("SELECT id FROM organizations ORDER BY id LIMIT 1").fetchone()
|
|
return int(row[0]) if row else 1
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def ensure_owner_from_env() -> None:
|
|
"""Optionally seed the first owner for unattended Docker deployments."""
|
|
email = os.environ.get("OWNER_EMAIL", "").strip().lower()
|
|
password = os.environ.get("OWNER_PASSWORD", "")
|
|
if not email or not password:
|
|
return
|
|
conn = db()
|
|
try:
|
|
if conn.execute("SELECT 1 FROM users LIMIT 1").fetchone():
|
|
return
|
|
name = os.environ.get("OWNER_NAME", "Workspace owner").strip()[:160] or "Workspace owner"
|
|
workspace = os.environ.get("WORKSPACE_NAME", WORKSPACE_NAME).strip()[:160] or WORKSPACE_NAME
|
|
conn.execute("UPDATE organizations SET name=?,updated_at=? WHERE id=1", (workspace, now()))
|
|
try:
|
|
hashed = password_hash(password)
|
|
except ValueError:
|
|
return
|
|
conn.execute("INSERT INTO users(organization_id,email,name,password_hash,role,active,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?)", (1, email, name, hashed, "owner", 1, now(), now()))
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def authenticated_user(handler: BaseHTTPRequestHandler) -> dict[str, Any] | None:
|
|
"""Resolve the signed-in user from the server-side session cookie."""
|
|
if not AUTH_REQUIRED:
|
|
organization_id = default_workspace_id()
|
|
user = {"id": None, "organization_id": organization_id, "email": "legacy-local", "name": "Local owner", "role": "owner", "active": True}
|
|
handler._auth_user = user # type: ignore[attr-defined]
|
|
handler._organization_id = organization_id # type: ignore[attr-defined]
|
|
handler._user_id = None # type: ignore[attr-defined]
|
|
return user
|
|
cookie = SimpleCookie()
|
|
try:
|
|
cookie.load(handler.headers.get("Cookie", ""))
|
|
except Exception:
|
|
return None
|
|
token = cookie.get(AUTH_COOKIE_NAME)
|
|
if not token or not token.value:
|
|
return None
|
|
token_hash = _session_hash(token.value)
|
|
conn = db()
|
|
try:
|
|
row = conn.execute("""SELECT u.*,s.expires_at AS session_expires_at,s.csrf_token AS session_csrf_token,o.name AS organization_name,o.created_at AS organization_created_at
|
|
FROM sessions s JOIN users u ON u.id=s.user_id JOIN organizations o ON o.id=u.organization_id
|
|
WHERE s.token_hash=?""", (token_hash,)).fetchone()
|
|
if not row or not row["active"]:
|
|
return None
|
|
if _expired(row["session_expires_at"]):
|
|
conn.execute("DELETE FROM sessions WHERE token_hash=?", (token_hash,)); conn.commit()
|
|
return None
|
|
conn.execute("UPDATE sessions SET last_seen_at=? WHERE token_hash=?", (now(), token_hash)); conn.commit()
|
|
user = user_view(row)
|
|
user["organization"] = {"id": int(row["organization_id"]), "name": row["organization_name"]}
|
|
handler._session_token_hash = token_hash # type: ignore[attr-defined]
|
|
handler._csrf_token = str(row["session_csrf_token"] or "") # type: ignore[attr-defined]
|
|
handler._auth_user = user # type: ignore[attr-defined]
|
|
handler._organization_id = int(row["organization_id"]) # type: ignore[attr-defined]
|
|
handler._user_id = int(row["id"]) # type: ignore[attr-defined]
|
|
return user
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def require_auth(handler: BaseHTTPRequestHandler, roles: set[str] | None = None) -> dict[str, Any] | None:
|
|
user = authenticated_user(handler)
|
|
if not user:
|
|
handler.send_json(401, {"error": "authentication_required"}) # type: ignore[attr-defined]
|
|
return None
|
|
if roles and str(user.get("role")) not in roles:
|
|
handler.send_json(403, {"error": "permission_denied", "required_roles": sorted(roles)}) # type: ignore[attr-defined]
|
|
return None
|
|
return user
|
|
|
|
|
|
def require_csrf(handler: BaseHTTPRequestHandler) -> bool:
|
|
"""Require the per-session CSRF token for browser state changes."""
|
|
if not AUTH_REQUIRED:
|
|
return True
|
|
expected = str(getattr(handler, "_csrf_token", "") or "")
|
|
provided = str(handler.headers.get("X-CSRF-Token", "") or "")
|
|
if expected and provided and hmac.compare_digest(expected, provided):
|
|
return True
|
|
handler.send_json(403, {"error": "csrf_token_required"}) # type: ignore[attr-defined]
|
|
return False
|
|
|
|
|
|
def same_org_user(conn: sqlite3.Connection, user_id: Any, organization_id: int) -> sqlite3.Row | None:
|
|
try:
|
|
value = int(user_id)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
return conn.execute("SELECT id,email,name,role,active FROM users WHERE id=? AND organization_id=?", (value, organization_id)).fetchone()
|
|
|
|
|
|
def role_allows(user: dict[str, Any] | None, minimum: str) -> bool:
|
|
return bool(user and ROLE_ORDER.get(str(user.get("role")), 0) >= ROLE_ORDER.get(minimum, 999))
|
|
|
|
|
|
def set_session_cookie(handler: BaseHTTPRequestHandler, token: str, max_age: int = AUTH_SESSION_TTL_SECONDS) -> None:
|
|
secure = "; Secure" if AUTH_COOKIE_SECURE or str(handler.headers.get("X-Forwarded-Proto", "")).lower() == "https" else ""
|
|
handler._set_cookie = f"{AUTH_COOKIE_NAME}={token}; Max-Age={int(max_age)}; Path=/; HttpOnly; SameSite=Lax{secure}" # type: ignore[attr-defined]
|
|
|
|
|
|
def clear_session_cookie(handler: BaseHTTPRequestHandler) -> None:
|
|
handler._set_cookie = f"{AUTH_COOKIE_NAME}=; Max-Age=0; Path=/; HttpOnly; SameSite=Lax" # type: ignore[attr-defined]
|
|
|
|
|
|
def create_session(handler: BaseHTTPRequestHandler, user_id: int) -> tuple[str, str]:
|
|
token = secrets.token_urlsafe(32)
|
|
csrf = secrets.token_urlsafe(24)
|
|
stamp = now()
|
|
conn = db()
|
|
try:
|
|
conn.execute("INSERT INTO sessions(user_id,token_hash,csrf_token,created_at,last_seen_at,expires_at,ip_address,user_agent) VALUES(?,?,?,?,?,?,?,?)", (user_id, _session_hash(token), csrf, stamp, stamp, _future_iso(AUTH_SESSION_TTL_SECONDS), str(handler.client_address[0] if handler.client_address else "")[:100], str(handler.headers.get("User-Agent", ""))[:300]))
|
|
conn.commit()
|
|
row = conn.execute("SELECT u.*,o.name AS organization_name FROM users u JOIN organizations o ON o.id=u.organization_id WHERE u.id=?", (user_id,)).fetchone()
|
|
finally:
|
|
conn.close()
|
|
handler._session_token_hash = _session_hash(token) # type: ignore[attr-defined]
|
|
handler._csrf_token = csrf # type: ignore[attr-defined]
|
|
if row:
|
|
value = user_view(row)
|
|
value["organization"] = {"id": int(row["organization_id"]), "name": row["organization_name"]}
|
|
handler._auth_user = value # type: ignore[attr-defined]
|
|
handler._organization_id = int(row["organization_id"]) # type: ignore[attr-defined]
|
|
handler._user_id = int(row["id"]) # type: ignore[attr-defined]
|
|
set_session_cookie(handler, token)
|
|
return token, csrf
|
|
|
|
|
|
def record_audit(action: str, *, organization_id: int | None = None, user_id: int | None = None,
|
|
entity_type: str = "", entity_id: int | None = None, details: Any = None,
|
|
ip_address: str = "") -> None:
|
|
try:
|
|
conn = db()
|
|
try:
|
|
encoded = json.dumps(_redact_log_value(details if details is not None else {}), ensure_ascii=False, default=str)[:20_000]
|
|
conn.execute("INSERT INTO audit_events(organization_id,user_id,action,entity_type,entity_id,details,ip_address,created_at) VALUES(?,?,?,?,?,?,?,?)", (int(organization_id or 1), user_id, str(action)[:200], str(entity_type)[:80], entity_id, encoded, str(ip_address)[:100], now()))
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
except Exception:
|
|
return
|
|
|
|
|
|
def backup_directory() -> Path:
|
|
return Path(BACKUP_DIR or (Path(DB_PATH).parent / "backups")).resolve()
|
|
|
|
|
|
def list_backups() -> list[dict[str, Any]]:
|
|
directory = backup_directory()
|
|
if not directory.is_dir():
|
|
return []
|
|
items: list[dict[str, Any]] = []
|
|
for path in sorted(directory.glob("prospects-*.db"), key=lambda item: item.stat().st_mtime, reverse=True):
|
|
try:
|
|
items.append({"name": path.name, "size_bytes": path.stat().st_size, "created_at": datetime.fromtimestamp(path.stat().st_mtime, timezone.utc).replace(microsecond=0).isoformat()})
|
|
except OSError:
|
|
continue
|
|
return items
|
|
|
|
|
|
def create_backup() -> dict[str, Any]:
|
|
"""Create an online SQLite backup and verify it before returning."""
|
|
directory = backup_directory()
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
filename = "prospects-" + datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + ".db"
|
|
target = directory / filename
|
|
source = sqlite3.connect(DB_PATH, timeout=30)
|
|
destination = sqlite3.connect(str(target), timeout=30)
|
|
try:
|
|
source.backup(destination)
|
|
result = destination.execute("PRAGMA integrity_check").fetchone()[0]
|
|
if str(result).lower() != "ok":
|
|
raise RuntimeError("backup_integrity_check_failed")
|
|
destination.commit()
|
|
finally:
|
|
destination.close(); source.close()
|
|
backups = sorted(directory.glob("prospects-*.db"), key=lambda item: item.stat().st_mtime, reverse=True)
|
|
for old in backups[BACKUP_RETENTION:]:
|
|
try: old.unlink()
|
|
except OSError: pass
|
|
return {"name": filename, "size_bytes": target.stat().st_size, "created_at": datetime.fromtimestamp(target.stat().st_mtime, timezone.utc).replace(microsecond=0).isoformat()}
|
|
|
|
|
|
def maybe_auto_backup() -> None:
|
|
"""Create a verified backup at most once per configured interval."""
|
|
if not BACKUP_INTERVAL_HOURS:
|
|
return
|
|
items = list_backups()
|
|
if items:
|
|
try:
|
|
latest = datetime.fromisoformat(str(items[0]["created_at"]).replace("Z", "+00:00"))
|
|
if (datetime.now(timezone.utc) - latest).total_seconds() < BACKUP_INTERVAL_HOURS * 3600:
|
|
return
|
|
except (TypeError, ValueError):
|
|
pass
|
|
try:
|
|
backup = create_backup()
|
|
record_log("system", "backup_auto_created", details=backup)
|
|
except Exception as exc:
|
|
record_log("system", "backup_auto_failed", level="error", error=str(exc)[:500])
|
|
|
|
|
|
def recover_stale_runs() -> None:
|
|
"""Recover abandoned runs once at process startup, not on every request."""
|
|
conn = db()
|
|
try:
|
|
stale = conn.execute("""SELECT id,organization_id,attempts FROM runs WHERE status IN ('queued','running')
|
|
AND julianday(CASE WHEN last_heartbeat_at='' THEN created_at ELSE last_heartbeat_at END) < julianday('now','-30 minutes')""").fetchall()
|
|
conn.execute("""UPDATE runs SET status='failed', stage='recovery_required', recovery_state='retryable', failure_count=failure_count+1,
|
|
error=CASE WHEN error='' THEN 'stale_run_recovered' ELSE error END,
|
|
completed_at=CASE WHEN completed_at='' THEN ? ELSE completed_at END
|
|
WHERE status IN ('queued','running')
|
|
AND julianday(CASE WHEN last_heartbeat_at='' THEN created_at ELSE last_heartbeat_at END) < julianday('now','-30 minutes')""", (now(),))
|
|
for row in stale:
|
|
conn.execute("INSERT INTO run_events(organization_id,run_id,stage,status,progress,message,details,created_at) VALUES(?,?,?,?,?,?,?,?)", (row["organization_id"], row["id"], "recovery_required", "failed", 100, "Run recovered after an interrupted worker", json.dumps({"attempts": row["attempts"]}), now()))
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def persist_website_candidates(prospect_id: int, candidates: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
"""Upsert candidate domains immediately, preserving accepted decisions.
|
|
|
|
Search-page extraction and Browser Use are both allowed to return partial
|
|
results. Persisting in one small transaction before any optional AI work
|
|
means a timeout, model error, or disconnected UI cannot discard links
|
|
that were already visible to the browser.
|
|
"""
|
|
conn = db()
|
|
try:
|
|
prospect = conn.execute("SELECT organization_id FROM prospects WHERE id=?", (prospect_id,)).fetchone()
|
|
organization_id = int(prospect["organization_id"] if prospect else 1)
|
|
for candidate in candidates:
|
|
url = str(candidate.get("url") or "").strip()
|
|
domain = str(candidate.get("domain") or clean_domain(url)).lower().strip()
|
|
if not url or not domain:
|
|
continue
|
|
evidence = candidate.get("evidence")
|
|
if not isinstance(evidence, list):
|
|
evidence = []
|
|
raw = dict(candidate)
|
|
source = str(candidate.get("source") or "browser_use").strip()[:80] or "browser_use"
|
|
conn.execute(
|
|
"""INSERT INTO website_candidates(organization_id,prospect_id,url,domain,confidence,evidence,source,status,discovered_at,raw_json)
|
|
VALUES(?,?,?,?,?,?,?,?,?,?)
|
|
ON CONFLICT(prospect_id,domain) DO UPDATE SET
|
|
url=excluded.url, confidence=MAX(website_candidates.confidence, excluded.confidence),
|
|
evidence=excluded.evidence, source=excluded.source, discovered_at=excluded.discovered_at,
|
|
raw_json=excluded.raw_json,
|
|
status=CASE WHEN website_candidates.status='accepted' THEN 'accepted' ELSE 'candidate' END""",
|
|
(organization_id, prospect_id, url[:1000], domain[:255], max(0, min(100, as_int(candidate.get("confidence")))),
|
|
json.dumps(evidence[:12], ensure_ascii=False), source, "candidate", now(),
|
|
json.dumps(raw, ensure_ascii=False, default=str)[:100_000]),
|
|
)
|
|
conn.commit()
|
|
rows = conn.execute("SELECT * FROM website_candidates WHERE prospect_id=? AND organization_id=? ORDER BY confidence DESC, id DESC", (prospect_id, organization_id)).fetchall()
|
|
return [dict(item) for item in rows]
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def verify_website_candidates(prospect: dict[str, Any], candidates: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
"""Attach a small, evidence-grounded reachability/identity check.
|
|
|
|
Search results are useful leads, not proof of ownership. A bounded
|
|
website scan confirms whether the domain responds and whether its title or
|
|
final URL contains business-name tokens. Failed checks leave the result
|
|
as a reviewable candidate instead of deleting it.
|
|
"""
|
|
name_tokens = [token for token in re.findall(r"[a-z0-9]+", str(prospect.get("name") or "").lower())
|
|
if len(token) >= 3 and token not in {"and", "the", "services", "service", "company", "pty", "ltd", "electrical"}]
|
|
output: list[dict[str, Any]] = []
|
|
for original in candidates:
|
|
candidate = dict(original)
|
|
url = str(candidate.get("url") or "").strip()
|
|
if not url:
|
|
output.append(candidate)
|
|
continue
|
|
try:
|
|
checked = scan_website(url, timeout_seconds=website_scan_timeout_seconds())
|
|
status = str(checked.get("website_status") or "unknown")
|
|
title = str(checked.get("website_title") or "").strip()
|
|
final_url = str(checked.get("website_final_url") or url).strip()
|
|
haystack = (title + " " + final_url).lower()
|
|
evidence = [str(item).strip() for item in (candidate.get("evidence") or []) if str(item).strip()]
|
|
matched = [token for token in name_tokens if token in haystack]
|
|
if matched:
|
|
evidence.append("website title or URL matches: " + ", ".join(matched[:4]))
|
|
candidate["confidence"] = min(100, max(int(candidate.get("confidence") or 0), int(candidate.get("confidence") or 0) + 8))
|
|
if checked.get("website_http_status"):
|
|
evidence.append(f"website responded HTTP {int(checked.get('website_http_status'))}")
|
|
if status in {"healthy", "redirect", "http_only", "slow"}:
|
|
evidence.append("domain is reachable for review")
|
|
elif checked.get("website_error"):
|
|
evidence.append("website check: " + str(checked.get("website_error"))[:160])
|
|
candidate["evidence"] = list(dict.fromkeys(evidence))[:12]
|
|
candidate["verification"] = {
|
|
"status": status,
|
|
"http_status": checked.get("website_http_status"),
|
|
"final_url": final_url[:1000],
|
|
"title": title[:300],
|
|
"identity_match": bool(matched),
|
|
"emails_found": len(checked.get("website_emails") or []),
|
|
}
|
|
except Exception as exc:
|
|
candidate.setdefault("evidence", []).append("website verification failed: " + str(exc)[:160])
|
|
candidate["verification"] = {"status": "unknown", "identity_match": False, "error": str(exc)[:200]}
|
|
output.append(candidate)
|
|
return output
|
|
|
|
|
|
def json_bytes(value: Any) -> bytes:
|
|
return json.dumps(value, ensure_ascii=False).encode("utf-8")
|
|
|
|
|
|
def clean_domain(url: str) -> str:
|
|
parsed = urlparse(url if "://" in url else "https://" + url)
|
|
host = (parsed.hostname or "").lower().strip().rstrip(".")
|
|
if host.startswith("www."):
|
|
host = host[4:]
|
|
return host
|
|
|
|
|
|
def normalize_url(value: Any) -> str:
|
|
text = str(value or "").strip()
|
|
if not text:
|
|
return ""
|
|
if text.startswith("//"):
|
|
text = "https:" + text
|
|
if not text.startswith(("http://", "https://")):
|
|
return ""
|
|
return text
|
|
|
|
|
|
def as_text(value: Any) -> str:
|
|
if value is None:
|
|
return ""
|
|
if isinstance(value, (list, tuple, set)):
|
|
return ", ".join(as_text(item) for item in value if as_text(item))
|
|
if isinstance(value, dict):
|
|
return ", ".join(as_text(item) for item in value.values() if as_text(item))
|
|
return " ".join(str(value).split()).strip()
|
|
|
|
|
|
def field_value(value: Any, *names: str) -> Any:
|
|
"""Read JSON fields from both tagged and default Go JSON casing."""
|
|
if not isinstance(value, dict):
|
|
return None
|
|
for name in names:
|
|
if name in value:
|
|
return value[name]
|
|
wanted = {name.lower() for name in names}
|
|
for key, item in value.items():
|
|
if str(key).lower() in wanted:
|
|
return item
|
|
return None
|
|
|
|
|
|
def as_float(value: Any) -> float | None:
|
|
try:
|
|
if value in (None, ""):
|
|
return None
|
|
return float(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def as_int(value: Any) -> int:
|
|
try:
|
|
if value in (None, ""):
|
|
return 0
|
|
return max(0, int(float(str(value).replace(",", ""))))
|
|
except (TypeError, ValueError):
|
|
return 0
|
|
|
|
|
|
EMAIL_RE = re.compile(r"[A-Z0-9._%+\-]+@[A-Z0-9.\-]+\.[A-Z]{2,63}", re.I)
|
|
FREE_EMAIL_DOMAINS = {"gmail.com", "googlemail.com", "outlook.com", "hotmail.com", "yahoo.com", "icloud.com"}
|
|
|
|
|
|
def normalize_public_emails(value: Any) -> list[str]:
|
|
found: list[str] = []
|
|
text = as_text(value)
|
|
for address in EMAIL_RE.findall(text):
|
|
address = address.strip(".,;:()[]{}<>").lower()
|
|
if address not in found:
|
|
found.append(address)
|
|
return found[:30]
|
|
|
|
|
|
def score_opportunity(item: dict[str, Any]) -> tuple[int, list[str]]:
|
|
"""Transparent, explainable triage score; it is not an AI verdict."""
|
|
score = 20
|
|
reasons: list[str] = []
|
|
website = str(item.get("website") or "")
|
|
website_status = str(item.get("website_status") or "unknown")
|
|
emails = normalize_public_emails(item.get("emails"))
|
|
phone = str(item.get("phone") or "")
|
|
rating = as_float(item.get("rating"))
|
|
review_count = as_int(item.get("review_count"))
|
|
if not website or website_status in {"no_detected_website", "missing"}:
|
|
score += 35
|
|
reasons.append("No detected website")
|
|
elif website_status in {"broken", "unreachable", "error"}:
|
|
score += 30
|
|
reasons.append("Website appears unavailable or broken")
|
|
elif website_status == "slow":
|
|
score += 15
|
|
reasons.append("Website response is slow")
|
|
elif website_status == "http_only":
|
|
score += 10
|
|
reasons.append("Website has no HTTPS")
|
|
elif website_status == "redirect":
|
|
score += 5
|
|
reasons.append("Website relies on redirects")
|
|
if emails:
|
|
score += 12
|
|
reasons.append("Public email found")
|
|
if any(address.split("@", 1)[-1] in FREE_EMAIL_DOMAINS for address in emails):
|
|
score += 8
|
|
reasons.append("Free-mail address used")
|
|
if phone:
|
|
score += 10
|
|
reasons.append("Phone listed")
|
|
if rating is not None and rating >= 4:
|
|
score += 10
|
|
reasons.append("Strong public rating")
|
|
if review_count >= 10:
|
|
score += 10
|
|
reasons.append("Established review activity")
|
|
if as_text(item.get("category") or item.get("type")):
|
|
score += 5
|
|
return min(100, score), reasons
|
|
|
|
|
|
def score_place(item: dict[str, Any], website: str, phone: str, rating: float | None, review_count: int) -> int:
|
|
item = {**item, "website": website, "phone": phone, "rating": rating, "review_count": review_count,
|
|
"website_status": "no_detected_website" if not website else "unknown"}
|
|
return score_opportunity(item)[0]
|
|
|
|
|
|
def map_gmaps_result(item: dict[str, Any], query: str, search_category: str = "") -> dict[str, Any]:
|
|
"""Normalize gosom/google-maps-scraper's place object to our catalogue."""
|
|
name = as_text(item.get("title") or item.get("name") or item.get("business_name")) or "Unnamed place"
|
|
website = normalize_url(item.get("website") or item.get("website_url"))
|
|
maps_url = normalize_url(item.get("link") or item.get("maps_url") or item.get("google_maps_url"))
|
|
address = as_text(item.get("complete_address") or item.get("address"))
|
|
phone = as_text(item.get("phone") or item.get("phone_number"))
|
|
category = as_text(item.get("category") or item.get("type") or item.get("categories"))
|
|
description = as_text(item.get("descriptions") or item.get("description") or category)
|
|
email_values = normalize_public_emails(item.get("emails") or item.get("email"))
|
|
emails = ", ".join(email_values)
|
|
rating = as_float(item.get("review_rating") if item.get("review_rating") is not None else item.get("rating"))
|
|
review_count = as_int(item.get("review_count") if item.get("review_count") is not None else item.get("reviews"))
|
|
source_id = as_text(item.get("place_id") or item.get("data_id") or item.get("cid") or maps_url)
|
|
if not source_id:
|
|
source_id = hashlib.sha1((name + "|" + address).encode("utf-8")).hexdigest()[:20]
|
|
domain = clean_domain(website) if website else "maps:" + source_id
|
|
latitude = as_float(item.get("latitude"))
|
|
longitude = as_float(item.get("longitude"))
|
|
return {
|
|
"name": name[:200],
|
|
"website": website,
|
|
"domain": domain[:255],
|
|
"description": description[:1000],
|
|
"query": query,
|
|
"source": "google_maps",
|
|
"source_id": source_id[:255],
|
|
"category": category[:255],
|
|
"search_category": as_text(search_category)[:255],
|
|
"address": address[:1000],
|
|
"phone": phone[:120],
|
|
"emails": emails[:500],
|
|
"rating": rating,
|
|
"review_count": review_count,
|
|
"maps_url": maps_url,
|
|
"latitude": latitude,
|
|
"longitude": longitude,
|
|
"listing_status": as_text(item.get("status"))[:120],
|
|
"website_status": "unknown" if website else "no_detected_website",
|
|
"email_evidence": json.dumps([{"email": value, "source_url": website or maps_url, "found_at": now(), "method": "maps_worker", "confidence": "medium"} for value in email_values], ensure_ascii=False),
|
|
"score_reasons": "[]",
|
|
"opportunity_score": score_place(item, website, phone, rating, review_count),
|
|
"raw_json": json.dumps(item, ensure_ascii=False)[:100_000],
|
|
}
|
|
|
|
|
|
def _looks_like_place(value: dict[str, Any]) -> bool:
|
|
keys = {str(key).lower() for key in value}
|
|
return bool(keys.intersection({"title", "name", "business_name"})) and bool(
|
|
keys.intersection({"address", "complete_address", "website", "phone", "place_id", "link"})
|
|
)
|
|
|
|
|
|
def extract_gmaps_results(raw: Any) -> list[dict[str, Any]]:
|
|
"""Handle both current API payloads and older nested result formats."""
|
|
output: list[dict[str, Any]] = []
|
|
seen: set[str] = set()
|
|
|
|
def visit(value: Any) -> None:
|
|
if isinstance(value, list):
|
|
for item in value:
|
|
visit(item)
|
|
elif isinstance(value, dict):
|
|
if _looks_like_place(value):
|
|
marker = as_text(value.get("place_id") or value.get("data_id") or value.get("cid") or value.get("link") or value.get("title") or value.get("name"))
|
|
if marker and marker not in seen:
|
|
seen.add(marker)
|
|
output.append(value)
|
|
return
|
|
for key in ("results", "data", "items", "places", "result"):
|
|
if key in value:
|
|
visit(value[key])
|
|
if not any(key in value for key in ("results", "data", "items", "places", "result")):
|
|
for child in value.values():
|
|
if isinstance(child, (list, dict)):
|
|
visit(child)
|
|
|
|
visit(raw)
|
|
return output
|
|
|
|
|
|
def _upstream_error(prefix: str, code: int, detail: str) -> str:
|
|
"""Return an actionable, provider-specific upstream error string."""
|
|
lowered = detail.lower()
|
|
if code == 403 and ("error 1010" in lowered or "error_code\":1010" in lowered or "error code: 1010" in lowered):
|
|
return f"{prefix}_cloudflare_1010:{detail}"
|
|
return f"{prefix}_http_{code}:{detail}"
|
|
|
|
|
|
def _log_http_kind(error_prefix: str) -> str:
|
|
return "ai" if error_prefix == "ai" else "search" if error_prefix == "gmaps" else "api"
|
|
|
|
|
|
def _safe_log_url(url: str) -> str:
|
|
parsed = urlparse(url)
|
|
return f"{parsed.scheme}://{parsed.netloc}{parsed.path}"[:700]
|
|
|
|
|
|
def http_json(method: str, url: str, payload: Any = None, headers: dict[str, str] | None = None, timeout: float = 20, error_prefix: str = "upstream") -> Any:
|
|
started = time.monotonic()
|
|
log_kind = _log_http_kind(error_prefix)
|
|
log_details = {"url": _safe_log_url(url), "timeout_seconds": timeout, "request": payload}
|
|
data = None if payload is None else json.dumps(payload).encode("utf-8")
|
|
request_headers = {"Accept": "application/json", "User-Agent": OUTBOUND_USER_AGENT, **(headers or {})}
|
|
if data is not None:
|
|
request_headers["Content-Type"] = "application/json"
|
|
request = urllib.request.Request(url, data=data, headers=request_headers, method=method.upper())
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
body = response.read()
|
|
response_status = int(getattr(response, "status", 200) or 200)
|
|
except urllib.error.HTTPError as exc:
|
|
detail = exc.read().decode("utf-8", "replace")[:300]
|
|
record_log(log_kind, "upstream_http_error", level="error", method=method, path=_safe_log_url(url), status=int(exc.code),
|
|
duration_ms=int((time.monotonic() - started) * 1000), details={**log_details, "response": detail}, error=_upstream_error(error_prefix, int(exc.code), detail))
|
|
raise RuntimeError(_upstream_error(error_prefix, int(exc.code), detail)) from exc
|
|
except urllib.error.URLError as exc:
|
|
record_log(log_kind, "upstream_unreachable", level="error", method=method, path=_safe_log_url(url),
|
|
duration_ms=int((time.monotonic() - started) * 1000), details=log_details, error=str(exc.reason)[:500])
|
|
raise RuntimeError(f"{error_prefix}_unreachable:{exc.reason}") from exc
|
|
try:
|
|
parsed = json.loads(body or b"{}")
|
|
record_log(log_kind, "upstream_request", method=method, path=_safe_log_url(url), status=response_status,
|
|
duration_ms=int((time.monotonic() - started) * 1000), details={**log_details, "response": parsed})
|
|
return parsed
|
|
except json.JSONDecodeError as exc:
|
|
record_log(log_kind, "upstream_invalid_json", level="error", method=method, path=_safe_log_url(url), status=response_status,
|
|
duration_ms=int((time.monotonic() - started) * 1000), details={**log_details, "response": body[:2_000].decode("utf-8", "replace")}, error=str(exc))
|
|
raise RuntimeError(f"{error_prefix}_invalid_json") from exc
|
|
|
|
|
|
def http_bytes(method: str, url: str, payload: Any = None, headers: dict[str, str] | None = None, timeout: float = 20, error_prefix: str = "upstream") -> bytes:
|
|
"""Make a small HTTP request where the response is not JSON (CSV download)."""
|
|
started = time.monotonic()
|
|
log_kind = _log_http_kind(error_prefix)
|
|
log_details = {"url": _safe_log_url(url), "timeout_seconds": timeout, "request": payload}
|
|
data = None if payload is None else json.dumps(payload).encode("utf-8")
|
|
request_headers = {"Accept": "*/*", "User-Agent": OUTBOUND_USER_AGENT, **(headers or {})}
|
|
if data is not None:
|
|
request_headers["Content-Type"] = "application/json"
|
|
request = urllib.request.Request(url, data=data, headers=request_headers, method=method.upper())
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
body = response.read()
|
|
response_status = int(getattr(response, "status", 200) or 200)
|
|
record_log(log_kind, "upstream_request", method=method, path=_safe_log_url(url), status=response_status,
|
|
duration_ms=int((time.monotonic() - started) * 1000), details={**log_details, "response_bytes": len(body)})
|
|
return body
|
|
except urllib.error.HTTPError as exc:
|
|
detail = exc.read().decode("utf-8", "replace")[:300]
|
|
record_log(log_kind, "upstream_http_error", level="error", method=method, path=_safe_log_url(url), status=int(exc.code),
|
|
duration_ms=int((time.monotonic() - started) * 1000), details={**log_details, "response": detail}, error=_upstream_error(error_prefix, int(exc.code), detail))
|
|
raise RuntimeError(_upstream_error(error_prefix, int(exc.code), detail)) from exc
|
|
except urllib.error.URLError as exc:
|
|
record_log(log_kind, "upstream_unreachable", level="error", method=method, path=_safe_log_url(url),
|
|
duration_ms=int((time.monotonic() - started) * 1000), details=log_details, error=str(exc.reason)[:500])
|
|
raise RuntimeError(f"{error_prefix}_unreachable:{exc.reason}") from exc
|
|
|
|
|
|
def ai_runtime_settings() -> dict[str, str]:
|
|
saved: dict[str, str] = {}
|
|
try:
|
|
conn = db()
|
|
saved = {str(r["key"]): str(r["value"]) for r in conn.execute("SELECT key,value FROM settings")}
|
|
conn.close()
|
|
except Exception:
|
|
pass
|
|
provider = saved.get("AI_PROVIDER", AI_PROVIDER).strip().lower() or "custom"
|
|
base = saved.get("AI_BASE_URL", AI_BASE_URL).strip().rstrip("/")
|
|
if provider == "nous":
|
|
base = NOUS_BASE_URL
|
|
return {"provider": provider, "base_url": base, "api_key": saved.get("AI_API_KEY", AI_API_KEY).strip(), "model": saved.get("AI_MODEL", AI_MODEL).strip()}
|
|
|
|
|
|
def nous_models() -> list[dict[str, Any]]:
|
|
"""Return the live Nous catalog when reachable, with stable free fallbacks."""
|
|
global _nous_models_cache, _nous_models_cached_at
|
|
# Avoid making every Settings refresh wait on an external catalog request.
|
|
# A successful catalog is reused for ten minutes; fallback data is also
|
|
# cached after a failed request and can be replaced on the next refresh.
|
|
if time.monotonic() - _nous_models_cached_at < 600:
|
|
return _nous_models_cache
|
|
try:
|
|
response = http_json("GET", NOUS_BASE_URL + "/models", timeout=10, error_prefix="ai")
|
|
raw = response.get("data", response) if isinstance(response, dict) else response
|
|
models: list[dict[str, Any]] = []
|
|
if isinstance(raw, list):
|
|
for item in raw:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
model_id = str(item.get("id") or "").strip()
|
|
if not model_id:
|
|
continue
|
|
pricing = item.get("pricing") if isinstance(item.get("pricing"), dict) else {}
|
|
prices = [str(pricing[k]).strip() for k in ("prompt", "completion", "input", "output") if k in pricing]
|
|
free = bool(prices) and all(price in {"0", "0.0", "0.00"} for price in prices)
|
|
models.append({"id": model_id, "label": model_id, "free": free, "context_length": item.get("context_length")})
|
|
free = [item for item in models if item.get("free")]
|
|
if free:
|
|
_nous_models_cache = free
|
|
_nous_models_cached_at = time.monotonic()
|
|
return free
|
|
except Exception:
|
|
pass
|
|
_nous_models_cached_at = time.monotonic()
|
|
return _nous_models_cache
|
|
|
|
|
|
def gmaps_headers() -> dict[str, str]:
|
|
if GMAPS_API_MODE == "saas":
|
|
if not GMAPS_API_KEY:
|
|
raise RuntimeError("gmaps_api_key_missing_for_saas_mode")
|
|
return {"X-API-Key": GMAPS_API_KEY}
|
|
return {}
|
|
|
|
|
|
def download_standard_results(job_id: str) -> list[dict[str, Any]]:
|
|
"""Download and parse the CSV produced by the standard web worker."""
|
|
body = http_bytes(
|
|
"GET",
|
|
GMAPS_URL + "/api/v1/jobs/" + quote_plus(str(job_id)) + "/download",
|
|
headers=gmaps_headers(),
|
|
timeout=30,
|
|
error_prefix="gmaps",
|
|
)
|
|
try:
|
|
text = body.decode("utf-8-sig")
|
|
return [dict(row) for row in csv.DictReader(io.StringIO(text))]
|
|
except (UnicodeDecodeError, csv.Error) as exc:
|
|
raise RuntimeError("gmaps_csv_invalid") from exc
|
|
|
|
|
|
def submit_gmaps_job(query: str, include_email: bool = False, extra_reviews: bool = False) -> str:
|
|
if not GMAPS_URL:
|
|
raise RuntimeError("gmaps_not_configured")
|
|
include_email = True
|
|
lang = os.environ.get("GMAPS_LANG", "en")
|
|
maps_runtime = gmaps_runtime_settings()
|
|
depth = int(maps_runtime["max_depth"])
|
|
request_timeout = int(maps_runtime["request_timeout_seconds"])
|
|
if GMAPS_API_MODE == "saas":
|
|
endpoint = "/api/v1/scrape"
|
|
payload = {
|
|
"keyword": query,
|
|
"lang": lang,
|
|
"max_depth": depth,
|
|
"email": bool(include_email),
|
|
"extra_reviews": bool(extra_reviews),
|
|
"timeout": max(1, min(7200, request_timeout)),
|
|
}
|
|
else:
|
|
# Standard gosom web mode uses the local job API. It has no API-key
|
|
# authentication and returns a UUID in {"id": ...} with HTTP 201.
|
|
endpoint = "/api/v1/jobs"
|
|
payload = {
|
|
"name": query[:180],
|
|
"keywords": [query],
|
|
"lang": lang,
|
|
"zoom": 15,
|
|
"depth": depth,
|
|
"email": bool(include_email),
|
|
"extra_reviews": bool(extra_reviews),
|
|
"max_time": max(60, min(7200, request_timeout)),
|
|
}
|
|
response = http_json("POST", GMAPS_URL + endpoint, payload, gmaps_headers(), timeout=30, error_prefix="gmaps")
|
|
job_id = field_value(response, "job_id", "id", "JobID", "ID")
|
|
if not job_id:
|
|
raise RuntimeError("gmaps_job_id_missing")
|
|
return str(job_id)
|
|
|
|
|
|
def poll_gmaps_job(job_id: str, progress: Callable[[str, int, int], None] | None = None) -> list[dict[str, Any]]:
|
|
if not GMAPS_URL:
|
|
raise RuntimeError("gmaps_not_configured")
|
|
maps_runtime = gmaps_runtime_settings()
|
|
deadline = time.monotonic() + max(60, int(maps_runtime["request_timeout_seconds"]) + 60)
|
|
poll_seconds = float(maps_runtime["poll_seconds"])
|
|
if GMAPS_API_MODE == "saas":
|
|
terminal_ok = {"completed", "succeeded", "success"}
|
|
else:
|
|
terminal_ok = {"ok", "completed", "succeeded", "success"}
|
|
terminal_bad = {"failed", "cancelled", "canceled", "discarded"}
|
|
while time.monotonic() < deadline:
|
|
response = http_json("GET", GMAPS_URL + "/api/v1/jobs/" + quote_plus(str(job_id)), headers=gmaps_headers(), timeout=30, error_prefix="gmaps")
|
|
status = str(field_value(response, "status", "Status") or "pending").lower()
|
|
result_count = as_int(field_value(response, "result_count", "ResultCount"))
|
|
if progress:
|
|
progress(status, result_count, 0)
|
|
if status in terminal_ok:
|
|
if "results" in response:
|
|
return extract_gmaps_results(response.get("results"))
|
|
if GMAPS_API_MODE == "standard":
|
|
return download_standard_results(job_id)
|
|
return extract_gmaps_results(response)
|
|
if status in terminal_bad:
|
|
error = field_value(response, "error", "Error")
|
|
raise RuntimeError("gmaps_job_" + status + (":" + as_text(error) if error else ""))
|
|
time.sleep(poll_seconds)
|
|
# The worker can finish writing its CSV just after its status endpoint
|
|
# exceeds our polling window. Recover a completed/partial result when it
|
|
# is available rather than discarding the run as a false failure.
|
|
if GMAPS_API_MODE == "standard":
|
|
try:
|
|
recovered = download_standard_results(job_id)
|
|
if recovered:
|
|
return recovered
|
|
except Exception:
|
|
pass
|
|
raise RuntimeError("gmaps_job_timeout")
|
|
|
|
|
|
# The browser fallback is bounded and does not bypass challenge pages. It is
|
|
# retained for local testing or when the user deliberately sets backend=browser.
|
|
SEARCH_DEADLINE_SECONDS = 25
|
|
ENGINE_TIMEOUT_MS = 7000
|
|
|
|
|
|
def blocked(text: str) -> bool:
|
|
lowered = text.lower()
|
|
return any(marker in lowered for marker in (
|
|
"unusual traffic", "recaptcha", "captcha", "automated queries",
|
|
"access denied", "sorry...", "detected unusual traffic",
|
|
))
|
|
|
|
|
|
def normalize_result_href(raw_href: str) -> str:
|
|
href = (raw_href or "").strip()
|
|
if href.startswith("//"):
|
|
href = "https:" + href
|
|
if href.startswith("/"):
|
|
return ""
|
|
parsed = urlparse(href)
|
|
if parsed.hostname and parsed.hostname.endswith("duckduckgo.com"):
|
|
params = parse_qs(parsed.query)
|
|
for key in ("uddg", "url", "u"):
|
|
target = params.get(key, [""])[0]
|
|
if target.startswith(("http://", "https://")):
|
|
href = unquote(target)
|
|
break
|
|
return href
|
|
|
|
|
|
def extract_search_results(page: Any, selector: str, title_selector: str | None,
|
|
excluded_hosts: set[str], requested: int) -> list[dict[str, Any]]:
|
|
output: list[dict[str, Any]] = []
|
|
seen_domains: set[str] = set()
|
|
|
|
def add_anchor(anchor: Any) -> None:
|
|
if len(output) >= requested:
|
|
return
|
|
try:
|
|
title_node = anchor.locator(title_selector) if title_selector else anchor
|
|
title = " ".join((title_node.inner_text() or "").split())[:200]
|
|
if not title:
|
|
title = " ".join(((anchor.get_attribute("aria-label") or anchor.get_attribute("title") or "").split()))[:200]
|
|
href = normalize_result_href(anchor.get_attribute("href") or "")
|
|
except Exception:
|
|
return
|
|
domain = clean_domain(href)
|
|
if not title or len(title) < 3 or not href.startswith(("http://", "https://")):
|
|
return
|
|
if not domain or domain in excluded_hosts or domain in seen_domains:
|
|
return
|
|
seen_domains.add(domain)
|
|
output.append({"title": title, "website": href, "name": title})
|
|
|
|
try:
|
|
anchors = page.locator(selector)
|
|
for i in range(min(anchors.count(), requested * 4)):
|
|
add_anchor(anchors.nth(i))
|
|
except Exception:
|
|
pass
|
|
if not output:
|
|
try:
|
|
anchors = page.locator("a")
|
|
for i in range(min(anchors.count(), 250)):
|
|
add_anchor(anchors.nth(i))
|
|
if len(output) >= requested:
|
|
break
|
|
except Exception:
|
|
pass
|
|
return output
|
|
|
|
|
|
def google_search(query: str, requested: int, run_id: int | None = None) -> list[dict[str, Any]]:
|
|
try:
|
|
from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeout
|
|
except ImportError as exc:
|
|
raise RuntimeError("playwright_not_installed") from exc
|
|
executable = os.environ.get("PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH", "").strip()
|
|
options: dict[str, Any] = {"headless": True}
|
|
if executable:
|
|
options["executable_path"] = executable
|
|
engines = [
|
|
("brave", "https://search.brave.com/search?q=" + quote_plus(query), "a.heading-serpresult,a.result-header", None, {"search.brave.com"}),
|
|
("startpage", "https://www.startpage.com/sp/search?query=" + quote_plus(query), "a.w-gl__result-title", None, {"startpage.com"}),
|
|
("duckduckgo", "https://html.duckduckgo.com/html/?q=" + quote_plus(query), "a.result__a,a[data-testid='result-title-a']", None, {"duckduckgo.com"}),
|
|
("bing", "https://www.bing.com/search?q=" + quote_plus(query) + "&count=" + str(requested), "li.b_algo h2 a", "h2", {"bing.com", "bingj.com"}),
|
|
("google", "https://www.google.com/search?q=" + quote_plus(query) + "&num=" + str(requested), "a:has(h3)", "h3", {"google.com", "google.co.za"}),
|
|
]
|
|
errors: list[str] = []
|
|
deadline = time.monotonic() + SEARCH_DEADLINE_SECONDS
|
|
with sync_playwright() as p:
|
|
browser = p.chromium.launch(**options)
|
|
try:
|
|
for engine, url, selector, title_selector, excluded_hosts in engines:
|
|
remaining = deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
errors.append("search_deadline_exceeded")
|
|
break
|
|
context = None
|
|
try:
|
|
record_log("search", "search_engine_started", run_id=run_id, details={"engine": engine, "query": query, "requested": requested})
|
|
context = browser.new_context(locale="en-ZA", java_script_enabled=True, service_workers="block", viewport={"width": 1280, "height": 900})
|
|
page = context.new_page()
|
|
page.set_default_timeout(ENGINE_TIMEOUT_MS)
|
|
navigation_timed_out = False
|
|
try:
|
|
page.goto(url, wait_until="domcontentloaded", timeout=min(ENGINE_TIMEOUT_MS, max(1000, int(remaining * 1000))))
|
|
except PlaywrightTimeout:
|
|
navigation_timed_out = True
|
|
page.wait_for_timeout(min(500, max(0, int((deadline - time.monotonic()) * 1000))))
|
|
body = page.locator("body").inner_text()
|
|
html = page.content()
|
|
if blocked(body) or blocked(html):
|
|
errors.append(engine + "_blocked")
|
|
record_log("search", "search_engine_blocked", level="warn", run_id=run_id, details={"engine": engine, "query": query})
|
|
continue
|
|
output = extract_search_results(page, selector, title_selector, excluded_hosts, requested)
|
|
if output:
|
|
record_log("search", "search_engine_results", run_id=run_id, details={"engine": engine, "query": query, "count": len(output), "results": output})
|
|
return output
|
|
errors.append(engine + ("_timeout" if navigation_timed_out else "_empty"))
|
|
record_log("search", "search_engine_empty", level="warn", run_id=run_id, details={"engine": engine, "query": query, "navigation_timed_out": navigation_timed_out})
|
|
except PlaywrightTimeout:
|
|
errors.append(engine + "_timeout")
|
|
record_log("search", "search_engine_timeout", level="warn", run_id=run_id, details={"engine": engine, "query": query})
|
|
except Exception as exc:
|
|
errors.append(engine + "_error")
|
|
record_log("search", "search_engine_error", level="error", run_id=run_id, details={"engine": engine, "query": query}, error=str(exc)[:500])
|
|
finally:
|
|
if context is not None:
|
|
try:
|
|
context.close()
|
|
except Exception:
|
|
pass
|
|
finally:
|
|
browser.close()
|
|
message = "search_engines_unavailable:" + ",".join(errors)
|
|
record_log("search", "search_failed", level="error", run_id=run_id, details={"query": query, "errors": errors}, error=message)
|
|
raise RuntimeError(message)
|
|
|
|
|
|
def validate_payload(payload: dict[str, Any]) -> tuple[str, str, str, int, bool, bool]:
|
|
category = str(payload.get("category", "")).strip()
|
|
keywords = str(payload.get("keywords", "")).strip()
|
|
location = str(payload.get("location", "South Africa")).strip() or "South Africa"
|
|
if not category or not keywords:
|
|
raise ValueError("category_and_keywords_required")
|
|
try:
|
|
requested = max(1, min(MAX_RESULTS, int(payload.get("max_results", 10))))
|
|
except (TypeError, ValueError) as exc:
|
|
raise ValueError("invalid_max_results") from exc
|
|
query = f"{keywords} {category} {location}".strip()
|
|
# Email discovery is part of this app's core use case. The server forces
|
|
# it on so a client/UI change cannot accidentally create a run without
|
|
# attempting public email extraction.
|
|
include_email = True
|
|
extra_reviews = bool(payload.get("extra_reviews", False))
|
|
return category, query, location, requested, include_email, extra_reviews
|
|
|
|
|
|
def update_run(run_id: int, **fields: Any) -> None:
|
|
if not fields:
|
|
return
|
|
allowed = {"found", "added", "status", "error", "source", "external_job_id", "stage", "progress", "completed_at", "attempts", "last_heartbeat_at", "cancel_requested", "failure_count", "recovery_state"}
|
|
safe = {key: value for key, value in fields.items() if key in allowed}
|
|
if not safe:
|
|
return
|
|
safe.setdefault("last_heartbeat_at", now())
|
|
assignments = ", ".join(f"{key}=?" for key in safe)
|
|
conn = db()
|
|
try:
|
|
row = conn.execute("SELECT organization_id,status,stage,progress FROM runs WHERE id=?", (run_id,)).fetchone()
|
|
conn.execute(f"UPDATE runs SET {assignments} WHERE id=?", (*safe.values(), run_id))
|
|
if row and any(key in safe for key in ("stage", "status", "progress", "error")):
|
|
conn.execute("INSERT INTO run_events(organization_id,run_id,stage,status,progress,message,details,created_at) VALUES(?,?,?,?,?,?,?,?)", (row["organization_id"], run_id, str(safe.get("stage", row["stage"])), str(safe.get("status", row["status"])), int(safe.get("progress", row["progress"]) or 0), str(safe.get("error") or ""), json.dumps({key: value for key, value in safe.items() if key != "last_heartbeat_at"}, ensure_ascii=False, default=str)[:20_000], now()))
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
if any(key in safe for key in ("stage", "status", "error")):
|
|
record_log("system", "run_state_updated", level="error" if safe.get("status") == "failed" else "info",
|
|
run_id=run_id, details={key: value for key, value in safe.items() if key in {"stage", "status", "progress", "found", "added", "error", "external_job_id"}},
|
|
error=str(safe.get("error") or ""))
|
|
|
|
|
|
def run_cancel_requested(run_id: int) -> bool:
|
|
conn = db()
|
|
try:
|
|
row = conn.execute("SELECT cancel_requested FROM runs WHERE id=?", (run_id,)).fetchone()
|
|
return bool(row and row[0])
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def _json_list(value: Any) -> list[Any]:
|
|
try:
|
|
parsed = json.loads(str(value or "[]"))
|
|
return parsed if isinstance(parsed, list) else []
|
|
except (TypeError, ValueError, json.JSONDecodeError):
|
|
return []
|
|
|
|
|
|
def enrich_place(item: dict[str, Any]) -> dict[str, Any]:
|
|
"""Add website health, public email evidence, and explainable scoring."""
|
|
prior_evidence = _json_list(item.get("email_evidence"))
|
|
if item.get("website"):
|
|
scan_started = time.monotonic()
|
|
record_log("search", "website_scan_started", prospect_id=as_int(item.get("id")) or None,
|
|
details={"business": item.get("name"), "url": _safe_log_url(str(item.get("website"))), "timeout_seconds": website_scan_timeout_seconds()})
|
|
try:
|
|
item.update(scan_website(str(item["website"]), timeout_seconds=website_scan_timeout_seconds()))
|
|
record_log("search", "website_scan_completed", prospect_id=as_int(item.get("id")) or None,
|
|
duration_ms=int((time.monotonic() - scan_started) * 1000),
|
|
details={"business": item.get("name"), "url": _safe_log_url(str(item.get("website"))), "status": item.get("website_status"), "emails_found": len(item.get("website_emails") or [])})
|
|
except Exception as exc: # a single hostile site must not abort a run
|
|
item.update({"website_status": "unknown", "website_checked_at": now(), "website_error": str(exc)[:300], "website_emails": [], "email_evidence": []})
|
|
record_log("search", "website_scan_failed", level="error", prospect_id=as_int(item.get("id")) or None,
|
|
duration_ms=int((time.monotonic() - scan_started) * 1000),
|
|
details={"business": item.get("name"), "url": _safe_log_url(str(item.get("website")))}, error=str(exc)[:500])
|
|
else:
|
|
item.setdefault("website_status", "no_detected_website")
|
|
item.setdefault("website_emails", [])
|
|
item.setdefault("email_evidence", [])
|
|
existing_emails = normalize_public_emails(item.get("emails"))
|
|
new_emails = normalize_public_emails(item.get("website_emails"))
|
|
all_emails = list(dict.fromkeys(existing_emails + new_emails))[:30]
|
|
item["emails"] = ", ".join(all_emails)
|
|
evidence = prior_evidence
|
|
scanned_evidence = item.get("email_evidence")
|
|
if isinstance(scanned_evidence, list):
|
|
evidence.extend(scanned_evidence)
|
|
# The Maps worker evidence was serialized before the website scan. Keep it
|
|
# only once, then append the scanner's page-level evidence.
|
|
seen_evidence: set[tuple[str, str]] = set()
|
|
compact: list[dict[str, Any]] = []
|
|
for record in evidence:
|
|
if not isinstance(record, dict):
|
|
continue
|
|
marker = (str(record.get("email", "")), str(record.get("source_url", "")))
|
|
if marker[0] and marker not in seen_evidence:
|
|
seen_evidence.add(marker)
|
|
compact.append(record)
|
|
item["email_evidence"] = json.dumps(compact[:50], ensure_ascii=False)
|
|
score, reasons = score_opportunity(item)
|
|
item["opportunity_score"] = score
|
|
item["score_reasons"] = json.dumps(reasons, ensure_ascii=False)
|
|
return item
|
|
|
|
|
|
def insert_catalogue(run_id: int, query: str, raw_results: list[dict[str, Any]], requested: int,
|
|
progress: Callable[[int, int], None] | None = None,
|
|
search_category: str = "") -> tuple[list[dict[str, Any]], int]:
|
|
mapped = [map_gmaps_result(item, query, search_category) for item in raw_results[:requested]]
|
|
conn = db()
|
|
added = 0
|
|
try:
|
|
run_row = conn.execute("SELECT organization_id,created_by FROM runs WHERE id=?", (run_id,)).fetchone()
|
|
organization_id = int(run_row["organization_id"] if run_row else 1)
|
|
created_by = run_row["created_by"] if run_row else None
|
|
suppressed_domains = {str(row[0]).lower() for row in conn.execute("SELECT value FROM suppressions WHERE organization_id=? AND kind='domain'", (organization_id,))}
|
|
suppressed_emails = {str(row[0]).lower() for row in conn.execute("SELECT value FROM suppressions WHERE organization_id=? AND kind='email'", (organization_id,))}
|
|
total = len(mapped)
|
|
for index, item in enumerate(mapped, 1):
|
|
# A hostile or slow public site must not invalidate an otherwise
|
|
# successful Maps run. Keep the raw listing and mark enrichment as
|
|
# unknown; it can be rescanned later from the explorer.
|
|
try:
|
|
item = enrich_place(item)
|
|
except Exception as exc:
|
|
item.update({"website_status": "unknown", "website_checked_at": now(), "website_error": "analysis_failed:" + str(exc)[:240], "email_evidence": item.get("email_evidence", "[]"), "score_reasons": "[]"})
|
|
item["opportunity_score"] = score_opportunity(item)[0]
|
|
# Release SQLite's read/write transaction before the progress
|
|
# callback opens its own connection to update the run row. Without
|
|
# this commit, a multi-result enrichment can deadlock forever in
|
|
# the website_analysis stage while waiting on the catalogue read.
|
|
conn.commit()
|
|
if progress:
|
|
progress(index, total)
|
|
emails = set(normalize_public_emails(item.get("emails")))
|
|
if item["domain"].lower() in suppressed_domains or emails.intersection(suppressed_emails):
|
|
record_log("system", "prospect_suppressed", run_id=run_id, details={"name": item.get("name"), "domain": item.get("domain"), "email_match": bool(emails.intersection(suppressed_emails))})
|
|
continue
|
|
try:
|
|
cur = conn.execute("""INSERT OR IGNORE INTO prospects(
|
|
organization_id,created_by,name,website,domain,description,query,discovered_at,source,source_id,
|
|
category,search_category,address,phone,emails,rating,review_count,maps_url,latitude,
|
|
longitude,listing_status,website_status,website_http_status,website_final_url,
|
|
website_latency_ms,website_https,website_title,website_technology,website_has_viewport,
|
|
website_tls_expires_at,website_checked_at,website_error,email_evidence,score_reasons,
|
|
ai_probability,ai_recommendation,ai_reasoning,ai_checked_at,ai_subject,ai_message,ai_offer,opportunity_score,raw_json
|
|
) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""", (
|
|
organization_id, created_by, item["name"], item["website"], item["domain"], item["description"], item["query"], now(),
|
|
item["source"], item["source_id"], item["category"], item.get("search_category", ""), item["address"], item["phone"], item["emails"],
|
|
item["rating"], item["review_count"], item["maps_url"], item["latitude"], item["longitude"],
|
|
item.get("listing_status", ""), item.get("website_status", "unknown"), item.get("website_http_status"),
|
|
item.get("website_final_url", ""), item.get("website_latency_ms", 0), item.get("website_https", 0),
|
|
item.get("website_title", ""), item.get("website_technology", ""), item.get("website_has_viewport", 0),
|
|
item.get("website_tls_expires_at", ""), item.get("website_checked_at", ""), item.get("website_error", ""),
|
|
item.get("email_evidence", "[]"), item.get("score_reasons", "[]"), None, "", "", "", "", "", "",
|
|
item["opportunity_score"], item["raw_json"],
|
|
))
|
|
except (sqlite3.Error, TypeError, ValueError):
|
|
# Keep processing the remaining listings if one malformed
|
|
# provider row cannot be written.
|
|
continue
|
|
if cur.rowcount:
|
|
added += 1
|
|
prospect_id = int(cur.lastrowid)
|
|
conn.execute("INSERT INTO score_history(organization_id,prospect_id,user_id,score_type,score,reasons,model,created_at) VALUES(?,?,?,?,?,?,?,?)", (organization_id, prospect_id, created_by, "opportunity", int(item.get("opportunity_score") or 0), item.get("score_reasons", "[]"), "deterministic", now()))
|
|
conn.commit()
|
|
record_log("system", "prospect_added", run_id=run_id, prospect_id=prospect_id, organization_id=organization_id, user_id=created_by, details={"name": item.get("name"), "domain": item.get("domain"), "score": item.get("opportunity_score")})
|
|
else:
|
|
conn.commit()
|
|
record_log("system", "prospect_duplicate", run_id=run_id, details={"name": item.get("name"), "domain": item.get("domain")})
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
return mapped, added
|
|
|
|
|
|
def run_discovery(run_id: int, payload: dict[str, Any]) -> None:
|
|
category, query, location, requested, include_email, extra_reviews = validate_payload(payload)
|
|
# Serialize local runs so two personal browser jobs do not fight over the
|
|
# same machine's Chromium/RAM. The Maps worker itself remains async.
|
|
with _discover_lock:
|
|
record_log("search", "discovery_started", run_id=run_id, details={"query": query, "location": location, "requested": requested, "backend": backend_mode(), "email_collection": include_email})
|
|
update_run(run_id, status="running", stage="connecting", progress=5, source=backend_mode())
|
|
try:
|
|
if backend_mode() == "gmaps":
|
|
external_job_id = submit_gmaps_job(query, include_email, extra_reviews)
|
|
update_run(run_id, external_job_id=external_job_id, stage="scraping", progress=10)
|
|
|
|
def on_progress(status: str, found: int, _unused: int) -> None:
|
|
progress = 15 if status == "pending" else 45 if status == "running" else 75
|
|
update_run(run_id, stage="scraping", progress=progress, found=found)
|
|
|
|
raw_results = poll_gmaps_job(external_job_id, on_progress)
|
|
else:
|
|
update_run(run_id, stage="browser_search", progress=20)
|
|
raw_results = google_search(query, requested, run_id=run_id)
|
|
if run_cancel_requested(run_id):
|
|
update_run(run_id, status="cancelled", stage="cancelled", progress=100, completed_at=now(), error="cancelled_by_user")
|
|
record_log("search", "discovery_cancelled", run_id=run_id)
|
|
return
|
|
update_run(run_id, stage="website_analysis", progress=76, found=len(raw_results))
|
|
|
|
def enrichment_progress(index: int, total: int) -> None:
|
|
if total:
|
|
update_run(run_id, stage="website_analysis", progress=76 + int(index * 19 / total), found=index)
|
|
|
|
results, added = insert_catalogue(run_id, query, raw_results, requested, enrichment_progress, search_category=category)
|
|
if run_cancel_requested(run_id):
|
|
update_run(run_id, status="cancelled", stage="cancelled", progress=100, completed_at=now(), error="cancelled_by_user")
|
|
record_log("search", "discovery_cancelled", run_id=run_id)
|
|
return
|
|
update_run(run_id, found=len(results), added=added, status="succeeded", stage="complete", progress=100, completed_at=now(), error="")
|
|
record_log("search", "discovery_completed", run_id=run_id, details={"found": len(results), "added": added, "backend": backend_mode()})
|
|
except Exception as exc:
|
|
message = str(exc).strip()[:500] or "discovery_failed"
|
|
update_run(run_id, status="failed", stage="failed", progress=100, error=message, completed_at=now())
|
|
record_log("search", "discovery_failed", level="error", run_id=run_id, details={"backend": backend_mode()}, error=message)
|
|
|
|
|
|
def create_discovery(payload: dict[str, Any], organization_id: int | None = None, created_by: int | None = None, retry_of: int | None = None) -> dict[str, Any]:
|
|
category, query, location, requested, include_email, extra_reviews = validate_payload(payload)
|
|
request_json = json.dumps({
|
|
"category": category,
|
|
"keywords": str(payload.get("keywords", "")).strip(),
|
|
"location": location,
|
|
"max_results": requested,
|
|
"include_email": include_email,
|
|
"extra_reviews": extra_reviews,
|
|
}, ensure_ascii=False)
|
|
conn = db()
|
|
try:
|
|
org = int(organization_id or 1)
|
|
cur = conn.execute("""INSERT INTO runs(organization_id,created_by,query,location,requested,status,created_at,source,stage,request_json,attempts,last_heartbeat_at,retry_of)
|
|
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)""", (org, created_by, query, location, requested, "queued", now(), backend_mode(), "queued", request_json, 1, now(), retry_of))
|
|
run_id = int(cur.lastrowid)
|
|
conn.execute("INSERT INTO run_events(organization_id,run_id,user_id,stage,status,progress,message,details,created_at) VALUES(?,?,?,?,?,?,?,?,?)", (org, run_id, created_by, "queued", "queued", 0, "Run queued", "{}", now()))
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
worker = threading.Thread(target=run_discovery, args=(run_id, payload), name=f"prospect-run-{run_id}", daemon=True)
|
|
worker.start()
|
|
return {"id": run_id, "status": "queued", "source": backend_mode(), "query": query, "requested": requested}
|
|
|
|
|
|
def scheduler_loop() -> None:
|
|
while True:
|
|
try:
|
|
maybe_auto_backup()
|
|
conn = db(); due = conn.execute("SELECT * FROM schedules WHERE enabled=1 AND next_run_at<=?", (now(),)).fetchall()
|
|
for row in due:
|
|
payload = {"category": row["category"], "keywords": row["keywords"], "location": row["location"], "max_results": row["max_results"], "extra_reviews": False}
|
|
run = create_discovery(payload, organization_id=int(row["organization_id"] or 1), created_by=row["created_by"])
|
|
conn.execute("UPDATE schedules SET next_run_at=?,last_run_id=? WHERE id=?", (datetime.now(timezone.utc).replace(microsecond=0).timestamp() and datetime.fromtimestamp(datetime.now(timezone.utc).timestamp()+int(row["interval_hours"])*3600,timezone.utc).isoformat(), run["id"], row["id"]))
|
|
conn.commit(); conn.close()
|
|
except Exception:
|
|
try: conn.close()
|
|
except Exception: pass
|
|
time.sleep(30)
|
|
|
|
|
|
def ai_generate_message(prospect: dict[str, Any], offer: str, tone: str = "warm", goal: str = "conversation", promotion: dict[str, Any] | None = None, mockup: dict[str, Any] | None = None) -> dict[str, Any]:
|
|
"""Generate an evidence-grounded, human-editable outreach draft.
|
|
|
|
``tone`` and ``goal`` are deliberately small, validated controls rather
|
|
than an unrestricted prompt field. This keeps drafts consistent while
|
|
still letting the owner choose how direct the first contact should feel.
|
|
"""
|
|
runtime = ai_runtime_settings()
|
|
ai_base_url = runtime["base_url"]
|
|
ai_api_key = runtime["api_key"]
|
|
ai_model = runtime["model"]
|
|
if runtime["provider"] == "nous" and not ai_api_key:
|
|
raise RuntimeError("nous_api_key_required_for_standalone_app")
|
|
if not ai_base_url or not ai_model:
|
|
raise RuntimeError("ai_not_configured_set_AI_BASE_URL_and_AI_MODEL")
|
|
evidence_ids = ["prospect_record"]
|
|
if prospect.get("maps_url"):
|
|
evidence_ids.append("public_maps_listing")
|
|
if prospect.get("website"):
|
|
evidence_ids.append("website_scan")
|
|
if prospect.get("emails"):
|
|
evidence_ids.append("public_email_listing")
|
|
if prospect.get("phone"):
|
|
evidence_ids.append("public_phone_listing")
|
|
if mockup and mockup.get("share_url"):
|
|
evidence_ids.append("mockup_preview")
|
|
tone_options = {"warm": "warm and approachable", "direct": "direct and concise", "consultative": "consultative and helpful"}
|
|
goal_options = {"conversation": "start a low-pressure conversation", "review": "offer a short website/hosting review", "quote": "invite them to request pricing or a quote"}
|
|
tone_key = str(tone or "warm").strip().lower()
|
|
goal_key = str(goal or "conversation").strip().lower()
|
|
if tone_key not in tone_options:
|
|
tone_key = "warm"
|
|
if goal_key not in goal_options:
|
|
goal_key = "conversation"
|
|
tone_instruction = tone_options[tone_key]
|
|
goal_instruction = goal_options[goal_key]
|
|
profile = company_profile_settings()
|
|
profile_for_prompt = {
|
|
"name": profile.get("name"),
|
|
"website_url": profile.get("website_url"),
|
|
"description": profile.get("description"),
|
|
"services": profile.get("services"),
|
|
"packages": profile.get("packages"),
|
|
"tone": profile.get("tone"),
|
|
"notes": profile.get("notes"),
|
|
"website_title": profile.get("website_title"),
|
|
"website_content": profile.get("website_content"),
|
|
}
|
|
evidence = {
|
|
"evidence_ids": evidence_ids,
|
|
"business_name": prospect.get("name"),
|
|
"search_category": prospect.get("search_category") or prospect.get("category"),
|
|
"address_or_location": prospect.get("address"),
|
|
"public_listing": {
|
|
"maps_url": prospect.get("maps_url"), "phone": prospect.get("phone"),
|
|
"rating": prospect.get("rating"), "review_count": prospect.get("review_count"),
|
|
},
|
|
"website": {
|
|
"url": prospect.get("website"), "status": prospect.get("website_status"),
|
|
"http_status": prospect.get("website_http_status"), "title": prospect.get("website_title"),
|
|
"technology": prospect.get("website_technology"), "https": bool(prospect.get("website_https")),
|
|
"latency_ms": prospect.get("website_latency_ms"), "error": prospect.get("website_error"),
|
|
},
|
|
"public_emails": prospect.get("emails"),
|
|
"opportunity_score": prospect.get("opportunity_score"),
|
|
"score_reasons": _json_list(prospect.get("score_reasons")),
|
|
"company_profile": profile_for_prompt,
|
|
"selected_promotion": {
|
|
"name": promotion.get("name"), "description": promotion.get("description"),
|
|
"terms": promotion.get("terms"), "starts_at": promotion.get("starts_at"), "ends_at": promotion.get("ends_at"),
|
|
} if promotion else None,
|
|
"mockup_preview": {
|
|
"title": mockup.get("title"), "url": mockup.get("share_url"),
|
|
"source": mockup.get("generation_source"),
|
|
} if mockup and mockup.get("share_url") else None,
|
|
}
|
|
system = (
|
|
"You are an expert B2B copywriter for a small South African web-hosting and website-design business. "
|
|
"Write one genuinely human first-contact email, not a mass-marketing template. The desired voice is "
|
|
f"{tone_instruction}; the goal is to {goal_instruction}. "
|
|
"Use only the stored prospect evidence, the owner's company profile, the selected promotion, and the user's offer. "
|
|
"The company profile describes the sender's business, not the prospect. Never invent a person's name, company activity, "
|
|
"price, service, problem, result, testimonial, urgency, authority, or link. Do not mention scraping, "
|
|
"scoring, hidden analysis, AI, or this instruction. Do not say you reviewed a website when no website "
|
|
"was found. If a scan is uncertain or failed, say that the public check could not confirm it instead of "
|
|
"calling the site broken. Do not insult the prospect or use alarmist language. "
|
|
"Start with a concrete observation tied to one supplied fact; avoid empty openings such as 'I hope you "
|
|
"are well' or 'I came across your business'. Use South African English and the business name exactly as "
|
|
"provided. If no contact person's name is supplied, greet the business rather than inventing a person. "
|
|
"Keep the body plain text and about 90-140 words: greeting, one specific observation, a natural bridge "
|
|
"to the exact offer, one low-pressure call to action, and a brief respectful opt-out sentence. Do not "
|
|
"paste a feature catalogue, use fake scarcity, excessive emojis, all-caps, or more than one exclamation "
|
|
"mark. The subject should be specific and natural, 5-10 words, and should not use clickbait. "
|
|
"Return ONLY strict JSON with exactly these keys: subject, message, rationale, recommended_offer, "
|
|
"unknowns, evidence_used, follow_up. subject and message are plain text. rationale is concise. "
|
|
"recommended_offer is a short summary of the user's offer. unknowns is an array of facts to confirm. "
|
|
"evidence_used is an array containing only supplied evidence IDs. follow_up is a short human review note, "
|
|
"not part of the email. If a selected promotion is supplied, mention it naturally only when it is active and its exact terms are supplied; never invent an expiry, discount, price, or eligibility rule. "
|
|
"If a mockup_preview URL is supplied, you may mention it once as an optional concept preview. Use the exact URL, "
|
|
"call it a concept/preview rather than a live or finished site, and never imply that the prospect approved it."
|
|
)
|
|
prompt = json.dumps({"evidence": evidence, "user_offer_or_instruction": offer[:2000], "tone": tone_key, "goal": goal_key}, ensure_ascii=False)
|
|
headers = {"Authorization": "Bearer " + ai_api_key} if ai_api_key else {}
|
|
request_payload: dict[str, Any] = {
|
|
"model": ai_model,
|
|
"temperature": 0.4,
|
|
"max_tokens": 900,
|
|
"messages": [{"role": "system", "content": system}, {"role": "user", "content": prompt}],
|
|
}
|
|
# Nous exposes an OpenAI-compatible chat endpoint, but not every free model
|
|
# accepts OpenAI's optional response_format parameter. The prompt still
|
|
# requires strict JSON and the parser below validates the response.
|
|
if runtime["provider"] != "nous":
|
|
request_payload["response_format"] = {"type": "json_object"}
|
|
result = http_json("POST", ai_base_url + "/chat/completions", request_payload, headers=headers, timeout=90, error_prefix="ai")
|
|
content = (((result.get("choices") or [{}])[0].get("message") or {}).get("content") or "") if isinstance(result, dict) else ""
|
|
parsed = _parse_ai_object(content)
|
|
if not isinstance(parsed, dict) or not str(parsed.get("subject") or "").strip() or not str(parsed.get("message") or "").strip():
|
|
raise RuntimeError("ai_incomplete_message")
|
|
def text_list(value: Any, limit: int = 6, size: int = 400) -> list[str]:
|
|
if not isinstance(value, list):
|
|
return []
|
|
return [str(item).strip()[:size] for item in value if str(item).strip()][:limit]
|
|
used = [item for item in text_list(parsed.get("evidence_used"), limit=10, size=80) if item in evidence_ids]
|
|
message_text = str(parsed["message"]).strip()[:8000]
|
|
mockup_url = str(mockup.get("share_url") or "").strip() if mockup else ""
|
|
# The link is an explicit part of the requested workflow. If the model
|
|
# omits it despite receiving the evidence, append a clear editable line.
|
|
if mockup_url and mockup_url not in message_text:
|
|
message_text = (message_text.rstrip() + "\n\nI also prepared a concept preview for you: " + mockup_url).strip()[:8000]
|
|
if mockup_url and "mockup_preview" not in used:
|
|
used.append("mockup_preview")
|
|
return {
|
|
"subject": str(parsed["subject"]).strip()[:200],
|
|
"message": message_text,
|
|
"rationale": str(parsed.get("rationale") or "").strip()[:2000],
|
|
"recommended_offer": str(parsed.get("recommended_offer") or offer).strip()[:2000],
|
|
"unknowns": text_list(parsed.get("unknowns")),
|
|
"evidence_used": used,
|
|
"follow_up": str(parsed.get("follow_up") or "Review the draft and confirm the offer before sending.").strip()[:1000],
|
|
"mockup_url": mockup_url,
|
|
}
|
|
|
|
|
|
def _parse_ai_object(content: Any) -> dict[str, Any]:
|
|
"""Parse a JSON object from models that occasionally add markdown fences."""
|
|
text = str(content or "").strip()
|
|
if text.startswith("```"):
|
|
text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text, flags=re.I | re.S).strip()
|
|
try:
|
|
value = json.loads(text)
|
|
except (TypeError, ValueError, json.JSONDecodeError):
|
|
start, end = text.find("{"), text.rfind("}")
|
|
if start < 0 or end <= start:
|
|
raise RuntimeError("ai_invalid_json_response")
|
|
try:
|
|
value = json.loads(text[start:end + 1])
|
|
except (TypeError, ValueError, json.JSONDecodeError) as exc:
|
|
raise RuntimeError("ai_invalid_json_response") from exc
|
|
if not isinstance(value, dict):
|
|
raise RuntimeError("ai_invalid_json_response")
|
|
return value
|
|
|
|
|
|
def ai_assess_prospect(prospect: dict[str, Any]) -> dict[str, Any]:
|
|
"""Assess whether a prospect is worth human review/contact."""
|
|
runtime = ai_runtime_settings()
|
|
if runtime["provider"] == "nous" and not runtime["api_key"]:
|
|
raise RuntimeError("nous_api_key_required_for_standalone_app")
|
|
if not runtime["base_url"] or not runtime["model"]:
|
|
raise RuntimeError("ai_not_configured_set_AI_BASE_URL_and_AI_MODEL")
|
|
profile = company_profile_settings()
|
|
evidence = {
|
|
"business": prospect.get("name"), "category": prospect.get("category"),
|
|
"address": prospect.get("address"), "phone": prospect.get("phone"),
|
|
"website": prospect.get("website"), "website_status": prospect.get("website_status"),
|
|
"website_http_status": prospect.get("website_http_status"),
|
|
"website_title": prospect.get("website_title"), "technology": prospect.get("website_technology"),
|
|
"emails": prospect.get("emails"), "rating": prospect.get("rating"),
|
|
"review_count": prospect.get("review_count"), "opportunity_score": prospect.get("opportunity_score"),
|
|
"score_reasons": _json_list(prospect.get("score_reasons")),
|
|
"sender_company_profile": {
|
|
"name": profile.get("name"), "description": profile.get("description"),
|
|
"services": profile.get("services"), "packages": profile.get("packages"),
|
|
"website_url": profile.get("website_url"), "website_content": profile.get("website_content"),
|
|
},
|
|
}
|
|
system = (
|
|
"You qualify prospects for a South African web hosting and web design business. "
|
|
"Assess only the supplied public evidence. A high probability means the business appears active "
|
|
"and has a plausible need that matches the sender company's services. Use the sender company profile to judge fit, "
|
|
"but do not invent facts, "
|
|
"intent, budget, decision-maker names, or contact details. Return strict JSON with exactly these "
|
|
"keys: probability (integer 0-100), recommendation (one of contact, review, do_not_contact), "
|
|
"reasoning (one concise paragraph)."
|
|
)
|
|
prompt = json.dumps({"prospect": evidence}, ensure_ascii=False)
|
|
headers = {"Authorization": "Bearer " + runtime["api_key"]} if runtime["api_key"] else {}
|
|
request_payload: dict[str, Any] = {
|
|
"model": runtime["model"], "temperature": 0.1, "max_tokens": 450,
|
|
"messages": [{"role": "system", "content": system}, {"role": "user", "content": prompt}],
|
|
}
|
|
if runtime["provider"] != "nous":
|
|
request_payload["response_format"] = {"type": "json_object"}
|
|
result = http_json("POST", runtime["base_url"] + "/chat/completions", request_payload, headers=headers, timeout=90, error_prefix="ai")
|
|
content = (((result.get("choices") or [{}])[0].get("message") or {}).get("content") or "") if isinstance(result, dict) else ""
|
|
parsed = _parse_ai_object(content)
|
|
try:
|
|
probability = max(0, min(100, int(round(float(parsed.get("probability", 0))))))
|
|
except (TypeError, ValueError):
|
|
raise RuntimeError("ai_invalid_probability")
|
|
recommendation = str(parsed.get("recommendation") or "").strip().lower().replace(" ", "_")
|
|
aliases = {
|
|
"yes": "contact", "contact_now": "contact", "high": "contact",
|
|
"maybe": "review", "needs_review": "review", "medium": "review",
|
|
"no": "do_not_contact", "do_not_contact_now": "do_not_contact", "low": "do_not_contact",
|
|
}
|
|
recommendation = aliases.get(recommendation, recommendation)
|
|
if recommendation not in {"contact", "review", "do_not_contact"}:
|
|
recommendation = "contact" if probability >= 70 else "review" if probability >= 45 else "do_not_contact"
|
|
return {
|
|
"ai_probability": probability,
|
|
"ai_recommendation": recommendation,
|
|
"ai_reasoning": str(parsed.get("reasoning") or "").strip()[:3000],
|
|
}
|
|
|
|
|
|
def _mockup_text(value: Any, limit: int = 6000) -> str:
|
|
return re.sub(r"\s+", " ", str(value or "")).strip()[:limit]
|
|
|
|
|
|
def sanitize_mockup_html(value: Any) -> str:
|
|
"""Remove active or unsafe markup before a generated preview is published."""
|
|
text = str(value or "").strip()
|
|
if text.startswith("```"):
|
|
text = re.sub(r"^```(?:html)?\s*|\s*```$", "", text, flags=re.I | re.S).strip()
|
|
text = re.sub(r"<script\b[^>]*>.*?</script\s*>", "", text, flags=re.I | re.S)
|
|
text = re.sub(r"<iframe\b[^>]*>.*?</iframe\s*>", "", text, flags=re.I | re.S)
|
|
text = re.sub(r"<form\b[^>]*>.*?</form\s*>", "", text, flags=re.I | re.S)
|
|
text = re.sub(r"<link\b[^>]*>", "", text, flags=re.I | re.S)
|
|
text = re.sub(r"<img\b[^>]*>", "", text, flags=re.I | re.S)
|
|
text = re.sub(r"<meta\b[^>]*(?:http-equiv\s*=\s*['\"]?refresh|content\s*=\s*['\"]?\d+\s*;\s*url)[^>]*>", "", text, flags=re.I | re.S)
|
|
text = re.sub(r"<(?:object|embed|video|audio|base)\b[^>]*>.*?</(?:object|embed|video|audio|base)\s*>", "", text, flags=re.I | re.S)
|
|
text = re.sub(r"\s+on[a-z]+\s*=\s*(?:\"[^\"]*\"|'[^']*'|[^\s>]+)", "", text, flags=re.I)
|
|
text = re.sub(r"\s+(?:href|src)\s*=\s*(['\"])\s*javascript:[^'\"]*\1", "", text, flags=re.I)
|
|
text = re.sub(r"url\s*\(\s*['\"]?\s*javascript:[^)]+\)", "", text, flags=re.I)
|
|
if not re.search(r"<html\b|<main\b|<body\b", text, flags=re.I):
|
|
return ""
|
|
return text[:250_000]
|
|
|
|
|
|
def fallback_mockup_html(prospect: dict[str, Any], snapshot: dict[str, Any] | None = None) -> str:
|
|
"""Produce a polished no-AI fallback so the button always gives a preview."""
|
|
snapshot = snapshot or {}
|
|
name = html_escape(_mockup_text(prospect.get("name"), 160) or "Your business")
|
|
category = html_escape(_mockup_text(prospect.get("category") or prospect.get("search_category"), 120) or "Local business")
|
|
address = html_escape(_mockup_text(prospect.get("address"), 240))
|
|
phone = _mockup_text(prospect.get("phone"), 80)
|
|
email = _mockup_text(prospect.get("emails"), 160).split(",")[0].strip()
|
|
website = html_escape(_mockup_text(prospect.get("website") or snapshot.get("website_url"), 400))
|
|
title = html_escape(_mockup_text(snapshot.get("website_title"), 160) or name)
|
|
description = html_escape(_mockup_text(prospect.get("description"), 500) or f"A clearer, faster online home for {name}.")
|
|
phone_link = html_escape(re.sub(r"[^0-9+]+", "", phone))
|
|
email_link = html_escape(email)
|
|
contact = " · ".join(item for item in [address, html_escape(phone), html_escape(email)] if item)
|
|
return f'''<!doctype html>
|
|
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><meta name="theme-color" content="#101a2d"><title>{name} · Website concept</title>
|
|
<style>
|
|
:root{{--ink:#17233a;--muted:#66738a;--brand:#635bff;--brand2:#16a77a;--surface:#fff;--soft:#f4f7fb;--line:#e6ebf2;--max:1120px;font-family:Inter,ui-sans-serif,system-ui,-apple-system,"Segoe UI",sans-serif;color:var(--ink);background:var(--soft);line-height:1.5}}*{{box-sizing:border-box}}body{{margin:0}}a{{color:inherit}}.preview{{position:sticky;top:0;z-index:3;text-align:center;padding:8px 14px;background:#111d2e;color:#dfe6f5;font:700 11px/1.2 system-ui;letter-spacing:.03em}}.nav{{display:flex;align-items:center;justify-content:space-between;gap:20px;max-width:var(--max);margin:auto;padding:20px 24px}}.brand{{font-size:18px;font-weight:850;letter-spacing:-.03em}}.brand i{{display:inline-block;width:10px;height:10px;margin-right:8px;border-radius:50%;background:var(--brand)}}.navlinks{{display:flex;gap:22px;color:var(--muted);font-size:13px;font-weight:700}}.navlinks a{{text-decoration:none}}.btn{{display:inline-flex;align-items:center;justify-content:center;min-height:44px;padding:0 17px;border-radius:10px;background:var(--brand);color:#fff;text-decoration:none;font-size:13px;font-weight:800;box-shadow:0 8px 18px #635bff36}}.hero{{background:linear-gradient(135deg,#f1f2ff 0,#fff 62%);border-top:1px solid var(--line);border-bottom:1px solid var(--line)}}.hero-inner{{display:grid;grid-template-columns:1.05fr .95fr;align-items:center;gap:50px;max-width:var(--max);margin:auto;padding:88px 24px}}.eyebrow{{color:var(--brand);font-size:11px;font-weight:900;letter-spacing:.17em;text-transform:uppercase}}h1{{max-width:650px;margin:12px 0;font-size:clamp(37px,5vw,66px);line-height:1.03;letter-spacing:-.06em}}.lead{{max-width:590px;color:var(--muted);font-size:18px}}.hero-actions{{display:flex;flex-wrap:wrap;gap:10px;margin-top:28px}}.ghost{{display:inline-flex;align-items:center;justify-content:center;min-height:44px;padding:0 17px;border:1px solid #ccd5e3;border-radius:10px;color:var(--ink);text-decoration:none;font-size:13px;font-weight:800;background:#fff}}.hero-card{{min-height:320px;padding:20px;border:1px solid #e0e4ff;border-radius:24px;background:linear-gradient(145deg,#e9e8ff,#fff 60%);box-shadow:0 25px 60px #2a386014;display:flex;flex-direction:column;justify-content:flex-end}}.hero-card-top{{width:60%;height:14px;border-radius:20px;background:#b9b4ff;margin-bottom:auto}}.hero-card-panel{{padding:18px;border-radius:15px;background:#ffffffbd;border:1px solid #fff}}.hero-card-panel b{{display:block;font-size:22px}}.hero-card-panel span{{display:block;color:var(--muted);font-size:12px;margin-top:4px}}.section{{max-width:var(--max);margin:auto;padding:72px 24px}}.section h2{{margin:8px 0 12px;font-size:34px;letter-spacing:-.05em}}.section-intro{{max-width:620px;color:var(--muted)}}.cards{{display:grid;grid-template-columns:repeat(3,1fr);gap:15px;margin-top:28px}}.card{{padding:24px;border:1px solid var(--line);border-radius:16px;background:var(--surface);box-shadow:0 9px 28px #27365308}}.icon{{width:38px;height:38px;display:grid;place-items:center;border-radius:12px;background:#ebeaff;color:var(--brand);font-weight:900}}.card h3{{margin:18px 0 5px;font-size:16px}}.card p{{margin:0;color:var(--muted);font-size:13px}}.contact{{background:#111d2e;color:#fff}}.contact .section-intro{{color:#aeb9ce}}.contact-grid{{display:grid;grid-template-columns:1fr auto;gap:30px;align-items:center}}.contact-links{{display:flex;flex-wrap:wrap;gap:10px}}.contact-links a{{color:#fff;text-decoration:none;border:1px solid #ffffff3a;border-radius:9px;padding:10px 13px;font-size:12px}}footer{{max-width:var(--max);margin:auto;padding:24px;color:#8a96a8;font-size:11px;text-align:center}}@media(max-width:760px){{.navlinks{{display:none}}.hero-inner{{grid-template-columns:1fr;padding:58px 20px 50px;gap:32px}}h1{{font-size:44px}}.lead{{font-size:16px}}.hero-card{{min-height:230px}}.section{{padding:52px 20px}}.cards{{grid-template-columns:1fr}}.contact-grid{{grid-template-columns:1fr}}.contact-links{{display:grid}}.contact-links a{{display:block}}}}
|
|
</style></head><body><div class="preview">WEBSITE CONCEPT PREVIEW · prepared for review</div><header><nav class="nav"><div class="brand"><i></i>{name}</div><div class="navlinks"><a href="#services">Services</a><a href="#about">About</a><a href="#contact">Contact</a></div><a class="btn" href="#contact">Get in touch</a></nav></header><main><section class="hero"><div class="hero-inner"><div><span class="eyebrow">{category}</span><h1>{name}, made easier to find.</h1><p class="lead">{description}</p><div class="hero-actions"><a class="btn" href="#contact">Start a conversation</a><a class="ghost" href="#services">Explore services</a></div></div><div class="hero-card"><div class="hero-card-top"></div><div class="hero-card-panel"><b>{title}</b><span>A modern, mobile-first way to present what you do.</span></div></div></div></section><section class="section" id="services"><span class="eyebrow">WHAT WE DO</span><h2>Clear services. Better first impressions.</h2><p class="section-intro">Visitors can quickly understand how {name} helps, then take the next step from any device.</p><div class="cards"><article class="card"><div class="icon">01</div><h3>Simple navigation</h3><p>Important information is easy to find on mobile and desktop.</p></article><article class="card"><div class="icon">02</div><h3>Trust at a glance</h3><p>Strong hierarchy and useful contact details make the business feel established.</p></article><article class="card"><div class="icon">03</div><h3>Built for action</h3><p>Clear calls to action help interested visitors make contact.</p></article></div></section><section class="section" id="about"><span class="eyebrow">ABOUT {category.upper()}</span><h2>A stronger home for your business.</h2><p class="section-intro">This concept uses the public information currently available and leaves space for your exact services, photos, proof, and preferred contact flow.</p></section><section class="contact"><div class="section contact-grid" id="contact"><div><span class="eyebrow" style="color:#a9a1ff">LET'S TALK</span><h2>Ready to make it easier for customers to choose you?</h2><p class="section-intro">A focused website can turn interest into a call, message, or enquiry.</p></div><div class="contact-links">{f'<a href="tel:{phone_link}">{html_escape(phone)}</a>' if phone else ''}{f'<a href="mailto:{email_link}">{html_escape(email)}</a>' if email else ''}{f'<a href="{website}" target="_blank" rel="noopener">Current website ↗</a>' if website else ''}</div></div></section></main><footer>{contact or name} · website concept preview</footer></body></html>'''
|
|
|
|
|
|
def generate_mockup_html(prospect: dict[str, Any], snapshot: dict[str, Any] | None = None, instructions: str = "") -> tuple[str, str, str]:
|
|
"""Generate a self-contained preview and return (html, source, model)."""
|
|
snapshot = snapshot or {}
|
|
runtime = ai_runtime_settings()
|
|
fallback = fallback_mockup_html(prospect, snapshot)
|
|
if not runtime["base_url"] or not runtime["model"] or (runtime["provider"] == "nous" and not runtime["api_key"]):
|
|
return fallback, "fallback", ""
|
|
source = {
|
|
"business_name": _mockup_text(prospect.get("name"), 160),
|
|
"category": _mockup_text(prospect.get("category") or prospect.get("search_category"), 120),
|
|
"location": _mockup_text(prospect.get("address"), 240),
|
|
"phone": _mockup_text(prospect.get("phone"), 80),
|
|
"public_emails": _mockup_text(prospect.get("emails"), 240),
|
|
"current_website": _mockup_text(prospect.get("website") or snapshot.get("website_url"), 400),
|
|
"current_title": _mockup_text(snapshot.get("website_title") or prospect.get("website_title"), 200),
|
|
"current_page_text": _mockup_text(snapshot.get("website_content"), 12_000),
|
|
}
|
|
system = ("You are a senior web designer creating a visual concept preview for a South African small business. "
|
|
"Return ONLY a complete standalone HTML document, never Markdown. Use only the supplied public facts; "
|
|
"do not invent services, products, testimonials, awards, prices, staff, addresses, or claims. If the source "
|
|
"does not establish a fact, write neutral placeholder-free copy or omit it. Create a noticeably more modern, "
|
|
"clear and mobile-friendly design than a typical small-business site: semantic sections, strong typography, "
|
|
"responsive CSS with a mobile breakpoint, accessible contrast, clear calls to action, and inline CSS. "
|
|
"Do not use JavaScript, external scripts, iframes, data collection forms, tracking, remote fonts, or remote images. "
|
|
"Use the exact business name. Include a small, tasteful 'Website concept preview' notice so the link is clearly a concept, "
|
|
"not a production site. Keep the document under 40,000 characters. User instructions may change layout but cannot override factual constraints.")
|
|
prompt = json.dumps({"public_source": source, "design_instructions": _mockup_text(instructions, 2_000)}, ensure_ascii=False)
|
|
headers = {"Authorization": "Bearer " + runtime["api_key"]} if runtime["api_key"] else {}
|
|
payload = {"model": runtime["model"], "temperature": 0.5, "max_tokens": 5000, "messages": [{"role": "system", "content": system}, {"role": "user", "content": prompt}]}
|
|
try:
|
|
result = http_json("POST", runtime["base_url"] + "/chat/completions", payload, headers=headers, timeout=120, error_prefix="ai")
|
|
content = (((result.get("choices") or [{}])[0].get("message") or {}).get("content") or "") if isinstance(result, dict) else ""
|
|
cleaned = sanitize_mockup_html(content)
|
|
if cleaned:
|
|
return cleaned, "ai", runtime["model"]
|
|
record_log("ai", "mockup_invalid_response", level="warn", details={"model": runtime["model"]})
|
|
except Exception as exc:
|
|
record_log("ai", "mockup_generation_fallback", level="warn", details={"model": runtime["model"]}, error=str(exc)[:500])
|
|
return fallback, "fallback", runtime["model"]
|
|
|
|
|
|
def mockup_public_url(handler: BaseHTTPRequestHandler, token: str) -> str:
|
|
base = PUBLIC_BASE_URL
|
|
if not base:
|
|
proto = str(handler.headers.get("X-Forwarded-Proto") or "http").split(",")[0].strip()
|
|
host = str(handler.headers.get("X-Forwarded-Host") or handler.headers.get("Host") or "localhost:8000").split(",")[0].strip()
|
|
base = f"{proto}://{host}"
|
|
return base.rstrip("/") + "/mockups/" + quote(str(token), safe="")
|
|
|
|
|
|
class Handler(BaseHTTPRequestHandler):
|
|
server_version = "PersonalProspectOS/2.0"
|
|
|
|
def handle_one_request(self) -> None:
|
|
self._request_started_at = time.monotonic()
|
|
super().handle_one_request()
|
|
|
|
def send_json(self, status: int, value: Any) -> None:
|
|
body = json_bytes(value)
|
|
self.send_response(status)
|
|
self.send_header("Content-Type", "application/json; charset=utf-8")
|
|
self.send_header("Cache-Control", "no-store")
|
|
pending_cookie = getattr(self, "_set_cookie", "")
|
|
if pending_cookie:
|
|
self.send_header("Set-Cookie", pending_cookie)
|
|
self._set_cookie = ""
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
if str(getattr(self, "path", "")).startswith("/api/"):
|
|
error_value = value.get("error") if isinstance(value, dict) else ""
|
|
record_log(
|
|
"api", f"{getattr(self, 'command', 'HTTP')} {urlparse(self.path).path}",
|
|
level="error" if status >= 400 else "info", method=getattr(self, "command", "HTTP"),
|
|
path=urlparse(self.path).path, status=status,
|
|
duration_ms=int((time.monotonic() - getattr(self, "_request_started_at", time.monotonic())) * 1000),
|
|
details={"response_status": status}, error=str(error_value or "")[:500],
|
|
organization_id=getattr(self, "_organization_id", 1), user_id=getattr(self, "_user_id", None),
|
|
)
|
|
|
|
def read_json(self) -> dict[str, Any]:
|
|
length = min(int(self.headers.get("Content-Length", "0") or 0), 100_000)
|
|
value = json.loads(self.rfile.read(length) or b"{}")
|
|
if not isinstance(value, dict):
|
|
raise ValueError("json_object_required")
|
|
return value
|
|
|
|
def do_GET(self) -> None:
|
|
parsed = urlparse(self.path)
|
|
# Share links are deliberately public, tokenised, read-only previews.
|
|
# Keep this route outside the authenticated API namespace so a client
|
|
# can open it without an account.
|
|
public_mockup = re.fullmatch(r"/mockups/([A-Za-z0-9_-]{20,80})", parsed.path)
|
|
if public_mockup:
|
|
return self.serve_mockup(public_mockup.group(1))
|
|
if not parsed.path.startswith("/api/"):
|
|
self.serve_static(parsed.path)
|
|
return
|
|
if parsed.path == "/api/auth/status":
|
|
user = authenticated_user(self)
|
|
conn = db()
|
|
try:
|
|
setup_required = not bool(conn.execute("SELECT 1 FROM users WHERE active=1 LIMIT 1").fetchone())
|
|
workspace = conn.execute("SELECT * FROM organizations ORDER BY id LIMIT 1").fetchone()
|
|
return self.send_json(200, {"authenticated": bool(user), "setup_required": setup_required, "user": user_view(user) if user else None, "workspace": workspace_view(workspace)})
|
|
finally:
|
|
conn.close()
|
|
if parsed.path == "/api/auth/setup":
|
|
conn = db()
|
|
try:
|
|
return self.send_json(200, {"setup_required": not bool(conn.execute("SELECT 1 FROM users WHERE active=1 LIMIT 1").fetchone())})
|
|
finally:
|
|
conn.close()
|
|
if not public_auth_path(parsed.path) and not require_auth(self):
|
|
return
|
|
if parsed.path == "/api/auth/me":
|
|
return self.send_json(200, {"user": getattr(self, "_auth_user", None), "csrf_token": getattr(self, "_csrf_token", "")})
|
|
conn = db()
|
|
try:
|
|
if parsed.path == "/api/health":
|
|
configured = bool(GMAPS_URL) if backend_mode() == "gmaps" else True
|
|
if GMAPS_API_MODE == "saas":
|
|
configured = configured and bool(GMAPS_API_KEY)
|
|
return self.send_json(200, {"status": "ok", "source": backend_mode(), "configured": configured, "maps_service": bool(GMAPS_URL), "maps_api_mode": GMAPS_API_MODE, "maps_api_key": bool(GMAPS_API_KEY)})
|
|
if parsed.path == "/api/config":
|
|
auth_required = GMAPS_API_MODE == "saas"
|
|
ready = backend_mode() == "browser" or (bool(GMAPS_URL) and (not auth_required or bool(GMAPS_API_KEY)))
|
|
ai_runtime = ai_runtime_settings()
|
|
browser_runtime = browser_use_settings()
|
|
return self.send_json(200, {
|
|
"backend": backend_mode(),
|
|
"gmaps_url": bool(GMAPS_URL),
|
|
"gmaps_api_mode": GMAPS_API_MODE,
|
|
"gmaps_auth_required": auth_required,
|
|
"gmaps_api_key": bool(GMAPS_API_KEY),
|
|
"ready": ready,
|
|
"email_extraction_default": True,
|
|
"email_extraction_always": True,
|
|
"browser_use_enabled": browser_runtime["enabled"],
|
|
"ai_provider": ai_runtime["provider"],
|
|
"ai_configured": bool(ai_runtime["base_url"] and ai_runtime["model"] and (ai_runtime["provider"] != "nous" or ai_runtime["api_key"])),
|
|
})
|
|
if parsed.path == "/api/ai/models":
|
|
provider = (parse_qs(parsed.query).get("provider", [AI_PROVIDER])[0] or AI_PROVIDER).strip().lower()
|
|
if provider == "nous":
|
|
return self.send_json(200, {"provider": "nous", "base_url": NOUS_BASE_URL, "models": nous_models()})
|
|
return self.send_json(200, {"provider": provider or "custom", "models": []})
|
|
if parsed.path == "/api/logs":
|
|
params = parse_qs(parsed.query)
|
|
kind = (params.get("kind", [""])[0] or "").strip().lower()
|
|
level = (params.get("level", [""])[0] or "").strip().lower()
|
|
query = (params.get("q", [""])[0] or "").strip().lower()
|
|
try:
|
|
limit = max(1, min(500, int(params.get("limit", ["200"])[0])))
|
|
except (TypeError, ValueError):
|
|
limit = 200
|
|
clauses: list[str] = ["organization_id=?"]
|
|
values: list[Any] = [getattr(self, "_organization_id", 1)]
|
|
if kind and kind != "all":
|
|
clauses.append("kind=?"); values.append(kind)
|
|
if level and level != "all":
|
|
clauses.append("level=?"); values.append(level)
|
|
if query:
|
|
clauses.append("(lower(event) LIKE ? OR lower(path) LIKE ? OR lower(error) LIKE ? OR lower(details) LIKE ?)")
|
|
needle = "%" + query[:120] + "%"; values.extend([needle, needle, needle, needle])
|
|
where = " WHERE " + " AND ".join(clauses) if clauses else ""
|
|
rows = conn.execute("SELECT * FROM logs" + where + " ORDER BY id DESC LIMIT ?", (*values, limit)).fetchall()
|
|
items: list[dict[str, Any]] = []
|
|
for row in rows:
|
|
item = dict(row)
|
|
try:
|
|
item["details"] = json.loads(item.get("details") or "{}")
|
|
except (TypeError, ValueError, json.JSONDecodeError):
|
|
item["details"] = {"raw": str(item.get("details") or "")[:20_000]}
|
|
items.append(item)
|
|
total = int(conn.execute("SELECT COUNT(*) FROM logs" + where, tuple(values)).fetchone()[0])
|
|
counts = {str(row[0]): int(row[1]) for row in conn.execute("SELECT kind,COUNT(*) FROM logs WHERE organization_id=? GROUP BY kind", (getattr(self, "_organization_id", 1),))}
|
|
error_count = int(conn.execute("SELECT COUNT(*) FROM logs WHERE organization_id=? AND level='error'", (getattr(self, "_organization_id", 1),)).fetchone()[0])
|
|
counts["errors"] = error_count
|
|
return self.send_json(200, {"items": items, "total": total, "counts": counts})
|
|
if parsed.path == "/api/settings":
|
|
saved = _saved_settings()
|
|
ai_runtime = ai_runtime_settings()
|
|
models = nous_models() if ai_runtime["provider"] == "nous" else []
|
|
browser_runtime = browser_use_settings()
|
|
maps_runtime = gmaps_runtime_settings()
|
|
scan_timeout = website_scan_timeout_seconds()
|
|
return self.send_json(200, {
|
|
"maps": {
|
|
"backend": backend_mode(), "worker_url": GMAPS_URL,
|
|
"ready": backend_mode() == "browser" or bool(GMAPS_URL),
|
|
**maps_runtime,
|
|
},
|
|
"ai": {"provider": ai_runtime["provider"], "base_url": ai_runtime["base_url"], "model": ai_runtime["model"], "configured": bool(ai_runtime["base_url"] and ai_runtime["model"] and (ai_runtime["provider"] != "nous" or ai_runtime["api_key"])), "models": models},
|
|
"browser_use": browser_runtime,
|
|
"email_collection": {"always_on": True, "public_only": True},
|
|
"limits": {"max_results": MAX_RESULTS, "website_scan_timeout_seconds": scan_timeout},
|
|
"company_profile": company_profile_settings(saved),
|
|
})
|
|
if parsed.path == "/api/prospects":
|
|
params = parse_qs(parsed.query)
|
|
q = (params.get("q", [""])[0]).strip().lower()
|
|
status = params.get("status", [""])[0].strip().lower()
|
|
rows = conn.execute("SELECT * FROM prospects WHERE organization_id=? ORDER BY opportunity_score DESC, id DESC", (getattr(self, "_organization_id", 1),)).fetchall()
|
|
items = []
|
|
for row in rows:
|
|
haystack = " ".join(str(row[key] or "") for key in ("name", "domain", "description", "address", "category", "search_category", "phone", "emails")).lower()
|
|
if q and q not in haystack:
|
|
continue
|
|
if status and row["review_status"] != status:
|
|
continue
|
|
items.append(dict(row))
|
|
return self.send_json(200, {"items": items, "total": len(items)})
|
|
if parsed.path == "/api/export.csv":
|
|
params = parse_qs(parsed.query); q = params.get("q", [""])[0].strip().lower()
|
|
rows = [dict(r) for r in conn.execute("SELECT * FROM prospects WHERE organization_id=? ORDER BY opportunity_score DESC, id DESC", (getattr(self, "_organization_id", 1),))]
|
|
if q: rows = [r for r in rows if q in " ".join(str(r.get(k) or "") for k in ("name","domain","address","category","search_category","emails")).lower()]
|
|
out = io.StringIO(); fields = ["id","name","search_category","category","address","phone","emails","domain","website","website_status","opportunity_score","score_reasons","review_status","last_seen"]
|
|
writer = csv.DictWriter(out, fieldnames=fields); writer.writeheader(); writer.writerows({k:r.get(k) for k in fields} for r in rows)
|
|
body = out.getvalue().encode(); self.send_response(200); self.send_header("Content-Type","text/csv; charset=utf-8"); self.send_header("Content-Disposition","attachment; filename=prospects.csv"); self.send_header("Content-Length",str(len(body))); self.end_headers(); self.wfile.write(body); return
|
|
if parsed.path == "/api/suppressions":
|
|
return self.send_json(200, {"items": [dict(r) for r in conn.execute("SELECT * FROM suppressions WHERE organization_id=? ORDER BY id DESC", (getattr(self, "_organization_id", 1),))]})
|
|
if parsed.path == "/api/schedules":
|
|
return self.send_json(200, {"items": [dict(r) for r in conn.execute("SELECT * FROM schedules WHERE organization_id=? ORDER BY id DESC", (getattr(self, "_organization_id", 1),))]})
|
|
if parsed.path == "/api/promotions":
|
|
rows = conn.execute("SELECT * FROM promotions WHERE organization_id=? ORDER BY active DESC, updated_at DESC, id DESC", (getattr(self, "_organization_id", 1),)).fetchall()
|
|
return self.send_json(200, {"items": [dict(row) for row in rows]})
|
|
if parsed.path == "/api/workspace":
|
|
organization_id = getattr(self, "_organization_id", 1)
|
|
workspace = conn.execute("SELECT * FROM organizations WHERE id=?", (organization_id,)).fetchone()
|
|
counts = {"users": int(conn.execute("SELECT COUNT(*) FROM users WHERE organization_id=?", (organization_id,)).fetchone()[0]), "prospects": int(conn.execute("SELECT COUNT(*) FROM prospects WHERE organization_id=?", (organization_id,)).fetchone()[0])}
|
|
return self.send_json(200, {"workspace": workspace_view(workspace), "counts": counts})
|
|
if parsed.path == "/api/users":
|
|
rows = conn.execute("SELECT id,organization_id,email,name,role,active,last_login_at,created_at,updated_at FROM users WHERE organization_id=? ORDER BY name,email", (getattr(self, "_organization_id", 1),)).fetchall()
|
|
return self.send_json(200, {"items": [user_view(row) for row in rows]})
|
|
if parsed.path == "/api/audit":
|
|
if not role_allows(getattr(self, "_auth_user", None), "admin"):
|
|
return self.send_json(403, {"error": "permission_denied"})
|
|
params = parse_qs(parsed.query)
|
|
try:
|
|
limit = max(1, min(500, int(params.get("limit", ["200"])[0])))
|
|
except (TypeError, ValueError):
|
|
limit = 200
|
|
rows = conn.execute("""SELECT a.*,u.email AS user_email,u.name AS user_name
|
|
FROM audit_events a LEFT JOIN users u ON u.id=a.user_id
|
|
WHERE a.organization_id=? ORDER BY a.id DESC LIMIT ?""", (getattr(self, "_organization_id", 1), limit)).fetchall()
|
|
items = []
|
|
for row in rows:
|
|
item = dict(row)
|
|
try: item["details"] = json.loads(item.get("details") or "{}")
|
|
except (TypeError, ValueError, json.JSONDecodeError): pass
|
|
items.append(item)
|
|
return self.send_json(200, {"items": items})
|
|
if parsed.path == "/api/backups":
|
|
if not role_allows(getattr(self, "_auth_user", None), "admin"):
|
|
return self.send_json(403, {"error": "permission_denied"})
|
|
return self.send_json(200, {"items": list_backups()})
|
|
match = re.fullmatch(r"/api/prospects/(\d+)/website-candidates", parsed.path)
|
|
if match:
|
|
prospect_id = int(match.group(1))
|
|
if not conn.execute("SELECT 1 FROM prospects WHERE id=? AND organization_id=?", (prospect_id, getattr(self, "_organization_id", 1))).fetchone():
|
|
return self.send_json(404, {"error": "not_found"})
|
|
rows = conn.execute("SELECT * FROM website_candidates WHERE prospect_id=? AND organization_id=? ORDER BY confidence DESC, id DESC", (prospect_id, getattr(self, "_organization_id", 1))).fetchall()
|
|
return self.send_json(200, {"items": [dict(row) for row in rows]})
|
|
match = re.fullmatch(r"/api/prospects/(\d+)/mockups", parsed.path)
|
|
if match:
|
|
prospect_id = int(match.group(1))
|
|
organization_id = getattr(self, "_organization_id", 1)
|
|
if not conn.execute("SELECT 1 FROM prospects WHERE id=? AND organization_id=?", (prospect_id, organization_id)).fetchone():
|
|
return self.send_json(404, {"error": "not_found"})
|
|
rows = conn.execute("""SELECT id,prospect_id,token,title,source_url,ai_model,generation_source,
|
|
instructions,status,created_at,updated_at FROM mockups
|
|
WHERE prospect_id=? AND organization_id=? ORDER BY id DESC""", (prospect_id, organization_id)).fetchall()
|
|
items = []
|
|
for row in rows:
|
|
item = dict(row)
|
|
item["share_url"] = mockup_public_url(self, item["token"])
|
|
items.append(item)
|
|
return self.send_json(200, {"items": items})
|
|
match = re.fullmatch(r"/api/prospects/(\d+)/(notes|tasks|history)", parsed.path)
|
|
if match:
|
|
prospect_id, resource, organization_id = int(match.group(1)), match.group(2), getattr(self, "_organization_id", 1)
|
|
if not conn.execute("SELECT 1 FROM prospects WHERE id=? AND organization_id=?", (prospect_id, organization_id)).fetchone():
|
|
return self.send_json(404, {"error": "not_found"})
|
|
if resource == "notes":
|
|
rows = conn.execute("""SELECT n.*,u.email AS user_email,u.name AS user_name
|
|
FROM prospect_notes n LEFT JOIN users u ON u.id=n.user_id
|
|
WHERE n.prospect_id=? AND n.organization_id=? ORDER BY n.id DESC""", (prospect_id, organization_id)).fetchall()
|
|
return self.send_json(200, {"items": [dict(row) for row in rows]})
|
|
if resource == "tasks":
|
|
rows = conn.execute("""SELECT t.*,u.email AS assigned_email,c.email AS created_by_email
|
|
FROM prospect_tasks t LEFT JOIN users u ON u.id=t.assigned_user_id LEFT JOIN users c ON c.id=t.created_by
|
|
WHERE t.prospect_id=? AND t.organization_id=? ORDER BY CASE t.status WHEN 'open' THEN 0 ELSE 1 END,t.due_at,t.id DESC""", (prospect_id, organization_id)).fetchall()
|
|
return self.send_json(200, {"items": [dict(row) for row in rows]})
|
|
scores = [dict(row) for row in conn.execute("SELECT * FROM score_history WHERE prospect_id=? AND organization_id=? ORDER BY id DESC LIMIT 100", (prospect_id, organization_id)).fetchall()]
|
|
assessments = [dict(row) for row in conn.execute("SELECT * FROM ai_assessment_history WHERE prospect_id=? AND organization_id=? ORDER BY id DESC LIMIT 100", (prospect_id, organization_id)).fetchall()]
|
|
return self.send_json(200, {"score_history": scores, "ai_history": assessments})
|
|
match = re.fullmatch(r"/api/prospects/(\d+)/domains", parsed.path)
|
|
if match:
|
|
row = conn.execute("SELECT * FROM prospects WHERE id=? AND organization_id=?", (int(match.group(1)), getattr(self, "_organization_id", 1))).fetchone()
|
|
if not row: return self.send_json(404, {"error":"not_found"})
|
|
return self.send_json(200, {"items": domain_candidates(row["name"], row["address"], row["category"])})
|
|
match = re.fullmatch(r"/api/prospects/(\d+)", parsed.path)
|
|
if match:
|
|
row = conn.execute("SELECT * FROM prospects WHERE id=? AND organization_id=?", (int(match.group(1)), getattr(self, "_organization_id", 1))).fetchone()
|
|
if not row: return self.send_json(404, {"error":"not_found"})
|
|
return self.send_json(200, dict(row))
|
|
if parsed.path == "/api/runs":
|
|
rows = [dict(row) for row in conn.execute("SELECT * FROM runs WHERE organization_id=? ORDER BY id DESC LIMIT 50", (getattr(self, "_organization_id", 1),))]
|
|
return self.send_json(200, {"items": rows})
|
|
match = re.fullmatch(r"/api/runs/(\d+)", parsed.path)
|
|
if match:
|
|
row = conn.execute("SELECT * FROM runs WHERE id=? AND organization_id=?", (int(match.group(1)), getattr(self, "_organization_id", 1))).fetchone()
|
|
if not row:
|
|
return self.send_json(404, {"error": "not_found"})
|
|
return self.send_json(200, dict(row))
|
|
match = re.fullmatch(r"/api/runs/(\d+)/events", parsed.path)
|
|
if match:
|
|
rows = conn.execute("SELECT * FROM run_events WHERE run_id=? AND organization_id=? ORDER BY id DESC LIMIT 200", (int(match.group(1)), getattr(self, "_organization_id", 1))).fetchall()
|
|
return self.send_json(200, {"items": [dict(row) for row in rows]})
|
|
if parsed.path == "/api/stats":
|
|
organization_id = getattr(self, "_organization_id", 1)
|
|
total = conn.execute("SELECT COUNT(*) FROM prospects WHERE organization_id=?", (organization_id,)).fetchone()[0]
|
|
pending = conn.execute("SELECT COUNT(*) FROM prospects WHERE organization_id=? AND review_status='pending'", (organization_id,)).fetchone()[0]
|
|
approved = conn.execute("SELECT COUNT(*) FROM prospects WHERE organization_id=? AND review_status='approved'", (organization_id,)).fetchone()[0]
|
|
contacted = conn.execute("SELECT COUNT(*) FROM prospects WHERE organization_id=? AND review_status='contacted'", (organization_id,)).fetchone()[0]
|
|
running = conn.execute("SELECT COUNT(*) FROM runs WHERE organization_id=? AND status IN ('queued','running')", (organization_id,)).fetchone()[0]
|
|
return self.send_json(200, {"total": total, "pending": pending, "approved": approved, "contacted": contacted, "running": running})
|
|
return self.send_json(404, {"error": "not_found"})
|
|
finally:
|
|
conn.close()
|
|
|
|
def serve_static(self, path: str) -> None:
|
|
relative = "index.html" if path in {"", "/"} else path.lstrip("/")
|
|
root = (Path(__file__).parent / "web").resolve()
|
|
if not root.is_dir():
|
|
root = (Path(__file__).parent.parent / "web").resolve()
|
|
candidate = (root / relative).resolve()
|
|
if root not in candidate.parents and candidate != root:
|
|
return self.send_json(400, {"error": "invalid_path"})
|
|
if not candidate.is_file():
|
|
return self.send_json(404, {"error": "not_found"})
|
|
content = candidate.read_bytes()
|
|
content_type = "text/plain; charset=utf-8"
|
|
if candidate.suffix == ".html": content_type = "text/html; charset=utf-8"
|
|
elif candidate.suffix == ".js": content_type = "text/javascript; charset=utf-8"
|
|
elif candidate.suffix == ".css": content_type = "text/css; charset=utf-8"
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", content_type)
|
|
self.send_header("Cache-Control", "no-cache")
|
|
self.send_header("Content-Length", str(len(content)))
|
|
self.end_headers()
|
|
self.wfile.write(content)
|
|
|
|
def serve_mockup(self, token: str) -> None:
|
|
"""Serve a published, sanitised concept preview by opaque token."""
|
|
conn = db()
|
|
try:
|
|
row = conn.execute("SELECT html FROM mockups WHERE token=? AND status='published' LIMIT 1", (token,)).fetchone()
|
|
finally:
|
|
conn.close()
|
|
if not row:
|
|
body = b"Preview not found"
|
|
self.send_response(404)
|
|
self.send_header("Content-Type", "text/plain; charset=utf-8")
|
|
self.send_header("Cache-Control", "no-store")
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
return
|
|
body = str(row["html"] or "").encode("utf-8", "replace")
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "text/html; charset=utf-8")
|
|
self.send_header("X-Robots-Tag", "noindex, nofollow")
|
|
self.send_header("Content-Security-Policy", "default-src 'none'; style-src 'unsafe-inline'; img-src data:; font-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'")
|
|
self.send_header("Cache-Control", "public, max-age=300")
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
|
|
def do_POST(self) -> None:
|
|
path = urlparse(self.path).path
|
|
if path == "/api/auth/setup":
|
|
conn: sqlite3.Connection | None = None
|
|
try:
|
|
payload = self.read_json()
|
|
conn = db()
|
|
if conn.execute("SELECT 1 FROM users WHERE active=1 LIMIT 1").fetchone():
|
|
return self.send_json(409, {"error": "owner_already_configured"})
|
|
email = str(payload.get("email") or "").strip().lower()
|
|
name = str(payload.get("name") or "").strip()[:160] or "Workspace owner"
|
|
password = str(payload.get("password") or "")
|
|
workspace_name = str(payload.get("workspace_name") or WORKSPACE_NAME).strip()[:160] or WORKSPACE_NAME
|
|
if not re.fullmatch(r"[^@\s]+@[^@\s]+\.[^@\s]+", email):
|
|
return self.send_json(400, {"error": "valid_email_required"})
|
|
hashed = password_hash(password)
|
|
stamp = now()
|
|
conn.execute("UPDATE organizations SET name=?,updated_at=? WHERE id=1", (workspace_name, stamp))
|
|
cur = conn.execute("INSERT INTO users(organization_id,email,name,password_hash,role,active,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?)", (1, email, name, hashed, "owner", 1, stamp, stamp))
|
|
conn.commit()
|
|
user_id = int(cur.lastrowid)
|
|
conn.close(); conn = None
|
|
_, csrf = create_session(self, user_id)
|
|
user = getattr(self, "_auth_user", None)
|
|
record_audit("owner_setup_completed", organization_id=1, user_id=user_id, entity_type="user", entity_id=user_id, ip_address=str(self.client_address[0] if self.client_address else ""))
|
|
return self.send_json(201, {"authenticated": True, "user": user, "csrf_token": csrf})
|
|
except sqlite3.IntegrityError:
|
|
return self.send_json(409, {"error": "email_already_registered"})
|
|
except (ValueError, TypeError, json.JSONDecodeError) as exc:
|
|
return self.send_json(400, {"error": str(exc)})
|
|
finally:
|
|
if conn is not None:
|
|
conn.close()
|
|
if path == "/api/auth/login":
|
|
conn: sqlite3.Connection | None = None
|
|
try:
|
|
payload = self.read_json()
|
|
email = str(payload.get("email") or "").strip().lower()
|
|
password = str(payload.get("password") or "")
|
|
conn = db()
|
|
row = conn.execute("SELECT * FROM users WHERE email=? ORDER BY id LIMIT 1", (email,)).fetchone()
|
|
if not row or not row["active"] or (row["locked_until"] and not _expired(row["locked_until"])):
|
|
return self.send_json(401, {"error": "invalid_email_or_password"})
|
|
if not password_matches(password, row["password_hash"]):
|
|
failures = int(row["failed_login_count"] or 0) + 1
|
|
lock_until = _future_iso(900) if failures >= 5 else ""
|
|
conn.execute("UPDATE users SET failed_login_count=?,locked_until=?,updated_at=? WHERE id=?", (failures, lock_until, now(), row["id"]))
|
|
conn.commit()
|
|
return self.send_json(401, {"error": "invalid_email_or_password"})
|
|
conn.execute("UPDATE users SET failed_login_count=0,locked_until='',last_login_at=?,updated_at=? WHERE id=?", (now(), now(), row["id"]))
|
|
conn.commit()
|
|
user_id = int(row["id"])
|
|
conn.close(); conn = None
|
|
_, csrf = create_session(self, user_id)
|
|
user = getattr(self, "_auth_user", None)
|
|
record_audit("login_succeeded", organization_id=int(row["organization_id"]), user_id=user_id, entity_type="user", entity_id=user_id, ip_address=str(self.client_address[0] if self.client_address else ""))
|
|
return self.send_json(200, {"authenticated": True, "user": user, "csrf_token": csrf})
|
|
except (ValueError, TypeError, json.JSONDecodeError) as exc:
|
|
return self.send_json(400, {"error": str(exc)})
|
|
finally:
|
|
if conn is not None:
|
|
conn.close()
|
|
if path == "/api/auth/logout":
|
|
user = require_auth(self)
|
|
if not user:
|
|
return
|
|
if not require_csrf(self):
|
|
return
|
|
token_hash = getattr(self, "_session_token_hash", "")
|
|
if token_hash:
|
|
conn = db()
|
|
try:
|
|
conn.execute("DELETE FROM sessions WHERE token_hash=?", (token_hash,)); conn.commit()
|
|
finally:
|
|
conn.close()
|
|
record_audit("logout", organization_id=int(user["organization_id"]), user_id=user.get("id"), entity_type="user", entity_id=user.get("id"), ip_address=str(self.client_address[0] if self.client_address else ""))
|
|
clear_session_cookie(self)
|
|
return self.send_json(200, {"logged_out": True})
|
|
if not public_auth_path(path) and not require_auth(self):
|
|
return
|
|
if path not in {"/api/auth/login", "/api/auth/setup"} and not require_csrf(self):
|
|
return
|
|
if path == "/api/logs":
|
|
try:
|
|
payload = self.read_json()
|
|
action = str(payload.get("action") or "").strip()
|
|
if not action:
|
|
return self.send_json(400, {"error": "action_required"})
|
|
level = str(payload.get("level") or "info").strip().lower()
|
|
if level not in {"info", "warn", "error"}:
|
|
level = "info"
|
|
details = payload.get("details") if isinstance(payload.get("details"), (dict, list, str, int, float, bool)) else {}
|
|
record_log("action", action, level=level, method="CLIENT", path=str(payload.get("path") or "")[:500], details=details,
|
|
error=str(payload.get("error") or "")[:500], organization_id=getattr(self, "_organization_id", 1), user_id=getattr(self, "_user_id", None))
|
|
return self.send_json(202, {"logged": True})
|
|
except (ValueError, TypeError, json.JSONDecodeError) as exc:
|
|
return self.send_json(400, {"error": str(exc)})
|
|
if path == "/api/users":
|
|
if not role_allows(getattr(self, "_auth_user", None), "admin"):
|
|
return self.send_json(403, {"error": "permission_denied"})
|
|
conn: sqlite3.Connection | None = None
|
|
try:
|
|
payload = self.read_json()
|
|
email = str(payload.get("email") or "").strip().lower()
|
|
name = str(payload.get("name") or "").strip()[:160]
|
|
password = str(payload.get("password") or "")
|
|
role = str(payload.get("role") or "reviewer").strip().lower()
|
|
if not re.fullmatch(r"[^@\s]+@[^@\s]+\.[^@\s]+", email):
|
|
return self.send_json(400, {"error": "valid_email_required"})
|
|
if role not in ALLOWED_ROLES or role == "owner":
|
|
return self.send_json(400, {"error": "invalid_role"})
|
|
if not name:
|
|
return self.send_json(400, {"error": "name_required"})
|
|
hashed = password_hash(password)
|
|
organization_id = int(getattr(self, "_organization_id", 1)); stamp = now()
|
|
conn = db()
|
|
cur = conn.execute("INSERT INTO users(organization_id,email,name,password_hash,role,active,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?)", (organization_id, email, name, hashed, role, 1, stamp, stamp))
|
|
conn.commit(); user_id = int(cur.lastrowid)
|
|
row = conn.execute("SELECT id,organization_id,email,name,role,active,last_login_at,created_at,updated_at FROM users WHERE id=?", (user_id,)).fetchone()
|
|
record_audit("user_created", organization_id=organization_id, user_id=getattr(self, "_user_id", None), entity_type="user", entity_id=user_id, details={"email": email, "role": role})
|
|
return self.send_json(201, user_view(row))
|
|
except sqlite3.IntegrityError:
|
|
return self.send_json(409, {"error": "email_already_registered"})
|
|
except (ValueError, TypeError, json.JSONDecodeError) as exc:
|
|
return self.send_json(400, {"error": str(exc)})
|
|
finally:
|
|
if conn is not None: conn.close()
|
|
if path == "/api/backups":
|
|
if not role_allows(getattr(self, "_auth_user", None), "admin"):
|
|
return self.send_json(403, {"error": "permission_denied"})
|
|
try:
|
|
backup = create_backup()
|
|
record_audit("backup_created", organization_id=getattr(self, "_organization_id", 1), user_id=getattr(self, "_user_id", None), entity_type="backup", details=backup)
|
|
return self.send_json(201, backup)
|
|
except Exception as exc:
|
|
record_log("system", "backup_failed", level="error", organization_id=getattr(self, "_organization_id", 1), user_id=getattr(self, "_user_id", None), error=str(exc)[:500])
|
|
return self.send_json(500, {"error": "backup_failed", "detail": str(exc)[:300]})
|
|
workspace_patch = path == "/api/workspace"
|
|
if workspace_patch:
|
|
if not role_allows(getattr(self, "_auth_user", None), "admin"):
|
|
return self.send_json(403, {"error": "permission_denied"})
|
|
try:
|
|
payload = self.read_json(); name = str(payload.get("name") or "").strip()[:160]
|
|
if not name: return self.send_json(400, {"error": "workspace_name_required"})
|
|
conn = db(); conn.execute("UPDATE organizations SET name=?,updated_at=? WHERE id=?", (name, now(), getattr(self, "_organization_id", 1))); conn.commit()
|
|
row = conn.execute("SELECT * FROM organizations WHERE id=?", (getattr(self, "_organization_id", 1),)).fetchone(); conn.close()
|
|
record_audit("workspace_updated", organization_id=getattr(self, "_organization_id", 1), user_id=getattr(self, "_user_id", None), entity_type="workspace", entity_id=getattr(self, "_organization_id", 1), details={"name": name})
|
|
return self.send_json(200, {"workspace": workspace_view(row)})
|
|
except Exception as exc:
|
|
return self.send_json(400, {"error": str(exc)})
|
|
if self.path == "/api/settings":
|
|
if not role_allows(getattr(self, "_auth_user", None), "admin"):
|
|
return self.send_json(403, {"error": "permission_denied"})
|
|
try:
|
|
payload = self.read_json()
|
|
ai = payload.get("ai") if isinstance(payload.get("ai"), dict) else {}
|
|
browser = payload.get("browser_use") if isinstance(payload.get("browser_use"), dict) else {}
|
|
maps = payload.get("maps") if isinstance(payload.get("maps"), dict) else {}
|
|
limits = payload.get("limits") if isinstance(payload.get("limits"), dict) else {}
|
|
company = payload.get("company_profile") if isinstance(payload.get("company_profile"), dict) else {}
|
|
conn = db()
|
|
saved_keys: list[str] = []
|
|
|
|
def save_setting(key: str, value: Any) -> None:
|
|
max_length = 20_000 if key.startswith("COMPANY_") else 500
|
|
text_value = str(value).strip()[:max_length]
|
|
conn.execute("INSERT INTO settings(key,value,updated_at) VALUES(?,?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value,updated_at=excluded.updated_at", (key, text_value, now()))
|
|
saved_keys.append(key)
|
|
|
|
for key in ("AI_PROVIDER", "AI_BASE_URL", "AI_API_KEY", "AI_MODEL"):
|
|
if key in ai:
|
|
# An empty password means “keep the existing credential”
|
|
# so a normal Settings save cannot erase a working key.
|
|
if key == "AI_API_KEY" and not str(ai.get(key) or "").strip():
|
|
continue
|
|
save_setting(key, ai.get(key) or "")
|
|
|
|
if "enabled" in browser:
|
|
enabled_value = str(browser.get("enabled") or "").strip().lower() not in {"0", "false", "no", "off", "disabled"}
|
|
save_setting("BROWSER_USE_ENABLED", "true" if enabled_value else "false")
|
|
if "max_steps" in browser:
|
|
save_setting("BROWSER_USE_MAX_STEPS", max(3, min(50, int(browser.get("max_steps")))))
|
|
if "timeout_seconds" in browser:
|
|
save_setting("BROWSER_USE_TIMEOUT_SECONDS", max(30, min(1800, int(browser.get("timeout_seconds")))))
|
|
if "llm_timeout_seconds" in browser:
|
|
save_setting("BROWSER_USE_LLM_TIMEOUT_SECONDS", max(10, min(300, int(browser.get("llm_timeout_seconds")))))
|
|
|
|
if "request_timeout_seconds" in maps:
|
|
save_setting("GMAPS_REQUEST_TIMEOUT", max(60, min(7200, int(maps.get("request_timeout_seconds")))))
|
|
if "poll_seconds" in maps:
|
|
save_setting("GMAPS_POLL_SECONDS", max(0.5, min(60.0, float(maps.get("poll_seconds")))))
|
|
if "max_depth" in maps:
|
|
save_setting("GMAPS_MAX_DEPTH", max(1, min(10, int(maps.get("max_depth")))))
|
|
if "website_scan_timeout_seconds" in limits:
|
|
save_setting("WEBSITE_SCAN_TIMEOUT", max(2, min(60, int(limits.get("website_scan_timeout_seconds")))))
|
|
|
|
for field, key in COMPANY_PROFILE_KEYS.items():
|
|
# Website content/title/timestamp are written by the
|
|
# bounded Learn button, but accepting them here makes a
|
|
# full profile export/import round-trip cleanly.
|
|
if field in company:
|
|
save_setting(key, str(company.get(field) or "")[:20_000])
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
record_log("action", "settings_updated", details={"keys": saved_keys})
|
|
record_audit("settings_updated", organization_id=getattr(self, "_organization_id", 1), user_id=getattr(self, "_user_id", None), entity_type="settings", details={"keys": saved_keys})
|
|
ai_runtime = ai_runtime_settings()
|
|
return self.send_json(200, {
|
|
"saved": True,
|
|
"saved_keys": saved_keys,
|
|
"ai": {"provider": ai_runtime["provider"], "base_url": ai_runtime["base_url"], "model": ai_runtime["model"], "configured": bool(ai_runtime["base_url"] and ai_runtime["model"] and (ai_runtime["provider"] != "nous" or ai_runtime["api_key"]))},
|
|
"browser_use": browser_use_settings(),
|
|
"maps": gmaps_runtime_settings(),
|
|
"limits": {"website_scan_timeout_seconds": website_scan_timeout_seconds()},
|
|
"company_profile": company_profile_settings(_saved_settings()),
|
|
})
|
|
except Exception as exc:
|
|
return self.send_json(400, {"error": str(exc)})
|
|
assignment_match = re.fullmatch(r"/api/prospects/(\d+)/assignment", path)
|
|
if assignment_match:
|
|
if not role_allows(getattr(self, "_auth_user", None), "researcher"):
|
|
return self.send_json(403, {"error": "permission_denied"})
|
|
conn: sqlite3.Connection | None = None
|
|
try:
|
|
prospect_id = int(assignment_match.group(1)); payload = self.read_json(); assignee = payload.get("assigned_user_id")
|
|
organization_id = int(getattr(self, "_organization_id", 1)); conn = db()
|
|
prospect = conn.execute("SELECT id FROM prospects WHERE id=? AND organization_id=?", (prospect_id, organization_id)).fetchone()
|
|
if not prospect: return self.send_json(404, {"error": "not_found"})
|
|
if assignee in (None, "", 0, "0"): assigned_id = None
|
|
else:
|
|
assigned = same_org_user(conn, assignee, organization_id)
|
|
if not assigned or not assigned["active"]: return self.send_json(400, {"error": "assigned_user_not_found"})
|
|
assigned_id = int(assigned["id"])
|
|
conn.execute("UPDATE prospects SET assigned_user_id=?,updated_at=? WHERE id=? AND organization_id=?", (assigned_id, now(), prospect_id, organization_id)); conn.commit()
|
|
row = conn.execute("SELECT * FROM prospects WHERE id=? AND organization_id=?", (prospect_id, getattr(self, "_organization_id", 1))).fetchone()
|
|
record_audit("prospect_assigned", organization_id=organization_id, user_id=getattr(self, "_user_id", None), entity_type="prospect", entity_id=prospect_id, details={"assigned_user_id": assigned_id})
|
|
return self.send_json(200, dict(row))
|
|
except (ValueError, TypeError, json.JSONDecodeError) as exc: return self.send_json(400, {"error": str(exc)})
|
|
finally:
|
|
if conn is not None: conn.close()
|
|
note_match = re.fullmatch(r"/api/prospects/(\d+)/notes", path)
|
|
if note_match:
|
|
if not role_allows(getattr(self, "_auth_user", None), "reviewer"): return self.send_json(403, {"error": "permission_denied"})
|
|
conn: sqlite3.Connection | None = None
|
|
try:
|
|
prospect_id = int(note_match.group(1)); payload = self.read_json(); body = str(payload.get("body") or "").strip()[:10_000]
|
|
if not body: return self.send_json(400, {"error": "note_body_required"})
|
|
organization_id = int(getattr(self, "_organization_id", 1)); conn = db()
|
|
if not conn.execute("SELECT 1 FROM prospects WHERE id=? AND organization_id=?", (prospect_id, organization_id)).fetchone(): return self.send_json(404, {"error": "not_found"})
|
|
stamp = now(); cur = conn.execute("INSERT INTO prospect_notes(organization_id,prospect_id,user_id,body,created_at,updated_at) VALUES(?,?,?,?,?,?)", (organization_id, prospect_id, getattr(self, "_user_id", None), body, stamp, stamp)); conn.commit()
|
|
row = conn.execute("SELECT * FROM prospect_notes WHERE id=?", (cur.lastrowid,)).fetchone()
|
|
record_audit("prospect_note_created", organization_id=organization_id, user_id=getattr(self, "_user_id", None), entity_type="prospect", entity_id=prospect_id)
|
|
return self.send_json(201, dict(row))
|
|
except (ValueError, TypeError, json.JSONDecodeError) as exc: return self.send_json(400, {"error": str(exc)})
|
|
finally:
|
|
if conn is not None: conn.close()
|
|
task_match = re.fullmatch(r"/api/prospects/(\d+)/tasks", path)
|
|
if task_match:
|
|
if not role_allows(getattr(self, "_auth_user", None), "reviewer"): return self.send_json(403, {"error": "permission_denied"})
|
|
conn: sqlite3.Connection | None = None
|
|
try:
|
|
prospect_id = int(task_match.group(1)); payload = self.read_json(); title = str(payload.get("title") or "").strip()[:240]
|
|
if not title: return self.send_json(400, {"error": "task_title_required"})
|
|
organization_id = int(getattr(self, "_organization_id", 1)); conn = db()
|
|
if not conn.execute("SELECT 1 FROM prospects WHERE id=? AND organization_id=?", (prospect_id, organization_id)).fetchone(): return self.send_json(404, {"error": "not_found"})
|
|
assigned_id = payload.get("assigned_user_id")
|
|
if assigned_id not in (None, "", 0, "0"):
|
|
assigned = same_org_user(conn, assigned_id, organization_id)
|
|
if not assigned or not assigned["active"]: return self.send_json(400, {"error": "assigned_user_not_found"})
|
|
assigned_id = int(assigned["id"])
|
|
else: assigned_id = None
|
|
status = str(payload.get("status") or "open").strip().lower()
|
|
if status not in {"open", "in_progress", "done", "cancelled"}: status = "open"
|
|
stamp = now(); cur = conn.execute("INSERT INTO prospect_tasks(organization_id,prospect_id,created_by,assigned_user_id,title,description,due_at,status,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?)", (organization_id, prospect_id, getattr(self, "_user_id", None), assigned_id, title, str(payload.get("description") or "").strip()[:10_000], str(payload.get("due_at") or "").strip()[:60], status, stamp, stamp)); conn.commit()
|
|
row = conn.execute("SELECT * FROM prospect_tasks WHERE id=?", (cur.lastrowid,)).fetchone()
|
|
record_audit("prospect_task_created", organization_id=organization_id, user_id=getattr(self, "_user_id", None), entity_type="prospect", entity_id=prospect_id, details={"task_id": int(cur.lastrowid)})
|
|
return self.send_json(201, dict(row))
|
|
except (ValueError, TypeError, json.JSONDecodeError) as exc: return self.send_json(400, {"error": str(exc)})
|
|
finally:
|
|
if conn is not None: conn.close()
|
|
retry_match = re.fullmatch(r"/api/runs/(\d+)/retry", path)
|
|
if retry_match:
|
|
if not role_allows(getattr(self, "_auth_user", None), "researcher"): return self.send_json(403, {"error": "permission_denied"})
|
|
conn: sqlite3.Connection | None = None
|
|
try:
|
|
run_id = int(retry_match.group(1)); organization_id = int(getattr(self, "_organization_id", 1)); conn = db(); row = conn.execute("SELECT * FROM runs WHERE id=? AND organization_id=?", (run_id, organization_id)).fetchone()
|
|
if not row: return self.send_json(404, {"error": "not_found"})
|
|
if row["status"] not in {"failed", "cancelled"} and row["recovery_state"] != "retryable": return self.send_json(400, {"error": "run_not_retryable"})
|
|
try: payload = json.loads(row["request_json"] or "{}")
|
|
except (TypeError, ValueError, json.JSONDecodeError): return self.send_json(400, {"error": "run_request_missing"})
|
|
conn.close(); conn = None
|
|
result = create_discovery(payload, organization_id=organization_id, created_by=getattr(self, "_user_id", None), retry_of=run_id)
|
|
record_audit("run_retried", organization_id=organization_id, user_id=getattr(self, "_user_id", None), entity_type="run", entity_id=run_id, details={"new_run_id": result["id"]})
|
|
return self.send_json(202, result)
|
|
except (ValueError, TypeError, json.JSONDecodeError) as exc: return self.send_json(400, {"error": str(exc)})
|
|
finally:
|
|
if conn is not None: conn.close()
|
|
cancel_match = re.fullmatch(r"/api/runs/(\d+)/cancel", path)
|
|
if cancel_match:
|
|
if not role_allows(getattr(self, "_auth_user", None), "researcher"): return self.send_json(403, {"error": "permission_denied"})
|
|
run_id = int(cancel_match.group(1)); conn = db()
|
|
try:
|
|
row = conn.execute("SELECT * FROM runs WHERE id=? AND organization_id=?", (run_id, getattr(self, "_organization_id", 1))).fetchone()
|
|
if not row: return self.send_json(404, {"error": "not_found"})
|
|
if row["status"] in {"succeeded", "failed", "cancelled"}: return self.send_json(400, {"error": "run_not_active"})
|
|
conn.execute("UPDATE runs SET cancel_requested=1 WHERE id=? AND organization_id=?", (run_id, getattr(self, "_organization_id", 1))); conn.commit()
|
|
record_audit("run_cancel_requested", organization_id=getattr(self, "_organization_id", 1), user_id=getattr(self, "_user_id", None), entity_type="run", entity_id=run_id)
|
|
return self.send_json(202, {"id": run_id, "cancel_requested": True})
|
|
finally: conn.close()
|
|
if urlparse(self.path).path == "/api/company-profile/learn":
|
|
if not role_allows(getattr(self, "_auth_user", None), "admin"):
|
|
return self.send_json(403, {"error": "permission_denied"})
|
|
conn: sqlite3.Connection | None = None
|
|
try:
|
|
payload = self.read_json()
|
|
learned = fetch_company_page(str(payload.get("url") or "").strip())
|
|
conn = db()
|
|
for field, value in learned.items():
|
|
key = COMPANY_PROFILE_KEYS[field]
|
|
conn.execute("INSERT INTO settings(key,value,updated_at) VALUES(?,?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value,updated_at=excluded.updated_at", (key, str(value)[:20_000], now()))
|
|
conn.commit()
|
|
profile = company_profile_settings(_saved_settings())
|
|
return self.send_json(200, {"saved": True, "company_profile": profile})
|
|
except (ValueError, TypeError, json.JSONDecodeError) as exc:
|
|
return self.send_json(400, {"error": str(exc)})
|
|
except RuntimeError as exc:
|
|
return self.send_json(502, {"error": str(exc)})
|
|
except Exception as exc:
|
|
return self.send_json(502, {"error": "company_profile_learn_failed:" + str(exc)[:240]})
|
|
finally:
|
|
if conn is not None:
|
|
conn.close()
|
|
if self.path == "/api/promotions":
|
|
if not role_allows(getattr(self, "_auth_user", None), "admin"):
|
|
return self.send_json(403, {"error": "permission_denied"})
|
|
conn: sqlite3.Connection | None = None
|
|
try:
|
|
payload = self.read_json()
|
|
name = str(payload.get("name") or "").strip()[:160]
|
|
description = str(payload.get("description") or "").strip()[:2_000]
|
|
if not name or not description:
|
|
return self.send_json(400, {"error": "promotion_name_and_description_required"})
|
|
active = 0 if str(payload.get("active", True)).strip().lower() in {"0", "false", "no", "off"} else 1
|
|
stamp = now()
|
|
conn = db()
|
|
cur = conn.execute("INSERT INTO promotions(organization_id,created_by,name,description,terms,starts_at,ends_at,active,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?)", (getattr(self, "_organization_id", 1), getattr(self, "_user_id", None), name, description, str(payload.get("terms") or "").strip()[:2_000], str(payload.get("starts_at") or "").strip()[:40], str(payload.get("ends_at") or "").strip()[:40], active, stamp, stamp))
|
|
conn.commit()
|
|
row = conn.execute("SELECT * FROM promotions WHERE id=?", (cur.lastrowid,)).fetchone()
|
|
record_audit("promotion_created", organization_id=getattr(self, "_organization_id", 1), user_id=getattr(self, "_user_id", None), entity_type="promotion", entity_id=int(cur.lastrowid), details={"name": name})
|
|
return self.send_json(201, dict(row))
|
|
except (ValueError, TypeError, json.JSONDecodeError) as exc:
|
|
return self.send_json(400, {"error": str(exc)})
|
|
except Exception as exc:
|
|
return self.send_json(400, {"error": str(exc)})
|
|
finally:
|
|
if conn is not None:
|
|
conn.close()
|
|
if urlparse(self.path).path == "/api/ai/test":
|
|
try:
|
|
runtime = ai_runtime_settings()
|
|
if not runtime["base_url"] or not runtime["model"]:
|
|
return self.send_json(400, {"error": "ai_not_configured_set_provider_and_model"})
|
|
if runtime["provider"] == "nous" and not runtime["api_key"]:
|
|
return self.send_json(400, {"error": "nous_api_key_required_for_standalone_app"})
|
|
headers = {"Authorization": "Bearer " + runtime["api_key"]} if runtime["api_key"] else {}
|
|
result = http_json("POST", runtime["base_url"] + "/chat/completions", {"model": runtime["model"], "temperature": 0, "max_tokens": 8, "messages": [{"role": "user", "content": "Reply with exactly OK."}]}, headers=headers, timeout=30, error_prefix="ai")
|
|
content = (((result.get("choices") or [{}])[0].get("message") or {}).get("content") or "") if isinstance(result, dict) else ""
|
|
if not str(content).strip():
|
|
return self.send_json(502, {"error": "ai_empty_response"})
|
|
return self.send_json(200, {"ok": True, "provider": runtime["provider"], "model": runtime["model"], "reply": str(content).strip()[:200]})
|
|
except RuntimeError as exc:
|
|
return self.send_json(502, {"error": str(exc)})
|
|
except Exception as exc:
|
|
return self.send_json(502, {"error": "ai_test_failed:" + str(exc)[:200]})
|
|
candidates_match = re.fullmatch(r"/api/prospects/(\d+)/website-candidates", urlparse(self.path).path)
|
|
if candidates_match:
|
|
if not role_allows(getattr(self, "_auth_user", None), "researcher"):
|
|
return self.send_json(403, {"error": "permission_denied"})
|
|
prospect_id = int(candidates_match.group(1))
|
|
conn = db()
|
|
try:
|
|
row = conn.execute("SELECT * FROM prospects WHERE id=? AND organization_id=?", (prospect_id, getattr(self, "_organization_id", 1))).fetchone()
|
|
finally:
|
|
conn.close()
|
|
if not row:
|
|
return self.send_json(404, {"error": "not_found"})
|
|
browser_runtime = browser_use_settings()
|
|
if not browser_runtime["enabled"]:
|
|
return self.send_json(503, {"error": "browser_use_disabled"})
|
|
try:
|
|
runtime = ai_runtime_settings()
|
|
record_log("browser", "website_investigation_started", prospect_id=prospect_id,
|
|
details={"business": row["name"], "model": runtime["model"], "public_only": True,
|
|
"max_steps": browser_runtime["max_steps"], "timeout_seconds": browser_runtime["timeout_seconds"],
|
|
"llm_timeout_seconds": browser_runtime["llm_timeout_seconds"]})
|
|
|
|
def browser_log(kind: str, event: str, details: dict[str, Any]) -> None:
|
|
record_log(kind, event, prospect_id=prospect_id, details=details)
|
|
|
|
# First use deterministic DOM extraction. This does not need
|
|
# an AI key and is deliberately run before Browser Use so a
|
|
# model timeout or malformed final response cannot erase a
|
|
# link that was already visible on Bing.
|
|
direct_candidates = direct_search_candidates(
|
|
dict(row), user_agent=OUTBOUND_USER_AGENT, log_callback=browser_log,
|
|
requested=5, timeout_seconds=min(180, int(browser_runtime["timeout_seconds"])),
|
|
)
|
|
if direct_candidates:
|
|
# Save the raw search links first. Verification is a
|
|
# separate bounded step and must never erase a useful
|
|
# result if a public site is slow or unavailable.
|
|
persist_website_candidates(prospect_id, direct_candidates)
|
|
verified_candidates = verify_website_candidates(dict(row), direct_candidates)
|
|
saved_rows = persist_website_candidates(prospect_id, verified_candidates)
|
|
for candidate in verified_candidates:
|
|
record_log("search", "website_candidate_verified", prospect_id=prospect_id,
|
|
details={"domain": candidate.get("domain"), "confidence": candidate.get("confidence"),
|
|
"verification": candidate.get("verification", {})})
|
|
record_log("browser", "website_candidates_persisted", prospect_id=prospect_id,
|
|
details={"source": "browser_dom", "candidate_count": len(direct_candidates),
|
|
"domains": [item.get("domain") for item in direct_candidates]})
|
|
return self.send_json(200, {"items": saved_rows, "count": len(saved_rows), "source": "browser_dom"})
|
|
|
|
# Keep Browser Use as a bounded fallback for cases where the
|
|
# public result pages yielded no parseable links.
|
|
record_log("browser", "browser_use_fallback_started", prospect_id=prospect_id,
|
|
details={"reason": "direct_search_no_candidates", "model": runtime["model"]})
|
|
candidates = investigate_websites(
|
|
dict(row), base_url=runtime["base_url"], api_key=runtime["api_key"],
|
|
model=runtime["model"], user_agent=OUTBOUND_USER_AGENT, log_callback=browser_log,
|
|
enabled=browser_runtime["enabled"], max_steps=browser_runtime["max_steps"],
|
|
timeout_seconds=browser_runtime["timeout_seconds"], llm_timeout_seconds=browser_runtime["llm_timeout_seconds"],
|
|
)
|
|
record_log("browser", "website_investigation_completed", prospect_id=prospect_id,
|
|
details={"candidate_count": len(candidates), "model": runtime["model"], "source": "browser_use"})
|
|
saved_rows = persist_website_candidates(prospect_id, candidates)
|
|
return self.send_json(200, {"items": saved_rows, "count": len(saved_rows), "source": "browser_use"})
|
|
except RuntimeError as exc:
|
|
record_log("browser", "website_investigation_failed", level="error", prospect_id=prospect_id, error=str(exc))
|
|
return self.send_json(503, {"error": str(exc)})
|
|
except Exception as exc:
|
|
record_log("browser", "website_investigation_failed", level="error", prospect_id=prospect_id, error=str(exc)[:500])
|
|
return self.send_json(502, {"error": "browser_use_failed:" + str(exc)[:240]})
|
|
finally:
|
|
try:
|
|
conn.close()
|
|
except Exception:
|
|
pass
|
|
mockup_match = re.fullmatch(r"/api/prospects/(\d+)/mockup", urlparse(self.path).path)
|
|
if mockup_match:
|
|
if not role_allows(getattr(self, "_auth_user", None), "reviewer"):
|
|
return self.send_json(403, {"error": "permission_denied"})
|
|
conn: sqlite3.Connection | None = None
|
|
prospect_id = int(mockup_match.group(1))
|
|
try:
|
|
conn = db()
|
|
organization_id = getattr(self, "_organization_id", 1)
|
|
row = conn.execute("SELECT * FROM prospects WHERE id=? AND organization_id=?", (prospect_id, organization_id)).fetchone()
|
|
if not row:
|
|
return self.send_json(404, {"error": "not_found"})
|
|
payload = self.read_json()
|
|
instructions = str(payload.get("instructions") or "").strip()[:2_000]
|
|
prospect = dict(row)
|
|
source_url = str(prospect.get("website") or prospect.get("website_final_url") or "").strip()
|
|
snapshot: dict[str, Any] = {}
|
|
if source_url:
|
|
try:
|
|
snapshot = fetch_company_page(source_url)
|
|
except Exception as exc:
|
|
# A preview must still be useful when the current site
|
|
# is slow, blocked, or listed as unhealthy.
|
|
record_log("search", "mockup_source_fetch_failed", level="warn", prospect_id=prospect_id,
|
|
details={"url": _safe_log_url(source_url)}, error=str(exc)[:500])
|
|
html, generation_source, model = generate_mockup_html(prospect, snapshot, instructions)
|
|
token = secrets.token_urlsafe(24)
|
|
stamp = now()
|
|
title = (_mockup_text(prospect.get("name"), 140) or "Business") + " · Website concept"
|
|
cur = conn.execute("""INSERT INTO mockups
|
|
(organization_id,prospect_id,created_by,token,title,source_url,html,ai_model,generation_source,instructions,status,created_at,updated_at)
|
|
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)""", (organization_id, prospect_id, getattr(self, "_user_id", None), token,
|
|
title, str(snapshot.get("website_url") or source_url)[:500], html, model, generation_source, instructions, "published", stamp, stamp))
|
|
conn.execute("UPDATE prospects SET ai_mockup_id=?,updated_at=? WHERE id=? AND organization_id=?", (int(cur.lastrowid), stamp, prospect_id, organization_id))
|
|
conn.commit()
|
|
share_url = mockup_public_url(self, token)
|
|
record_log("ai", "mockup_generated", prospect_id=prospect_id,
|
|
details={"mockup_id": int(cur.lastrowid), "source": generation_source, "model": model,
|
|
"source_url": _safe_log_url(str(snapshot.get("website_url") or source_url)), "share_url": share_url})
|
|
record_audit("mockup_generated", organization_id=organization_id, user_id=getattr(self, "_user_id", None),
|
|
entity_type="mockup", entity_id=int(cur.lastrowid), details={"prospect_id": prospect_id, "source": generation_source, "model": model})
|
|
return self.send_json(201, {"id": int(cur.lastrowid), "prospect_id": prospect_id, "token": token,
|
|
"title": title, "source_url": str(snapshot.get("website_url") or source_url),
|
|
"generation_source": generation_source, "ai_model": model,
|
|
"created_at": stamp, "share_url": share_url})
|
|
except RuntimeError as exc:
|
|
return self.send_json(503, {"error": str(exc)})
|
|
except (ValueError, TypeError, json.JSONDecodeError) as exc:
|
|
return self.send_json(400, {"error": str(exc)})
|
|
except sqlite3.Error as exc:
|
|
record_log("ai", "mockup_persist_failed", level="error", prospect_id=prospect_id, error=str(exc)[:500])
|
|
return self.send_json(500, {"error": "mockup_persist_failed"})
|
|
finally:
|
|
if conn is not None:
|
|
conn.close()
|
|
assess_match = re.fullmatch(r"/api/prospects/(\d+)/assess", urlparse(self.path).path)
|
|
if assess_match:
|
|
if not role_allows(getattr(self, "_auth_user", None), "reviewer"):
|
|
return self.send_json(403, {"error": "permission_denied"})
|
|
conn = None
|
|
try:
|
|
conn = db(); prospect_id = int(assess_match.group(1))
|
|
row = conn.execute("SELECT * FROM prospects WHERE id=? AND organization_id=?", (prospect_id, getattr(self, "_organization_id", 1))).fetchone()
|
|
if not row:
|
|
return self.send_json(404, {"error": "not_found"})
|
|
assessment = ai_assess_prospect(dict(row))
|
|
stamp = now()
|
|
conn.execute(
|
|
"UPDATE prospects SET ai_probability=?,ai_recommendation=?,ai_reasoning=?,ai_checked_at=?,updated_at=? WHERE id=? AND organization_id=?",
|
|
(assessment["ai_probability"], assessment["ai_recommendation"], assessment["ai_reasoning"], stamp, stamp, prospect_id, getattr(self, "_organization_id", 1)),
|
|
)
|
|
runtime = ai_runtime_settings()
|
|
conn.execute("INSERT INTO ai_assessment_history(organization_id,prospect_id,user_id,probability,recommendation,reasoning,model,created_at) VALUES(?,?,?,?,?,?,?,?)", (getattr(self, "_organization_id", 1), prospect_id, getattr(self, "_user_id", None), int(assessment["ai_probability"] or 0), assessment["ai_recommendation"], assessment["ai_reasoning"], runtime["model"], stamp))
|
|
conn.execute("INSERT INTO score_history(organization_id,prospect_id,user_id,score_type,score,reasons,model,created_at) VALUES(?,?,?,?,?,?,?,?)", (getattr(self, "_organization_id", 1), prospect_id, getattr(self, "_user_id", None), "ai", int(assessment["ai_probability"] or 0), json.dumps([assessment["ai_reasoning"]], ensure_ascii=False), runtime["model"], stamp))
|
|
conn.commit()
|
|
record_audit("prospect_ai_assessed", organization_id=getattr(self, "_organization_id", 1), user_id=getattr(self, "_user_id", None), entity_type="prospect", entity_id=prospect_id, details={"probability": assessment["ai_probability"], "recommendation": assessment["ai_recommendation"]})
|
|
return self.send_json(200, {**assessment, "ai_checked_at": stamp, "prospect_id": prospect_id})
|
|
except RuntimeError as exc:
|
|
return self.send_json(503, {"error": str(exc)})
|
|
except (ValueError, TypeError, json.JSONDecodeError) as exc:
|
|
return self.send_json(400, {"error": str(exc)})
|
|
finally:
|
|
if conn is not None:
|
|
conn.close()
|
|
message_match = re.fullmatch(r"/api/prospects/(\d+)/message", urlparse(self.path).path)
|
|
if message_match:
|
|
if not role_allows(getattr(self, "_auth_user", None), "reviewer"):
|
|
return self.send_json(403, {"error": "permission_denied"})
|
|
conn = None
|
|
try:
|
|
conn = db(); row = conn.execute("SELECT * FROM prospects WHERE id=? AND organization_id=?", (int(message_match.group(1)), getattr(self, "_organization_id", 1))).fetchone()
|
|
if not row: return self.send_json(404, {"error":"not_found"})
|
|
payload = self.read_json(); offer = str(payload.get("offer") or "").strip()
|
|
if not offer: return self.send_json(400, {"error":"offer_required"})
|
|
promotion = None
|
|
promotion_id = payload.get("promotion_id")
|
|
if promotion_id not in (None, "", 0, "0"):
|
|
try:
|
|
promotion_id = int(promotion_id)
|
|
except (TypeError, ValueError):
|
|
return self.send_json(400, {"error": "invalid_promotion_id"})
|
|
promotion_row = conn.execute("SELECT * FROM promotions WHERE id=? AND organization_id=?", (promotion_id, getattr(self, "_organization_id", 1))).fetchone()
|
|
if not promotion_row:
|
|
return self.send_json(404, {"error": "promotion_not_found"})
|
|
if not promotion_row["active"]:
|
|
return self.send_json(400, {"error": "promotion_inactive"})
|
|
promotion = dict(promotion_row)
|
|
else:
|
|
promotion_id = None
|
|
include_value = payload.get("include_mockup", True)
|
|
include_mockup = not (include_value is False or str(include_value).strip().lower() in {"0", "false", "off", "no"})
|
|
mockup = None
|
|
mockup_id = payload.get("mockup_id")
|
|
if include_mockup:
|
|
if mockup_id not in (None, "", 0, "0"):
|
|
try:
|
|
mockup_id = int(mockup_id)
|
|
except (TypeError, ValueError):
|
|
return self.send_json(400, {"error": "invalid_mockup_id"})
|
|
mockup_row = conn.execute("SELECT * FROM mockups WHERE id=? AND prospect_id=? AND organization_id=? AND status='published'", (mockup_id, int(message_match.group(1)), getattr(self, "_organization_id", 1))).fetchone()
|
|
else:
|
|
mockup_row = conn.execute("SELECT * FROM mockups WHERE prospect_id=? AND organization_id=? AND status='published' ORDER BY id DESC LIMIT 1", (int(message_match.group(1)), getattr(self, "_organization_id", 1))).fetchone()
|
|
if mockup_row:
|
|
mockup = dict(mockup_row)
|
|
mockup["share_url"] = mockup_public_url(self, mockup["token"])
|
|
mockup_id = int(mockup["id"])
|
|
else:
|
|
mockup_id = None
|
|
else:
|
|
mockup_id = None
|
|
draft = ai_generate_message(dict(row), offer, tone=str(payload.get("tone") or "warm"), goal=str(payload.get("goal") or "conversation"), promotion=promotion, mockup=mockup)
|
|
stamp = now(); conn.execute("UPDATE prospects SET ai_subject=?,ai_message=?,ai_offer=?,ai_promotion_id=?,ai_mockup_id=?,ai_reasoning=?,ai_unknowns=?,ai_evidence_ids=?,ai_follow_up=?,ai_checked_at=?,updated_at=? WHERE id=? AND organization_id=?", (draft["subject"],draft["message"],offer,promotion_id,mockup_id,draft["rationale"],json.dumps(draft.get("unknowns") or [], ensure_ascii=False),json.dumps(draft.get("evidence_used") or [], ensure_ascii=False),draft.get("follow_up") or "",stamp,stamp,int(message_match.group(1)),getattr(self, "_organization_id", 1))); conn.commit()
|
|
record_audit("prospect_message_generated", organization_id=getattr(self, "_organization_id", 1), user_id=getattr(self, "_user_id", None), entity_type="prospect", entity_id=int(message_match.group(1)), details={"promotion_id": promotion_id})
|
|
return self.send_json(200, {**draft,"ai_checked_at":stamp,"prospect_id":int(message_match.group(1))})
|
|
except RuntimeError as exc: return self.send_json(503, {"error":str(exc)})
|
|
except (ValueError, TypeError, json.JSONDecodeError) as exc: return self.send_json(400, {"error":str(exc)})
|
|
finally:
|
|
if conn is not None: conn.close()
|
|
scan_match = re.fullmatch(r"/api/prospects/(\d+)/scan", urlparse(self.path).path)
|
|
if scan_match:
|
|
if not role_allows(getattr(self, "_auth_user", None), "reviewer"):
|
|
return self.send_json(403, {"error": "permission_denied"})
|
|
conn = None
|
|
try:
|
|
conn=db(); row=conn.execute("SELECT * FROM prospects WHERE id=? AND organization_id=?",(int(scan_match.group(1)), getattr(self, "_organization_id", 1))).fetchone()
|
|
if not row: return self.send_json(404,{"error":"not_found"})
|
|
updated=enrich_place(dict(row)); cols=["website_status","website_http_status","website_final_url","website_latency_ms","website_https","website_title","website_technology","website_has_viewport","website_tls_expires_at","website_checked_at","website_error","emails","email_evidence","score_reasons","opportunity_score"]
|
|
prospect_id = int(scan_match.group(1)); conn.execute("UPDATE prospects SET "+",".join(c+"=?" for c in cols)+",updated_at=? WHERE id=? AND organization_id=?",tuple(updated.get(c) for c in cols)+(now(), prospect_id, getattr(self, "_organization_id", 1))); conn.execute("INSERT INTO score_history(organization_id,prospect_id,user_id,score_type,score,reasons,model,created_at) VALUES(?,?,?,?,?,?,?,?)", (getattr(self, "_organization_id", 1), prospect_id, getattr(self, "_user_id", None), "opportunity", int(updated.get("opportunity_score") or 0), updated.get("score_reasons", "[]"), "website_scan", now())); conn.commit(); record_audit("prospect_scanned", organization_id=getattr(self, "_organization_id", 1), user_id=getattr(self, "_user_id", None), entity_type="prospect", entity_id=prospect_id); return self.send_json(200,dict(conn.execute("SELECT * FROM prospects WHERE id=? AND organization_id=?",(prospect_id, getattr(self, "_organization_id", 1))).fetchone()))
|
|
except Exception as exc: return self.send_json(500,{"error":str(exc)})
|
|
finally:
|
|
if conn is not None: conn.close()
|
|
if self.path == "/api/suppressions":
|
|
if not role_allows(getattr(self, "_auth_user", None), "admin"):
|
|
return self.send_json(403, {"error": "permission_denied"})
|
|
try:
|
|
p=self.read_json(); kind=str(p.get("kind","domain")).lower(); value=str(p.get("value","")).strip().lower()
|
|
if kind not in {"domain","email"} or not value: return self.send_json(400,{"error":"kind_and_value_required"})
|
|
conn=db(); conn.execute("INSERT OR IGNORE INTO suppressions(organization_id,created_by,kind,value,reason,created_at) VALUES(?,?,?,?,?,?)",(getattr(self, "_organization_id", 1), getattr(self, "_user_id", None), kind,value,str(p.get("reason","manual"))[:200],now())); conn.commit(); row=conn.execute("SELECT * FROM suppressions WHERE organization_id=? AND kind=? AND value=?",(getattr(self, "_organization_id", 1),kind,value)).fetchone(); conn.close(); record_audit("suppression_created", organization_id=getattr(self, "_organization_id", 1), user_id=getattr(self, "_user_id", None), entity_type="suppression", entity_id=int(row["id"]) if row else None, details={"kind":kind,"value":value}); return self.send_json(201,dict(row))
|
|
except Exception as exc: return self.send_json(400,{"error":str(exc)})
|
|
if self.path == "/api/schedules":
|
|
if not role_allows(getattr(self, "_auth_user", None), "researcher"):
|
|
return self.send_json(403, {"error": "permission_denied"})
|
|
try:
|
|
p=self.read_json(); category=str(p.get("category") or "").strip(); keywords=str(p.get("keywords") or "").strip(); location=str(p.get("location") or "South Africa").strip(); interval=max(1,min(168,int(p.get("interval_hours",24)))); max_results=max(1,min(MAX_RESULTS,int(p.get("max_results",25))))
|
|
if not category or not keywords: return self.send_json(400,{"error":"category_and_keywords_required"})
|
|
next_at=datetime.fromtimestamp(datetime.now(timezone.utc).timestamp()+interval*3600,timezone.utc).isoformat(); conn=db(); cur=conn.execute("INSERT INTO schedules(organization_id,created_by,category,keywords,location,max_results,interval_hours,enabled,next_run_at,created_at) VALUES(?,?,?,?,?,?,?,?,?,?)",(getattr(self, "_organization_id", 1), getattr(self, "_user_id", None), category,keywords,location,max_results,interval,1,next_at,now())); conn.commit(); row=conn.execute("SELECT * FROM schedules WHERE id=?",(cur.lastrowid,)).fetchone(); conn.close(); return self.send_json(201,dict(row))
|
|
except Exception as exc: return self.send_json(400,{"error":str(exc)})
|
|
if self.path != "/api/discover":
|
|
return self.send_json(404, {"error": "not_found"})
|
|
if not role_allows(getattr(self, "_auth_user", None), "researcher"):
|
|
return self.send_json(403, {"error": "permission_denied"})
|
|
try:
|
|
result = create_discovery(self.read_json(), organization_id=getattr(self, "_organization_id", 1), created_by=getattr(self, "_user_id", None))
|
|
return self.send_json(202, result)
|
|
except ValueError as exc:
|
|
return self.send_json(400, {"error": str(exc)})
|
|
except (TypeError, json.JSONDecodeError) as exc:
|
|
return self.send_json(400, {"error": str(exc)})
|
|
|
|
def do_PATCH(self) -> None:
|
|
if not require_auth(self):
|
|
return
|
|
if not require_csrf(self):
|
|
return
|
|
user_match = re.fullmatch(r"/api/users/(\d+)", urlparse(self.path).path)
|
|
if user_match:
|
|
if not role_allows(getattr(self, "_auth_user", None), "admin"): return self.send_json(403, {"error": "permission_denied"})
|
|
conn: sqlite3.Connection | None = None
|
|
try:
|
|
user_id = int(user_match.group(1)); payload = self.read_json(); organization_id = int(getattr(self, "_organization_id", 1)); conn = db()
|
|
existing = conn.execute("SELECT * FROM users WHERE id=? AND organization_id=?", (user_id, organization_id)).fetchone()
|
|
if not existing: return self.send_json(404, {"error": "not_found"})
|
|
fields: dict[str, Any] = {}
|
|
if "name" in payload:
|
|
name = str(payload.get("name") or "").strip()[:160]
|
|
if not name: return self.send_json(400, {"error": "name_required"})
|
|
fields["name"] = name
|
|
if "role" in payload:
|
|
role = str(payload.get("role") or "").strip().lower()
|
|
if role not in ALLOWED_ROLES or role == "owner": return self.send_json(400, {"error": "invalid_role"})
|
|
fields["role"] = role
|
|
if "active" in payload: fields["active"] = 1 if bool(payload.get("active")) else 0
|
|
if "password" in payload and str(payload.get("password") or ""):
|
|
fields["password_hash"] = password_hash(str(payload.get("password")))
|
|
if existing["role"] == "owner" and (fields.get("role") != "owner" or fields.get("active") == 0):
|
|
owner_count = conn.execute("SELECT COUNT(*) FROM users WHERE organization_id=? AND role='owner' AND active=1", (organization_id,)).fetchone()[0]
|
|
if owner_count <= 1: return self.send_json(400, {"error": "last_owner_required"})
|
|
if not fields: return self.send_json(400, {"error": "no_changes"})
|
|
fields["updated_at"] = now(); conn.execute("UPDATE users SET " + ",".join(f"{key}=?" for key in fields) + " WHERE id=? AND organization_id=?", (*fields.values(), user_id, organization_id)); conn.commit()
|
|
row = conn.execute("SELECT id,organization_id,email,name,role,active,last_login_at,created_at,updated_at FROM users WHERE id=?", (user_id,)).fetchone()
|
|
record_audit("user_updated", organization_id=organization_id, user_id=getattr(self, "_user_id", None), entity_type="user", entity_id=user_id, details={"fields": list(fields)})
|
|
return self.send_json(200, user_view(row))
|
|
except (ValueError, TypeError, json.JSONDecodeError) as exc: return self.send_json(400, {"error": str(exc)})
|
|
finally:
|
|
if conn is not None: conn.close()
|
|
task_match = re.fullmatch(r"/api/prospects/(\d+)/tasks/(\d+)", urlparse(self.path).path)
|
|
if task_match:
|
|
if not role_allows(getattr(self, "_auth_user", None), "reviewer"): return self.send_json(403, {"error": "permission_denied"})
|
|
conn: sqlite3.Connection | None = None
|
|
try:
|
|
prospect_id, task_id = int(task_match.group(1)), int(task_match.group(2)); payload = self.read_json(); organization_id = int(getattr(self, "_organization_id", 1)); conn = db()
|
|
existing = conn.execute("SELECT * FROM prospect_tasks WHERE id=? AND prospect_id=? AND organization_id=?", (task_id, prospect_id, organization_id)).fetchone()
|
|
if not existing: return self.send_json(404, {"error": "not_found"})
|
|
fields: dict[str, Any] = {}
|
|
for key in ("title", "description", "due_at"):
|
|
if key in payload: fields[key] = str(payload.get(key) or "").strip()[:10_000 if key != "title" else 240]
|
|
if "title" in fields and not fields["title"]: return self.send_json(400, {"error": "task_title_required"})
|
|
if "status" in payload:
|
|
status = str(payload.get("status") or "open").strip().lower()
|
|
if status not in {"open", "in_progress", "done", "cancelled"}: return self.send_json(400, {"error": "invalid_task_status"})
|
|
fields["status"] = status; fields["completed_at"] = now() if status == "done" else ""
|
|
if "assigned_user_id" in payload:
|
|
assignee = payload.get("assigned_user_id")
|
|
if assignee in (None, "", 0, "0"): fields["assigned_user_id"] = None
|
|
else:
|
|
assigned = same_org_user(conn, assignee, organization_id)
|
|
if not assigned or not assigned["active"]: return self.send_json(400, {"error": "assigned_user_not_found"})
|
|
fields["assigned_user_id"] = int(assigned["id"])
|
|
if not fields: return self.send_json(400, {"error": "no_changes"})
|
|
fields["updated_at"] = now(); conn.execute("UPDATE prospect_tasks SET " + ",".join(f"{key}=?" for key in fields) + " WHERE id=?", (*fields.values(), task_id)); conn.commit()
|
|
row = conn.execute("SELECT * FROM prospect_tasks WHERE id=?", (task_id,)).fetchone(); record_audit("prospect_task_updated", organization_id=organization_id, user_id=getattr(self, "_user_id", None), entity_type="prospect", entity_id=prospect_id, details={"task_id": task_id, "fields": list(fields)})
|
|
return self.send_json(200, dict(row))
|
|
except (ValueError, TypeError, json.JSONDecodeError) as exc: return self.send_json(400, {"error": str(exc)})
|
|
finally:
|
|
if conn is not None: conn.close()
|
|
promotion_match = re.fullmatch(r"/api/promotions/(\d+)", urlparse(self.path).path)
|
|
if promotion_match:
|
|
if not role_allows(getattr(self, "_auth_user", None), "admin"):
|
|
return self.send_json(403, {"error": "permission_denied"})
|
|
conn: sqlite3.Connection | None = None
|
|
try:
|
|
payload = self.read_json()
|
|
conn = db()
|
|
existing = conn.execute("SELECT * FROM promotions WHERE id=? AND organization_id=?", (int(promotion_match.group(1)), getattr(self, "_organization_id", 1))).fetchone()
|
|
if not existing:
|
|
return self.send_json(404, {"error": "not_found"})
|
|
fields: dict[str, Any] = {}
|
|
for field in ("name", "description", "terms", "starts_at", "ends_at"):
|
|
if field in payload:
|
|
fields[field] = str(payload.get(field) or "").strip()[:2_000]
|
|
if "name" in fields and not fields["name"]:
|
|
return self.send_json(400, {"error": "promotion_name_required"})
|
|
if "description" in fields and not fields["description"]:
|
|
return self.send_json(400, {"error": "promotion_description_required"})
|
|
if "active" in payload:
|
|
fields["active"] = 0 if str(payload.get("active")).strip().lower() in {"0", "false", "no", "off"} else 1
|
|
fields["updated_at"] = now()
|
|
assignments = ",".join(f"{key}=?" for key in fields)
|
|
conn.execute("UPDATE promotions SET " + assignments + " WHERE id=? AND organization_id=?", (*fields.values(), int(promotion_match.group(1)), getattr(self, "_organization_id", 1)))
|
|
conn.commit()
|
|
row = conn.execute("SELECT * FROM promotions WHERE id=? AND organization_id=?", (int(promotion_match.group(1)), getattr(self, "_organization_id", 1))).fetchone()
|
|
record_audit("promotion_updated", organization_id=getattr(self, "_organization_id", 1), user_id=getattr(self, "_user_id", None), entity_type="promotion", entity_id=int(promotion_match.group(1)), details={"fields": list(fields)})
|
|
return self.send_json(200, dict(row))
|
|
except (ValueError, TypeError, json.JSONDecodeError) as exc:
|
|
return self.send_json(400, {"error": str(exc)})
|
|
finally:
|
|
if conn is not None:
|
|
conn.close()
|
|
candidate_match = re.fullmatch(r"/api/prospects/(\d+)/website-candidates/(\d+)", urlparse(self.path).path)
|
|
if candidate_match:
|
|
if not role_allows(getattr(self, "_auth_user", None), "researcher"):
|
|
return self.send_json(403, {"error": "permission_denied"})
|
|
prospect_id, candidate_id = (int(candidate_match.group(1)), int(candidate_match.group(2)))
|
|
conn: sqlite3.Connection | None = None
|
|
try:
|
|
payload = self.read_json()
|
|
status = str(payload.get("status", "")).strip().lower()
|
|
if status not in {"candidate", "accepted", "rejected"}:
|
|
return self.send_json(400, {"error": "invalid_website_candidate_status"})
|
|
conn = db()
|
|
candidate = conn.execute(
|
|
"SELECT * FROM website_candidates WHERE id=? AND prospect_id=? AND organization_id=?",
|
|
(candidate_id, prospect_id, getattr(self, "_organization_id", 1)),
|
|
).fetchone()
|
|
prospect = conn.execute("SELECT * FROM prospects WHERE id=? AND organization_id=?", (prospect_id, getattr(self, "_organization_id", 1))).fetchone()
|
|
if not candidate or not prospect:
|
|
return self.send_json(404, {"error": "not_found"})
|
|
if status != "accepted":
|
|
conn.execute("UPDATE website_candidates SET status=? WHERE id=?", (status, candidate_id))
|
|
conn.commit()
|
|
updated_candidate = conn.execute("SELECT * FROM website_candidates WHERE id=?", (candidate_id,)).fetchone()
|
|
record_audit("website_candidate_reviewed", organization_id=getattr(self, "_organization_id", 1), user_id=getattr(self, "_user_id", None), entity_type="prospect", entity_id=prospect_id, details={"candidate_id": candidate_id, "status": status})
|
|
return self.send_json(200, {"candidate": dict(updated_candidate)})
|
|
|
|
# The candidate is now a real catalogue domain. Check the
|
|
# catalogue before scanning so SQLite's unique domain index
|
|
# produces an actionable response instead of a generic 500.
|
|
duplicate = conn.execute(
|
|
"SELECT id FROM prospects WHERE domain=? AND id<>? AND organization_id=?",
|
|
(str(candidate["domain"]).lower(), prospect_id, getattr(self, "_organization_id", 1)),
|
|
).fetchone()
|
|
if duplicate:
|
|
return self.send_json(409, {"error": "domain_already_catalogued", "prospect_id": int(duplicate["id"])})
|
|
candidate_url = normalize_url(candidate["url"])
|
|
if not candidate_url:
|
|
return self.send_json(400, {"error": "invalid_candidate_url"})
|
|
|
|
# Website probing is network-bound. Release the SQLite handle
|
|
# while it runs, then persist the selected URL and evidence in
|
|
# one short transaction.
|
|
base_item = {**dict(prospect), "website": candidate_url, "domain": str(candidate["domain"]).lower()}
|
|
conn.close()
|
|
conn = None
|
|
updated = enrich_place(base_item)
|
|
cols = [
|
|
"website", "domain", "website_status", "website_http_status", "website_final_url",
|
|
"website_latency_ms", "website_https", "website_title", "website_technology",
|
|
"website_has_viewport", "website_tls_expires_at", "website_checked_at", "website_error",
|
|
"emails", "email_evidence", "score_reasons", "opportunity_score",
|
|
]
|
|
conn = db()
|
|
conn.execute(
|
|
"UPDATE website_candidates SET status='candidate' WHERE prospect_id=? AND id<>? AND status='accepted'",
|
|
(prospect_id, candidate_id),
|
|
)
|
|
conn.execute("UPDATE website_candidates SET status='accepted' WHERE id=? AND prospect_id=?", (candidate_id, prospect_id))
|
|
conn.execute(
|
|
"UPDATE prospects SET " + ",".join(f"{column}=?" for column in cols) + " WHERE id=?",
|
|
tuple(updated.get(column) for column in cols) + (prospect_id,),
|
|
)
|
|
conn.commit()
|
|
updated_candidate = conn.execute("SELECT * FROM website_candidates WHERE id=?", (candidate_id,)).fetchone()
|
|
updated_prospect = conn.execute("SELECT * FROM prospects WHERE id=?", (prospect_id,)).fetchone()
|
|
record_audit("website_candidate_accepted", organization_id=getattr(self, "_organization_id", 1), user_id=getattr(self, "_user_id", None), entity_type="prospect", entity_id=prospect_id, details={"candidate_id": candidate_id, "domain": candidate["domain"]})
|
|
return self.send_json(200, {"candidate": dict(updated_candidate), "prospect": dict(updated_prospect)})
|
|
except sqlite3.IntegrityError:
|
|
if conn is not None:
|
|
conn.rollback()
|
|
return self.send_json(409, {"error": "domain_already_catalogued"})
|
|
except (ValueError, TypeError, json.JSONDecodeError) as exc:
|
|
return self.send_json(400, {"error": str(exc)})
|
|
except Exception as exc:
|
|
return self.send_json(502, {"error": "website_candidate_accept_failed:" + str(exc)[:240]})
|
|
finally:
|
|
if conn is not None:
|
|
conn.close()
|
|
match = re.fullmatch(r"/api/prospects/(\d+)/review", urlparse(self.path).path)
|
|
if not match:
|
|
return self.send_json(404, {"error": "not_found"})
|
|
if not role_allows(getattr(self, "_auth_user", None), "reviewer"):
|
|
return self.send_json(403, {"error": "permission_denied"})
|
|
conn: sqlite3.Connection | None = None
|
|
try:
|
|
payload = self.read_json()
|
|
status = str(payload.get("status", "")).strip().lower()
|
|
if status not in ALLOWED_REVIEW:
|
|
return self.send_json(400, {"error": "invalid_review_status"})
|
|
conn = db()
|
|
cur = conn.execute("UPDATE prospects SET review_status=?,review_notes=?,updated_at=? WHERE id=? AND organization_id=?", (status, str(payload.get("notes", ""))[:1000], now(), int(match.group(1)), getattr(self, "_organization_id", 1)))
|
|
conn.commit()
|
|
if not cur.rowcount:
|
|
return self.send_json(404, {"error": "not_found"})
|
|
row = conn.execute("SELECT * FROM prospects WHERE id=? AND organization_id=?", (int(match.group(1)), getattr(self, "_organization_id", 1))).fetchone()
|
|
record_audit("prospect_review_updated", organization_id=getattr(self, "_organization_id", 1), user_id=getattr(self, "_user_id", None), entity_type="prospect", entity_id=int(match.group(1)), details={"status": status})
|
|
return self.send_json(200, dict(row))
|
|
except (ValueError, TypeError, json.JSONDecodeError) as exc:
|
|
return self.send_json(400, {"error": str(exc)})
|
|
finally:
|
|
if conn is not None:
|
|
conn.close()
|
|
|
|
def do_DELETE(self) -> None:
|
|
if not require_auth(self):
|
|
return
|
|
if not require_csrf(self):
|
|
return
|
|
promotion_match = re.fullmatch(r"/api/promotions/(\d+)", urlparse(self.path).path)
|
|
if promotion_match:
|
|
if not role_allows(getattr(self, "_auth_user", None), "admin"):
|
|
return self.send_json(403, {"error": "permission_denied"})
|
|
conn = db()
|
|
try:
|
|
cur = conn.execute("DELETE FROM promotions WHERE id=? AND organization_id=?", (int(promotion_match.group(1)), getattr(self, "_organization_id", 1)))
|
|
conn.commit()
|
|
record_audit("promotion_deleted", organization_id=getattr(self, "_organization_id", 1), user_id=getattr(self, "_user_id", None), entity_type="promotion", entity_id=int(promotion_match.group(1)))
|
|
return self.send_json(200, {"deleted": bool(cur.rowcount)})
|
|
finally:
|
|
conn.close()
|
|
match = re.fullmatch(r"/api/suppressions/(\d+)", urlparse(self.path).path)
|
|
if not match: return self.send_json(404, {"error":"not_found"})
|
|
if not role_allows(getattr(self, "_auth_user", None), "admin"):
|
|
return self.send_json(403, {"error": "permission_denied"})
|
|
conn=db()
|
|
try:
|
|
cur=conn.execute("DELETE FROM suppressions WHERE id=? AND organization_id=?",(int(match.group(1)), getattr(self, "_organization_id", 1))); conn.commit()
|
|
return self.send_json(200,{"deleted":bool(cur.rowcount)})
|
|
finally: conn.close()
|
|
|
|
|
|
def serve() -> None:
|
|
host = os.environ.get("HOST", "0.0.0.0")
|
|
port = int(os.environ.get("PORT", "8000"))
|
|
db()
|
|
ensure_owner_from_env()
|
|
recover_stale_runs()
|
|
threading.Thread(target=scheduler_loop, name="prospect-scheduler", daemon=True).start()
|
|
print(f"Personal ProspectOS listening on http://{host}:{port} (backend={backend_mode()})", flush=True)
|
|
ThreadingHTTPServer((host, port), Handler).serve_forever()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
serve()
|