root/MarketingTool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

deploy Nous v19 update

Browse Source
This commit is contained in:
Marco0300
2026-09-05 21:53:29 +02:00
parent 9da3df08ce
commit c2f14edd29
11 changed files with 537 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/server.py
+238 -28
View File
@@ -27,8 +27,10 @@ from urllib.parse import parse_qs, quote_plus, unquote, urlparse
try:
from intelligence import domain_candidates, scan_website
from browser_investigator import investigate_websites
except ImportError: # tests import the module as api.server
from api.intelligence import domain_candidates, scan_website
from api.browser_investigator import investigate_websites
DB_PATH = os.environ.get("PROSPECT_DB", "/data/prospects.db")
MAX_RESULTS = 50
@@ -52,6 +54,7 @@ AI_PROVIDER = os.environ.get("AI_PROVIDER", "custom").strip().lower()
# reaches the provider. Identify this application consistently on provider
# requests instead of relying on that default.
OUTBOUND_USER_AGENT = os.environ.get("OUTBOUND_USER_AGENT", "ProspectOS/2.0 (+personal-use)").strip() or "ProspectOS/2.0 (+personal-use)"
BROWSER_USE_ENABLED = os.environ.get("BROWSER_USE_ENABLED", "1").strip().lower() not in {"0", "false", "no", "off"}
NOUS_BASE_URL = "https://inference-api.nousresearch.com/v1"
NOUS_FREE_MODELS = [
{"id": "meituan/longcat-2.0:free", "label": "Meituan · LongCat 2.0", "free": True},
@@ -70,6 +73,19 @@ _schema_lock = threading.Lock()
_schema_path: str | None = None
def browser_use_limits() -> tuple[int, int]:
"""Return bounded Browser Use steps/timeout even if .env is malformed."""
try:
steps = int(os.environ.get("BROWSER_USE_MAX_STEPS", "10"))
except (TypeError, ValueError):
steps = 10
try:
timeout = int(os.environ.get("BROWSER_USE_TIMEOUT_SECONDS", "120"))
except (TypeError, ValueError):
timeout = 120
return max(3, min(15, steps)), max(30, min(300, timeout))
def now() -> str:
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
@@ -106,6 +122,7 @@ def db() -> sqlite3.Connection:
source TEXT NOT NULL DEFAULT 'google_maps',
source_id TEXT NOT NULL DEFAULT '',
category TEXT NOT NULL DEFAULT '',
search_category TEXT NOT NULL DEFAULT '',
address TEXT NOT NULL DEFAULT '',
phone TEXT NOT NULL DEFAULT '',
emails TEXT NOT NULL DEFAULT '',
@@ -135,6 +152,9 @@ def db() -> sqlite3.Connection:
ai_subject TEXT NOT NULL DEFAULT '',
ai_message TEXT NOT NULL DEFAULT '',
ai_offer TEXT NOT NULL DEFAULT '',
ai_unknowns TEXT NOT NULL DEFAULT '[]',
ai_evidence_ids TEXT NOT NULL DEFAULT '[]',
ai_follow_up TEXT NOT NULL DEFAULT '',
opportunity_score INTEGER NOT NULL DEFAULT 0,
raw_json TEXT NOT NULL DEFAULT '{}'
)""")
@@ -172,6 +192,19 @@ def db() -> sqlite3.Connection:
conn.execute("""CREATE TABLE IF NOT EXISTS settings (
key TEXT PRIMARY KEY, value TEXT NOT NULL DEFAULT '', updated_at TEXT NOT NULL
)""")
conn.execute("""CREATE TABLE IF NOT EXISTS website_candidates (
id INTEGER PRIMARY KEY AUTOINCREMENT,
prospect_id INTEGER NOT NULL,
url TEXT NOT NULL,
domain TEXT NOT NULL,
confidence INTEGER NOT NULL DEFAULT 0,
evidence TEXT NOT NULL DEFAULT '[]',
source TEXT NOT NULL DEFAULT 'browser_use',
status TEXT NOT NULL DEFAULT 'candidate',
discovered_at TEXT NOT NULL,
raw_json TEXT NOT NULL DEFAULT '{}',
UNIQUE(prospect_id, domain)
)""")
# Migrate databases created by v1-v5 without destroying the user's
# catalogue. SQLite permits adding nullable/defaulted columns in place.
@@ -179,6 +212,7 @@ def db() -> sqlite3.Connection:
"source": "TEXT NOT NULL DEFAULT 'google_maps'",
"source_id": "TEXT NOT NULL DEFAULT ''",
"category": "TEXT NOT NULL DEFAULT ''",
"search_category": "TEXT NOT NULL DEFAULT ''",
"address": "TEXT NOT NULL DEFAULT ''",
"phone": "TEXT NOT NULL DEFAULT ''",
"emails": "TEXT NOT NULL DEFAULT ''",
@@ -208,6 +242,9 @@ def db() -> sqlite3.Connection:
"ai_subject": "TEXT NOT NULL DEFAULT ''",
"ai_message": "TEXT NOT NULL DEFAULT ''",
"ai_offer": "TEXT NOT NULL DEFAULT ''",
"ai_unknowns": "TEXT NOT NULL DEFAULT '[]'",
"ai_evidence_ids": "TEXT NOT NULL DEFAULT '[]'",
"ai_follow_up": "TEXT NOT NULL DEFAULT ''",
"opportunity_score": "INTEGER NOT NULL DEFAULT 0",
"raw_json": "TEXT NOT NULL DEFAULT '{}'",
}
@@ -230,6 +267,7 @@ def db() -> sqlite3.Connection:
conn.execute("CREATE INDEX IF NOT EXISTS idx_prospects_review ON prospects(review_status)")
conn.execute("CREATE INDEX IF NOT EXISTS idx_prospects_score ON prospects(opportunity_score DESC)")
conn.execute("CREATE INDEX IF NOT EXISTS idx_runs_created ON runs(id DESC)")
conn.execute("CREATE INDEX IF NOT EXISTS idx_website_candidates_prospect ON website_candidates(prospect_id, confidence DESC)")
conn.commit()
_schema_path = DB_PATH
_schema_lock.release()
@@ -380,7 +418,7 @@ def score_place(item: dict[str, Any], website: str, phone: str, rating: float |
return score_opportunity(item)[0]
def map_gmaps_result(item: dict[str, Any], query: str) -> dict[str, Any]:
def map_gmaps_result(item: dict[str, Any], query: str, search_category: str = "") -> dict[str, Any]:
"""Normalize gosom/google-maps-scraper's place object to our catalogue."""
name = as_text(item.get("title") or item.get("name") or item.get("business_name")) or "Unnamed place"
website = normalize_url(item.get("website") or item.get("website_url"))
@@ -408,6 +446,7 @@ def map_gmaps_result(item: dict[str, Any], query: str) -> dict[str, Any]:
"source": "google_maps",
"source_id": source_id[:255],
"category": category[:255],
"search_category": as_text(search_category)[:255],
"address": address[:1000],
"phone": phone[:120],
"emails": emails[:500],
@@ -869,8 +908,9 @@ def enrich_place(item: dict[str, Any]) -> dict[str, Any]:
def insert_catalogue(run_id: int, query: str, raw_results: list[dict[str, Any]], requested: int,
progress: Callable[[int, int], None] | None = None) -> tuple[list[dict[str, Any]], int]:
mapped = [map_gmaps_result(item, query) for item in raw_results[:requested]]
progress: Callable[[int, int], None] | None = None,
search_category: str = "") -> tuple[list[dict[str, Any]], int]:
mapped = [map_gmaps_result(item, query, search_category) for item in raw_results[:requested]]
conn = db()
added = 0
try:
@@ -899,14 +939,14 @@ def insert_catalogue(run_id: int, query: str, raw_results: list[dict[str, Any]],
try:
cur = conn.execute("""INSERT OR IGNORE INTO prospects(
name,website,domain,description,query,discovered_at,source,source_id,
category,address,phone,emails,rating,review_count,maps_url,latitude,
category,search_category,address,phone,emails,rating,review_count,maps_url,latitude,
longitude,listing_status,website_status,website_http_status,website_final_url,
website_latency_ms,website_https,website_title,website_technology,website_has_viewport,
website_tls_expires_at,website_checked_at,website_error,email_evidence,score_reasons,
ai_probability,ai_recommendation,ai_reasoning,ai_checked_at,ai_subject,ai_message,ai_offer,opportunity_score,raw_json
) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""", (
) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""", (
item["name"], item["website"], item["domain"], item["description"], item["query"], now(),
item["source"], item["source_id"], item["category"], item["address"], item["phone"], item["emails"],
item["source"], item["source_id"], item["category"], item.get("search_category", ""), item["address"], item["phone"], item["emails"],
item["rating"], item["review_count"], item["maps_url"], item["latitude"], item["longitude"],
item.get("listing_status", ""), item.get("website_status", "unknown"), item.get("website_http_status"),
item.get("website_final_url", ""), item.get("website_latency_ms", 0), item.get("website_https", 0),
@@ -952,7 +992,7 @@ def run_discovery(run_id: int, payload: dict[str, Any]) -> None:
if total:
update_run(run_id, stage="website_analysis", progress=76 + int(index * 19 / total), found=index)
results, added = insert_catalogue(run_id, query, raw_results, requested, enrichment_progress)
results, added = insert_catalogue(run_id, query, raw_results, requested, enrichment_progress, search_category=category)
update_run(run_id, found=len(results), added=added, status="succeeded", stage="complete", progress=100, completed_at=now(), error="")
except Exception as exc:
message = str(exc).strip()[:500] or "discovery_failed"
@@ -998,7 +1038,7 @@ def scheduler_loop() -> None:
def ai_generate_message(prospect: dict[str, Any], offer: str) -> dict[str, Any]:
"""Generate an evidence-grounded draft through an OpenAI-compatible endpoint."""
"""Generate an evidence-grounded, human-editable outreach draft."""
runtime = ai_runtime_settings()
ai_base_url = runtime["base_url"]
ai_api_key = runtime["api_key"]
@@ -1007,24 +1047,57 @@ def ai_generate_message(prospect: dict[str, Any], offer: str) -> dict[str, Any]:
raise RuntimeError("nous_api_key_required_for_standalone_app")
if not ai_base_url or not ai_model:
raise RuntimeError("ai_not_configured_set_AI_BASE_URL_and_AI_MODEL")
evidence_ids = ["prospect_record"]
if prospect.get("maps_url"):
evidence_ids.append("public_maps_listing")
if prospect.get("website"):
evidence_ids.append("website_scan")
if prospect.get("emails"):
evidence_ids.append("public_email_listing")
if prospect.get("phone"):
evidence_ids.append("public_phone_listing")
evidence = {
"business": prospect.get("name"), "category": prospect.get("category"),
"location": prospect.get("address"), "website": prospect.get("website"),
"website_status": prospect.get("website_status"), "website_title": prospect.get("website_title"),
"technology": prospect.get("website_technology"), "emails": prospect.get("emails"),
"phone": prospect.get("phone"), "score": prospect.get("opportunity_score"),
"evidence_ids": evidence_ids,
"business_name": prospect.get("name"),
"search_category": prospect.get("search_category") or prospect.get("category"),
"address_or_location": prospect.get("address"),
"public_listing": {
"maps_url": prospect.get("maps_url"), "phone": prospect.get("phone"),
"rating": prospect.get("rating"), "review_count": prospect.get("review_count"),
},
"website": {
"url": prospect.get("website"), "status": prospect.get("website_status"),
"http_status": prospect.get("website_http_status"), "title": prospect.get("website_title"),
"technology": prospect.get("website_technology"), "https": bool(prospect.get("website_https")),
"latency_ms": prospect.get("website_latency_ms"),
},
"public_emails": prospect.get("emails"),
"opportunity_score": prospect.get("opportunity_score"),
"score_reasons": _json_list(prospect.get("score_reasons")),
}
system = ("You write concise, respectful B2B prospecting drafts for a South African web hosting and design business. "
"Use only the supplied evidence; never invent facts, prices, names, services, or claims. "
"Do not mention scraping or hidden analysis. Return strict JSON with keys subject, message, rationale. "
"The message must be plain text, personalised, transparent, and include a polite opt-out line.")
system = (
"You are a senior B2B copywriter for a South African web-hosting and website-design business. "
"Create a short, respectful, genuinely personalised first-contact email from the supplied evidence. "
"Use only stored facts and the user's offer; never invent a person's name, company activity, price, "
"service, problem, result, testimonial, urgency, or link. Do not mention scraping, scoring, hidden "
"analysis, or imply certainty where the evidence is uncertain. Do not insult the prospect or call a "
"site bad; use neutral wording such as 'may be having trouble loading' when appropriate. "
"If there is no website, do not say you reviewed their website. If there is no mockup URL in the offer, "
"do not invent one. Keep the email around 90-150 words, with a natural greeting using the business name, "
"one specific evidence-based observation, the exact offer/instructions supplied by the user, one simple "
"call to action, and a polite opt-out sentence. Avoid spammy hype, excessive emojis, all-caps, and fake "
"personal familiarity. Return ONLY strict JSON with exactly these keys: subject, message, rationale, "
"recommended_offer, unknowns, evidence_used, follow_up. subject is 5-10 natural words. message is plain "
"text. rationale is concise. recommended_offer is a short summary of the user's offer. unknowns is an "
"array of facts that should be confirmed. evidence_used is an array containing only the supplied evidence "
"IDs. follow_up is a short human review note, not part of the email."
)
prompt = json.dumps({"evidence": evidence, "user_offer_or_instruction": offer[:2000]}, ensure_ascii=False)
headers = {"Authorization": "Bearer " + ai_api_key} if ai_api_key else {}
request_payload: dict[str, Any] = {
"model": ai_model,
"temperature": 0.4,
"max_tokens": 700,
"max_tokens": 900,
"messages": [{"role": "system", "content": system}, {"role": "user", "content": prompt}],
}
# Nous exposes an OpenAI-compatible chat endpoint, but not every free model
@@ -1034,13 +1107,23 @@ def ai_generate_message(prospect: dict[str, Any], offer: str) -> dict[str, Any]:
request_payload["response_format"] = {"type": "json_object"}
result = http_json("POST", ai_base_url + "/chat/completions", request_payload, headers=headers, timeout=90, error_prefix="ai")
content = (((result.get("choices") or [{}])[0].get("message") or {}).get("content") or "") if isinstance(result, dict) else ""
try:
parsed = json.loads(content)
except (TypeError, ValueError, json.JSONDecodeError) as exc:
raise RuntimeError("ai_invalid_json_response") from exc
parsed = _parse_ai_object(content)
if not isinstance(parsed, dict) or not str(parsed.get("subject") or "").strip() or not str(parsed.get("message") or "").strip():
raise RuntimeError("ai_incomplete_message")
return {"subject": str(parsed["subject"])[:200], "message": str(parsed["message"])[:8000], "rationale": str(parsed.get("rationale") or "")[:2000]}
def text_list(value: Any, limit: int = 6, size: int = 400) -> list[str]:
if not isinstance(value, list):
return []
return [str(item).strip()[:size] for item in value if str(item).strip()][:limit]
used = [item for item in text_list(parsed.get("evidence_used"), limit=10, size=80) if item in evidence_ids]
return {
"subject": str(parsed["subject"]).strip()[:200],
"message": str(parsed["message"]).strip()[:8000],
"rationale": str(parsed.get("rationale") or "").strip()[:2000],
"recommended_offer": str(parsed.get("recommended_offer") or offer).strip()[:2000],
"unknowns": text_list(parsed.get("unknowns")),
"evidence_used": used,
"follow_up": str(parsed.get("follow_up") or "Review the draft and confirm the offer before sending.").strip()[:1000],
}
def _parse_ai_object(content: Any) -> dict[str, Any]:
@@ -1163,6 +1246,7 @@ class Handler(BaseHTTPRequestHandler):
"ready": ready,
"email_extraction_default": True,
"email_extraction_always": True,
"browser_use_enabled": BROWSER_USE_ENABLED,
"ai_provider": ai_runtime["provider"],
"ai_configured": bool(ai_runtime["base_url"] and ai_runtime["model"] and (ai_runtime["provider"] != "nous" or ai_runtime["api_key"])),
})
@@ -1174,7 +1258,8 @@ class Handler(BaseHTTPRequestHandler):
if parsed.path == "/api/settings":
ai_runtime = ai_runtime_settings()
models = nous_models() if ai_runtime["provider"] == "nous" else []
return self.send_json(200, {"maps": {"backend": backend_mode(), "worker_url": GMAPS_URL, "ready": backend_mode() == "browser" or bool(GMAPS_URL)}, "ai": {"provider": ai_runtime["provider"], "base_url": ai_runtime["base_url"], "model": ai_runtime["model"], "configured": bool(ai_runtime["base_url"] and ai_runtime["model"] and (ai_runtime["provider"] != "nous" or ai_runtime["api_key"])), "models": models}, "email_collection": {"always_on": True, "public_only": True}, "limits": {"max_results": MAX_RESULTS, "website_scan_timeout_seconds": 8}})
browser_steps, browser_timeout = browser_use_limits()
return self.send_json(200, {"maps": {"backend": backend_mode(), "worker_url": GMAPS_URL, "ready": backend_mode() == "browser" or bool(GMAPS_URL)}, "ai": {"provider": ai_runtime["provider"], "base_url": ai_runtime["base_url"], "model": ai_runtime["model"], "configured": bool(ai_runtime["base_url"] and ai_runtime["model"] and (ai_runtime["provider"] != "nous" or ai_runtime["api_key"])), "models": models}, "browser_use": {"enabled": BROWSER_USE_ENABLED, "headless": True, "public_only": True, "max_steps": browser_steps, "timeout_seconds": browser_timeout}, "email_collection": {"always_on": True, "public_only": True}, "limits": {"max_results": MAX_RESULTS, "website_scan_timeout_seconds": 8}})
if parsed.path == "/api/prospects":
params = parse_qs(parsed.query)
q = (params.get("q", [""])[0]).strip().lower()
@@ -1182,7 +1267,7 @@ class Handler(BaseHTTPRequestHandler):
rows = conn.execute("SELECT * FROM prospects ORDER BY opportunity_score DESC, id DESC").fetchall()
items = []
for row in rows:
haystack = " ".join(str(row[key] or "") for key in ("name", "domain", "description", "address", "category", "phone", "emails")).lower()
haystack = " ".join(str(row[key] or "") for key in ("name", "domain", "description", "address", "category", "search_category", "phone", "emails")).lower()
if q and q not in haystack:
continue
if status and row["review_status"] != status:
@@ -1192,14 +1277,21 @@ class Handler(BaseHTTPRequestHandler):
if parsed.path == "/api/export.csv":
params = parse_qs(parsed.query); q = params.get("q", [""])[0].strip().lower()
rows = [dict(r) for r in conn.execute("SELECT * FROM prospects ORDER BY opportunity_score DESC, id DESC")]
if q: rows = [r for r in rows if q in " ".join(str(r.get(k) or "") for k in ("name","domain","address","category","emails")).lower()]
out = io.StringIO(); fields = ["id","name","category","address","phone","emails","domain","website","website_status","opportunity_score","score_reasons","review_status","last_seen"]
if q: rows = [r for r in rows if q in " ".join(str(r.get(k) or "") for k in ("name","domain","address","category","search_category","emails")).lower()]
out = io.StringIO(); fields = ["id","name","search_category","category","address","phone","emails","domain","website","website_status","opportunity_score","score_reasons","review_status","last_seen"]
writer = csv.DictWriter(out, fieldnames=fields); writer.writeheader(); writer.writerows({k:r.get(k) for k in fields} for r in rows)
body = out.getvalue().encode(); self.send_response(200); self.send_header("Content-Type","text/csv; charset=utf-8"); self.send_header("Content-Disposition","attachment; filename=prospects.csv"); self.send_header("Content-Length",str(len(body))); self.end_headers(); self.wfile.write(body); return
if parsed.path == "/api/suppressions":
return self.send_json(200, {"items": [dict(r) for r in conn.execute("SELECT * FROM suppressions ORDER BY id DESC")]})
if parsed.path == "/api/schedules":
return self.send_json(200, {"items": [dict(r) for r in conn.execute("SELECT * FROM schedules ORDER BY id DESC")]})
match = re.fullmatch(r"/api/prospects/(\d+)/website-candidates", parsed.path)
if match:
prospect_id = int(match.group(1))
if not conn.execute("SELECT 1 FROM prospects WHERE id=?", (prospect_id,)).fetchone():
return self.send_json(404, {"error": "not_found"})
rows = conn.execute("SELECT * FROM website_candidates WHERE prospect_id=? ORDER BY confidence DESC, id DESC", (prospect_id,)).fetchall()
return self.send_json(200, {"items": [dict(row) for row in rows]})
match = re.fullmatch(r"/api/prospects/(\d+)/domains", parsed.path)
if match:
row = conn.execute("SELECT * FROM prospects WHERE id=?", (int(match.group(1)),)).fetchone()
@@ -1283,6 +1375,50 @@ class Handler(BaseHTTPRequestHandler):
return self.send_json(502, {"error": str(exc)})
except Exception as exc:
return self.send_json(502, {"error": "ai_test_failed:" + str(exc)[:200]})
candidates_match = re.fullmatch(r"/api/prospects/(\d+)/website-candidates", urlparse(self.path).path)
if candidates_match:
prospect_id = int(candidates_match.group(1))
conn = db()
try:
row = conn.execute("SELECT * FROM prospects WHERE id=?", (prospect_id,)).fetchone()
finally:
conn.close()
if not row:
return self.send_json(404, {"error": "not_found"})
if not BROWSER_USE_ENABLED:
return self.send_json(503, {"error": "browser_use_disabled"})
try:
runtime = ai_runtime_settings()
candidates = investigate_websites(
dict(row), base_url=runtime["base_url"], api_key=runtime["api_key"],
model=runtime["model"], user_agent=OUTBOUND_USER_AGENT,
)
conn = db()
for candidate in candidates:
encoded_evidence = json.dumps(candidate.get("evidence") or [], ensure_ascii=False)
encoded_raw = json.dumps(candidate, ensure_ascii=False)
conn.execute(
"""INSERT INTO website_candidates(prospect_id,url,domain,confidence,evidence,source,status,discovered_at,raw_json)
VALUES(?,?,?,?,?,?,?,?,?)
ON CONFLICT(prospect_id,domain) DO UPDATE SET
url=excluded.url, confidence=excluded.confidence, evidence=excluded.evidence,
source=excluded.source, discovered_at=excluded.discovered_at, raw_json=excluded.raw_json,
status=CASE WHEN website_candidates.status='accepted' THEN 'accepted' ELSE 'candidate' END""",
(prospect_id, candidate["url"], candidate["domain"], candidate["confidence"], encoded_evidence,
"browser_use", "candidate", now(), encoded_raw),
)
conn.commit()
rows = conn.execute("SELECT * FROM website_candidates WHERE prospect_id=? ORDER BY confidence DESC, id DESC", (prospect_id,)).fetchall()
return self.send_json(200, {"items": [dict(item) for item in rows], "count": len(rows)})
except RuntimeError as exc:
return self.send_json(503, {"error": str(exc)})
except Exception as exc:
return self.send_json(502, {"error": "browser_use_failed:" + str(exc)[:240]})
finally:
try:
conn.close()
except Exception:
pass
assess_match = re.fullmatch(r"/api/prospects/(\d+)/assess", urlparse(self.path).path)
if assess_match:
conn = None
@@ -1315,7 +1451,7 @@ class Handler(BaseHTTPRequestHandler):
payload = self.read_json(); offer = str(payload.get("offer") or "").strip()
if not offer: return self.send_json(400, {"error":"offer_required"})
draft = ai_generate_message(dict(row), offer)
stamp = now(); conn.execute("UPDATE prospects SET ai_subject=?,ai_message=?,ai_offer=?,ai_reasoning=?,ai_checked_at=? WHERE id=?", (draft["subject"],draft["message"],offer,draft["rationale"],stamp,int(message_match.group(1)))); conn.commit()
stamp = now(); conn.execute("UPDATE prospects SET ai_subject=?,ai_message=?,ai_offer=?,ai_reasoning=?,ai_unknowns=?,ai_evidence_ids=?,ai_follow_up=?,ai_checked_at=? WHERE id=?", (draft["subject"],draft["message"],offer,draft["rationale"],json.dumps(draft.get("unknowns") or [], ensure_ascii=False),json.dumps(draft.get("evidence_used") or [], ensure_ascii=False),draft.get("follow_up") or "",stamp,int(message_match.group(1)))); conn.commit()
return self.send_json(200, {**draft,"ai_checked_at":stamp,"prospect_id":int(message_match.group(1))})
except RuntimeError as exc: return self.send_json(503, {"error":str(exc)})
except (ValueError, TypeError, json.JSONDecodeError) as exc: return self.send_json(400, {"error":str(exc)})
@@ -1355,6 +1491,80 @@ class Handler(BaseHTTPRequestHandler):
return self.send_json(400, {"error": str(exc)})
def do_PATCH(self) -> None:
candidate_match = re.fullmatch(r"/api/prospects/(\d+)/website-candidates/(\d+)", urlparse(self.path).path)
if candidate_match:
prospect_id, candidate_id = (int(candidate_match.group(1)), int(candidate_match.group(2)))
conn: sqlite3.Connection | None = None
try:
payload = self.read_json()
status = str(payload.get("status", "")).strip().lower()
if status not in {"candidate", "accepted", "rejected"}:
return self.send_json(400, {"error": "invalid_website_candidate_status"})
conn = db()
candidate = conn.execute(
"SELECT * FROM website_candidates WHERE id=? AND prospect_id=?",
(candidate_id, prospect_id),
).fetchone()
prospect = conn.execute("SELECT * FROM prospects WHERE id=?", (prospect_id,)).fetchone()
if not candidate or not prospect:
return self.send_json(404, {"error": "not_found"})
if status != "accepted":
conn.execute("UPDATE website_candidates SET status=? WHERE id=?", (status, candidate_id))
conn.commit()
updated_candidate = conn.execute("SELECT * FROM website_candidates WHERE id=?", (candidate_id,)).fetchone()
return self.send_json(200, {"candidate": dict(updated_candidate)})
# The candidate is now a real catalogue domain. Check the
# catalogue before scanning so SQLite's unique domain index
# produces an actionable response instead of a generic 500.
duplicate = conn.execute(
"SELECT id FROM prospects WHERE domain=? AND id<>?",
(str(candidate["domain"]).lower(), prospect_id),
).fetchone()
if duplicate:
return self.send_json(409, {"error": "domain_already_catalogued", "prospect_id": int(duplicate["id"])})
candidate_url = normalize_url(candidate["url"])
if not candidate_url:
return self.send_json(400, {"error": "invalid_candidate_url"})
# Website probing is network-bound. Release the SQLite handle
# while it runs, then persist the selected URL and evidence in
# one short transaction.
base_item = {**dict(prospect), "website": candidate_url, "domain": str(candidate["domain"]).lower()}
conn.close()
conn = None
updated = enrich_place(base_item)
cols = [
"website", "domain", "website_status", "website_http_status", "website_final_url",
"website_latency_ms", "website_https", "website_title", "website_technology",
"website_has_viewport", "website_tls_expires_at", "website_checked_at", "website_error",
"emails", "email_evidence", "score_reasons", "opportunity_score",
]
conn = db()
conn.execute(
"UPDATE website_candidates SET status='candidate' WHERE prospect_id=? AND id<>? AND status='accepted'",
(prospect_id, candidate_id),
)
conn.execute("UPDATE website_candidates SET status='accepted' WHERE id=? AND prospect_id=?", (candidate_id, prospect_id))
conn.execute(
"UPDATE prospects SET " + ",".join(f"{column}=?" for column in cols) + " WHERE id=?",
tuple(updated.get(column) for column in cols) + (prospect_id,),
)
conn.commit()
updated_candidate = conn.execute("SELECT * FROM website_candidates WHERE id=?", (candidate_id,)).fetchone()
updated_prospect = conn.execute("SELECT * FROM prospects WHERE id=?", (prospect_id,)).fetchone()
return self.send_json(200, {"candidate": dict(updated_candidate), "prospect": dict(updated_prospect)})
except sqlite3.IntegrityError:
if conn is not None:
conn.rollback()
return self.send_json(409, {"error": "domain_already_catalogued"})
except (ValueError, TypeError, json.JSONDecodeError) as exc:
return self.send_json(400, {"error": str(exc)})
except Exception as exc:
return self.send_json(502, {"error": "website_candidate_accept_failed:" + str(exc)[:240]})
finally:
if conn is not None:
conn.close()
match = re.fullmatch(r"/api/prospects/(\d+)/review", urlparse(self.path).path)
if not match:
return self.send_json(404, {"error": "not_found"})