root/MarketingTool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

deploy Maps v6 update

Browse Source
This commit is contained in:
Marco0300
2026-09-05 17:06:17 +02:00
parent 3d2b87b2df
commit 5e14ddf92c
10 changed files with 673 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/server.py
+341 -21
View File
@@ -25,6 +25,11 @@ from pathlib import Path
from typing import Any, Callable
from urllib.parse import parse_qs, quote_plus, unquote, urlparse
try:
from intelligence import domain_candidates, scan_website
except ImportError: # tests import the module as api.server
from api.intelligence import domain_candidates, scan_website
DB_PATH = os.environ.get("PROSPECT_DB", "/data/prospects.db")
MAX_RESULTS = 50
GMAPS_POLL_SECONDS = float(os.environ.get("GMAPS_POLL_SECONDS", "2"))
@@ -35,8 +40,13 @@ GMAPS_URL = os.environ.get("GMAPS_URL", "").strip().rstrip("/")
# X-API-Key header; keep support for it only when explicitly selected.
GMAPS_API_MODE = os.environ.get("GMAPS_API_MODE", "standard").strip().lower()
GMAPS_API_KEY = os.environ.get("GMAPS_API_KEY", "").strip()
AI_BASE_URL = os.environ.get("AI_BASE_URL", "").strip().rstrip("/")
AI_API_KEY = os.environ.get("AI_API_KEY", "").strip()
AI_MODEL = os.environ.get("AI_MODEL", "").strip()
ALLOWED_REVIEW = {"pending", "approved", "rejected", "contacted"}
_discover_lock = threading.Lock()
_schema_lock = threading.Lock()
_schema_path: str | None = None
def now() -> str:
@@ -51,10 +61,17 @@ def backend_mode() -> str:
def db() -> sqlite3.Connection:
global _schema_path
os.makedirs(os.path.dirname(DB_PATH) or ".", exist_ok=True)
conn = sqlite3.connect(DB_PATH, timeout=30)
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA busy_timeout=30000")
if _schema_path == DB_PATH:
return conn
_schema_lock.acquire()
if _schema_path == DB_PATH:
_schema_lock.release()
return conn
conn.execute("""CREATE TABLE IF NOT EXISTS prospects (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
@@ -78,6 +95,25 @@ def db() -> sqlite3.Connection:
longitude REAL,
listing_status TEXT NOT NULL DEFAULT '',
website_status TEXT NOT NULL DEFAULT 'unknown',
website_http_status INTEGER,
website_final_url TEXT NOT NULL DEFAULT '',
website_latency_ms INTEGER NOT NULL DEFAULT 0,
website_https INTEGER NOT NULL DEFAULT 0,
website_title TEXT NOT NULL DEFAULT '',
website_technology TEXT NOT NULL DEFAULT '',
website_has_viewport INTEGER NOT NULL DEFAULT 0,
website_tls_expires_at TEXT NOT NULL DEFAULT '',
website_checked_at TEXT NOT NULL DEFAULT '',
website_error TEXT NOT NULL DEFAULT '',
email_evidence TEXT NOT NULL DEFAULT '[]',
score_reasons TEXT NOT NULL DEFAULT '[]',
ai_probability REAL,
ai_recommendation TEXT NOT NULL DEFAULT '',
ai_reasoning TEXT NOT NULL DEFAULT '',
ai_checked_at TEXT NOT NULL DEFAULT '',
ai_subject TEXT NOT NULL DEFAULT '',
ai_message TEXT NOT NULL DEFAULT '',
ai_offer TEXT NOT NULL DEFAULT '',
opportunity_score INTEGER NOT NULL DEFAULT 0,
raw_json TEXT NOT NULL DEFAULT '{}'
)""")
@@ -98,6 +134,20 @@ def db() -> sqlite3.Connection:
completed_at TEXT NOT NULL DEFAULT '',
request_json TEXT NOT NULL DEFAULT '{}'
)""")
conn.execute("""CREATE TABLE IF NOT EXISTS suppressions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
kind TEXT NOT NULL,
value TEXT NOT NULL,
reason TEXT NOT NULL DEFAULT '',
created_at TEXT NOT NULL,
UNIQUE(kind, value)
)""")
conn.execute("""CREATE TABLE IF NOT EXISTS schedules (
id INTEGER PRIMARY KEY AUTOINCREMENT, category TEXT NOT NULL, keywords TEXT NOT NULL,
location TEXT NOT NULL DEFAULT 'South Africa', max_results INTEGER NOT NULL DEFAULT 25,
interval_hours INTEGER NOT NULL DEFAULT 24, enabled INTEGER NOT NULL DEFAULT 1,
next_run_at TEXT NOT NULL, last_run_id INTEGER, created_at TEXT NOT NULL
)""")
# Migrate databases created by v1-v5 without destroying the user's
# catalogue. SQLite permits adding nullable/defaulted columns in place.
@@ -115,6 +165,25 @@ def db() -> sqlite3.Connection:
"longitude": "REAL",
"listing_status": "TEXT NOT NULL DEFAULT ''",
"website_status": "TEXT NOT NULL DEFAULT 'unknown'",
"website_http_status": "INTEGER",
"website_final_url": "TEXT NOT NULL DEFAULT ''",
"website_latency_ms": "INTEGER NOT NULL DEFAULT 0",
"website_https": "INTEGER NOT NULL DEFAULT 0",
"website_title": "TEXT NOT NULL DEFAULT ''",
"website_technology": "TEXT NOT NULL DEFAULT ''",
"website_has_viewport": "INTEGER NOT NULL DEFAULT 0",
"website_tls_expires_at": "TEXT NOT NULL DEFAULT ''",
"website_checked_at": "TEXT NOT NULL DEFAULT ''",
"website_error": "TEXT NOT NULL DEFAULT ''",
"email_evidence": "TEXT NOT NULL DEFAULT '[]'",
"score_reasons": "TEXT NOT NULL DEFAULT '[]'",
"ai_probability": "REAL",
"ai_recommendation": "TEXT NOT NULL DEFAULT ''",
"ai_reasoning": "TEXT NOT NULL DEFAULT ''",
"ai_checked_at": "TEXT NOT NULL DEFAULT ''",
"ai_subject": "TEXT NOT NULL DEFAULT ''",
"ai_message": "TEXT NOT NULL DEFAULT ''",
"ai_offer": "TEXT NOT NULL DEFAULT ''",
"opportunity_score": "INTEGER NOT NULL DEFAULT 0",
"raw_json": "TEXT NOT NULL DEFAULT '{}'",
}
@@ -137,17 +206,26 @@ def db() -> sqlite3.Connection:
conn.execute("CREATE INDEX IF NOT EXISTS idx_prospects_review ON prospects(review_status)")
conn.execute("CREATE INDEX IF NOT EXISTS idx_prospects_score ON prospects(opportunity_score DESC)")
conn.execute("CREATE INDEX IF NOT EXISTS idx_runs_created ON runs(id DESC)")
# A Maps scrape can legitimately take several minutes. Only recover jobs
# that have been abandoned for half an hour, never after 35 seconds.
conn.execute("""UPDATE runs SET status='failed', stage='failed',
error=CASE WHEN error='' THEN 'stale_run_recovered' ELSE error END,
completed_at=CASE WHEN completed_at='' THEN ? ELSE completed_at END
WHERE status IN ('queued','running')
AND julianday(created_at) < julianday('now','-30 minutes')""", (now(),))
conn.commit()
_schema_path = DB_PATH
_schema_lock.release()
return conn
def recover_stale_runs() -> None:
"""Recover abandoned runs once at process startup, not on every request."""
conn = db()
try:
conn.execute("""UPDATE runs SET status='failed', stage='failed',
error=CASE WHEN error='' THEN 'stale_run_recovered' ELSE error END,
completed_at=CASE WHEN completed_at='' THEN ? ELSE completed_at END
WHERE status IN ('queued','running')
AND julianday(created_at) < julianday('now','-30 minutes')""", (now(),))
conn.commit()
finally:
conn.close()
def json_bytes(value: Any) -> bytes:
return json.dumps(value, ensure_ascii=False).encode("utf-8")
@@ -213,20 +291,69 @@ def as_int(value: Any) -> int:
return 0
def score_place(item: dict[str, Any], website: str, phone: str, rating: float | None, review_count: int) -> int:
"""Small transparent triage score; it is not an AI verdict."""
EMAIL_RE = re.compile(r"[A-Z0-9._%+\-]+@[A-Z0-9.\-]+\.[A-Z]{2,63}", re.I)
FREE_EMAIL_DOMAINS = {"gmail.com", "googlemail.com", "outlook.com", "hotmail.com", "yahoo.com", "icloud.com"}
def normalize_public_emails(value: Any) -> list[str]:
found: list[str] = []
text = as_text(value)
for address in EMAIL_RE.findall(text):
address = address.strip(".,;:()[]{}<>").lower()
if address not in found:
found.append(address)
return found[:30]
def score_opportunity(item: dict[str, Any]) -> tuple[int, list[str]]:
"""Transparent, explainable triage score; it is not an AI verdict."""
score = 20
if not website:
reasons: list[str] = []
website = str(item.get("website") or "")
website_status = str(item.get("website_status") or "unknown")
emails = normalize_public_emails(item.get("emails"))
phone = str(item.get("phone") or "")
rating = as_float(item.get("rating"))
review_count = as_int(item.get("review_count"))
if not website or website_status in {"no_detected_website", "missing"}:
score += 35
reasons.append("No detected website")
elif website_status in {"broken", "unreachable", "error"}:
score += 30
reasons.append("Website appears unavailable or broken")
elif website_status == "slow":
score += 15
reasons.append("Website response is slow")
elif website_status == "http_only":
score += 10
reasons.append("Website has no HTTPS")
elif website_status == "redirect":
score += 5
reasons.append("Website relies on redirects")
if emails:
score += 12
reasons.append("Public email found")
if any(address.split("@", 1)[-1] in FREE_EMAIL_DOMAINS for address in emails):
score += 8
reasons.append("Free-mail address used")
if phone:
score += 10
reasons.append("Phone listed")
if rating is not None and rating >= 4:
score += 10
reasons.append("Strong public rating")
if review_count >= 10:
score += 10
reasons.append("Established review activity")
if as_text(item.get("category") or item.get("type")):
score += 5
return min(100, score)
return min(100, score), reasons
def score_place(item: dict[str, Any], website: str, phone: str, rating: float | None, review_count: int) -> int:
item = {**item, "website": website, "phone": phone, "rating": rating, "review_count": review_count,
"website_status": "no_detected_website" if not website else "unknown"}
return score_opportunity(item)[0]
def map_gmaps_result(item: dict[str, Any], query: str) -> dict[str, Any]:
@@ -238,7 +365,8 @@ def map_gmaps_result(item: dict[str, Any], query: str) -> dict[str, Any]:
phone = as_text(item.get("phone") or item.get("phone_number"))
category = as_text(item.get("category") or item.get("type") or item.get("categories"))
description = as_text(item.get("descriptions") or item.get("description") or category)
emails = as_text(item.get("emails") or item.get("email"))
email_values = normalize_public_emails(item.get("emails") or item.get("email"))
emails = ", ".join(email_values)
rating = as_float(item.get("review_rating") if item.get("review_rating") is not None else item.get("rating"))
review_count = as_int(item.get("review_count") if item.get("review_count") is not None else item.get("reviews"))
source_id = as_text(item.get("place_id") or item.get("data_id") or item.get("cid") or maps_url)
@@ -265,7 +393,9 @@ def map_gmaps_result(item: dict[str, Any], query: str) -> dict[str, Any]:
"latitude": latitude,
"longitude": longitude,
"listing_status": as_text(item.get("status"))[:120],
"website_status": "present" if website else "missing",
"website_status": "unknown" if website else "no_detected_website",
"email_evidence": json.dumps([{"email": value, "source_url": website or maps_url, "found_at": now(), "method": "maps_worker", "confidence": "medium"} for value in email_values], ensure_ascii=False),
"score_reasons": "[]",
"opportunity_score": score_place(item, website, phone, rating, review_count),
"raw_json": json.dumps(item, ensure_ascii=False)[:100_000],
}
@@ -369,6 +499,7 @@ def download_standard_results(job_id: str) -> list[dict[str, Any]]:
def submit_gmaps_job(query: str, include_email: bool = False, extra_reviews: bool = False) -> str:
if not GMAPS_URL:
raise RuntimeError("gmaps_not_configured")
include_email = True
lang = os.environ.get("GMAPS_LANG", "en")
depth = max(1, min(10, int(os.environ.get("GMAPS_MAX_DEPTH", "1"))))
if GMAPS_API_MODE == "saas":
@@ -575,7 +706,10 @@ def validate_payload(payload: dict[str, Any]) -> tuple[str, str, str, int, bool,
except (TypeError, ValueError) as exc:
raise ValueError("invalid_max_results") from exc
query = f"{keywords} {category} {location}".strip()
include_email = bool(payload.get("include_email", False))
# Email discovery is part of this app's core use case. The server forces
# it on so a client/UI change cannot accidentally create a run without
# attempting public email extraction.
include_email = True
extra_reviews = bool(payload.get("extra_reviews", False))
return category, query, location, requested, include_email, extra_reviews
@@ -596,21 +730,85 @@ def update_run(run_id: int, **fields: Any) -> None:
conn.close()
def insert_catalogue(run_id: int, query: str, raw_results: list[dict[str, Any]], requested: int) -> tuple[list[dict[str, Any]], int]:
def _json_list(value: Any) -> list[Any]:
try:
parsed = json.loads(str(value or "[]"))
return parsed if isinstance(parsed, list) else []
except (TypeError, ValueError, json.JSONDecodeError):
return []
def enrich_place(item: dict[str, Any]) -> dict[str, Any]:
"""Add website health, public email evidence, and explainable scoring."""
prior_evidence = _json_list(item.get("email_evidence"))
if item.get("website"):
try:
item.update(scan_website(str(item["website"])))
except Exception as exc: # a single hostile site must not abort a run
item.update({"website_status": "unknown", "website_checked_at": now(), "website_error": str(exc)[:300], "website_emails": [], "email_evidence": []})
else:
item.setdefault("website_status", "no_detected_website")
item.setdefault("website_emails", [])
item.setdefault("email_evidence", [])
existing_emails = normalize_public_emails(item.get("emails"))
new_emails = normalize_public_emails(item.get("website_emails"))
all_emails = list(dict.fromkeys(existing_emails + new_emails))[:30]
item["emails"] = ", ".join(all_emails)
evidence = prior_evidence
scanned_evidence = item.get("email_evidence")
if isinstance(scanned_evidence, list):
evidence.extend(scanned_evidence)
# The Maps worker evidence was serialized before the website scan. Keep it
# only once, then append the scanner's page-level evidence.
seen_evidence: set[tuple[str, str]] = set()
compact: list[dict[str, Any]] = []
for record in evidence:
if not isinstance(record, dict):
continue
marker = (str(record.get("email", "")), str(record.get("source_url", "")))
if marker[0] and marker not in seen_evidence:
seen_evidence.add(marker)
compact.append(record)
item["email_evidence"] = json.dumps(compact[:50], ensure_ascii=False)
score, reasons = score_opportunity(item)
item["opportunity_score"] = score
item["score_reasons"] = json.dumps(reasons, ensure_ascii=False)
return item
def insert_catalogue(run_id: int, query: str, raw_results: list[dict[str, Any]], requested: int,
progress: Callable[[int, int], None] | None = None) -> tuple[list[dict[str, Any]], int]:
mapped = [map_gmaps_result(item, query) for item in raw_results[:requested]]
conn = db()
added = 0
try:
for item in mapped:
suppressed_domains = {str(row[0]).lower() for row in conn.execute("SELECT value FROM suppressions WHERE kind='domain'")}
suppressed_emails = {str(row[0]).lower() for row in conn.execute("SELECT value FROM suppressions WHERE kind='email'")}
total = len(mapped)
for index, item in enumerate(mapped, 1):
item = enrich_place(item)
if progress:
progress(index, total)
emails = set(normalize_public_emails(item.get("emails")))
if item["domain"].lower() in suppressed_domains or emails.intersection(suppressed_emails):
continue
cur = conn.execute("""INSERT OR IGNORE INTO prospects(
name,website,domain,description,query,discovered_at,source,source_id,
category,address,phone,emails,rating,review_count,maps_url,latitude,
longitude,listing_status,website_status,opportunity_score,raw_json
) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""", (
longitude,listing_status,website_status,website_http_status,website_final_url,
website_latency_ms,website_https,website_title,website_technology,website_has_viewport,
website_tls_expires_at,website_checked_at,website_error,email_evidence,score_reasons,
ai_probability,ai_recommendation,ai_reasoning,ai_checked_at,ai_subject,ai_message,ai_offer,opportunity_score,raw_json
) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""", (
item["name"], item["website"], item["domain"], item["description"], item["query"], now(),
item["source"], item["source_id"], item["category"], item["address"], item["phone"], item["emails"],
item["rating"], item["review_count"], item["maps_url"], item["latitude"], item["longitude"],
item["listing_status"], item["website_status"], item["opportunity_score"], item["raw_json"],
item.get("listing_status", ""), item.get("website_status", "unknown"), item.get("website_http_status"),
item.get("website_final_url", ""), item.get("website_latency_ms", 0), item.get("website_https", 0),
item.get("website_title", ""), item.get("website_technology", ""), item.get("website_has_viewport", 0),
item.get("website_tls_expires_at", ""), item.get("website_checked_at", ""), item.get("website_error", ""),
item.get("email_evidence", "[]"), item.get("score_reasons", "[]"), None, "", "", "", "", "", "",
item["opportunity_score"], item["raw_json"],
))
if cur.rowcount:
added += 1
@@ -639,7 +837,13 @@ def run_discovery(run_id: int, payload: dict[str, Any]) -> None:
else:
update_run(run_id, stage="browser_search", progress=20)
raw_results = google_search(query, requested)
results, added = insert_catalogue(run_id, query, raw_results, requested)
update_run(run_id, stage="website_analysis", progress=76, found=len(raw_results))
def enrichment_progress(index: int, total: int) -> None:
if total:
update_run(run_id, stage="website_analysis", progress=76 + int(index * 19 / total), found=index)
results, added = insert_catalogue(run_id, query, raw_results, requested, enrichment_progress)
update_run(run_id, found=len(results), added=added, status="succeeded", stage="complete", progress=100, completed_at=now(), error="")
except Exception as exc:
message = str(exc).strip()[:500] or "discovery_failed"
@@ -669,6 +873,50 @@ def create_discovery(payload: dict[str, Any]) -> dict[str, Any]:
return {"id": run_id, "status": "queued", "source": backend_mode(), "query": query, "requested": requested}
def scheduler_loop() -> None:
while True:
try:
conn = db(); due = conn.execute("SELECT * FROM schedules WHERE enabled=1 AND next_run_at<=?", (now(),)).fetchall()
for row in due:
payload = {"category": row["category"], "keywords": row["keywords"], "location": row["location"], "max_results": row["max_results"], "extra_reviews": False}
run = create_discovery(payload)
conn.execute("UPDATE schedules SET next_run_at=?,last_run_id=? WHERE id=?", (datetime.now(timezone.utc).replace(microsecond=0).timestamp() and datetime.fromtimestamp(datetime.now(timezone.utc).timestamp()+int(row["interval_hours"])*3600,timezone.utc).isoformat(), run["id"], row["id"]))
conn.commit(); conn.close()
except Exception:
try: conn.close()
except Exception: pass
time.sleep(30)
def ai_generate_message(prospect: dict[str, Any], offer: str) -> dict[str, Any]:
"""Generate an evidence-grounded draft through an OpenAI-compatible endpoint."""
if not AI_BASE_URL or not AI_MODEL:
raise RuntimeError("ai_not_configured_set_AI_BASE_URL_and_AI_MODEL")
evidence = {
"business": prospect.get("name"), "category": prospect.get("category"),
"location": prospect.get("address"), "website": prospect.get("website"),
"website_status": prospect.get("website_status"), "website_title": prospect.get("website_title"),
"technology": prospect.get("website_technology"), "emails": prospect.get("emails"),
"phone": prospect.get("phone"), "score": prospect.get("opportunity_score"),
"score_reasons": _json_list(prospect.get("score_reasons")),
}
system = ("You write concise, respectful B2B prospecting drafts for a South African web hosting and design business. "
"Use only the supplied evidence; never invent facts, prices, names, services, or claims. "
"Do not mention scraping or hidden analysis. Return strict JSON with keys subject, message, rationale. "
"The message must be plain text, personalised, transparent, and include a polite opt-out line.")
prompt = json.dumps({"evidence": evidence, "user_offer_or_instruction": offer[:2000]}, ensure_ascii=False)
headers = {"Authorization": "Bearer " + AI_API_KEY} if AI_API_KEY else {}
result = http_json("POST", AI_BASE_URL + "/chat/completions", {"model": AI_MODEL, "temperature": 0.4, "response_format": {"type": "json_object"}, "messages": [{"role": "system", "content": system}, {"role": "user", "content": prompt}]}, headers=headers, timeout=60)
content = (((result.get("choices") or [{}])[0].get("message") or {}).get("content") or "") if isinstance(result, dict) else ""
try:
parsed = json.loads(content)
except (TypeError, ValueError, json.JSONDecodeError) as exc:
raise RuntimeError("ai_invalid_json_response") from exc
if not isinstance(parsed, dict) or not str(parsed.get("subject") or "").strip() or not str(parsed.get("message") or "").strip():
raise RuntimeError("ai_incomplete_message")
return {"subject": str(parsed["subject"])[:200], "message": str(parsed["message"])[:8000], "rationale": str(parsed.get("rationale") or "")[:2000]}
class Handler(BaseHTTPRequestHandler):
server_version = "PersonalProspectOS/2.0"
@@ -710,7 +958,9 @@ class Handler(BaseHTTPRequestHandler):
"gmaps_auth_required": auth_required,
"gmaps_api_key": bool(GMAPS_API_KEY),
"ready": ready,
"email_extraction_default": False,
"email_extraction_default": True,
"email_extraction_always": True,
"ai_configured": bool(AI_BASE_URL and AI_MODEL),
})
if parsed.path == "/api/prospects":
params = parse_qs(parsed.query)
@@ -726,6 +976,27 @@ class Handler(BaseHTTPRequestHandler):
continue
items.append(dict(row))
return self.send_json(200, {"items": items, "total": len(items)})
if parsed.path == "/api/export.csv":
params = parse_qs(parsed.query); q = params.get("q", [""])[0].strip().lower()
rows = [dict(r) for r in conn.execute("SELECT * FROM prospects ORDER BY opportunity_score DESC, id DESC")]
if q: rows = [r for r in rows if q in " ".join(str(r.get(k) or "") for k in ("name","domain","address","category","emails")).lower()]
out = io.StringIO(); fields = ["id","name","category","address","phone","emails","domain","website","website_status","opportunity_score","score_reasons","review_status","last_seen"]
writer = csv.DictWriter(out, fieldnames=fields); writer.writeheader(); writer.writerows({k:r.get(k) for k in fields} for r in rows)
body = out.getvalue().encode(); self.send_response(200); self.send_header("Content-Type","text/csv; charset=utf-8"); self.send_header("Content-Disposition","attachment; filename=prospects.csv"); self.send_header("Content-Length",str(len(body))); self.end_headers(); self.wfile.write(body); return
if parsed.path == "/api/suppressions":
return self.send_json(200, {"items": [dict(r) for r in conn.execute("SELECT * FROM suppressions ORDER BY id DESC")]})
if parsed.path == "/api/schedules":
return self.send_json(200, {"items": [dict(r) for r in conn.execute("SELECT * FROM schedules ORDER BY id DESC")]})
match = re.fullmatch(r"/api/prospects/(\d+)/domains", parsed.path)
if match:
row = conn.execute("SELECT * FROM prospects WHERE id=?", (int(match.group(1)),)).fetchone()
if not row: return self.send_json(404, {"error":"not_found"})
return self.send_json(200, {"items": domain_candidates(row["name"], row["address"], row["category"])})
match = re.fullmatch(r"/api/prospects/(\d+)", parsed.path)
if match:
row = conn.execute("SELECT * FROM prospects WHERE id=?", (int(match.group(1)),)).fetchone()
if not row: return self.send_json(404, {"error":"not_found"})
return self.send_json(200, dict(row))
if parsed.path == "/api/runs":
rows = [dict(row) for row in conn.execute("SELECT * FROM runs ORDER BY id DESC LIMIT 50")]
return self.send_json(200, {"items": rows})
@@ -769,6 +1040,44 @@ class Handler(BaseHTTPRequestHandler):
self.wfile.write(content)
def do_POST(self) -> None:
message_match = re.fullmatch(r"/api/prospects/(\d+)/message", urlparse(self.path).path)
if message_match:
conn = None
try:
conn = db(); row = conn.execute("SELECT * FROM prospects WHERE id=?", (int(message_match.group(1)),)).fetchone()
if not row: return self.send_json(404, {"error":"not_found"})
payload = self.read_json(); offer = str(payload.get("offer") or "").strip()
if not offer: return self.send_json(400, {"error":"offer_required"})
draft = ai_generate_message(dict(row), offer)
stamp = now(); conn.execute("UPDATE prospects SET ai_subject=?,ai_message=?,ai_offer=?,ai_reasoning=?,ai_checked_at=? WHERE id=?", (draft["subject"],draft["message"],offer,draft["rationale"],stamp,int(message_match.group(1)))); conn.commit()
return self.send_json(200, {**draft,"ai_checked_at":stamp,"prospect_id":int(message_match.group(1))})
except RuntimeError as exc: return self.send_json(503, {"error":str(exc)})
except (ValueError, TypeError, json.JSONDecodeError) as exc: return self.send_json(400, {"error":str(exc)})
finally:
if conn is not None: conn.close()
scan_match = re.fullmatch(r"/api/prospects/(\d+)/scan", urlparse(self.path).path)
if scan_match:
conn = None
try:
conn=db(); row=conn.execute("SELECT * FROM prospects WHERE id=?",(int(scan_match.group(1)),)).fetchone()
if not row: return self.send_json(404,{"error":"not_found"})
updated=enrich_place(dict(row)); cols=["website_status","website_http_status","website_final_url","website_latency_ms","website_https","website_title","website_technology","website_has_viewport","website_tls_expires_at","website_checked_at","website_error","emails","email_evidence","score_reasons","opportunity_score"]
conn.execute("UPDATE prospects SET "+",".join(c+"=?" for c in cols)+" WHERE id=?",tuple(updated.get(c) for c in cols)+(int(scan_match.group(1)),)); conn.commit(); return self.send_json(200,dict(conn.execute("SELECT * FROM prospects WHERE id=?",(int(scan_match.group(1)),)).fetchone()))
except Exception as exc: return self.send_json(500,{"error":str(exc)})
finally:
if conn is not None: conn.close()
if self.path == "/api/suppressions":
try:
p=self.read_json(); kind=str(p.get("kind","domain")).lower(); value=str(p.get("value","")).strip().lower()
if kind not in {"domain","email"} or not value: return self.send_json(400,{"error":"kind_and_value_required"})
conn=db(); conn.execute("INSERT OR IGNORE INTO suppressions(kind,value,reason,created_at) VALUES(?,?,?,?)",(kind,value,str(p.get("reason","manual"))[:200],now())); conn.commit(); row=conn.execute("SELECT * FROM suppressions WHERE kind=? AND value=?",(kind,value)).fetchone(); conn.close(); return self.send_json(201,dict(row))
except Exception as exc: return self.send_json(400,{"error":str(exc)})
if self.path == "/api/schedules":
try:
p=self.read_json(); category=str(p.get("category") or "").strip(); keywords=str(p.get("keywords") or "").strip(); location=str(p.get("location") or "South Africa").strip(); interval=max(1,min(168,int(p.get("interval_hours",24)))); max_results=max(1,min(MAX_RESULTS,int(p.get("max_results",25))))
if not category or not keywords: return self.send_json(400,{"error":"category_and_keywords_required"})
next_at=datetime.fromtimestamp(datetime.now(timezone.utc).timestamp()+interval*3600,timezone.utc).isoformat(); conn=db(); cur=conn.execute("INSERT INTO schedules(category,keywords,location,max_results,interval_hours,enabled,next_run_at,created_at) VALUES(?,?,?,?,?,?,?,?)",(category,keywords,location,max_results,interval,1,next_at,now())); conn.commit(); row=conn.execute("SELECT * FROM schedules WHERE id=?",(cur.lastrowid,)).fetchone(); conn.close(); return self.send_json(201,dict(row))
except Exception as exc: return self.send_json(400,{"error":str(exc)})
if self.path != "/api/discover":
return self.send_json(404, {"error": "not_found"})
try:
@@ -802,11 +1111,22 @@ class Handler(BaseHTTPRequestHandler):
if conn is not None:
conn.close()
def do_DELETE(self) -> None:
match = re.fullmatch(r"/api/suppressions/(\d+)", urlparse(self.path).path)
if not match: return self.send_json(404, {"error":"not_found"})
conn=db()
try:
cur=conn.execute("DELETE FROM suppressions WHERE id=?",(int(match.group(1)),)); conn.commit()
return self.send_json(200,{"deleted":bool(cur.rowcount)})
finally: conn.close()
def serve() -> None:
host = os.environ.get("HOST", "0.0.0.0")
port = int(os.environ.get("PORT", "8000"))
db()
recover_stale_runs()
Thread(target=scheduler_loop, name="prospect-scheduler", daemon=True).start()
print(f"Personal ProspectOS listening on http://{host}:{port} (backend={backend_mode()})", flush=True)
ThreadingHTTPServer((host, port), Handler).serve_forever()