deploy Nous v19 update

api/Dockerfile

@@ -4,7 +4,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends chromium && rm
 WORKDIR /app
 COPY api/requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
-COPY api/server.py api/intelligence.py ./
+COPY api/server.py api/intelligence.py api/browser_investigator.py ./
 COPY web /app/web
 RUN useradd --system --create-home app && mkdir -p /data && chown -R app:app /app /data
 USER app

api/browser_investigator.py

@@ -0,0 +1,173 @@
+"""Optional Browser Use investigator for public website candidates.
+
+This module is deliberately isolated and imports browser-use only when called.
+It performs bounded, headless public research. It does not log in, solve
+CAPTCHAs, rotate proxies, or continue after an access challenge.
+"""
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import re
+from typing import Any
+from urllib.parse import urlparse
+
+
+_BLOCKED_HOSTS = {
+    "facebook.com", "instagram.com", "linkedin.com", "youtube.com", "x.com",
+    "twitter.com", "google.com", "google.co.za", "bing.com", "brave.com",
+    "duckduckgo.com", "startpage.com", "yelp.com", "yellowpages.com",
+}
+
+
+def _normalise_url(value: Any) -> str:
+    text = str(value or "").strip()
+    if not text:
+        return ""
+    if text.startswith("//"):
+        text = "https:" + text
+    if not re.match(r"^https?://", text, re.I):
+        text = "https://" + text
+    parsed = urlparse(text)
+    if parsed.scheme not in {"http", "https"} or not parsed.hostname:
+        return ""
+    host = parsed.hostname.lower().rstrip(".")
+    if host in {"localhost", "127.0.0.1", "::1"} or host.endswith(".local"):
+        return ""
+    if any(host == blocked or host.endswith("." + blocked) for blocked in _BLOCKED_HOSTS):
+        return ""
+    return text[:1000]
+
+
+def _parse_agent_result(raw: Any) -> list[dict[str, Any]]:
+    """Parse and normalise the agent's final JSON candidate list."""
+    text = str(raw or "").strip()
+    if text.startswith("```"):
+        text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text, flags=re.I | re.S).strip()
+    try:
+        payload = json.loads(text)
+    except (TypeError, ValueError, json.JSONDecodeError):
+        start, end = text.find("{"), text.rfind("}")
+        if start < 0 or end <= start:
+            return []
+        try:
+            payload = json.loads(text[start:end + 1])
+        except (TypeError, ValueError, json.JSONDecodeError):
+            return []
+    if not isinstance(payload, dict):
+        return []
+    raw_candidates = payload.get("candidates")
+    if not isinstance(raw_candidates, list):
+        return []
+    output: list[dict[str, Any]] = []
+    seen: set[str] = set()
+    for item in raw_candidates[:10]:
+        if not isinstance(item, dict):
+            continue
+        url = _normalise_url(item.get("url") or item.get("website") or item.get("domain"))
+        if not url:
+            continue
+        domain = (urlparse(url).hostname or "").lower().removeprefix("www.")
+        if not domain or domain in seen:
+            continue
+        seen.add(domain)
+        try:
+            confidence = max(0, min(100, int(round(float(item.get("confidence", 0))))))
+        except (TypeError, ValueError):
+            confidence = 0
+        evidence = item.get("evidence")
+        if not isinstance(evidence, list):
+            evidence = []
+        output.append({
+            "url": url,
+            "domain": domain,
+            "confidence": confidence,
+            "evidence": [str(value).strip()[:240] for value in evidence if str(value).strip()][:8],
+            "status": str(item.get("status") or "candidate").strip()[:80],
+        })
+    return sorted(output, key=lambda item: item["confidence"], reverse=True)
+
+
+def investigate_websites(
+    prospect: dict[str, Any],
+    *,
+    base_url: str,
+    api_key: str,
+    model: str,
+    user_agent: str,
+) -> list[dict[str, Any]]:
+    """Run one bounded Browser Use investigation and return candidates."""
+    if os.environ.get("BROWSER_USE_ENABLED", "1").strip().lower() in {"0", "false", "no", "off"}:
+        raise RuntimeError("browser_use_disabled")
+    if not base_url or not model:
+        raise RuntimeError("browser_use_requires_ai_provider_and_model")
+    if not api_key:
+        raise RuntimeError("browser_use_requires_ai_api_key")
+    try:
+        from browser_use import Agent, BrowserProfile, ChatOpenAI
+    except ImportError as exc:
+        raise RuntimeError("browser_use_not_installed_rebuild_container") from exc
+
+    name = str(prospect.get("name") or "").strip()
+    address = str(prospect.get("address") or "").strip()
+    category = str(prospect.get("category") or "").strip()
+    phone = str(prospect.get("phone") or "").strip()
+    emails = str(prospect.get("emails") or "").strip()
+    task = f"""
+You are a bounded public-data website investigator. Find possible official websites for this one business:
+Business name: {name}
+Category/services: {category}
+Address/location: {address}
+Phone: {phone}
+Public email(s): {emails}
+
+Use only public, unauthenticated pages. Search at most three result pages and inspect at most five candidate domains.
+Try exact business-name and location/phone searches. Prefer the official business website over social networks,
+directories, review sites, and map pages. Do not log in, submit forms, solve CAPTCHAs, bypass access challenges,
+or use a proxy. If a site blocks the browser, record it and move on.
+
+Return ONLY JSON in this shape:
+{{"candidates":[{{"url":"https://example.co.za","confidence":0,"evidence":["name match","phone match"],"status":"candidate"}}]}}
+Confidence must be an integer from 0 to 100. Include only plausible official website URLs and never invent a URL.
+""".strip()
+
+    async def run() -> Any:
+        browser_profile = BrowserProfile(
+            headless=True,
+            executable_path=os.environ.get("PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH", "/usr/bin/chromium"),
+            minimum_wait_page_load_time=0.5,
+            wait_between_actions=0.7,
+            enable_default_extensions=False,
+            captcha_solver=False,
+        )
+        llm = ChatOpenAI(
+            model=model,
+            api_key=api_key,
+            base_url=base_url,
+            default_headers={"User-Agent": user_agent},
+            temperature=0.1,
+            frequency_penalty=None,
+            max_completion_tokens=900,
+            max_retries=1,
+            timeout=45,
+        )
+        agent = Agent(
+            task=task,
+            llm=llm,
+            browser_profile=browser_profile,
+            use_vision=False,
+            flash_mode=True,
+            use_judge=False,
+            enable_planning=False,
+            enable_signal_handler=False,
+        )
+        return await agent.run(max_steps=max(3, min(15, int(os.environ.get("BROWSER_USE_MAX_STEPS", "10")))))
+
+    timeout = max(30, min(300, int(os.environ.get("BROWSER_USE_TIMEOUT_SECONDS", "120"))))
+    try:
+        history = asyncio.run(asyncio.wait_for(run(), timeout=timeout))
+    except asyncio.TimeoutError as exc:
+        raise RuntimeError("browser_use_timeout") from exc
+    final = history.final_result() if history is not None and hasattr(history, "final_result") else ""
+    return _parse_agent_result(final)

api/requirements.txt

@@ -1 +1,2 @@
 playwright==1.52.0
+browser-use==0.13.10

api/server.py

@@ -27,8 +27,10 @@ from urllib.parse import parse_qs, quote_plus, unquote, urlparse
 
 try:
     from intelligence import domain_candidates, scan_website
+    from browser_investigator import investigate_websites
 except ImportError:  # tests import the module as api.server
     from api.intelligence import domain_candidates, scan_website
+    from api.browser_investigator import investigate_websites
 
 DB_PATH = os.environ.get("PROSPECT_DB", "/data/prospects.db")
 MAX_RESULTS = 50
@@ -52,6 +54,7 @@ AI_PROVIDER = os.environ.get("AI_PROVIDER", "custom").strip().lower()
 # reaches the provider. Identify this application consistently on provider
 # requests instead of relying on that default.
 OUTBOUND_USER_AGENT = os.environ.get("OUTBOUND_USER_AGENT", "ProspectOS/2.0 (+personal-use)").strip() or "ProspectOS/2.0 (+personal-use)"
+BROWSER_USE_ENABLED = os.environ.get("BROWSER_USE_ENABLED", "1").strip().lower() not in {"0", "false", "no", "off"}
 NOUS_BASE_URL = "https://inference-api.nousresearch.com/v1"
 NOUS_FREE_MODELS = [
     {"id": "meituan/longcat-2.0:free", "label": "Meituan · LongCat 2.0", "free": True},
@@ -70,6 +73,19 @@ _schema_lock = threading.Lock()
 _schema_path: str | None = None
 
 
+def browser_use_limits() -> tuple[int, int]:
+    """Return bounded Browser Use steps/timeout even if .env is malformed."""
+    try:
+        steps = int(os.environ.get("BROWSER_USE_MAX_STEPS", "10"))
+    except (TypeError, ValueError):
+        steps = 10
+    try:
+        timeout = int(os.environ.get("BROWSER_USE_TIMEOUT_SECONDS", "120"))
+    except (TypeError, ValueError):
+        timeout = 120
+    return max(3, min(15, steps)), max(30, min(300, timeout))
+
+
 def now() -> str:
     return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
 
@@ -106,6 +122,7 @@ def db() -> sqlite3.Connection:
         source TEXT NOT NULL DEFAULT 'google_maps',
         source_id TEXT NOT NULL DEFAULT '',
         category TEXT NOT NULL DEFAULT '',
+        search_category TEXT NOT NULL DEFAULT '',
         address TEXT NOT NULL DEFAULT '',
         phone TEXT NOT NULL DEFAULT '',
         emails TEXT NOT NULL DEFAULT '',
@@ -135,6 +152,9 @@ def db() -> sqlite3.Connection:
         ai_subject TEXT NOT NULL DEFAULT '',
         ai_message TEXT NOT NULL DEFAULT '',
         ai_offer TEXT NOT NULL DEFAULT '',
+        ai_unknowns TEXT NOT NULL DEFAULT '[]',
+        ai_evidence_ids TEXT NOT NULL DEFAULT '[]',
+        ai_follow_up TEXT NOT NULL DEFAULT '',
         opportunity_score INTEGER NOT NULL DEFAULT 0,
         raw_json TEXT NOT NULL DEFAULT '{}'
     )""")
@@ -172,6 +192,19 @@ def db() -> sqlite3.Connection:
     conn.execute("""CREATE TABLE IF NOT EXISTS settings (
         key TEXT PRIMARY KEY, value TEXT NOT NULL DEFAULT '', updated_at TEXT NOT NULL
     )""")
+    conn.execute("""CREATE TABLE IF NOT EXISTS website_candidates (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        prospect_id INTEGER NOT NULL,
+        url TEXT NOT NULL,
+        domain TEXT NOT NULL,
+        confidence INTEGER NOT NULL DEFAULT 0,
+        evidence TEXT NOT NULL DEFAULT '[]',
+        source TEXT NOT NULL DEFAULT 'browser_use',
+        status TEXT NOT NULL DEFAULT 'candidate',
+        discovered_at TEXT NOT NULL,
+        raw_json TEXT NOT NULL DEFAULT '{}',
+        UNIQUE(prospect_id, domain)
+    )""")
 
     # Migrate databases created by v1-v5 without destroying the user's
     # catalogue. SQLite permits adding nullable/defaulted columns in place.
@@ -179,6 +212,7 @@ def db() -> sqlite3.Connection:
         "source": "TEXT NOT NULL DEFAULT 'google_maps'",
         "source_id": "TEXT NOT NULL DEFAULT ''",
         "category": "TEXT NOT NULL DEFAULT ''",
+        "search_category": "TEXT NOT NULL DEFAULT ''",
         "address": "TEXT NOT NULL DEFAULT ''",
         "phone": "TEXT NOT NULL DEFAULT ''",
         "emails": "TEXT NOT NULL DEFAULT ''",
@@ -208,6 +242,9 @@ def db() -> sqlite3.Connection:
         "ai_subject": "TEXT NOT NULL DEFAULT ''",
         "ai_message": "TEXT NOT NULL DEFAULT ''",
         "ai_offer": "TEXT NOT NULL DEFAULT ''",
+        "ai_unknowns": "TEXT NOT NULL DEFAULT '[]'",
+        "ai_evidence_ids": "TEXT NOT NULL DEFAULT '[]'",
+        "ai_follow_up": "TEXT NOT NULL DEFAULT ''",
         "opportunity_score": "INTEGER NOT NULL DEFAULT 0",
         "raw_json": "TEXT NOT NULL DEFAULT '{}'",
     }
@@ -230,6 +267,7 @@ def db() -> sqlite3.Connection:
     conn.execute("CREATE INDEX IF NOT EXISTS idx_prospects_review ON prospects(review_status)")
     conn.execute("CREATE INDEX IF NOT EXISTS idx_prospects_score ON prospects(opportunity_score DESC)")
     conn.execute("CREATE INDEX IF NOT EXISTS idx_runs_created ON runs(id DESC)")
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_website_candidates_prospect ON website_candidates(prospect_id, confidence DESC)")
     conn.commit()
     _schema_path = DB_PATH
     _schema_lock.release()
@@ -380,7 +418,7 @@ def score_place(item: dict[str, Any], website: str, phone: str, rating: float |
     return score_opportunity(item)[0]
 
 
-def map_gmaps_result(item: dict[str, Any], query: str) -> dict[str, Any]:
+def map_gmaps_result(item: dict[str, Any], query: str, search_category: str = "") -> dict[str, Any]:
     """Normalize gosom/google-maps-scraper's place object to our catalogue."""
     name = as_text(item.get("title") or item.get("name") or item.get("business_name")) or "Unnamed place"
     website = normalize_url(item.get("website") or item.get("website_url"))
@@ -408,6 +446,7 @@ def map_gmaps_result(item: dict[str, Any], query: str) -> dict[str, Any]:
         "source": "google_maps",
         "source_id": source_id[:255],
         "category": category[:255],
+        "search_category": as_text(search_category)[:255],
         "address": address[:1000],
         "phone": phone[:120],
         "emails": emails[:500],
@@ -869,8 +908,9 @@ def enrich_place(item: dict[str, Any]) -> dict[str, Any]:
 
 
 def insert_catalogue(run_id: int, query: str, raw_results: list[dict[str, Any]], requested: int,
-                     progress: Callable[[int, int], None] | None = None) -> tuple[list[dict[str, Any]], int]:
-    mapped = [map_gmaps_result(item, query) for item in raw_results[:requested]]
+                     progress: Callable[[int, int], None] | None = None,
+                     search_category: str = "") -> tuple[list[dict[str, Any]], int]:
+    mapped = [map_gmaps_result(item, query, search_category) for item in raw_results[:requested]]
     conn = db()
     added = 0
     try:
@@ -899,14 +939,14 @@ def insert_catalogue(run_id: int, query: str, raw_results: list[dict[str, Any]],
             try:
                 cur = conn.execute("""INSERT OR IGNORE INTO prospects(
                 name,website,domain,description,query,discovered_at,source,source_id,
-                category,address,phone,emails,rating,review_count,maps_url,latitude,
+                category,search_category,address,phone,emails,rating,review_count,maps_url,latitude,
                 longitude,listing_status,website_status,website_http_status,website_final_url,
                 website_latency_ms,website_https,website_title,website_technology,website_has_viewport,
                 website_tls_expires_at,website_checked_at,website_error,email_evidence,score_reasons,
                 ai_probability,ai_recommendation,ai_reasoning,ai_checked_at,ai_subject,ai_message,ai_offer,opportunity_score,raw_json
-            ) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""", (
+            ) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""", (
                 item["name"], item["website"], item["domain"], item["description"], item["query"], now(),
-                item["source"], item["source_id"], item["category"], item["address"], item["phone"], item["emails"],
+                item["source"], item["source_id"], item["category"], item.get("search_category", ""), item["address"], item["phone"], item["emails"],
                 item["rating"], item["review_count"], item["maps_url"], item["latitude"], item["longitude"],
                 item.get("listing_status", ""), item.get("website_status", "unknown"), item.get("website_http_status"),
                 item.get("website_final_url", ""), item.get("website_latency_ms", 0), item.get("website_https", 0),
@@ -952,7 +992,7 @@ def run_discovery(run_id: int, payload: dict[str, Any]) -> None:
                 if total:
                     update_run(run_id, stage="website_analysis", progress=76 + int(index * 19 / total), found=index)
 
-            results, added = insert_catalogue(run_id, query, raw_results, requested, enrichment_progress)
+            results, added = insert_catalogue(run_id, query, raw_results, requested, enrichment_progress, search_category=category)
             update_run(run_id, found=len(results), added=added, status="succeeded", stage="complete", progress=100, completed_at=now(), error="")
         except Exception as exc:
             message = str(exc).strip()[:500] or "discovery_failed"
@@ -998,7 +1038,7 @@ def scheduler_loop() -> None:
 
 
 def ai_generate_message(prospect: dict[str, Any], offer: str) -> dict[str, Any]:
-    """Generate an evidence-grounded draft through an OpenAI-compatible endpoint."""
+    """Generate an evidence-grounded, human-editable outreach draft."""
     runtime = ai_runtime_settings()
     ai_base_url = runtime["base_url"]
     ai_api_key = runtime["api_key"]
@@ -1007,24 +1047,57 @@ def ai_generate_message(prospect: dict[str, Any], offer: str) -> dict[str, Any]:
         raise RuntimeError("nous_api_key_required_for_standalone_app")
     if not ai_base_url or not ai_model:
         raise RuntimeError("ai_not_configured_set_AI_BASE_URL_and_AI_MODEL")
+    evidence_ids = ["prospect_record"]
+    if prospect.get("maps_url"):
+        evidence_ids.append("public_maps_listing")
+    if prospect.get("website"):
+        evidence_ids.append("website_scan")
+    if prospect.get("emails"):
+        evidence_ids.append("public_email_listing")
+    if prospect.get("phone"):
+        evidence_ids.append("public_phone_listing")
     evidence = {
-        "business": prospect.get("name"), "category": prospect.get("category"),
-        "location": prospect.get("address"), "website": prospect.get("website"),
-        "website_status": prospect.get("website_status"), "website_title": prospect.get("website_title"),
-        "technology": prospect.get("website_technology"), "emails": prospect.get("emails"),
-        "phone": prospect.get("phone"), "score": prospect.get("opportunity_score"),
+        "evidence_ids": evidence_ids,
+        "business_name": prospect.get("name"),
+        "search_category": prospect.get("search_category") or prospect.get("category"),
+        "address_or_location": prospect.get("address"),
+        "public_listing": {
+            "maps_url": prospect.get("maps_url"), "phone": prospect.get("phone"),
+            "rating": prospect.get("rating"), "review_count": prospect.get("review_count"),
+        },
+        "website": {
+            "url": prospect.get("website"), "status": prospect.get("website_status"),
+            "http_status": prospect.get("website_http_status"), "title": prospect.get("website_title"),
+            "technology": prospect.get("website_technology"), "https": bool(prospect.get("website_https")),
+            "latency_ms": prospect.get("website_latency_ms"),
+        },
+        "public_emails": prospect.get("emails"),
+        "opportunity_score": prospect.get("opportunity_score"),
         "score_reasons": _json_list(prospect.get("score_reasons")),
     }
-    system = ("You write concise, respectful B2B prospecting drafts for a South African web hosting and design business. "
-              "Use only the supplied evidence; never invent facts, prices, names, services, or claims. "
-              "Do not mention scraping or hidden analysis. Return strict JSON with keys subject, message, rationale. "
-              "The message must be plain text, personalised, transparent, and include a polite opt-out line.")
+    system = (
+        "You are a senior B2B copywriter for a South African web-hosting and website-design business. "
+        "Create a short, respectful, genuinely personalised first-contact email from the supplied evidence. "
+        "Use only stored facts and the user's offer; never invent a person's name, company activity, price, "
+        "service, problem, result, testimonial, urgency, or link. Do not mention scraping, scoring, hidden "
+        "analysis, or imply certainty where the evidence is uncertain. Do not insult the prospect or call a "
+        "site bad; use neutral wording such as 'may be having trouble loading' when appropriate. "
+        "If there is no website, do not say you reviewed their website. If there is no mockup URL in the offer, "
+        "do not invent one. Keep the email around 90-150 words, with a natural greeting using the business name, "
+        "one specific evidence-based observation, the exact offer/instructions supplied by the user, one simple "
+        "call to action, and a polite opt-out sentence. Avoid spammy hype, excessive emojis, all-caps, and fake "
+        "personal familiarity. Return ONLY strict JSON with exactly these keys: subject, message, rationale, "
+        "recommended_offer, unknowns, evidence_used, follow_up. subject is 5-10 natural words. message is plain "
+        "text. rationale is concise. recommended_offer is a short summary of the user's offer. unknowns is an "
+        "array of facts that should be confirmed. evidence_used is an array containing only the supplied evidence "
+        "IDs. follow_up is a short human review note, not part of the email."
+    )
     prompt = json.dumps({"evidence": evidence, "user_offer_or_instruction": offer[:2000]}, ensure_ascii=False)
     headers = {"Authorization": "Bearer " + ai_api_key} if ai_api_key else {}
     request_payload: dict[str, Any] = {
         "model": ai_model,
         "temperature": 0.4,
-        "max_tokens": 700,
+        "max_tokens": 900,
         "messages": [{"role": "system", "content": system}, {"role": "user", "content": prompt}],
     }
     # Nous exposes an OpenAI-compatible chat endpoint, but not every free model
@@ -1034,13 +1107,23 @@ def ai_generate_message(prospect: dict[str, Any], offer: str) -> dict[str, Any]:
         request_payload["response_format"] = {"type": "json_object"}
     result = http_json("POST", ai_base_url + "/chat/completions", request_payload, headers=headers, timeout=90, error_prefix="ai")
     content = (((result.get("choices") or [{}])[0].get("message") or {}).get("content") or "") if isinstance(result, dict) else ""
-    try:
-        parsed = json.loads(content)
-    except (TypeError, ValueError, json.JSONDecodeError) as exc:
-        raise RuntimeError("ai_invalid_json_response") from exc
+    parsed = _parse_ai_object(content)
     if not isinstance(parsed, dict) or not str(parsed.get("subject") or "").strip() or not str(parsed.get("message") or "").strip():
         raise RuntimeError("ai_incomplete_message")
-    return {"subject": str(parsed["subject"])[:200], "message": str(parsed["message"])[:8000], "rationale": str(parsed.get("rationale") or "")[:2000]}
+    def text_list(value: Any, limit: int = 6, size: int = 400) -> list[str]:
+        if not isinstance(value, list):
+            return []
+        return [str(item).strip()[:size] for item in value if str(item).strip()][:limit]
+    used = [item for item in text_list(parsed.get("evidence_used"), limit=10, size=80) if item in evidence_ids]
+    return {
+        "subject": str(parsed["subject"]).strip()[:200],
+        "message": str(parsed["message"]).strip()[:8000],
+        "rationale": str(parsed.get("rationale") or "").strip()[:2000],
+        "recommended_offer": str(parsed.get("recommended_offer") or offer).strip()[:2000],
+        "unknowns": text_list(parsed.get("unknowns")),
+        "evidence_used": used,
+        "follow_up": str(parsed.get("follow_up") or "Review the draft and confirm the offer before sending.").strip()[:1000],
+    }
 
 
 def _parse_ai_object(content: Any) -> dict[str, Any]:
@@ -1163,6 +1246,7 @@ class Handler(BaseHTTPRequestHandler):
                     "ready": ready,
                     "email_extraction_default": True,
                     "email_extraction_always": True,
+                    "browser_use_enabled": BROWSER_USE_ENABLED,
                     "ai_provider": ai_runtime["provider"],
                     "ai_configured": bool(ai_runtime["base_url"] and ai_runtime["model"] and (ai_runtime["provider"] != "nous" or ai_runtime["api_key"])),
                 })
@@ -1174,7 +1258,8 @@ class Handler(BaseHTTPRequestHandler):
             if parsed.path == "/api/settings":
                 ai_runtime = ai_runtime_settings()
                 models = nous_models() if ai_runtime["provider"] == "nous" else []
-                return self.send_json(200, {"maps": {"backend": backend_mode(), "worker_url": GMAPS_URL, "ready": backend_mode() == "browser" or bool(GMAPS_URL)}, "ai": {"provider": ai_runtime["provider"], "base_url": ai_runtime["base_url"], "model": ai_runtime["model"], "configured": bool(ai_runtime["base_url"] and ai_runtime["model"] and (ai_runtime["provider"] != "nous" or ai_runtime["api_key"])), "models": models}, "email_collection": {"always_on": True, "public_only": True}, "limits": {"max_results": MAX_RESULTS, "website_scan_timeout_seconds": 8}})
+                browser_steps, browser_timeout = browser_use_limits()
+                return self.send_json(200, {"maps": {"backend": backend_mode(), "worker_url": GMAPS_URL, "ready": backend_mode() == "browser" or bool(GMAPS_URL)}, "ai": {"provider": ai_runtime["provider"], "base_url": ai_runtime["base_url"], "model": ai_runtime["model"], "configured": bool(ai_runtime["base_url"] and ai_runtime["model"] and (ai_runtime["provider"] != "nous" or ai_runtime["api_key"])), "models": models}, "browser_use": {"enabled": BROWSER_USE_ENABLED, "headless": True, "public_only": True, "max_steps": browser_steps, "timeout_seconds": browser_timeout}, "email_collection": {"always_on": True, "public_only": True}, "limits": {"max_results": MAX_RESULTS, "website_scan_timeout_seconds": 8}})
             if parsed.path == "/api/prospects":
                 params = parse_qs(parsed.query)
                 q = (params.get("q", [""])[0]).strip().lower()
@@ -1182,7 +1267,7 @@ class Handler(BaseHTTPRequestHandler):
                 rows = conn.execute("SELECT * FROM prospects ORDER BY opportunity_score DESC, id DESC").fetchall()
                 items = []
                 for row in rows:
-                    haystack = " ".join(str(row[key] or "") for key in ("name", "domain", "description", "address", "category", "phone", "emails")).lower()
+                    haystack = " ".join(str(row[key] or "") for key in ("name", "domain", "description", "address", "category", "search_category", "phone", "emails")).lower()
                     if q and q not in haystack:
                         continue
                     if status and row["review_status"] != status:
@@ -1192,14 +1277,21 @@ class Handler(BaseHTTPRequestHandler):
             if parsed.path == "/api/export.csv":
                 params = parse_qs(parsed.query); q = params.get("q", [""])[0].strip().lower()
                 rows = [dict(r) for r in conn.execute("SELECT * FROM prospects ORDER BY opportunity_score DESC, id DESC")]
-                if q: rows = [r for r in rows if q in " ".join(str(r.get(k) or "") for k in ("name","domain","address","category","emails")).lower()]
-                out = io.StringIO(); fields = ["id","name","category","address","phone","emails","domain","website","website_status","opportunity_score","score_reasons","review_status","last_seen"]
+                if q: rows = [r for r in rows if q in " ".join(str(r.get(k) or "") for k in ("name","domain","address","category","search_category","emails")).lower()]
+                out = io.StringIO(); fields = ["id","name","search_category","category","address","phone","emails","domain","website","website_status","opportunity_score","score_reasons","review_status","last_seen"]
                 writer = csv.DictWriter(out, fieldnames=fields); writer.writeheader(); writer.writerows({k:r.get(k) for k in fields} for r in rows)
                 body = out.getvalue().encode(); self.send_response(200); self.send_header("Content-Type","text/csv; charset=utf-8"); self.send_header("Content-Disposition","attachment; filename=prospects.csv"); self.send_header("Content-Length",str(len(body))); self.end_headers(); self.wfile.write(body); return
             if parsed.path == "/api/suppressions":
                 return self.send_json(200, {"items": [dict(r) for r in conn.execute("SELECT * FROM suppressions ORDER BY id DESC")]})
             if parsed.path == "/api/schedules":
                 return self.send_json(200, {"items": [dict(r) for r in conn.execute("SELECT * FROM schedules ORDER BY id DESC")]})
+            match = re.fullmatch(r"/api/prospects/(\d+)/website-candidates", parsed.path)
+            if match:
+                prospect_id = int(match.group(1))
+                if not conn.execute("SELECT 1 FROM prospects WHERE id=?", (prospect_id,)).fetchone():
+                    return self.send_json(404, {"error": "not_found"})
+                rows = conn.execute("SELECT * FROM website_candidates WHERE prospect_id=? ORDER BY confidence DESC, id DESC", (prospect_id,)).fetchall()
+                return self.send_json(200, {"items": [dict(row) for row in rows]})
             match = re.fullmatch(r"/api/prospects/(\d+)/domains", parsed.path)
             if match:
                 row = conn.execute("SELECT * FROM prospects WHERE id=?", (int(match.group(1)),)).fetchone()
@@ -1283,6 +1375,50 @@ class Handler(BaseHTTPRequestHandler):
                 return self.send_json(502, {"error": str(exc)})
             except Exception as exc:
                 return self.send_json(502, {"error": "ai_test_failed:" + str(exc)[:200]})
+        candidates_match = re.fullmatch(r"/api/prospects/(\d+)/website-candidates", urlparse(self.path).path)
+        if candidates_match:
+            prospect_id = int(candidates_match.group(1))
+            conn = db()
+            try:
+                row = conn.execute("SELECT * FROM prospects WHERE id=?", (prospect_id,)).fetchone()
+            finally:
+                conn.close()
+            if not row:
+                return self.send_json(404, {"error": "not_found"})
+            if not BROWSER_USE_ENABLED:
+                return self.send_json(503, {"error": "browser_use_disabled"})
+            try:
+                runtime = ai_runtime_settings()
+                candidates = investigate_websites(
+                    dict(row), base_url=runtime["base_url"], api_key=runtime["api_key"],
+                    model=runtime["model"], user_agent=OUTBOUND_USER_AGENT,
+                )
+                conn = db()
+                for candidate in candidates:
+                    encoded_evidence = json.dumps(candidate.get("evidence") or [], ensure_ascii=False)
+                    encoded_raw = json.dumps(candidate, ensure_ascii=False)
+                    conn.execute(
+                        """INSERT INTO website_candidates(prospect_id,url,domain,confidence,evidence,source,status,discovered_at,raw_json)
+                           VALUES(?,?,?,?,?,?,?,?,?)
+                           ON CONFLICT(prospect_id,domain) DO UPDATE SET
+                             url=excluded.url, confidence=excluded.confidence, evidence=excluded.evidence,
+                             source=excluded.source, discovered_at=excluded.discovered_at, raw_json=excluded.raw_json,
+                             status=CASE WHEN website_candidates.status='accepted' THEN 'accepted' ELSE 'candidate' END""",
+                        (prospect_id, candidate["url"], candidate["domain"], candidate["confidence"], encoded_evidence,
+                         "browser_use", "candidate", now(), encoded_raw),
+                    )
+                conn.commit()
+                rows = conn.execute("SELECT * FROM website_candidates WHERE prospect_id=? ORDER BY confidence DESC, id DESC", (prospect_id,)).fetchall()
+                return self.send_json(200, {"items": [dict(item) for item in rows], "count": len(rows)})
+            except RuntimeError as exc:
+                return self.send_json(503, {"error": str(exc)})
+            except Exception as exc:
+                return self.send_json(502, {"error": "browser_use_failed:" + str(exc)[:240]})
+            finally:
+                try:
+                    conn.close()
+                except Exception:
+                    pass
         assess_match = re.fullmatch(r"/api/prospects/(\d+)/assess", urlparse(self.path).path)
         if assess_match:
             conn = None
@@ -1315,7 +1451,7 @@ class Handler(BaseHTTPRequestHandler):
                 payload = self.read_json(); offer = str(payload.get("offer") or "").strip()
                 if not offer: return self.send_json(400, {"error":"offer_required"})
                 draft = ai_generate_message(dict(row), offer)
-                stamp = now(); conn.execute("UPDATE prospects SET ai_subject=?,ai_message=?,ai_offer=?,ai_reasoning=?,ai_checked_at=? WHERE id=?", (draft["subject"],draft["message"],offer,draft["rationale"],stamp,int(message_match.group(1)))); conn.commit()
+                stamp = now(); conn.execute("UPDATE prospects SET ai_subject=?,ai_message=?,ai_offer=?,ai_reasoning=?,ai_unknowns=?,ai_evidence_ids=?,ai_follow_up=?,ai_checked_at=? WHERE id=?", (draft["subject"],draft["message"],offer,draft["rationale"],json.dumps(draft.get("unknowns") or [], ensure_ascii=False),json.dumps(draft.get("evidence_used") or [], ensure_ascii=False),draft.get("follow_up") or "",stamp,int(message_match.group(1)))); conn.commit()
                 return self.send_json(200, {**draft,"ai_checked_at":stamp,"prospect_id":int(message_match.group(1))})
             except RuntimeError as exc: return self.send_json(503, {"error":str(exc)})
             except (ValueError, TypeError, json.JSONDecodeError) as exc: return self.send_json(400, {"error":str(exc)})
@@ -1355,6 +1491,80 @@ class Handler(BaseHTTPRequestHandler):
             return self.send_json(400, {"error": str(exc)})
 
     def do_PATCH(self) -> None:
+        candidate_match = re.fullmatch(r"/api/prospects/(\d+)/website-candidates/(\d+)", urlparse(self.path).path)
+        if candidate_match:
+            prospect_id, candidate_id = (int(candidate_match.group(1)), int(candidate_match.group(2)))
+            conn: sqlite3.Connection | None = None
+            try:
+                payload = self.read_json()
+                status = str(payload.get("status", "")).strip().lower()
+                if status not in {"candidate", "accepted", "rejected"}:
+                    return self.send_json(400, {"error": "invalid_website_candidate_status"})
+                conn = db()
+                candidate = conn.execute(
+                    "SELECT * FROM website_candidates WHERE id=? AND prospect_id=?",
+                    (candidate_id, prospect_id),
+                ).fetchone()
+                prospect = conn.execute("SELECT * FROM prospects WHERE id=?", (prospect_id,)).fetchone()
+                if not candidate or not prospect:
+                    return self.send_json(404, {"error": "not_found"})
+                if status != "accepted":
+                    conn.execute("UPDATE website_candidates SET status=? WHERE id=?", (status, candidate_id))
+                    conn.commit()
+                    updated_candidate = conn.execute("SELECT * FROM website_candidates WHERE id=?", (candidate_id,)).fetchone()
+                    return self.send_json(200, {"candidate": dict(updated_candidate)})
+
+                # The candidate is now a real catalogue domain. Check the
+                # catalogue before scanning so SQLite's unique domain index
+                # produces an actionable response instead of a generic 500.
+                duplicate = conn.execute(
+                    "SELECT id FROM prospects WHERE domain=? AND id<>?",
+                    (str(candidate["domain"]).lower(), prospect_id),
+                ).fetchone()
+                if duplicate:
+                    return self.send_json(409, {"error": "domain_already_catalogued", "prospect_id": int(duplicate["id"])})
+                candidate_url = normalize_url(candidate["url"])
+                if not candidate_url:
+                    return self.send_json(400, {"error": "invalid_candidate_url"})
+
+                # Website probing is network-bound. Release the SQLite handle
+                # while it runs, then persist the selected URL and evidence in
+                # one short transaction.
+                base_item = {**dict(prospect), "website": candidate_url, "domain": str(candidate["domain"]).lower()}
+                conn.close()
+                conn = None
+                updated = enrich_place(base_item)
+                cols = [
+                    "website", "domain", "website_status", "website_http_status", "website_final_url",
+                    "website_latency_ms", "website_https", "website_title", "website_technology",
+                    "website_has_viewport", "website_tls_expires_at", "website_checked_at", "website_error",
+                    "emails", "email_evidence", "score_reasons", "opportunity_score",
+                ]
+                conn = db()
+                conn.execute(
+                    "UPDATE website_candidates SET status='candidate' WHERE prospect_id=? AND id<>? AND status='accepted'",
+                    (prospect_id, candidate_id),
+                )
+                conn.execute("UPDATE website_candidates SET status='accepted' WHERE id=? AND prospect_id=?", (candidate_id, prospect_id))
+                conn.execute(
+                    "UPDATE prospects SET " + ",".join(f"{column}=?" for column in cols) + " WHERE id=?",
+                    tuple(updated.get(column) for column in cols) + (prospect_id,),
+                )
+                conn.commit()
+                updated_candidate = conn.execute("SELECT * FROM website_candidates WHERE id=?", (candidate_id,)).fetchone()
+                updated_prospect = conn.execute("SELECT * FROM prospects WHERE id=?", (prospect_id,)).fetchone()
+                return self.send_json(200, {"candidate": dict(updated_candidate), "prospect": dict(updated_prospect)})
+            except sqlite3.IntegrityError:
+                if conn is not None:
+                    conn.rollback()
+                return self.send_json(409, {"error": "domain_already_catalogued"})
+            except (ValueError, TypeError, json.JSONDecodeError) as exc:
+                return self.send_json(400, {"error": str(exc)})
+            except Exception as exc:
+                return self.send_json(502, {"error": "website_candidate_accept_failed:" + str(exc)[:240]})
+            finally:
+                if conn is not None:
+                    conn.close()
         match = re.fullmatch(r"/api/prospects/(\d+)/review", urlparse(self.path).path)
         if not match:
             return self.send_json(404, {"error": "not_found"})