add Firecrawl discovery backend and settings

.env.example

@@ -12,6 +12,9 @@ GMAPS_HOST_PORT=8081
 GMAPS_API_MODE=standard
 GMAPS_API_KEY=
 
+# Firecrawl is selected from Settings and the key is stored write-only in the app.
+FIRECRAWL_API_KEY=
+
 # A single personal run may take several minutes.
 GMAPS_REQUEST_TIMEOUT=900
 GMAPS_POLL_SECONDS=2

api/server.py

@@ -397,13 +397,57 @@ def record_log(kind: str, event: str, *, level: str = "info", method: str = "",
         return
 
 
+def firecrawl_runtime_settings() -> dict[str, Any]:
+    saved = _saved_settings()
+    return {"api_key": saved.get("FIRECRAWL_API_KEY", os.environ.get("FIRECRAWL_API_KEY", "")).strip(), "base_url": "https://api.firecrawl.dev/v2", "configured": bool(saved.get("FIRECRAWL_API_KEY", os.environ.get("FIRECRAWL_API_KEY", "")).strip())}
+
+
 def backend_mode() -> str:
-    configured = os.environ.get("DISCOVERY_BACKEND", "").strip().lower()
-    if configured in {"gmaps", "browser"}:
+    configured = _saved_settings().get("DISCOVERY_BACKEND", os.environ.get("DISCOVERY_BACKEND", "")).strip().lower()
+    if configured in {"gmaps", "firecrawl", "browser"}:
         return configured
     return "gmaps" if os.environ.get("GMAPS_URL", "").strip() else "browser"
 
 
+def registrable_domain(value: str) -> str:
+    host = (urlparse(value).hostname or "").lower().strip(".")
+    if host.startswith("www."):
+        host = host[4:]
+    parts = host.split(".")
+    if len(parts) >= 3 and parts[-2:] in (["co", "za"], ["org", "za"], ["net", "za"]):
+        return ".".join(parts[-3:])
+    return ".".join(parts[-2:]) if len(parts) >= 2 else host
+
+
+def firecrawl_search(query: str, requested: int, api_key: str | None = None) -> list[dict[str, Any]]:
+    key = (api_key if api_key is not None else firecrawl_runtime_settings()["api_key"]).strip()
+    if not key:
+        raise RuntimeError("firecrawl_api_key_required")
+    limit = max(1, min(50, int(requested)))
+    payload = {"query": str(query).strip()[:500], "limit": limit, "sources": ["web"], "country": "ZA", "safe": True, "timeout": 30_000}
+    response = http_json("POST", "https://api.firecrawl.dev/v2/search", payload, headers={"Authorization": "Bearer " + key}, timeout=45, error_prefix="firecrawl")
+    data = response.get("data", {}) if isinstance(response, dict) else {}
+    raw = data.get("web", []) if isinstance(data, dict) else data
+    results: list[dict[str, Any]] = []
+    seen: set[str] = set()
+    for item in raw if isinstance(raw, list) else []:
+        if not isinstance(item, dict):
+            continue
+        website = normalize_url(item.get("url") or item.get("sourceURL")).rstrip("/")
+        if not website or not _safe_url(website):
+            continue
+        domain = registrable_domain(website)
+        if not domain or domain in seen:
+            continue
+        seen.add(domain)
+        results.append({"name": as_text(item.get("title"))[:240] or domain, "website": website, "domain": domain,
+                        "description": as_text(item.get("description") or item.get("snippet") or item.get("markdown"))[:4000],
+                        "source": "firecrawl", "source_url": website, "raw_json": item})
+        if len(results) >= limit:
+            break
+    return results
+
+
 def db() -> sqlite3.Connection:
     global _schema_path
     os.makedirs(os.path.dirname(DB_PATH) or ".", exist_ok=True)
@@ -1337,6 +1381,23 @@ def map_gmaps_result(item: dict[str, Any], query: str, search_category: str = ""
     }
 
 
+def map_firecrawl_result(item: dict[str, Any], query: str, search_category: str = "") -> dict[str, Any]:
+    name = as_text(item.get("name") or item.get("title")) or "Unnamed web result"
+    website = normalize_url(item.get("website") or item.get("url"))
+    domain = registrable_domain(website)
+    description = as_text(item.get("description"))
+    return {"name": name[:200], "website": website, "domain": domain[:255], "description": description[:1000],
+            "query": query, "source": "firecrawl", "source_id": domain[:255], "category": as_text(item.get("category"))[:255],
+            "search_category": as_text(search_category)[:255], "address": "", "phone": "", "emails": "", "rating": None,
+            "review_count": 0, "maps_url": "", "latitude": None, "longitude": None, "listing_status": "",
+            "website_status": "unknown" if website else "no_detected_website", "email_evidence": "[]", "score_reasons": "[]",
+            "opportunity_score": score_place(item, website, "", None, 0), "raw_json": json.dumps(item, ensure_ascii=False)[:100_000]}
+
+
+def map_discovery_result(item: dict[str, Any], query: str, search_category: str = "") -> dict[str, Any]:
+    return map_firecrawl_result(item, query, search_category) if item.get("source") == "firecrawl" else map_gmaps_result(item, query, search_category)
+
+
 def _looks_like_place(value: dict[str, Any]) -> bool:
     keys = {str(key).lower() for key in value}
     return bool(keys.intersection({"title", "name", "business_name"})) and bool(
@@ -1867,7 +1928,7 @@ def enrich_place(item: dict[str, Any]) -> dict[str, Any]:
 def insert_catalogue(run_id: int, query: str, raw_results: list[dict[str, Any]], requested: int,
                      progress: Callable[[int, int], None] | None = None,
                      search_category: str = "") -> tuple[list[dict[str, Any]], int]:
-    mapped = [map_gmaps_result(item, query, search_category) for item in raw_results[:requested]]
+    mapped = [map_discovery_result(item, query, search_category) for item in raw_results[:requested]]
     conn = db()
     added = 0
     try:
@@ -1952,6 +2013,9 @@ def run_discovery(run_id: int, payload: dict[str, Any]) -> None:
                     update_run(run_id, stage="scraping", progress=progress, found=found)
 
                 raw_results = poll_gmaps_job(external_job_id, on_progress)
+            elif backend_mode() == "firecrawl":
+                update_run(run_id, stage="firecrawl_search", progress=20)
+                raw_results = firecrawl_search(query, requested)
             else:
                 update_run(run_id, stage="browser_search", progress=20)
                 raw_results = google_search(query, requested, run_id=run_id)
@@ -3266,13 +3330,13 @@ class Handler(BaseHTTPRequestHandler):
         conn = db()
         try:
             if parsed.path == "/api/health":
-                configured = bool(GMAPS_URL) if backend_mode() == "gmaps" else True
+                configured = bool(GMAPS_URL) if backend_mode() == "gmaps" else firecrawl_runtime_settings()["configured"] if backend_mode() == "firecrawl" else True
                 if GMAPS_API_MODE == "saas":
                     configured = configured and bool(GMAPS_API_KEY)
                 return self.send_json(200, {"status": "ok", "source": backend_mode(), "configured": configured, "maps_service": bool(GMAPS_URL), "maps_api_mode": GMAPS_API_MODE, "maps_api_key": bool(GMAPS_API_KEY)})
             if parsed.path == "/api/config":
                 auth_required = GMAPS_API_MODE == "saas"
-                ready = backend_mode() == "browser" or (bool(GMAPS_URL) and (not auth_required or bool(GMAPS_API_KEY)))
+                ready = backend_mode() == "browser" or (backend_mode() == "firecrawl" and firecrawl_runtime_settings()["configured"]) or (backend_mode() == "gmaps" and bool(GMAPS_URL) and (not auth_required or bool(GMAPS_API_KEY)))
                 ai_runtime = ai_runtime_settings()
                 browser_runtime = browser_use_settings()
                 return self.send_json(200, {
@@ -3336,6 +3400,7 @@ class Handler(BaseHTTPRequestHandler):
                 maps_runtime = gmaps_runtime_settings()
                 scan_timeout = website_scan_timeout_seconds()
                 return self.send_json(200, {
+                    "discovery": {"backend": backend_mode(), "firecrawl_base_url": "https://api.firecrawl.dev/v2", "firecrawl_configured": firecrawl_runtime_settings()["configured"]},
                     "maps": {
                         "backend": backend_mode(), "worker_url": GMAPS_URL,
                         "ready": backend_mode() == "browser" or bool(GMAPS_URL),
@@ -3734,6 +3799,7 @@ class Handler(BaseHTTPRequestHandler):
                 return self.send_json(403, {"error": "permission_denied"})
             try:
                 payload = self.read_json()
+                discovery = payload.get("discovery") if isinstance(payload.get("discovery"), dict) else {}
                 ai = payload.get("ai") if isinstance(payload.get("ai"), dict) else {}
                 browser = payload.get("browser_use") if isinstance(payload.get("browser_use"), dict) else {}
                 maps = payload.get("maps") if isinstance(payload.get("maps"), dict) else {}
@@ -3748,6 +3814,14 @@ class Handler(BaseHTTPRequestHandler):
                     conn.execute("INSERT INTO settings(key,value,updated_at) VALUES(?,?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value,updated_at=excluded.updated_at", (key, text_value, now()))
                     saved_keys.append(key)
 
+                if "backend" in discovery:
+                    backend = str(discovery.get("backend") or "").strip().lower()
+                    if backend not in {"gmaps", "firecrawl", "browser"}:
+                        return self.send_json(400, {"error": "invalid_discovery_backend"})
+                    save_setting("DISCOVERY_BACKEND", backend)
+                if str(discovery.get("firecrawl_api_key") or "").strip():
+                    save_setting("FIRECRAWL_API_KEY", discovery.get("firecrawl_api_key"))
+
                 for key in ("AI_PROVIDER", "AI_BASE_URL", "AI_API_KEY", "AI_MODEL"):
                     if key in ai:
                         # An empty password means “keep the existing credential”
@@ -3792,6 +3866,7 @@ class Handler(BaseHTTPRequestHandler):
                 return self.send_json(200, {
                     "saved": True,
                     "saved_keys": saved_keys,
+                    "discovery": {"backend": backend_mode(), "firecrawl_base_url": "https://api.firecrawl.dev/v2", "firecrawl_configured": firecrawl_runtime_settings()["configured"]},
                     "ai": {"provider": ai_runtime["provider"], "base_url": ai_runtime["base_url"], "model": ai_runtime["model"], "configured": ai_is_configured(ai_runtime)},
                     "browser_use": browser_use_settings(),
                     "maps": gmaps_runtime_settings(),

docker-compose.yml

@@ -38,6 +38,7 @@ services:
       GMAPS_URL: ${GMAPS_URL:-http://gmaps:8080}
       GMAPS_API_MODE: ${GMAPS_API_MODE:-standard}
       GMAPS_API_KEY: ${GMAPS_API_KEY:-}
+      FIRECRAWL_API_KEY: ${FIRECRAWL_API_KEY:-}
       GMAPS_REQUEST_TIMEOUT: ${GMAPS_REQUEST_TIMEOUT:-900}
       GMAPS_POLL_SECONDS: ${GMAPS_POLL_SECONDS:-2}
       OUTBOUND_USER_AGENT: ${OUTBOUND_USER_AGENT:-ProspectOS/2.0 (+personal-use)}

tests/test_app.py

@@ -191,6 +191,32 @@ class BrowserOnlyAppTests(unittest.TestCase):
         saved = server.db().execute("SELECT ai_probability,ai_recommendation,ai_reasoning FROM prospects WHERE id=1").fetchone()
         self.assertEqual(tuple(saved), (84.0, "contact", "Active business with a clear website opportunity."))
 
+    def test_firecrawl_search_normalises_public_results(self):
+        response = {"success": True, "data": {"web": [{"title": "Cape Town Plumbing", "url": "https://plumber.example.co.za/", "description": "Local plumbing services"}]}}
+        with patch("api.server.http_json", return_value=response) as request, patch("api.server._safe_url", return_value=True):
+            results = server.firecrawl_search("plumbers Cape Town", 5, api_key="fc-test")
+        self.assertEqual(results[0]["website"], "https://plumber.example.co.za")
+        self.assertEqual(results[0]["source"], "firecrawl")
+        self.assertEqual(results[0]["description"], "Local plumbing services")
+        self.assertEqual(request.call_args.args[0], "POST")
+        self.assertIn("/v2/search", request.call_args.args[1])
+        self.assertEqual(request.call_args.kwargs["headers"]["Authorization"], "Bearer fc-test")
+
+    def test_firecrawl_backend_requires_key_and_is_exposed_without_secret(self):
+        with patch.dict(os.environ, {"DISCOVERY_BACKEND": "firecrawl", "FIRECRAWL_API_KEY": ""}, clear=False):
+            with self.assertRaisesRegex(RuntimeError, "firecrawl_api_key_required"):
+                server.firecrawl_search("plumbers Cape Town", 5)
+
+    def test_firecrawl_settings_persist_without_returning_secret(self):
+        status, result = self.request("POST", "/api/settings", {"discovery": {"backend": "firecrawl", "firecrawl_api_key": "fc-secret-test"}})
+        self.assertEqual(status, 200)
+        self.assertTrue(result["discovery"]["firecrawl_configured"])
+        self.assertNotIn("fc-secret-test", json.dumps(result))
+        status, settings = self.request("GET", "/api/settings")
+        self.assertEqual(status, 200)
+        self.assertEqual(settings["discovery"]["backend"], "firecrawl")
+        self.assertNotIn("fc-secret-test", json.dumps(settings))
+
     def test_live_site_snapshot_captures_design_signals_for_generation(self):
         class Response:
             headers = {"Content-Type": "text/html; charset=utf-8"}
@@ -213,6 +239,7 @@ class BrowserOnlyAppTests(unittest.TestCase):
         self.assertIn("layout-editorial", first)
         self.assertIn("layout-bento", second)
 
+    def test_mockup_fallback_is_saved_and_public_share_link_works(self):
         conn = server.db()
         conn.execute("INSERT INTO prospects(name,domain,category,address,phone,emails,website,website_status,discovered_at) VALUES(?,?,?,?,?,?,?,?,?)", ("Acme Plumbing", "maps:acme", "plumbers", "Cape Town", "+27101234567", "info@acme.co.za", "", "no_detected_website", server.now()))
         conn.commit(); conn.close()

web/app.js

@@ -34,6 +34,7 @@ function renderTeachAI(){const p=settings.company_profile||{};$('view').innerHTM
 function renderPromotions(){const items=Array.isArray(settings.promotions)?settings.promotions:[];$('view').innerHTML=layout('Promotions','Create offers once, then select an active promotion while drafting a prospect email. Inactive promotions stay in your history but cannot be used by the AI.',`<div class="promotions-grid"><article class="panel form-panel"><div class="panel-title"><div><span class="eyebrow">NEW PROMOTION</span><h2>Set up an offer</h2><p>Keep the exact wording, price, dates, and eligibility here so drafts stay accurate.</p></div></div><form id="promotionForm" class="promotion-form"><label>Promotion name<input id="promotionName" required placeholder="Free website rebuild"></label><label>What is included<textarea id="promotionDescription" rows="5" required placeholder="Free rebuild for businesses that move hosting to Gold Plus."></textarea></label><label>Terms or eligibility<textarea id="promotionTerms" rows="4" placeholder="New hosting customers only; confirm scope before sending."></textarea></label><div class="settings-fields"><label>Starts<input id="promotionStarts" type="date"></label><label>Ends<input id="promotionEnds" type="date"></label></div><label class="checkline"><input id="promotionActive" type="checkbox" checked> Active and available for drafts</label><div class="form-row"><button class="primary" type="submit">Save promotion</button><span id="promotionMessage" class="notice"></span></div></form></article><article class="panel"><div class="panel-title"><div><span class="eyebrow">YOUR OFFERS</span><h2>Saved promotions</h2><p>${items.filter(x=>x.active).length} active · ${items.length} total</p></div></div><div class="promotion-list">${items.map(item=>`<div class="promotion-card ${item.active?'active':'inactive'}"><div><div class="promotion-card-head"><b>${esc(item.name)}</b><label class="switch"><input type="checkbox" data-promotion-active="${item.id}" ${item.active?'checked':''}><span>${item.active?'Active':'Inactive'}</span></label></div><p>${esc(item.description)}</p>${item.terms?`<small>Terms: ${esc(item.terms)}</small>`:''}<small>${item.starts_at||item.ends_at?`Valid ${esc(item.starts_at||'now')} → ${esc(item.ends_at||'ongoing')}`:'No dates set'}</small></div><button class="secondary" data-delete-promotion="${item.id}" type="button">Delete</button></div>`).join('')||'<p class="muted">No promotions yet. Add your first offer on the left.</p>'}</div></article></div>`)}
 function renderSettings(){
   const ai=settings.ai||{};
+  const discovery=settings.discovery||{};
   const maps=settings.maps||{};
   const browser=settings.browser_use||{};
   const limits=settings.limits||{};
@@ -45,6 +46,7 @@ function renderSettings(){
   const options=modelList.map(m=>`<option value="${esc(m.id)}">${esc(m.label||m.id)}${m.free?' · free':''}</option>`).join('')+(selectedModel&&!modelList.some(m=>m.id===selectedModel)?`<option value="${esc(selectedModel)}">${esc(selectedModel)}</option>`:'');
   $('view').innerHTML=layout('Settings','All provider, browser, run, and safety controls live here. Daily discovery only asks for the search criteria.',`
     <div class="settings-grid">
+      <article class="panel wide-panel"><div class="panel-title"><div><span class="eyebrow">DISCOVERY SOURCE</span><h2>Where to search for prospects</h2><p>Firecrawl searches the public web and can replace Maps for criteria-first discovery. It does not provide Google place IDs, ratings, coordinates, or guaranteed addresses.</p></div><span class="status ${discovery.backend==='firecrawl'&&discovery.firecrawl_configured?'succeeded':'failed'}">${discovery.backend==='firecrawl'&&discovery.firecrawl_configured?'Firecrawl ready':discovery.backend==='firecrawl'?'Firecrawl needs a key':discovery.backend==='browser'?'Browser search':'Maps worker'}</span></div><form id="discoverySettingsForm" class="settings-fields"><label>Discovery backend<select id="discoveryBackend"><option value="gmaps" ${discovery.backend==='gmaps'?'selected':''}>Google Maps structured data</option><option value="firecrawl" ${discovery.backend==='firecrawl'?'selected':''}>Firecrawl web search</option><option value="browser" ${discovery.backend==='browser'?'selected':''}>Browser fallback</option></select></label><label>Firecrawl API key <input id="firecrawlKey" type="password" placeholder="Leave blank to keep the saved key" autocomplete="new-password"><small>Stored server-side and never returned to the browser.</small></label><div class="form-row"><button class="primary" type="submit">Save discovery settings</button><span id="discoverySettingsMessage" class="notice"></span></div></form></article>
       <article class="panel wide-panel">
         <div class="panel-title"><div><span class="eyebrow">RUNTIME CONTROLS</span><h2>Browser and discovery timing</h2><p>These values are saved in the local database and apply to new work immediately. Increase the Browser Use timeout when website investigations need more time.</p></div><span class="status ${maps.ready?'succeeded':'failed'}">${maps.ready?'Maps ready':'Maps not connected'}</span></div>
         <form id="runtimeSettingsForm" class="runtime-settings-form">
@@ -89,6 +91,11 @@ document.addEventListener('submit',async e=>{
     e.preventDefault();const m=$('discoverMessage');$('discoverBtn').disabled=true;m.textContent='Queueing…';
     try{const r=await api('/api/discover',{method:'POST',body:JSON.stringify({keywords:$('keywords').value,category:$('category').value,location:$('location').value,max_results:Number($('max-results').value),extra_reviews:$('extra-reviews').checked,include_email:true})});m.textContent=`Run #${r.id} started. View it in Runs.`;setTimeout(route,500)}catch(x){m.textContent=x.message}finally{$('discoverBtn').disabled=false}
   }
+  if(e.target.id==='discoverySettingsForm'){
+    e.preventDefault();const m=$('discoverySettingsMessage');m.textContent='Saving…';m.className='notice';
+    try{const discovery={backend:$('discoveryBackend').value};if($('firecrawlKey').value.trim())discovery.firecrawl_api_key=$('firecrawlKey').value.trim();const result=await api('/api/settings',{method:'POST',body:JSON.stringify({discovery})});settings.discovery=result.discovery||settings.discovery;m.textContent='Discovery settings saved.';m.className='notice ok';}
+    catch(x){m.textContent=x.message;m.className='notice error'}
+  }
   if(e.target.id==='settingsForm'){
     e.preventDefault();const m=$('settingsMessage');
     try{const ai={AI_PROVIDER:$('aiProvider').value,AI_BASE_URL:$('aiBase').value,AI_MODEL:$('aiModel').value};if($('aiKey').value)ai.AI_API_KEY=$('aiKey').value;await api('/api/settings',{method:'POST',body:JSON.stringify({ai})});m.textContent='AI settings saved.';m.className='notice ok';}catch(x){m.textContent=x.message;m.className='notice error'}