From 650ce399f997af50627c7c3043ba4c693ef2b962 Mon Sep 17 00:00:00 2001 From: Marco0300 Date: Thu, 17 Sep 2026 23:21:52 +0200 Subject: [PATCH] add Firecrawl discovery backend and settings --- .env.example | 3 ++ api/server.py | 85 +++++++++++++++++++++++++++++++++++++++++++--- docker-compose.yml | 1 + tests/test_app.py | 27 +++++++++++++++ web/app.js | 7 ++++ 5 files changed, 118 insertions(+), 5 deletions(-) diff --git a/.env.example b/.env.example index fd1f9ad..a0249e8 100644 --- a/.env.example +++ b/.env.example @@ -12,6 +12,9 @@ GMAPS_HOST_PORT=8081 GMAPS_API_MODE=standard GMAPS_API_KEY= +# Firecrawl is selected from Settings and the key is stored write-only in the app. +FIRECRAWL_API_KEY= + # A single personal run may take several minutes. GMAPS_REQUEST_TIMEOUT=900 GMAPS_POLL_SECONDS=2 diff --git a/api/server.py b/api/server.py index 958a99d..3f8a24d 100644 --- a/api/server.py +++ b/api/server.py @@ -397,13 +397,57 @@ def record_log(kind: str, event: str, *, level: str = "info", method: str = "", return +def firecrawl_runtime_settings() -> dict[str, Any]: + saved = _saved_settings() + return {"api_key": saved.get("FIRECRAWL_API_KEY", os.environ.get("FIRECRAWL_API_KEY", "")).strip(), "base_url": "https://api.firecrawl.dev/v2", "configured": bool(saved.get("FIRECRAWL_API_KEY", os.environ.get("FIRECRAWL_API_KEY", "")).strip())} + + def backend_mode() -> str: - configured = os.environ.get("DISCOVERY_BACKEND", "").strip().lower() - if configured in {"gmaps", "browser"}: + configured = _saved_settings().get("DISCOVERY_BACKEND", os.environ.get("DISCOVERY_BACKEND", "")).strip().lower() + if configured in {"gmaps", "firecrawl", "browser"}: return configured return "gmaps" if os.environ.get("GMAPS_URL", "").strip() else "browser" +def registrable_domain(value: str) -> str: + host = (urlparse(value).hostname or "").lower().strip(".") + if host.startswith("www."): + host = host[4:] + parts = host.split(".") + if len(parts) >= 3 and parts[-2:] in (["co", "za"], ["org", "za"], ["net", "za"]): + return ".".join(parts[-3:]) + return ".".join(parts[-2:]) if len(parts) >= 2 else host + + +def firecrawl_search(query: str, requested: int, api_key: str | None = None) -> list[dict[str, Any]]: + key = (api_key if api_key is not None else firecrawl_runtime_settings()["api_key"]).strip() + if not key: + raise RuntimeError("firecrawl_api_key_required") + limit = max(1, min(50, int(requested))) + payload = {"query": str(query).strip()[:500], "limit": limit, "sources": ["web"], "country": "ZA", "safe": True, "timeout": 30_000} + response = http_json("POST", "https://api.firecrawl.dev/v2/search", payload, headers={"Authorization": "Bearer " + key}, timeout=45, error_prefix="firecrawl") + data = response.get("data", {}) if isinstance(response, dict) else {} + raw = data.get("web", []) if isinstance(data, dict) else data + results: list[dict[str, Any]] = [] + seen: set[str] = set() + for item in raw if isinstance(raw, list) else []: + if not isinstance(item, dict): + continue + website = normalize_url(item.get("url") or item.get("sourceURL")).rstrip("/") + if not website or not _safe_url(website): + continue + domain = registrable_domain(website) + if not domain or domain in seen: + continue + seen.add(domain) + results.append({"name": as_text(item.get("title"))[:240] or domain, "website": website, "domain": domain, + "description": as_text(item.get("description") or item.get("snippet") or item.get("markdown"))[:4000], + "source": "firecrawl", "source_url": website, "raw_json": item}) + if len(results) >= limit: + break + return results + + def db() -> sqlite3.Connection: global _schema_path os.makedirs(os.path.dirname(DB_PATH) or ".", exist_ok=True) @@ -1337,6 +1381,23 @@ def map_gmaps_result(item: dict[str, Any], query: str, search_category: str = "" } +def map_firecrawl_result(item: dict[str, Any], query: str, search_category: str = "") -> dict[str, Any]: + name = as_text(item.get("name") or item.get("title")) or "Unnamed web result" + website = normalize_url(item.get("website") or item.get("url")) + domain = registrable_domain(website) + description = as_text(item.get("description")) + return {"name": name[:200], "website": website, "domain": domain[:255], "description": description[:1000], + "query": query, "source": "firecrawl", "source_id": domain[:255], "category": as_text(item.get("category"))[:255], + "search_category": as_text(search_category)[:255], "address": "", "phone": "", "emails": "", "rating": None, + "review_count": 0, "maps_url": "", "latitude": None, "longitude": None, "listing_status": "", + "website_status": "unknown" if website else "no_detected_website", "email_evidence": "[]", "score_reasons": "[]", + "opportunity_score": score_place(item, website, "", None, 0), "raw_json": json.dumps(item, ensure_ascii=False)[:100_000]} + + +def map_discovery_result(item: dict[str, Any], query: str, search_category: str = "") -> dict[str, Any]: + return map_firecrawl_result(item, query, search_category) if item.get("source") == "firecrawl" else map_gmaps_result(item, query, search_category) + + def _looks_like_place(value: dict[str, Any]) -> bool: keys = {str(key).lower() for key in value} return bool(keys.intersection({"title", "name", "business_name"})) and bool( @@ -1867,7 +1928,7 @@ def enrich_place(item: dict[str, Any]) -> dict[str, Any]: def insert_catalogue(run_id: int, query: str, raw_results: list[dict[str, Any]], requested: int, progress: Callable[[int, int], None] | None = None, search_category: str = "") -> tuple[list[dict[str, Any]], int]: - mapped = [map_gmaps_result(item, query, search_category) for item in raw_results[:requested]] + mapped = [map_discovery_result(item, query, search_category) for item in raw_results[:requested]] conn = db() added = 0 try: @@ -1952,6 +2013,9 @@ def run_discovery(run_id: int, payload: dict[str, Any]) -> None: update_run(run_id, stage="scraping", progress=progress, found=found) raw_results = poll_gmaps_job(external_job_id, on_progress) + elif backend_mode() == "firecrawl": + update_run(run_id, stage="firecrawl_search", progress=20) + raw_results = firecrawl_search(query, requested) else: update_run(run_id, stage="browser_search", progress=20) raw_results = google_search(query, requested, run_id=run_id) @@ -3266,13 +3330,13 @@ class Handler(BaseHTTPRequestHandler): conn = db() try: if parsed.path == "/api/health": - configured = bool(GMAPS_URL) if backend_mode() == "gmaps" else True + configured = bool(GMAPS_URL) if backend_mode() == "gmaps" else firecrawl_runtime_settings()["configured"] if backend_mode() == "firecrawl" else True if GMAPS_API_MODE == "saas": configured = configured and bool(GMAPS_API_KEY) return self.send_json(200, {"status": "ok", "source": backend_mode(), "configured": configured, "maps_service": bool(GMAPS_URL), "maps_api_mode": GMAPS_API_MODE, "maps_api_key": bool(GMAPS_API_KEY)}) if parsed.path == "/api/config": auth_required = GMAPS_API_MODE == "saas" - ready = backend_mode() == "browser" or (bool(GMAPS_URL) and (not auth_required or bool(GMAPS_API_KEY))) + ready = backend_mode() == "browser" or (backend_mode() == "firecrawl" and firecrawl_runtime_settings()["configured"]) or (backend_mode() == "gmaps" and bool(GMAPS_URL) and (not auth_required or bool(GMAPS_API_KEY))) ai_runtime = ai_runtime_settings() browser_runtime = browser_use_settings() return self.send_json(200, { @@ -3336,6 +3400,7 @@ class Handler(BaseHTTPRequestHandler): maps_runtime = gmaps_runtime_settings() scan_timeout = website_scan_timeout_seconds() return self.send_json(200, { + "discovery": {"backend": backend_mode(), "firecrawl_base_url": "https://api.firecrawl.dev/v2", "firecrawl_configured": firecrawl_runtime_settings()["configured"]}, "maps": { "backend": backend_mode(), "worker_url": GMAPS_URL, "ready": backend_mode() == "browser" or bool(GMAPS_URL), @@ -3734,6 +3799,7 @@ class Handler(BaseHTTPRequestHandler): return self.send_json(403, {"error": "permission_denied"}) try: payload = self.read_json() + discovery = payload.get("discovery") if isinstance(payload.get("discovery"), dict) else {} ai = payload.get("ai") if isinstance(payload.get("ai"), dict) else {} browser = payload.get("browser_use") if isinstance(payload.get("browser_use"), dict) else {} maps = payload.get("maps") if isinstance(payload.get("maps"), dict) else {} @@ -3748,6 +3814,14 @@ class Handler(BaseHTTPRequestHandler): conn.execute("INSERT INTO settings(key,value,updated_at) VALUES(?,?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value,updated_at=excluded.updated_at", (key, text_value, now())) saved_keys.append(key) + if "backend" in discovery: + backend = str(discovery.get("backend") or "").strip().lower() + if backend not in {"gmaps", "firecrawl", "browser"}: + return self.send_json(400, {"error": "invalid_discovery_backend"}) + save_setting("DISCOVERY_BACKEND", backend) + if str(discovery.get("firecrawl_api_key") or "").strip(): + save_setting("FIRECRAWL_API_KEY", discovery.get("firecrawl_api_key")) + for key in ("AI_PROVIDER", "AI_BASE_URL", "AI_API_KEY", "AI_MODEL"): if key in ai: # An empty password means “keep the existing credential” @@ -3792,6 +3866,7 @@ class Handler(BaseHTTPRequestHandler): return self.send_json(200, { "saved": True, "saved_keys": saved_keys, + "discovery": {"backend": backend_mode(), "firecrawl_base_url": "https://api.firecrawl.dev/v2", "firecrawl_configured": firecrawl_runtime_settings()["configured"]}, "ai": {"provider": ai_runtime["provider"], "base_url": ai_runtime["base_url"], "model": ai_runtime["model"], "configured": ai_is_configured(ai_runtime)}, "browser_use": browser_use_settings(), "maps": gmaps_runtime_settings(), diff --git a/docker-compose.yml b/docker-compose.yml index 408a978..8843a57 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,6 +38,7 @@ services: GMAPS_URL: ${GMAPS_URL:-http://gmaps:8080} GMAPS_API_MODE: ${GMAPS_API_MODE:-standard} GMAPS_API_KEY: ${GMAPS_API_KEY:-} + FIRECRAWL_API_KEY: ${FIRECRAWL_API_KEY:-} GMAPS_REQUEST_TIMEOUT: ${GMAPS_REQUEST_TIMEOUT:-900} GMAPS_POLL_SECONDS: ${GMAPS_POLL_SECONDS:-2} OUTBOUND_USER_AGENT: ${OUTBOUND_USER_AGENT:-ProspectOS/2.0 (+personal-use)} diff --git a/tests/test_app.py b/tests/test_app.py index d297197..cf0cecf 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -191,6 +191,32 @@ class BrowserOnlyAppTests(unittest.TestCase): saved = server.db().execute("SELECT ai_probability,ai_recommendation,ai_reasoning FROM prospects WHERE id=1").fetchone() self.assertEqual(tuple(saved), (84.0, "contact", "Active business with a clear website opportunity.")) + def test_firecrawl_search_normalises_public_results(self): + response = {"success": True, "data": {"web": [{"title": "Cape Town Plumbing", "url": "https://plumber.example.co.za/", "description": "Local plumbing services"}]}} + with patch("api.server.http_json", return_value=response) as request, patch("api.server._safe_url", return_value=True): + results = server.firecrawl_search("plumbers Cape Town", 5, api_key="fc-test") + self.assertEqual(results[0]["website"], "https://plumber.example.co.za") + self.assertEqual(results[0]["source"], "firecrawl") + self.assertEqual(results[0]["description"], "Local plumbing services") + self.assertEqual(request.call_args.args[0], "POST") + self.assertIn("/v2/search", request.call_args.args[1]) + self.assertEqual(request.call_args.kwargs["headers"]["Authorization"], "Bearer fc-test") + + def test_firecrawl_backend_requires_key_and_is_exposed_without_secret(self): + with patch.dict(os.environ, {"DISCOVERY_BACKEND": "firecrawl", "FIRECRAWL_API_KEY": ""}, clear=False): + with self.assertRaisesRegex(RuntimeError, "firecrawl_api_key_required"): + server.firecrawl_search("plumbers Cape Town", 5) + + def test_firecrawl_settings_persist_without_returning_secret(self): + status, result = self.request("POST", "/api/settings", {"discovery": {"backend": "firecrawl", "firecrawl_api_key": "fc-secret-test"}}) + self.assertEqual(status, 200) + self.assertTrue(result["discovery"]["firecrawl_configured"]) + self.assertNotIn("fc-secret-test", json.dumps(result)) + status, settings = self.request("GET", "/api/settings") + self.assertEqual(status, 200) + self.assertEqual(settings["discovery"]["backend"], "firecrawl") + self.assertNotIn("fc-secret-test", json.dumps(settings)) + def test_live_site_snapshot_captures_design_signals_for_generation(self): class Response: headers = {"Content-Type": "text/html; charset=utf-8"} @@ -213,6 +239,7 @@ class BrowserOnlyAppTests(unittest.TestCase): self.assertIn("layout-editorial", first) self.assertIn("layout-bento", second) + def test_mockup_fallback_is_saved_and_public_share_link_works(self): conn = server.db() conn.execute("INSERT INTO prospects(name,domain,category,address,phone,emails,website,website_status,discovered_at) VALUES(?,?,?,?,?,?,?,?,?)", ("Acme Plumbing", "maps:acme", "plumbers", "Cape Town", "+27101234567", "info@acme.co.za", "", "no_detected_website", server.now())) conn.commit(); conn.close() diff --git a/web/app.js b/web/app.js index ed55c1e..9cf7b14 100644 --- a/web/app.js +++ b/web/app.js @@ -34,6 +34,7 @@ function renderTeachAI(){const p=settings.company_profile||{};$('view').innerHTM function renderPromotions(){const items=Array.isArray(settings.promotions)?settings.promotions:[];$('view').innerHTML=layout('Promotions','Create offers once, then select an active promotion while drafting a prospect email. Inactive promotions stay in your history but cannot be used by the AI.',`
NEW PROMOTION

Set up an offer

Keep the exact wording, price, dates, and eligibility here so drafts stay accurate.

YOUR OFFERS

Saved promotions

${items.filter(x=>x.active).length} active · ${items.length} total

${items.map(item=>`
${esc(item.name)}

${esc(item.description)}

${item.terms?`Terms: ${esc(item.terms)}`:''}${item.starts_at||item.ends_at?`Valid ${esc(item.starts_at||'now')} → ${esc(item.ends_at||'ongoing')}`:'No dates set'}
`).join('')||'

No promotions yet. Add your first offer on the left.

'}
`)} function renderSettings(){ const ai=settings.ai||{}; + const discovery=settings.discovery||{}; const maps=settings.maps||{}; const browser=settings.browser_use||{}; const limits=settings.limits||{}; @@ -45,6 +46,7 @@ function renderSettings(){ const options=modelList.map(m=>``).join('')+(selectedModel&&!modelList.some(m=>m.id===selectedModel)?``:''); $('view').innerHTML=layout('Settings','All provider, browser, run, and safety controls live here. Daily discovery only asks for the search criteria.',`
+
DISCOVERY SOURCE

Where to search for prospects

Firecrawl searches the public web and can replace Maps for criteria-first discovery. It does not provide Google place IDs, ratings, coordinates, or guaranteed addresses.

${discovery.backend==='firecrawl'&&discovery.firecrawl_configured?'Firecrawl ready':discovery.backend==='firecrawl'?'Firecrawl needs a key':discovery.backend==='browser'?'Browser search':'Maps worker'}
RUNTIME CONTROLS

Browser and discovery timing

These values are saved in the local database and apply to new work immediately. Increase the Browser Use timeout when website investigations need more time.

${maps.ready?'Maps ready':'Maps not connected'}
@@ -89,6 +91,11 @@ document.addEventListener('submit',async e=>{ e.preventDefault();const m=$('discoverMessage');$('discoverBtn').disabled=true;m.textContent='Queueing…'; try{const r=await api('/api/discover',{method:'POST',body:JSON.stringify({keywords:$('keywords').value,category:$('category').value,location:$('location').value,max_results:Number($('max-results').value),extra_reviews:$('extra-reviews').checked,include_email:true})});m.textContent=`Run #${r.id} started. View it in Runs.`;setTimeout(route,500)}catch(x){m.textContent=x.message}finally{$('discoverBtn').disabled=false} } + if(e.target.id==='discoverySettingsForm'){ + e.preventDefault();const m=$('discoverySettingsMessage');m.textContent='Saving…';m.className='notice'; + try{const discovery={backend:$('discoveryBackend').value};if($('firecrawlKey').value.trim())discovery.firecrawl_api_key=$('firecrawlKey').value.trim();const result=await api('/api/settings',{method:'POST',body:JSON.stringify({discovery})});settings.discovery=result.discovery||settings.discovery;m.textContent='Discovery settings saved.';m.className='notice ok';} + catch(x){m.textContent=x.message;m.className='notice error'} + } if(e.target.id==='settingsForm'){ e.preventDefault();const m=$('settingsMessage'); try{const ai={AI_PROVIDER:$('aiProvider').value,AI_BASE_URL:$('aiBase').value,AI_MODEL:$('aiModel').value};if($('aiKey').value)ai.AI_API_KEY=$('aiKey').value;await api('/api/settings',{method:'POST',body:JSON.stringify({ai})});m.textContent='AI settings saved.';m.className='notice ok';}catch(x){m.textContent=x.message;m.className='notice error'}