root/MarketingTool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add Firecrawl discovery backend and settings

Browse Source
This commit is contained in:
Marco0300
2026-09-17 23:21:52 +02:00
parent 286673eb1d
commit 650ce399f9
5 changed files with 118 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/server.py
+80 -5
View File
@@ -397,13 +397,57 @@ def record_log(kind: str, event: str, *, level: str = "info", method: str = "",
return
def firecrawl_runtime_settings() -> dict[str, Any]:
saved = _saved_settings()
return {"api_key": saved.get("FIRECRAWL_API_KEY", os.environ.get("FIRECRAWL_API_KEY", "")).strip(), "base_url": "https://api.firecrawl.dev/v2", "configured": bool(saved.get("FIRECRAWL_API_KEY", os.environ.get("FIRECRAWL_API_KEY", "")).strip())}
def backend_mode() -> str:
configured = os.environ.get("DISCOVERY_BACKEND", "").strip().lower()
if configured in {"gmaps", "browser"}:
configured = _saved_settings().get("DISCOVERY_BACKEND", os.environ.get("DISCOVERY_BACKEND", "")).strip().lower()
if configured in {"gmaps", "firecrawl", "browser"}:
return configured
return "gmaps" if os.environ.get("GMAPS_URL", "").strip() else "browser"
def registrable_domain(value: str) -> str:
host = (urlparse(value).hostname or "").lower().strip(".")
if host.startswith("www."):
host = host[4:]
parts = host.split(".")
if len(parts) >= 3 and parts[-2:] in (["co", "za"], ["org", "za"], ["net", "za"]):
return ".".join(parts[-3:])
return ".".join(parts[-2:]) if len(parts) >= 2 else host
def firecrawl_search(query: str, requested: int, api_key: str | None = None) -> list[dict[str, Any]]:
key = (api_key if api_key is not None else firecrawl_runtime_settings()["api_key"]).strip()
if not key:
raise RuntimeError("firecrawl_api_key_required")
limit = max(1, min(50, int(requested)))
payload = {"query": str(query).strip()[:500], "limit": limit, "sources": ["web"], "country": "ZA", "safe": True, "timeout": 30_000}
response = http_json("POST", "https://api.firecrawl.dev/v2/search", payload, headers={"Authorization": "Bearer " + key}, timeout=45, error_prefix="firecrawl")
data = response.get("data", {}) if isinstance(response, dict) else {}
raw = data.get("web", []) if isinstance(data, dict) else data
results: list[dict[str, Any]] = []
seen: set[str] = set()
for item in raw if isinstance(raw, list) else []:
if not isinstance(item, dict):
continue
website = normalize_url(item.get("url") or item.get("sourceURL")).rstrip("/")
if not website or not _safe_url(website):
continue
domain = registrable_domain(website)
if not domain or domain in seen:
continue
seen.add(domain)
results.append({"name": as_text(item.get("title"))[:240] or domain, "website": website, "domain": domain,
"description": as_text(item.get("description") or item.get("snippet") or item.get("markdown"))[:4000],
"source": "firecrawl", "source_url": website, "raw_json": item})
if len(results) >= limit:
break
return results
def db() -> sqlite3.Connection:
global _schema_path
os.makedirs(os.path.dirname(DB_PATH) or ".", exist_ok=True)
@@ -1337,6 +1381,23 @@ def map_gmaps_result(item: dict[str, Any], query: str, search_category: str = ""
}
def map_firecrawl_result(item: dict[str, Any], query: str, search_category: str = "") -> dict[str, Any]:
name = as_text(item.get("name") or item.get("title")) or "Unnamed web result"
website = normalize_url(item.get("website") or item.get("url"))
domain = registrable_domain(website)
description = as_text(item.get("description"))
return {"name": name[:200], "website": website, "domain": domain[:255], "description": description[:1000],
"query": query, "source": "firecrawl", "source_id": domain[:255], "category": as_text(item.get("category"))[:255],
"search_category": as_text(search_category)[:255], "address": "", "phone": "", "emails": "", "rating": None,
"review_count": 0, "maps_url": "", "latitude": None, "longitude": None, "listing_status": "",
"website_status": "unknown" if website else "no_detected_website", "email_evidence": "[]", "score_reasons": "[]",
"opportunity_score": score_place(item, website, "", None, 0), "raw_json": json.dumps(item, ensure_ascii=False)[:100_000]}
def map_discovery_result(item: dict[str, Any], query: str, search_category: str = "") -> dict[str, Any]:
return map_firecrawl_result(item, query, search_category) if item.get("source") == "firecrawl" else map_gmaps_result(item, query, search_category)
def _looks_like_place(value: dict[str, Any]) -> bool:
keys = {str(key).lower() for key in value}
return bool(keys.intersection({"title", "name", "business_name"})) and bool(
@@ -1867,7 +1928,7 @@ def enrich_place(item: dict[str, Any]) -> dict[str, Any]:
def insert_catalogue(run_id: int, query: str, raw_results: list[dict[str, Any]], requested: int,
progress: Callable[[int, int], None] | None = None,
search_category: str = "") -> tuple[list[dict[str, Any]], int]:
mapped = [map_gmaps_result(item, query, search_category) for item in raw_results[:requested]]
mapped = [map_discovery_result(item, query, search_category) for item in raw_results[:requested]]
conn = db()
added = 0
try:
@@ -1952,6 +2013,9 @@ def run_discovery(run_id: int, payload: dict[str, Any]) -> None:
update_run(run_id, stage="scraping", progress=progress, found=found)
raw_results = poll_gmaps_job(external_job_id, on_progress)
elif backend_mode() == "firecrawl":
update_run(run_id, stage="firecrawl_search", progress=20)
raw_results = firecrawl_search(query, requested)
else:
update_run(run_id, stage="browser_search", progress=20)
raw_results = google_search(query, requested, run_id=run_id)
@@ -3266,13 +3330,13 @@ class Handler(BaseHTTPRequestHandler):
conn = db()
try:
if parsed.path == "/api/health":
configured = bool(GMAPS_URL) if backend_mode() == "gmaps" else True
configured = bool(GMAPS_URL) if backend_mode() == "gmaps" else firecrawl_runtime_settings()["configured"] if backend_mode() == "firecrawl" else True
if GMAPS_API_MODE == "saas":
configured = configured and bool(GMAPS_API_KEY)
return self.send_json(200, {"status": "ok", "source": backend_mode(), "configured": configured, "maps_service": bool(GMAPS_URL), "maps_api_mode": GMAPS_API_MODE, "maps_api_key": bool(GMAPS_API_KEY)})
if parsed.path == "/api/config":
auth_required = GMAPS_API_MODE == "saas"
ready = backend_mode() == "browser" or (bool(GMAPS_URL) and (not auth_required or bool(GMAPS_API_KEY)))
ready = backend_mode() == "browser" or (backend_mode() == "firecrawl" and firecrawl_runtime_settings()["configured"]) or (backend_mode() == "gmaps" and bool(GMAPS_URL) and (not auth_required or bool(GMAPS_API_KEY)))
ai_runtime = ai_runtime_settings()
browser_runtime = browser_use_settings()
return self.send_json(200, {
@@ -3336,6 +3400,7 @@ class Handler(BaseHTTPRequestHandler):
maps_runtime = gmaps_runtime_settings()
scan_timeout = website_scan_timeout_seconds()
return self.send_json(200, {
"discovery": {"backend": backend_mode(), "firecrawl_base_url": "https://api.firecrawl.dev/v2", "firecrawl_configured": firecrawl_runtime_settings()["configured"]},
"maps": {
"backend": backend_mode(), "worker_url": GMAPS_URL,
"ready": backend_mode() == "browser" or bool(GMAPS_URL),
@@ -3734,6 +3799,7 @@ class Handler(BaseHTTPRequestHandler):
return self.send_json(403, {"error": "permission_denied"})
try:
payload = self.read_json()
discovery = payload.get("discovery") if isinstance(payload.get("discovery"), dict) else {}
ai = payload.get("ai") if isinstance(payload.get("ai"), dict) else {}
browser = payload.get("browser_use") if isinstance(payload.get("browser_use"), dict) else {}
maps = payload.get("maps") if isinstance(payload.get("maps"), dict) else {}
@@ -3748,6 +3814,14 @@ class Handler(BaseHTTPRequestHandler):
conn.execute("INSERT INTO settings(key,value,updated_at) VALUES(?,?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value,updated_at=excluded.updated_at", (key, text_value, now()))
saved_keys.append(key)
if "backend" in discovery:
backend = str(discovery.get("backend") or "").strip().lower()
if backend not in {"gmaps", "firecrawl", "browser"}:
return self.send_json(400, {"error": "invalid_discovery_backend"})
save_setting("DISCOVERY_BACKEND", backend)
if str(discovery.get("firecrawl_api_key") or "").strip():
save_setting("FIRECRAWL_API_KEY", discovery.get("firecrawl_api_key"))
for key in ("AI_PROVIDER", "AI_BASE_URL", "AI_API_KEY", "AI_MODEL"):
if key in ai:
# An empty password means “keep the existing credential”
@@ -3792,6 +3866,7 @@ class Handler(BaseHTTPRequestHandler):
return self.send_json(200, {
"saved": True,
"saved_keys": saved_keys,
"discovery": {"backend": backend_mode(), "firecrawl_base_url": "https://api.firecrawl.dev/v2", "firecrawl_configured": firecrawl_runtime_settings()["configured"]},
"ai": {"provider": ai_runtime["provider"], "base_url": ai_runtime["base_url"], "model": ai_runtime["model"], "configured": ai_is_configured(ai_runtime)},
"browser_use": browser_use_settings(),
"maps": gmaps_runtime_settings(),