add safe public search-engine fallback

README.md

@@ -1,11 +1,12 @@
 # Browser-only ProspectOS
 
 This is the stripped-down replacement for the larger ProspectOS application.
-It has one live source: Google Search through a headless Playwright Chromium
-browser. Each discovery run builds a query from category, keywords, and
-location, filters results already present in the local catalogue by root
-domain, and stores only new prospects. The explorer supports search and
-human review states (`pending`, `approved`, `rejected`, `contacted`).
+It searches public engines (Google first, then Bing and DuckDuckGo if Google
+challenges the server) through a headless Playwright Chromium browser. Each
+discovery run builds a query from category, keywords, and location, filters
+results already present in the local catalogue by root domain, and stores only
+new prospects. The explorer supports search and human review states
+(`pending`, `approved`, `rejected`, `contacted`).
 
 There is deliberately no source registry, email scraper, AI provider, CRM,
 outreach sender, or automatic contact action.

api/server.py

@@ -82,18 +82,31 @@ def blocked(text: str) -> bool:
 
 
 def google_search(query: str, requested: int) -> list[dict[str, str]]:
+    """Search public engines through Playwright, with safe engine fallback.
+
+    Google frequently challenges datacenter IPs. We do not bypass that
+    challenge; we close the context and try Bing, then DuckDuckGo, still using
+    a real headless browser and the same bounded result extraction.
+    """
     try:
         from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeout
     except ImportError as exc:
         raise RuntimeError("playwright_not_installed") from exc
-    url = "https://www.google.com/search?q=" + quote_plus(query) + "&num=" + str(requested)
     executable = os.environ.get("PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH", "").strip()
     options: dict[str, Any] = {"headless": True}
     if executable:
         options["executable_path"] = executable
-    try:
+    engines = [
+        ("google", "https://www.google.com/search?q=" + quote_plus(query) + "&num=" + str(requested), "a:has(h3)", "h3", {"google.com", "google.co.za"}),
+        ("bing", "https://www.bing.com/search?q=" + quote_plus(query) + "&count=" + str(requested), "li.b_algo h2 a", "h2", {"bing.com", "bingj.com"}),
+        ("duckduckgo", "https://html.duckduckgo.com/html/?q=" + quote_plus(query), "a.result__a", None, {"duckduckgo.com"}),
+    ]
+    errors: list[str] = []
     with sync_playwright() as p:
         browser = p.chromium.launch(**options)
+        try:
+          for engine, url, selector, title_selector, excluded_hosts in engines:
+            context = None
             try:
                 context = browser.new_context(
                     locale="en-ZA",
@@ -108,15 +121,17 @@ def google_search(query: str, requested: int) -> list[dict[str, str]]:
                 body = page.locator("body").inner_text()
                 html = page.content()
                 if blocked(body) or blocked(html):
-                    raise RuntimeError("google_blocked")
+                    errors.append(engine + "_blocked")
+                    continue
                 output: list[dict[str, str]] = []
-                anchors = page.locator("a:has(h3)")
+                anchors = page.locator(selector)
                 for i in range(min(anchors.count(), requested * 3)):
                     anchor = anchors.nth(i)
-                    title = " ".join((anchor.locator("h3").inner_text() or "").split())[:200]
+                    title_node = anchor.locator(title_selector) if title_selector else anchor
+                    title = " ".join((title_node.inner_text() or "").split())[:200]
                     href = anchor.get_attribute("href") or ""
                     domain = clean_domain(href)
-                    if not title or not href or domain in {"google.com", "google.co.za"} or not domain:
+                    if not title or not href or domain in excluded_hosts or not domain:
                         continue
                     if not href.startswith(("http://", "https://")):
                         continue
@@ -124,19 +139,20 @@ def google_search(query: str, requested: int) -> list[dict[str, str]]:
                         output.append({"name": title, "website": href, "domain": domain})
                     if len(output) >= requested:
                         break
+                if output:
                     return output
-            finally:
-                try:
-                    context.close()
-                except Exception:
-                    pass
-                browser.close()
-    except RuntimeError:
-        raise
-    except PlaywrightTimeout as exc:
-        raise RuntimeError("browser_timeout") from exc
+                errors.append(engine + "_empty")
+            except PlaywrightTimeout:
+                errors.append(engine + "_timeout")
             except Exception as exc:
-        raise RuntimeError("browser_error") from exc
+                errors.append(engine + "_error")
+            finally:
+                if context is not None:
+                    try: context.close()
+                    except Exception: pass
+        finally:
+            browser.close()
+    raise RuntimeError("search_engines_unavailable:" + ",".join(errors[-3:]))
 
 
 def run_discovery(payload: dict[str, Any]) -> dict[str, Any]:

web/index.html

@@ -1,6 +1,6 @@
 <!doctype html>
 <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>ProspectOS · Browser Discovery</title><link rel="stylesheet" href="/style.css"></head>
-<body><header class="top"><div><span class="eyebrow">PROSPECTOS</span><h1>Browser discovery</h1><p>Find businesses through Google, remove anything already catalogued, then review the remaining prospects.</p></div><div class="source-pill">● Google · Playwright</div></header>
+<body><header class="top"><div><span class="eyebrow">PROSPECTOS</span><h1>Browser discovery</h1><p>Find businesses through public search engines, remove anything already catalogued, then review the remaining prospects.</p></div><div class="source-pill">● Browser search · Playwright</div></header>
 <main>
 <section class="grid stats"><article><span>Catalogued prospects</span><strong id="total">0</strong></article><article><span>Pending review</span><strong id="pending">0</strong></article><article><span>Approved</span><strong id="approved">0</strong></article><article><span>Last run</span><strong id="lastRun">—</strong></article></section>
 <section class="panel"><div class="section-head"><div><span class="eyebrow">DISCOVERY</span><h2>Find new prospects</h2><p>Search criteria are used only for this run. The system stores the business name, website, query, and discovery time.</p></div></div><form id="discoverForm" class="form-grid"><label>Category<input name="category" placeholder="Plumbers" required></label><label>Keywords<input name="keywords" placeholder="plumbing services" required></label><label>Location<input name="location" value="South Africa" required></label><label>Maximum results<select name="max_results"><option value="5">5</option><option value="10" selected>10</option><option value="25">25</option></select></label><div class="form-actions"><button class="primary" type="submit" id="discoverBtn">Run browser discovery</button><span id="discoverMessage" role="status"></span></div></form></section>