add resilient multi-engine browser discovery

api/server.py

@@ -18,7 +18,7 @@ from datetime import datetime, timezone
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from pathlib import Path
 from typing import Any
-from urllib.parse import parse_qs, quote_plus, urlparse
+from urllib.parse import parse_qs, quote_plus, unquote, urlparse
 
 DB_PATH = os.environ.get("PROSPECT_DB", "/data/prospects.db")
 MAX_RESULTS = 25
@@ -81,11 +81,79 @@ def blocked(text: str) -> bool:
     ))
 
 
+def normalize_result_href(raw_href: str) -> str:
+    """Turn a search-result href into the public destination URL.
+
+    Engines commonly use protocol-relative links or redirect wrappers (for
+    example DuckDuckGo's ``uddg`` parameter).  We unwrap those links before
+    applying the catalog/domain filters.
+    """
+    href = (raw_href or "").strip()
+    if href.startswith("//"):
+        href = "https:" + href
+    if href.startswith("/"):
+        return ""
+    parsed = urlparse(href)
+    if parsed.hostname and parsed.hostname.endswith("duckduckgo.com"):
+        params = parse_qs(parsed.query)
+        for key in ("uddg", "url", "u"):
+            target = params.get(key, [""])[0]
+            if target.startswith(("http://", "https://")):
+                href = unquote(target)
+                break
+    return href
+
+
+def extract_search_results(page: Any, selector: str, title_selector: str | None,
+                           excluded_hosts: set[str], requested: int) -> list[dict[str, str]]:
+    """Extract external result links, with a generic-anchor safety net."""
+    output: list[dict[str, str]] = []
+    seen_domains: set[str] = set()
+
+    def add_anchor(anchor: Any) -> None:
+        if len(output) >= requested:
+            return
+        try:
+            title_node = anchor.locator(title_selector) if title_selector else anchor
+            title = " ".join((title_node.inner_text() or "").split())[:200]
+            if not title:
+                title = " ".join(((anchor.get_attribute("aria-label") or anchor.get_attribute("title") or "").split()))[:200]
+            href = normalize_result_href(anchor.get_attribute("href") or "")
+        except Exception:
+            return
+        domain = clean_domain(href)
+        if not title or len(title) < 3 or not href.startswith(("http://", "https://")):
+            return
+        if not domain or domain in excluded_hosts or domain in seen_domains:
+            return
+        seen_domains.add(domain)
+        output.append({"name": title, "website": href, "domain": domain})
+
+    # Prefer stable engine-specific result selectors to avoid collecting nav
+    # and footer links.  If markup changed, fall back to generic anchors.
+    try:
+        anchors = page.locator(selector)
+        for i in range(min(anchors.count(), requested * 4)):
+            add_anchor(anchors.nth(i))
+    except Exception:
+        pass
+    if not output:
+        try:
+            anchors = page.locator("a")
+            for i in range(min(anchors.count(), 250)):
+                add_anchor(anchors.nth(i))
+                if len(output) >= requested:
+                    break
+        except Exception:
+            pass
+    return output
+
+
 def google_search(query: str, requested: int) -> list[dict[str, str]]:
     """Search public engines through Playwright, with safe engine fallback.
 
     Google frequently challenges datacenter IPs. We do not bypass that
-    challenge; we close the context and try Bing, then DuckDuckGo, still using
+    challenge; we close the context and try other public engines, still using
     a real headless browser and the same bounded result extraction.
     """
     try:
@@ -99,7 +167,9 @@ def google_search(query: str, requested: int) -> list[dict[str, str]]:
     engines = [
         ("google", "https://www.google.com/search?q=" + quote_plus(query) + "&num=" + str(requested), "a:has(h3)", "h3", {"google.com", "google.co.za"}),
         ("bing", "https://www.bing.com/search?q=" + quote_plus(query) + "&count=" + str(requested), "li.b_algo h2 a", "h2", {"bing.com", "bingj.com"}),
-        ("duckduckgo", "https://html.duckduckgo.com/html/?q=" + quote_plus(query), "a.result__a", None, {"duckduckgo.com"}),
+        ("duckduckgo", "https://html.duckduckgo.com/html/?q=" + quote_plus(query), "a.result__a,a[data-testid='result-title-a']", None, {"duckduckgo.com"}),
+        ("brave", "https://search.brave.com/search?q=" + quote_plus(query), "a.heading-serpresult,a.result-header", None, {"search.brave.com"}),
+        ("startpage", "https://www.startpage.com/sp/search?query=" + quote_plus(query), "a.w-gl__result-title", None, {"startpage.com"}),
     ]
     errors: list[str] = []
     with sync_playwright() as p:
@@ -113,35 +183,32 @@ def google_search(query: str, requested: int) -> list[dict[str, str]]:
                     java_script_enabled=True,
                     service_workers="block",
                     viewport={"width": 1280, "height": 900},
+                    user_agent=(
+                        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+                        "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+                    ),
                 )
                 page = context.new_page()
-                page.set_default_timeout(15000)
-                page.goto(url, wait_until="domcontentloaded", timeout=15000)
-                page.wait_for_timeout(250)
+                page.set_default_timeout(12000)
+                navigation_timed_out = False
+                # DOMContentLoaded gives us usable markup without waiting for
+                # analytics, ads, or other background requests.  If an engine
+                # keeps navigation open, still inspect the partially loaded
+                # page rather than discarding it as a hard failure.
+                try:
+                    page.goto(url, wait_until="domcontentloaded", timeout=15000)
+                except PlaywrightTimeout:
+                    navigation_timed_out = True
+                page.wait_for_timeout(1000)
                 body = page.locator("body").inner_text()
                 html = page.content()
                 if blocked(body) or blocked(html):
                     errors.append(engine + "_blocked")
                     continue
-                output: list[dict[str, str]] = []
-                anchors = page.locator(selector)
-                for i in range(min(anchors.count(), requested * 3)):
-                    anchor = anchors.nth(i)
-                    title_node = anchor.locator(title_selector) if title_selector else anchor
-                    title = " ".join((title_node.inner_text() or "").split())[:200]
-                    href = anchor.get_attribute("href") or ""
-                    domain = clean_domain(href)
-                    if not title or not href or domain in excluded_hosts or not domain:
-                        continue
-                    if not href.startswith(("http://", "https://")):
-                        continue
-                    if domain not in {item["domain"] for item in output}:
-                        output.append({"name": title, "website": href, "domain": domain})
-                    if len(output) >= requested:
-                        break
+                output = extract_search_results(page, selector, title_selector, excluded_hosts, requested)
                 if output:
                     return output
-                errors.append(engine + "_empty")
+                errors.append(engine + ("_timeout" if navigation_timed_out else "_empty"))
             except PlaywrightTimeout:
                 errors.append(engine + "_timeout")
             except Exception as exc:
@@ -152,7 +219,7 @@ def google_search(query: str, requested: int) -> list[dict[str, str]]:
                     except Exception: pass
         finally:
             browser.close()
-    raise RuntimeError("search_engines_unavailable:" + ",".join(errors[-3:]))
+    raise RuntimeError("search_engines_unavailable:" + ",".join(errors))
 
 
 def run_discovery(payload: dict[str, Any]) -> dict[str, Any]: