add internal Degoog prospect search fallback

api/browser_investigator.py

@@ -13,6 +13,8 @@ import json
 import os
 import re
 import time
+import urllib.error
+import urllib.request
 from typing import Any, Callable
 from urllib.parse import parse_qs, quote_plus, unquote, urlparse
 
@@ -275,6 +277,63 @@ def _safe_log(log_callback: Callable[[str, str, dict[str, Any]], Any] | None,
         return
 
 
+def _degoog_search_candidates(
+    prospect: dict[str, Any],
+    queries: list[str],
+    *,
+    requested: int,
+    timeout_seconds: int,
+    log_callback: Callable[[str, str, dict[str, Any]], Any] | None = None,
+) -> list[dict[str, Any]]:
+    """Use an internal Degoog instance as a bounded public-search source."""
+    base = os.environ.get("DEGOOG_URL", "").strip().rstrip("/")
+    enabled = os.environ.get("DEGOOG_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"}
+    if not enabled or not base:
+        return []
+    engines_env = [item.strip() for item in os.environ.get("DEGOOG_ENGINES", "").split(",") if item.strip()]
+    found: dict[str, dict[str, Any]] = {}
+    deadline = time.monotonic() + max(5, min(60, int(timeout_seconds)))
+    for query in queries:
+        if time.monotonic() >= deadline:
+            break
+        body: dict[str, Any] = {"query": query}
+        if engines_env:
+            body["engines"] = engines_env
+        request = urllib.request.Request(
+            base + "/api/search",
+            data=json.dumps(body).encode("utf-8"),
+            headers={"Content-Type": "application/json", "Accept": "application/json", "User-Agent": os.environ.get("OUTBOUND_USER_AGENT", "ProspectOS/2.0 (+personal-use)")},
+            method="POST",
+        )
+        try:
+            _safe_log(log_callback, "search", "degoog_search_started", {"query": query, "url": base + "/api/search"})
+            with urllib.request.urlopen(request, timeout=max(3, min(20, int(deadline - time.monotonic())))) as response:
+                payload = json.loads(response.read(1_000_000).decode("utf-8", "replace"))
+            raw_results = payload.get("results", []) if isinstance(payload, dict) else []
+            if not isinstance(raw_results, list):
+                continue
+            for item in raw_results[: requested * 5]:
+                if not isinstance(item, dict):
+                    continue
+                href = _normalise_search_href(item.get("url") or item.get("href") or item.get("link"))
+                if not href:
+                    continue
+                domain = _candidate_domain(href)
+                if not domain or domain in found:
+                    continue
+                title = str(item.get("title") or item.get("name") or "")[:300]
+                context = str(item.get("content") or item.get("snippet") or item.get("description") or "")[:1200]
+                score, evidence = _candidate_score(prospect, href, title, context)
+                if score < 48:
+                    continue
+                found[domain] = {"url": href, "domain": domain, "confidence": score, "evidence": evidence + ["visible on Degoog search results"], "status": "candidate", "source": "degoog", "search_engine": "degoog", "title": title, "snippet": context, "queries": [query]}
+                if len(found) >= requested:
+                    return sorted(found.values(), key=lambda item: item["confidence"], reverse=True)[:requested]
+        except (urllib.error.URLError, TimeoutError, ValueError, json.JSONDecodeError) as exc:
+            _safe_log(log_callback, "search", "degoog_search_error", {"query": query, "error": str(exc)[:300]})
+    results = sorted(found.values(), key=lambda item: item["confidence"], reverse=True)[:requested]
+    _safe_log(log_callback, "search", "degoog_search_completed", {"candidate_count": len(results), "candidates": results})
+    return results
 def direct_search_candidates(
     prospect: dict[str, Any],
     *,
@@ -298,6 +357,9 @@ def direct_search_candidates(
     queries = _search_query_variants(prospect)
     if not queries:
         return []
+    degoog_results = _degoog_search_candidates(prospect, queries, requested=requested, timeout_seconds=min(timeout_seconds, 60), log_callback=log_callback)
+    if degoog_results:
+        return degoog_results
     try:
         hard_timeout = max(15, min(180, int(timeout_seconds)))
     except (TypeError, ValueError):