add internal Degoog prospect search fallback

.env.example

@@ -31,6 +31,14 @@ BROWSER_USE_LLM_TIMEOUT_SECONDS=90
 ANONYMIZED_TELEMETRY=false
 LOG_RETENTION=20000
 
+# Optional internal Degoog website-search fallback. Keep disabled until you
+# install/review engines in the Degoog settings UI. Degoog is not published
+# on a host port; ProspectOS reaches it over the private Compose network.
+DEGOOG_ENABLED=false
+DEGOOG_URL=http://degoog:4444
+DEGOOG_ENGINES=
+DEGOOG_SETTINGS_PASSWORD=change-this-before-exposing
+
 # Authentication and workspace ownership. Leave OWNER_* blank to use the
 # first-run setup screen; setting them seeds the owner on an unattended start.
 AUTH_REQUIRED=true

README.md

@@ -132,7 +132,25 @@ error rather than remaining stuck forever.
 | `AI_API_KEY` | empty | Optional provider key; keep server-side |
 | `AI_MOCKUP_TIMEOUT_SECONDS` | `45` | Maximum AI wait for mockup copy before the deterministic premium renderer is used (20–300 seconds) |
 
-### Nous Research Portal
+### Degoog website-search fallback
+
+ProspectOS can optionally query an internal Degoog instance when looking for a
+possible official website for a Maps prospect. Maps remains the structured
+business source; Degoog is only a criteria/name-to-website discovery fallback.
+
+Enable it only after installing and reviewing Degoog engines in its settings:
+
+```bash
+DEGOOG_ENABLED=true docker compose up -d
+```
+
+Degoog is private to the Compose network and is not published on a host port.
+The adapter uses Degoog's JSON `POST /api/search` endpoint, applies the same
+bounded result and domain/identity filters as the browser-search path, and
+stores `source=degoog` in candidate provenance. Do not install unreviewed
+third-party Degoog extensions; the project supports extensions/transports that
+may execute server-side code.
+
 
 Choose **Nous Portal · free models** in Settings, paste a Nous API key, select a
 free model, save, and use **Test connection** before drafting a message. The

api/browser_investigator.py

@@ -13,6 +13,8 @@ import json
 import os
 import re
 import time
+import urllib.error
+import urllib.request
 from typing import Any, Callable
 from urllib.parse import parse_qs, quote_plus, unquote, urlparse
 
@@ -275,6 +277,63 @@ def _safe_log(log_callback: Callable[[str, str, dict[str, Any]], Any] | None,
         return
 
 
+def _degoog_search_candidates(
+    prospect: dict[str, Any],
+    queries: list[str],
+    *,
+    requested: int,
+    timeout_seconds: int,
+    log_callback: Callable[[str, str, dict[str, Any]], Any] | None = None,
+) -> list[dict[str, Any]]:
+    """Use an internal Degoog instance as a bounded public-search source."""
+    base = os.environ.get("DEGOOG_URL", "").strip().rstrip("/")
+    enabled = os.environ.get("DEGOOG_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"}
+    if not enabled or not base:
+        return []
+    engines_env = [item.strip() for item in os.environ.get("DEGOOG_ENGINES", "").split(",") if item.strip()]
+    found: dict[str, dict[str, Any]] = {}
+    deadline = time.monotonic() + max(5, min(60, int(timeout_seconds)))
+    for query in queries:
+        if time.monotonic() >= deadline:
+            break
+        body: dict[str, Any] = {"query": query}
+        if engines_env:
+            body["engines"] = engines_env
+        request = urllib.request.Request(
+            base + "/api/search",
+            data=json.dumps(body).encode("utf-8"),
+            headers={"Content-Type": "application/json", "Accept": "application/json", "User-Agent": os.environ.get("OUTBOUND_USER_AGENT", "ProspectOS/2.0 (+personal-use)")},
+            method="POST",
+        )
+        try:
+            _safe_log(log_callback, "search", "degoog_search_started", {"query": query, "url": base + "/api/search"})
+            with urllib.request.urlopen(request, timeout=max(3, min(20, int(deadline - time.monotonic())))) as response:
+                payload = json.loads(response.read(1_000_000).decode("utf-8", "replace"))
+            raw_results = payload.get("results", []) if isinstance(payload, dict) else []
+            if not isinstance(raw_results, list):
+                continue
+            for item in raw_results[: requested * 5]:
+                if not isinstance(item, dict):
+                    continue
+                href = _normalise_search_href(item.get("url") or item.get("href") or item.get("link"))
+                if not href:
+                    continue
+                domain = _candidate_domain(href)
+                if not domain or domain in found:
+                    continue
+                title = str(item.get("title") or item.get("name") or "")[:300]
+                context = str(item.get("content") or item.get("snippet") or item.get("description") or "")[:1200]
+                score, evidence = _candidate_score(prospect, href, title, context)
+                if score < 48:
+                    continue
+                found[domain] = {"url": href, "domain": domain, "confidence": score, "evidence": evidence + ["visible on Degoog search results"], "status": "candidate", "source": "degoog", "search_engine": "degoog", "title": title, "snippet": context, "queries": [query]}
+                if len(found) >= requested:
+                    return sorted(found.values(), key=lambda item: item["confidence"], reverse=True)[:requested]
+        except (urllib.error.URLError, TimeoutError, ValueError, json.JSONDecodeError) as exc:
+            _safe_log(log_callback, "search", "degoog_search_error", {"query": query, "error": str(exc)[:300]})
+    results = sorted(found.values(), key=lambda item: item["confidence"], reverse=True)[:requested]
+    _safe_log(log_callback, "search", "degoog_search_completed", {"candidate_count": len(results), "candidates": results})
+    return results
 def direct_search_candidates(
     prospect: dict[str, Any],
     *,
@@ -298,6 +357,9 @@ def direct_search_candidates(
     queries = _search_query_variants(prospect)
     if not queries:
         return []
+    degoog_results = _degoog_search_candidates(prospect, queries, requested=requested, timeout_seconds=min(timeout_seconds, 60), log_callback=log_callback)
+    if degoog_results:
+        return degoog_results
     try:
         hard_timeout = max(15, min(180, int(timeout_seconds)))
     except (TypeError, ValueError):

docker-compose.yml

@@ -1,4 +1,12 @@
 services:
+  degoog:
+    image: ghcr.io/degoog-org/degoog:latest
+    environment:
+      DEGOOG_SETTINGS_PASSWORDS: ${DEGOOG_SETTINGS_PASSWORD:-change-this-before-exposing}
+    volumes:
+      - degoog_data:/app/data
+    restart: unless-stopped
+
   gmaps:
     image: gosom/google-maps-scraper:latest
     command: ["-web", "-addr", ":8080", "-data-folder", "/gmapsdata"]
@@ -30,6 +38,9 @@ services:
       BROWSER_USE_MAX_STEPS: ${BROWSER_USE_MAX_STEPS:-10}
       BROWSER_USE_TIMEOUT_SECONDS: ${BROWSER_USE_TIMEOUT_SECONDS:-600}
       BROWSER_USE_LLM_TIMEOUT_SECONDS: ${BROWSER_USE_LLM_TIMEOUT_SECONDS:-90}
+      DEGOOG_ENABLED: ${DEGOOG_ENABLED:-false}
+      DEGOOG_URL: ${DEGOOG_URL:-http://degoog:4444}
+      DEGOOG_ENGINES: ${DEGOOG_ENGINES:-}
       ANONYMIZED_TELEMETRY: ${ANONYMIZED_TELEMETRY:-false}
       LOG_RETENTION: ${LOG_RETENTION:-20000}
       AUTH_REQUIRED: ${AUTH_REQUIRED:-true}
@@ -60,5 +71,6 @@ services:
       retries: 5
 
 volumes:
+  degoog_data:
   prospect_data:
   gmaps_data:

tests/test_app.py

@@ -100,7 +100,18 @@ class BrowserOnlyAppTests(unittest.TestCase):
         self.assertTrue(first_call.args[2]["email"])
         self.assertEqual(http_bytes.call_args.args[1], "http://gmaps:8080/api/v1/jobs/job-1/download")
 
-    def test_nous_provider_exposes_models_and_tests_connection(self):
+    def test_degoog_search_normalises_candidates_and_keeps_provenance(self):
+        class Response:
+            def __enter__(self): return self
+            def __exit__(self, *args): return None
+            def read(self, _limit=None):
+                return json.dumps({"results": [{"url": "https://www.acme.co.za/", "title": "Acme Plumbing", "content": "Acme Plumbing Cape Town"}]}).encode()
+        prospect = {"name": "Acme Plumbing", "address": "Cape Town", "category": "Plumbers", "phone": ""}
+        with patch.dict(os.environ, {"DEGOOG_ENABLED": "true", "DEGOOG_URL": "http://degoog:4444"}, clear=False), patch("api.browser_investigator.urllib.request.urlopen", return_value=Response()):
+            results = browser_investigator._degoog_search_candidates(prospect, ['"Acme Plumbing" Cape Town'], requested=3, timeout_seconds=10)
+        self.assertEqual(results[0]["source"], "degoog")
+        self.assertEqual(results[0]["domain"], "acme.co.za")
+
         free_models = [{"id": "meituan/longcat-2.0:free", "label": "Meituan · LongCat 2.0", "free": True}]
         with patch.object(server, "nous_models", return_value=free_models):
             status, body = self.request("GET", "/api/ai/models?provider=nous")