diff --git a/.env.example b/.env.example index db4e8d6..1705e4e 100644 --- a/.env.example +++ b/.env.example @@ -31,6 +31,14 @@ BROWSER_USE_LLM_TIMEOUT_SECONDS=90 ANONYMIZED_TELEMETRY=false LOG_RETENTION=20000 +# Optional internal Degoog website-search fallback. Keep disabled until you +# install/review engines in the Degoog settings UI. Degoog is not published +# on a host port; ProspectOS reaches it over the private Compose network. +DEGOOG_ENABLED=false +DEGOOG_URL=http://degoog:4444 +DEGOOG_ENGINES= +DEGOOG_SETTINGS_PASSWORD=change-this-before-exposing + # Authentication and workspace ownership. Leave OWNER_* blank to use the # first-run setup screen; setting them seeds the owner on an unattended start. AUTH_REQUIRED=true diff --git a/README.md b/README.md index c857a13..ee1143d 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,25 @@ error rather than remaining stuck forever. | `AI_API_KEY` | empty | Optional provider key; keep server-side | | `AI_MOCKUP_TIMEOUT_SECONDS` | `45` | Maximum AI wait for mockup copy before the deterministic premium renderer is used (20–300 seconds) | -### Nous Research Portal +### Degoog website-search fallback + +ProspectOS can optionally query an internal Degoog instance when looking for a +possible official website for a Maps prospect. Maps remains the structured +business source; Degoog is only a criteria/name-to-website discovery fallback. + +Enable it only after installing and reviewing Degoog engines in its settings: + +```bash +DEGOOG_ENABLED=true docker compose up -d +``` + +Degoog is private to the Compose network and is not published on a host port. +The adapter uses Degoog's JSON `POST /api/search` endpoint, applies the same +bounded result and domain/identity filters as the browser-search path, and +stores `source=degoog` in candidate provenance. Do not install unreviewed +third-party Degoog extensions; the project supports extensions/transports that +may execute server-side code. + Choose **Nous Portal · free models** in Settings, paste a Nous API key, select a free model, save, and use **Test connection** before drafting a message. The diff --git a/api/browser_investigator.py b/api/browser_investigator.py index d296f8f..ced75a0 100644 --- a/api/browser_investigator.py +++ b/api/browser_investigator.py @@ -13,6 +13,8 @@ import json import os import re import time +import urllib.error +import urllib.request from typing import Any, Callable from urllib.parse import parse_qs, quote_plus, unquote, urlparse @@ -275,6 +277,63 @@ def _safe_log(log_callback: Callable[[str, str, dict[str, Any]], Any] | None, return +def _degoog_search_candidates( + prospect: dict[str, Any], + queries: list[str], + *, + requested: int, + timeout_seconds: int, + log_callback: Callable[[str, str, dict[str, Any]], Any] | None = None, +) -> list[dict[str, Any]]: + """Use an internal Degoog instance as a bounded public-search source.""" + base = os.environ.get("DEGOOG_URL", "").strip().rstrip("/") + enabled = os.environ.get("DEGOOG_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"} + if not enabled or not base: + return [] + engines_env = [item.strip() for item in os.environ.get("DEGOOG_ENGINES", "").split(",") if item.strip()] + found: dict[str, dict[str, Any]] = {} + deadline = time.monotonic() + max(5, min(60, int(timeout_seconds))) + for query in queries: + if time.monotonic() >= deadline: + break + body: dict[str, Any] = {"query": query} + if engines_env: + body["engines"] = engines_env + request = urllib.request.Request( + base + "/api/search", + data=json.dumps(body).encode("utf-8"), + headers={"Content-Type": "application/json", "Accept": "application/json", "User-Agent": os.environ.get("OUTBOUND_USER_AGENT", "ProspectOS/2.0 (+personal-use)")}, + method="POST", + ) + try: + _safe_log(log_callback, "search", "degoog_search_started", {"query": query, "url": base + "/api/search"}) + with urllib.request.urlopen(request, timeout=max(3, min(20, int(deadline - time.monotonic())))) as response: + payload = json.loads(response.read(1_000_000).decode("utf-8", "replace")) + raw_results = payload.get("results", []) if isinstance(payload, dict) else [] + if not isinstance(raw_results, list): + continue + for item in raw_results[: requested * 5]: + if not isinstance(item, dict): + continue + href = _normalise_search_href(item.get("url") or item.get("href") or item.get("link")) + if not href: + continue + domain = _candidate_domain(href) + if not domain or domain in found: + continue + title = str(item.get("title") or item.get("name") or "")[:300] + context = str(item.get("content") or item.get("snippet") or item.get("description") or "")[:1200] + score, evidence = _candidate_score(prospect, href, title, context) + if score < 48: + continue + found[domain] = {"url": href, "domain": domain, "confidence": score, "evidence": evidence + ["visible on Degoog search results"], "status": "candidate", "source": "degoog", "search_engine": "degoog", "title": title, "snippet": context, "queries": [query]} + if len(found) >= requested: + return sorted(found.values(), key=lambda item: item["confidence"], reverse=True)[:requested] + except (urllib.error.URLError, TimeoutError, ValueError, json.JSONDecodeError) as exc: + _safe_log(log_callback, "search", "degoog_search_error", {"query": query, "error": str(exc)[:300]}) + results = sorted(found.values(), key=lambda item: item["confidence"], reverse=True)[:requested] + _safe_log(log_callback, "search", "degoog_search_completed", {"candidate_count": len(results), "candidates": results}) + return results def direct_search_candidates( prospect: dict[str, Any], *, @@ -298,6 +357,9 @@ def direct_search_candidates( queries = _search_query_variants(prospect) if not queries: return [] + degoog_results = _degoog_search_candidates(prospect, queries, requested=requested, timeout_seconds=min(timeout_seconds, 60), log_callback=log_callback) + if degoog_results: + return degoog_results try: hard_timeout = max(15, min(180, int(timeout_seconds))) except (TypeError, ValueError): diff --git a/docker-compose.yml b/docker-compose.yml index fb64b77..a0cc306 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,12 @@ services: + degoog: + image: ghcr.io/degoog-org/degoog:latest + environment: + DEGOOG_SETTINGS_PASSWORDS: ${DEGOOG_SETTINGS_PASSWORD:-change-this-before-exposing} + volumes: + - degoog_data:/app/data + restart: unless-stopped + gmaps: image: gosom/google-maps-scraper:latest command: ["-web", "-addr", ":8080", "-data-folder", "/gmapsdata"] @@ -30,6 +38,9 @@ services: BROWSER_USE_MAX_STEPS: ${BROWSER_USE_MAX_STEPS:-10} BROWSER_USE_TIMEOUT_SECONDS: ${BROWSER_USE_TIMEOUT_SECONDS:-600} BROWSER_USE_LLM_TIMEOUT_SECONDS: ${BROWSER_USE_LLM_TIMEOUT_SECONDS:-90} + DEGOOG_ENABLED: ${DEGOOG_ENABLED:-false} + DEGOOG_URL: ${DEGOOG_URL:-http://degoog:4444} + DEGOOG_ENGINES: ${DEGOOG_ENGINES:-} ANONYMIZED_TELEMETRY: ${ANONYMIZED_TELEMETRY:-false} LOG_RETENTION: ${LOG_RETENTION:-20000} AUTH_REQUIRED: ${AUTH_REQUIRED:-true} @@ -60,5 +71,6 @@ services: retries: 5 volumes: + degoog_data: prospect_data: gmaps_data: diff --git a/tests/test_app.py b/tests/test_app.py index bd007b5..94228bd 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -100,7 +100,18 @@ class BrowserOnlyAppTests(unittest.TestCase): self.assertTrue(first_call.args[2]["email"]) self.assertEqual(http_bytes.call_args.args[1], "http://gmaps:8080/api/v1/jobs/job-1/download") - def test_nous_provider_exposes_models_and_tests_connection(self): + def test_degoog_search_normalises_candidates_and_keeps_provenance(self): + class Response: + def __enter__(self): return self + def __exit__(self, *args): return None + def read(self, _limit=None): + return json.dumps({"results": [{"url": "https://www.acme.co.za/", "title": "Acme Plumbing", "content": "Acme Plumbing Cape Town"}]}).encode() + prospect = {"name": "Acme Plumbing", "address": "Cape Town", "category": "Plumbers", "phone": ""} + with patch.dict(os.environ, {"DEGOOG_ENABLED": "true", "DEGOOG_URL": "http://degoog:4444"}, clear=False), patch("api.browser_investigator.urllib.request.urlopen", return_value=Response()): + results = browser_investigator._degoog_search_candidates(prospect, ['"Acme Plumbing" Cape Town'], requested=3, timeout_seconds=10) + self.assertEqual(results[0]["source"], "degoog") + self.assertEqual(results[0]["domain"], "acme.co.za") + free_models = [{"id": "meituan/longcat-2.0:free", "label": "Meituan · LongCat 2.0", "free": True}] with patch.object(server, "nous_models", return_value=free_models): status, body = self.request("GET", "/api/ai/models?provider=nous")