feat: add self-hosted SearXNG research mode

2026-09-03 21:49:02 +02:00
parent 0f70674b16
commit d298a98723
11 changed files with 175 additions and 42 deletions
@@ -1,9 +1,8 @@
 """Fail-closed AI web-research providers for criteria-first discovery.

-The native Nous adapter is a locator only. It may ask an approved Firecrawl-
-compatible service for bounded search/scrape observations, but only structured
-HTTPS targets returned by the model are handed to discovery.py. The existing
-crawler performs the final SSRF validation and persists the evidence.
+Nous is the model/orchestrator. Search is provided by an internal-only SearXNG
+service and page reads use ProspectOS's own SSRF-safe website scanner. Firecrawl
+remains a backwards-compatible legacy path when explicitly configured.
 """
 from __future__ import annotations

@@ -11,10 +10,11 @@ import json
 import os
 import re
 import sqlite3
+from html import unescape
 from urllib.parse import urlparse
 from urllib.request import Request, urlopen

-from .website_scanner import validate_url
+from .website_scanner import scan_website, validate_url

 MAX_CANDIDATES = 50
 MAX_CRITERIA_BYTES = 8192
@@ -61,7 +61,7 @@ def _config():
            if row:
                from .provider_config import decrypt
                credentials = json.loads(decrypt(row["credentials_ciphertext"])) if row["credentials_ciphertext"] else {}
-                return {"provider": row["provider"], "model": row["model"], "nous_url": row["nous_base_url"], "nous_allowed": {urlparse(row["nous_base_url"]).hostname}, "nous_key": credentials.get("nous_api_key", ""), "firecrawl_url": row["firecrawl_base_url"], "firecrawl_allowed": {urlparse(row["firecrawl_base_url"]).hostname}, "firecrawl_key": credentials.get("firecrawl_api_key", "")}
+                return {"provider": row["provider"], "model": row["model"], "nous_url": row["nous_base_url"], "nous_allowed": {urlparse(row["nous_base_url"]).hostname}, "nous_key": credentials.get("nous_api_key", ""), "searxng_url": os.environ.get("SEARXNG_BASE_URL", "").strip(), "searxng_allowed": _hosts("SEARXNG_ALLOWED_HOSTS", "searxng"), "firecrawl_url": row["firecrawl_base_url"], "firecrawl_allowed": {urlparse(row["firecrawl_base_url"]).hostname}, "firecrawl_key": credentials.get("firecrawl_api_key", "")}
        except Exception:
            return {"provider": "", "model": "", "endpoint": "", "allowed": set(), "api_key": ""}
    provider = os.environ.get("AI_RESEARCH_PROVIDER", "").strip().lower()
@@ -73,7 +73,8 @@ def _config():
        return {"provider": provider, "model": os.environ.get("NOUS_MODEL", "Hermes-4-405B").strip(),
                "nous_url": os.environ.get("NOUS_BASE_URL", "https://inference-api.nousresearch.com/v1").strip(),
                "nous_allowed": _hosts("NOUS_ALLOWED_HOSTS", "inference-api.nousresearch.com"),
-                "nous_key": nous_key, "firecrawl_url": os.environ.get("FIRECRAWL_BASE_URL", "https://api.firecrawl.dev/v2").strip(),
+                "nous_key": nous_key, "searxng_url": os.environ.get("SEARXNG_BASE_URL", "").strip(),
+                "searxng_allowed": _hosts("SEARXNG_ALLOWED_HOSTS", "searxng"), "firecrawl_url": os.environ.get("FIRECRAWL_BASE_URL", "https://api.firecrawl.dev/v2").strip(),
                "firecrawl_allowed": _hosts("FIRECRAWL_ALLOWED_HOSTS", "api.firecrawl.dev"), "firecrawl_key": firecrawl_key}
    api_key = generic_key
    if provider == "openai_web_search": api_key = api_key or os.environ.get("OPENAI_API_KEY", "").strip()
@@ -85,9 +86,17 @@ def _config():
 def _endpoint():
    cfg = _config()
    if cfg["provider"] in NOUS_PROVIDER_IDS:
-        if not cfg["model"] or not cfg["nous_key"] or not cfg["firecrawl_key"]:
+        if not cfg["model"] or not cfg["nous_key"]:
            raise AIResearchConfigError("not_configured")
-        return cfg, _safe_endpoint(cfg["nous_url"], cfg["nous_allowed"]), _safe_endpoint(cfg["firecrawl_url"], cfg["firecrawl_allowed"])
+        nous = _safe_endpoint(cfg["nous_url"], cfg["nous_allowed"])
+        if cfg.get("searxng_url"):
+            parsed = urlparse(cfg["searxng_url"]); host = (parsed.hostname or "").lower().rstrip(".")
+            if parsed.scheme != "http" or host not in cfg["searxng_allowed"] or parsed.username or parsed.password or parsed.query or parsed.fragment:
+                raise AIResearchConfigError("unsafe_provider")
+            return cfg, nous, cfg["searxng_url"].rstrip("/")
+        if not cfg["firecrawl_key"]:
+            raise AIResearchConfigError("not_configured")
+        return cfg, nous, _safe_endpoint(cfg["firecrawl_url"], cfg["firecrawl_allowed"])
    if not cfg["provider"] or not cfg["endpoint"] or not cfg["model"]: raise AIResearchConfigError("not_configured")
    if cfg["provider"] not in APPROVED_PROVIDER_IDS: raise AIResearchConfigError("unapproved_provider")
    parsed = urlparse(cfg["endpoint"]); host = (parsed.hostname or "").lower().rstrip(".")
@@ -99,10 +108,13 @@ def _endpoint():
 def provider_status() -> dict[str, object]:
    cfg = _config()
    if cfg["provider"] in NOUS_PROVIDER_IDS:
-        try: _, nous_url, firecrawl_url = _endpoint()
+        try: _, nous_url, tool_url = _endpoint()
        except AIResearchConfigError as exc:
            return {"provider": cfg["provider"], "status": str(exc), "configured": False, "network_enabled": False, "outbound_calls": False}
-        return {"provider": cfg["provider"], "model": cfg["model"], "nous_host": urlparse(nous_url).hostname, "firecrawl_host": urlparse(firecrawl_url).hostname, "status": "ready", "configured": True, "network_enabled": True, "outbound_calls": True, "max_candidates": MAX_CANDIDATES, "max_tool_calls": MAX_TOOL_CALLS}
+        result = {"provider": cfg["provider"], "model": cfg["model"], "nous_host": urlparse(nous_url).hostname, "status": "ready", "configured": True, "network_enabled": True, "outbound_calls": True, "max_candidates": MAX_CANDIDATES, "max_tool_calls": MAX_TOOL_CALLS}
+        if cfg.get("searxng_url"): result.update({"search_provider": "searxng", "searxng_host": urlparse(tool_url).hostname, "scrape_provider": "native_crawler"})
+        else: result.update({"search_provider": "firecrawl_legacy", "firecrawl_host": urlparse(tool_url).hostname, "scrape_provider": "firecrawl_legacy"})
+        return result
    if not cfg["provider"] and not cfg["endpoint"]: return {"provider": "", "status": "not_configured", "configured": False, "network_enabled": False, "outbound_calls": False}
    if cfg["provider"] and cfg["provider"] not in APPROVED_PROVIDER_IDS: return {"provider": cfg["provider"], "status": "unapproved_provider", "configured": False, "network_enabled": False, "outbound_calls": False}
    try: _, endpoint, _ = _endpoint()
@@ -147,17 +159,35 @@ def _post(url: str, key: str, body_obj: dict, *, limit: int = MAX_RESPONSE_BYTES
    return payload


+def _page_text(html: str) -> str:
+    """Return small, non-executable page text for the model."""
+    text = re.sub(r"(?is)<(script|style|noscript).*?>.*?</\1>", " ", html or "")
+    text = re.sub(r"(?s)<[^>]*>", " ", text)
+    return re.sub(r"\s+", " ", unescape(text)).strip()[:MAX_TOOL_RESULT_BYTES]
+
+
 def _tool_result(cfg, name: str, arguments: str, remaining: int) -> dict:
    if remaining < 0: raise AIResearchConfigError("tool_budget_exhausted")
    try: args = json.loads(arguments or "{}")
    except json.JSONDecodeError as exc: raise AIResearchConfigError("invalid_tool_arguments") from exc
    if not isinstance(args, dict): raise AIResearchConfigError("invalid_tool_arguments")
-    base = _safe_endpoint(cfg["firecrawl_url"], cfg["firecrawl_allowed"])
    if name == "web_search":
        query = args.get("query")
        try: requested_limit = int(args.get("limit", MAX_SEARCH_RESULTS))
        except (TypeError, ValueError) as exc: raise AIResearchConfigError("invalid_tool_arguments") from exc
        if not isinstance(query, str) or not query.strip() or len(query.encode()) > 1000 or not 1 <= requested_limit <= MAX_SEARCH_RESULTS: raise AIResearchConfigError("invalid_tool_arguments")
+        if cfg.get("searxng_url"):
+            request = Request(cfg["searxng_url"].rstrip("/") + "/search" + "?q=" + __import__("urllib.parse", fromlist=["quote"]).quote(query.strip()) + "&format=json", headers={"Accept": "application/json"}, method="GET")
+            try:
+                with urlopen(request, timeout=TIMEOUT_SECONDS) as response: raw = response.read(MAX_TOOL_RESULT_BYTES + 1)
+            except Exception as exc: raise AIResearchConfigError("provider_unavailable") from exc
+            if len(raw) > MAX_TOOL_RESULT_BYTES: raise AIResearchConfigError("provider_response_too_large")
+            try: payload = json.loads(raw.decode("utf-8"))
+            except (UnicodeDecodeError, json.JSONDecodeError) as exc: raise AIResearchConfigError("invalid_provider_response") from exc
+            if not isinstance(payload, dict): raise AIResearchConfigError("invalid_provider_response")
+            results = [{"title": x.get("title", "")[:300], "url": x.get("url", ""), "snippet": x.get("content", "")[:500]} for x in payload.get("results", [])[:requested_limit] if isinstance(x, dict) and isinstance(x.get("url"), str)]
+            return {"type": "web_search_result", "data": results}
+        base = _safe_endpoint(cfg["firecrawl_url"], cfg["firecrawl_allowed"])
        payload = _post(base + "/search", cfg["firecrawl_key"], {"query": query.strip(), "limit": requested_limit}, limit=MAX_TOOL_RESULT_BYTES)
        return {"type": "web_search_result", "data": payload.get("data", payload.get("results", []))}
    if name == "scrape_website":
@@ -165,6 +195,11 @@ def _tool_result(cfg, name: str, arguments: str, remaining: int) -> dict:
        if not isinstance(target, str) or urlparse(target).scheme != "https": raise AIResearchConfigError("invalid_tool_arguments")
        try: safe = validate_url(target)
        except (TypeError, ValueError) as exc: raise AIResearchConfigError("unsafe_target_url") from exc
+        if cfg.get("searxng_url"):
+            scanned = scan_website(safe, max_bytes=MAX_TOOL_RESULT_BYTES)
+            if scanned.get("error_code"): raise AIResearchConfigError("scrape_" + str(scanned["error_code"]))
+            return {"type": "scrape_result", "url": scanned.get("final_url") or safe, "data": {"status": scanned.get("status"), "title": scanned.get("title", ""), "description": scanned.get("meta_description", ""), "headings": scanned.get("headings", [])[:20], "content": _page_text(scanned.get("html", ""))}}
+        base = _safe_endpoint(cfg["firecrawl_url"], cfg["firecrawl_allowed"])
        payload = _post(base + "/scrape", cfg["firecrawl_key"], {"url": safe, "formats": ["markdown"], "onlyMainContent": True}, limit=MAX_TOOL_RESULT_BYTES)
        return {"type": "scrape_result", "url": safe, "data": payload.get("data", payload)}
    raise AIResearchConfigError("unknown_tool")