feat: add self-hosted SearXNG research mode

.env.example

@@ -20,7 +20,7 @@ AUTOMATED_OUTREACH_ENABLED=false
 # they are never copied into API responses. Connectivity tests use bounded GET
 # requests only and report outbound_calls=false (no outreach is implemented).
 # The following variables are legacy/bootstrap fallback values only.
-AI_RESEARCH_PROVIDER=
+# AI_RESEARCH_PROVIDER is set below for the primary self-hosted mode.
 # NOUS_MODEL=Hermes-4-405B
 # NOUS_BASE_URL=https://inference-api.nousresearch.com/v1
 # NOUS_ALLOWED_HOSTS=inference-api.nousresearch.com
@@ -28,11 +28,17 @@ AI_RESEARCH_PROVIDER=
 # FIRECRAWL_BASE_URL=https://api.firecrawl.dev/v2
 # FIRECRAWL_ALLOWED_HOSTS=api.firecrawl.dev
 # FIRECRAWL_API_KEY=<Firecrawl API key; secret-manager only>
-AI_RESEARCH_PROVIDER=
+# Primary no-paid-scraper mode: Nous orchestrates, internal SearXNG searches,
+# and the API's SSRF-safe crawler reads pages. No Firecrawl key is required.
+AI_RESEARCH_PROVIDER=nous_portal
 NOUS_API_KEY=
 NOUS_MODEL=Hermes-4-405B
 NOUS_BASE_URL=https://inference-api.nousresearch.com/v1
 NOUS_ALLOWED_HOSTS=inference-api.nousresearch.com
+SEARXNG_BASE_URL=http://searxng:8080
+SEARXNG_ALLOWED_HOSTS=searxng
+SEARXNG_SECRET_KEY=
+# Optional legacy fallback only; never required by self-hosted mode.
 FIRECRAWL_API_KEY=
 FIRECRAWL_BASE_URL=https://api.firecrawl.dev/v2
 FIRECRAWL_ALLOWED_HOSTS=api.firecrawl.dev

apps/api/README.md

@@ -31,13 +31,12 @@ The provider status is available at authenticated `GET
 `/api/v1/discovery/provider-status` alias is retained). Configure only on the
 server. The native Nous adapter uses OpenAI-compatible Chat Completions at
 `https://inference-api.nousresearch.com/v1/chat/completions` and strict
-`web_search`/`scrape_website` tools backed by an allowlisted Firecrawl-compatible
+`web_search`/`scrape_website` tools. In the primary no-paid-scraper mode,
-API. Configure server-side `NOUS_API_KEY`, `NOUS_MODEL`, `NOUS_BASE_URL`,
+`web_search` uses internal SearXNG (`SEARXNG_BASE_URL`, normally
-`NOUS_ALLOWED_HOSTS`, `FIRECRAWL_API_KEY`, `FIRECRAWL_BASE_URL`, and
+`http://searxng:8080`) and `scrape_website` uses the existing SSRF-safe scanner;
-`FIRECRAWL_ALLOWED_HOSTS` with `AI_RESEARCH_PROVIDER=nous_portal`. Tool calls,
+configure server-side `NOUS_API_KEY`, `NOUS_MODEL`, `NOUS_BASE_URL`,
-responses, criteria, and results are bounded; page text is untrusted; only
+`NOUS_ALLOWED_HOSTS`, `SEARXNG_ALLOWED_HOSTS`, and `SEARXNG_SECRET_KEY`.
-structured HTTPS targets are accepted and the existing SSRF-safe crawler fetches
+Firecrawl settings are optional legacy compatibility only.
-and persists evidence. Status is fail-closed and never returns secrets.
 
 
 All protected endpoints require the server-side session cookie. Every query is constrained by the authenticated user's `organization_id`; IDs from another tenant behave as not found and must not disclose whether a record exists.

apps/api/app/ai_research.py

@@ -1,9 +1,8 @@
 """Fail-closed AI web-research providers for criteria-first discovery.
 
-The native Nous adapter is a locator only. It may ask an approved Firecrawl-
+Nous is the model/orchestrator. Search is provided by an internal-only SearXNG
-compatible service for bounded search/scrape observations, but only structured
+service and page reads use ProspectOS's own SSRF-safe website scanner. Firecrawl
-HTTPS targets returned by the model are handed to discovery.py. The existing
+remains a backwards-compatible legacy path when explicitly configured.
-crawler performs the final SSRF validation and persists the evidence.
 """
 from __future__ import annotations
 
@@ -11,10 +10,11 @@ import json
 import os
 import re
 import sqlite3
+from html import unescape
 from urllib.parse import urlparse
 from urllib.request import Request, urlopen
 
-from .website_scanner import validate_url
+from .website_scanner import scan_website, validate_url
 
 MAX_CANDIDATES = 50
 MAX_CRITERIA_BYTES = 8192
@@ -61,7 +61,7 @@ def _config():
             if row:
                 from .provider_config import decrypt
                 credentials = json.loads(decrypt(row["credentials_ciphertext"])) if row["credentials_ciphertext"] else {}
-                return {"provider": row["provider"], "model": row["model"], "nous_url": row["nous_base_url"], "nous_allowed": {urlparse(row["nous_base_url"]).hostname}, "nous_key": credentials.get("nous_api_key", ""), "firecrawl_url": row["firecrawl_base_url"], "firecrawl_allowed": {urlparse(row["firecrawl_base_url"]).hostname}, "firecrawl_key": credentials.get("firecrawl_api_key", "")}
+                return {"provider": row["provider"], "model": row["model"], "nous_url": row["nous_base_url"], "nous_allowed": {urlparse(row["nous_base_url"]).hostname}, "nous_key": credentials.get("nous_api_key", ""), "searxng_url": os.environ.get("SEARXNG_BASE_URL", "").strip(), "searxng_allowed": _hosts("SEARXNG_ALLOWED_HOSTS", "searxng"), "firecrawl_url": row["firecrawl_base_url"], "firecrawl_allowed": {urlparse(row["firecrawl_base_url"]).hostname}, "firecrawl_key": credentials.get("firecrawl_api_key", "")}
         except Exception:
             return {"provider": "", "model": "", "endpoint": "", "allowed": set(), "api_key": ""}
     provider = os.environ.get("AI_RESEARCH_PROVIDER", "").strip().lower()
@@ -73,7 +73,8 @@ def _config():
         return {"provider": provider, "model": os.environ.get("NOUS_MODEL", "Hermes-4-405B").strip(),
                 "nous_url": os.environ.get("NOUS_BASE_URL", "https://inference-api.nousresearch.com/v1").strip(),
                 "nous_allowed": _hosts("NOUS_ALLOWED_HOSTS", "inference-api.nousresearch.com"),
-                "nous_key": nous_key, "firecrawl_url": os.environ.get("FIRECRAWL_BASE_URL", "https://api.firecrawl.dev/v2").strip(),
+                "nous_key": nous_key, "searxng_url": os.environ.get("SEARXNG_BASE_URL", "").strip(),
+                "searxng_allowed": _hosts("SEARXNG_ALLOWED_HOSTS", "searxng"), "firecrawl_url": os.environ.get("FIRECRAWL_BASE_URL", "https://api.firecrawl.dev/v2").strip(),
                 "firecrawl_allowed": _hosts("FIRECRAWL_ALLOWED_HOSTS", "api.firecrawl.dev"), "firecrawl_key": firecrawl_key}
     api_key = generic_key
     if provider == "openai_web_search": api_key = api_key or os.environ.get("OPENAI_API_KEY", "").strip()
@@ -85,9 +86,17 @@ def _config():
 def _endpoint():
     cfg = _config()
     if cfg["provider"] in NOUS_PROVIDER_IDS:
-        if not cfg["model"] or not cfg["nous_key"] or not cfg["firecrawl_key"]:
+        if not cfg["model"] or not cfg["nous_key"]:
             raise AIResearchConfigError("not_configured")
-        return cfg, _safe_endpoint(cfg["nous_url"], cfg["nous_allowed"]), _safe_endpoint(cfg["firecrawl_url"], cfg["firecrawl_allowed"])
+        nous = _safe_endpoint(cfg["nous_url"], cfg["nous_allowed"])
+        if cfg.get("searxng_url"):
+            parsed = urlparse(cfg["searxng_url"]); host = (parsed.hostname or "").lower().rstrip(".")
+            if parsed.scheme != "http" or host not in cfg["searxng_allowed"] or parsed.username or parsed.password or parsed.query or parsed.fragment:
+                raise AIResearchConfigError("unsafe_provider")
+            return cfg, nous, cfg["searxng_url"].rstrip("/")
+        if not cfg["firecrawl_key"]:
+            raise AIResearchConfigError("not_configured")
+        return cfg, nous, _safe_endpoint(cfg["firecrawl_url"], cfg["firecrawl_allowed"])
     if not cfg["provider"] or not cfg["endpoint"] or not cfg["model"]: raise AIResearchConfigError("not_configured")
     if cfg["provider"] not in APPROVED_PROVIDER_IDS: raise AIResearchConfigError("unapproved_provider")
     parsed = urlparse(cfg["endpoint"]); host = (parsed.hostname or "").lower().rstrip(".")
@@ -99,10 +108,13 @@ def _endpoint():
 def provider_status() -> dict[str, object]:
     cfg = _config()
     if cfg["provider"] in NOUS_PROVIDER_IDS:
-        try: _, nous_url, firecrawl_url = _endpoint()
+        try: _, nous_url, tool_url = _endpoint()
         except AIResearchConfigError as exc:
             return {"provider": cfg["provider"], "status": str(exc), "configured": False, "network_enabled": False, "outbound_calls": False}
-        return {"provider": cfg["provider"], "model": cfg["model"], "nous_host": urlparse(nous_url).hostname, "firecrawl_host": urlparse(firecrawl_url).hostname, "status": "ready", "configured": True, "network_enabled": True, "outbound_calls": True, "max_candidates": MAX_CANDIDATES, "max_tool_calls": MAX_TOOL_CALLS}
+        result = {"provider": cfg["provider"], "model": cfg["model"], "nous_host": urlparse(nous_url).hostname, "status": "ready", "configured": True, "network_enabled": True, "outbound_calls": True, "max_candidates": MAX_CANDIDATES, "max_tool_calls": MAX_TOOL_CALLS}
+        if cfg.get("searxng_url"): result.update({"search_provider": "searxng", "searxng_host": urlparse(tool_url).hostname, "scrape_provider": "native_crawler"})
+        else: result.update({"search_provider": "firecrawl_legacy", "firecrawl_host": urlparse(tool_url).hostname, "scrape_provider": "firecrawl_legacy"})
+        return result
     if not cfg["provider"] and not cfg["endpoint"]: return {"provider": "", "status": "not_configured", "configured": False, "network_enabled": False, "outbound_calls": False}
     if cfg["provider"] and cfg["provider"] not in APPROVED_PROVIDER_IDS: return {"provider": cfg["provider"], "status": "unapproved_provider", "configured": False, "network_enabled": False, "outbound_calls": False}
     try: _, endpoint, _ = _endpoint()
@@ -147,17 +159,35 @@ def _post(url: str, key: str, body_obj: dict, *, limit: int = MAX_RESPONSE_BYTES
     return payload
 
 
+def _page_text(html: str) -> str:
+    """Return small, non-executable page text for the model."""
+    text = re.sub(r"(?is)<(script|style|noscript).*?>.*?</\1>", " ", html or "")
+    text = re.sub(r"(?s)<[^>]*>", " ", text)
+    return re.sub(r"\s+", " ", unescape(text)).strip()[:MAX_TOOL_RESULT_BYTES]
+
+
 def _tool_result(cfg, name: str, arguments: str, remaining: int) -> dict:
     if remaining < 0: raise AIResearchConfigError("tool_budget_exhausted")
     try: args = json.loads(arguments or "{}")
     except json.JSONDecodeError as exc: raise AIResearchConfigError("invalid_tool_arguments") from exc
     if not isinstance(args, dict): raise AIResearchConfigError("invalid_tool_arguments")
-    base = _safe_endpoint(cfg["firecrawl_url"], cfg["firecrawl_allowed"])
     if name == "web_search":
         query = args.get("query")
         try: requested_limit = int(args.get("limit", MAX_SEARCH_RESULTS))
         except (TypeError, ValueError) as exc: raise AIResearchConfigError("invalid_tool_arguments") from exc
         if not isinstance(query, str) or not query.strip() or len(query.encode()) > 1000 or not 1 <= requested_limit <= MAX_SEARCH_RESULTS: raise AIResearchConfigError("invalid_tool_arguments")
+        if cfg.get("searxng_url"):
+            request = Request(cfg["searxng_url"].rstrip("/") + "/search" + "?q=" + __import__("urllib.parse", fromlist=["quote"]).quote(query.strip()) + "&format=json", headers={"Accept": "application/json"}, method="GET")
+            try:
+                with urlopen(request, timeout=TIMEOUT_SECONDS) as response: raw = response.read(MAX_TOOL_RESULT_BYTES + 1)
+            except Exception as exc: raise AIResearchConfigError("provider_unavailable") from exc
+            if len(raw) > MAX_TOOL_RESULT_BYTES: raise AIResearchConfigError("provider_response_too_large")
+            try: payload = json.loads(raw.decode("utf-8"))
+            except (UnicodeDecodeError, json.JSONDecodeError) as exc: raise AIResearchConfigError("invalid_provider_response") from exc
+            if not isinstance(payload, dict): raise AIResearchConfigError("invalid_provider_response")
+            results = [{"title": x.get("title", "")[:300], "url": x.get("url", ""), "snippet": x.get("content", "")[:500]} for x in payload.get("results", [])[:requested_limit] if isinstance(x, dict) and isinstance(x.get("url"), str)]
+            return {"type": "web_search_result", "data": results}
+        base = _safe_endpoint(cfg["firecrawl_url"], cfg["firecrawl_allowed"])
         payload = _post(base + "/search", cfg["firecrawl_key"], {"query": query.strip(), "limit": requested_limit}, limit=MAX_TOOL_RESULT_BYTES)
         return {"type": "web_search_result", "data": payload.get("data", payload.get("results", []))}
     if name == "scrape_website":
@@ -165,6 +195,11 @@ def _tool_result(cfg, name: str, arguments: str, remaining: int) -> dict:
         if not isinstance(target, str) or urlparse(target).scheme != "https": raise AIResearchConfigError("invalid_tool_arguments")
         try: safe = validate_url(target)
         except (TypeError, ValueError) as exc: raise AIResearchConfigError("unsafe_target_url") from exc
+        if cfg.get("searxng_url"):
+            scanned = scan_website(safe, max_bytes=MAX_TOOL_RESULT_BYTES)
+            if scanned.get("error_code"): raise AIResearchConfigError("scrape_" + str(scanned["error_code"]))
+            return {"type": "scrape_result", "url": scanned.get("final_url") or safe, "data": {"status": scanned.get("status"), "title": scanned.get("title", ""), "description": scanned.get("meta_description", ""), "headings": scanned.get("headings", [])[:20], "content": _page_text(scanned.get("html", ""))}}
+        base = _safe_endpoint(cfg["firecrawl_url"], cfg["firecrawl_allowed"])
         payload = _post(base + "/scrape", cfg["firecrawl_key"], {"url": safe, "formats": ["markdown"], "onlyMainContent": True}, limit=MAX_TOOL_RESULT_BYTES)
         return {"type": "scrape_result", "url": safe, "data": payload.get("data", payload)}
     raise AIResearchConfigError("unknown_tool")

apps/api/app/config.py

@@ -2,6 +2,7 @@
 from dataclasses import dataclass
 import os
 from pathlib import Path
+from urllib.parse import urlparse
 
 
 class ConfigError(ValueError):
@@ -17,6 +18,8 @@ class Config:
     log_level: str
     ai_research_provider: str
     ai_research_model: str
+    nous_base_url: str = ""
+    searxng_base_url: str = ""
 
 
 def _env(values, key, default=""):
@@ -42,6 +45,15 @@ def load_config(values=None):
     log_level = _env(values, "LOG_LEVEL", "INFO").upper()
     if log_level not in {"QUIET", "ERROR", "WARNING", "INFO", "DEBUG"}:
         raise ConfigError("LOG_LEVEL is invalid")
-    return Config(app_env, data_dir, secret, False, log_level,
+    provider = _env(values, "AI_RESEARCH_PROVIDER").lower()
-                  _env(values, "AI_RESEARCH_PROVIDER").lower(),
+    nous_url = _env(values, "NOUS_BASE_URL", "https://inference-api.nousresearch.com/v1")
-                  _env(values, "AI_RESEARCH_PROVIDER_MODEL"))
+    searx_url = _env(values, "SEARXNG_BASE_URL", "http://searxng:8080")
+    if provider in {"nous_portal", "nous_portal_web_research"}:
+        parsed = urlparse(nous_url); allowed = {x.strip().lower() for x in _env(values, "NOUS_ALLOWED_HOSTS", "inference-api.nousresearch.com").split(",") if x.strip()}
+        if parsed.scheme != "https" or parsed.hostname not in allowed or parsed.username or parsed.password or parsed.query or parsed.fragment:
+            raise ConfigError("NOUS_BASE_URL is unsafe")
+        parsed = urlparse(searx_url); allowed = {x.strip().lower() for x in _env(values, "SEARXNG_ALLOWED_HOSTS", "searxng").split(",") if x.strip()}
+        if parsed.scheme != "http" or parsed.hostname not in allowed or parsed.username or parsed.password or parsed.query or parsed.fragment:
+            raise ConfigError("SEARXNG_BASE_URL is unsafe")
+    return Config(app_env, data_dir, secret, False, log_level, provider,
+                  _env(values, "AI_RESEARCH_PROVIDER_MODEL") or _env(values, "NOUS_MODEL", "Hermes-4-405B"), nous_url, searx_url)

apps/api/app/main.py

@@ -496,7 +496,7 @@ class ApiHandler(BaseHTTPRequestHandler):
                     except Exception: old = {}
                     for name in ("nous_api_key", "firecrawl_api_key"):
                         if name not in credentials and name in old: credentials[name] = old[name]
-                if config["enabled"] and any(name not in credentials for name in ("nous_api_key", "firecrawl_api_key")):
+                if config["enabled"] and ("nous_api_key" not in credentials or ("SEARXNG_BASE_URL" not in os.environ and "firecrawl_api_key" not in credentials)):
                     return self.send_json(400, {"error": "provider_credentials_required"})
                 ciphertext = encrypt_provider_secret(json.dumps(credentials, sort_keys=True)) if credentials else ""
                 fingerprint = hashlib.sha256(json.dumps(credentials, sort_keys=True).encode()).hexdigest() if credentials else ""

apps/api/app/provider_config.py

@@ -86,7 +86,7 @@ def validate_payload(payload: dict) -> dict:
     for field, default in urls.items():
         value = str(payload.get(field, default)).strip().rstrip("/")
         parsed = urlparse(value)
-        if parsed.scheme != "https" or not parsed.hostname or parsed.username or parsed.password or parsed.fragment or parsed.query:
+        if (parsed.scheme != "https" and not (field == "firecrawl_base_url" and parsed.scheme == "http" and parsed.hostname == "searxng")) or not parsed.hostname or parsed.username or parsed.password or parsed.fragment or parsed.query:
             raise ValueError("unsafe_provider_url")
         urls[field] = value
     credentials = payload.get("credentials", {})

apps/api/tests/test_ai_research.py

@@ -11,7 +11,7 @@ class AIResearchTests(unittest.TestCase):
         "AI_RESEARCH_PROVIDER", "AI_RESEARCH_PROVIDER_MODEL", "AI_RESEARCH_PROVIDER_URL",
         "AI_RESEARCH_PROVIDER_ALLOWED_HOSTS", "AI_RESEARCH_PROVIDER_API_KEY", "OPENAI_API_KEY",
         "NOUS_API_KEY", "NOUS_MODEL", "NOUS_BASE_URL", "NOUS_ALLOWED_HOSTS",
-        "FIRECRAWL_API_KEY", "FIRECRAWL_BASE_URL", "FIRECRAWL_ALLOWED_HOSTS",
+        "FIRECRAWL_API_KEY", "FIRECRAWL_BASE_URL", "FIRECRAWL_ALLOWED_HOSTS", "SEARXNG_BASE_URL", "SEARXNG_ALLOWED_HOSTS",
     )
 
     def tearDown(self):
@@ -164,6 +164,27 @@ class AIResearchTests(unittest.TestCase):
         with self.assertRaisesRegex(AIResearchConfigError, "not_configured"):
             research({"keywords": ["solar"]}, 5)
 
+    def test_self_hosted_searxng_search_and_native_scrape_are_bounded(self):
+        self.configure_nous(); os.environ.pop("FIRECRAWL_API_KEY", None)
+        os.environ.update({"SEARXNG_BASE_URL": "http://searxng:8080", "SEARXNG_ALLOWED_HOSTS": "searxng"})
+        responses = [
+            self.response({"choices": [{"message": {"tool_calls": [{"id": "s", "type": "function", "function": {"name": "web_search", "arguments": '{"query":"solar","limit":1}'}}]}}]}),
+            self.response({"results": [{"title": "Solar", "url": "https://solar.example", "content": "snippet"}]}),
+            self.response({"choices": [{"message": {"tool_calls": [{"id": "p", "type": "function", "function": {"name": "scrape_website", "arguments": '{"url":"https://solar.example"}'}}]}}]}),
+            self.response({"choices": [{"message": {"content": '{"targets":[{"url":"https://solar.example"}]}'}}]}),
+        ]
+        scan = {"status": 200, "final_url": "https://solar.example", "title": "Solar", "meta_description": "", "headings": [], "html": "<script>ignore</script><h1>Solar</h1><p>Public page</p>", "error_code": None}
+        with patch("app.ai_research.urlopen", side_effect=responses), patch("app.ai_research.validate_url", side_effect=lambda url, **_: url), patch("app.ai_research.scan_website", return_value=scan) as scanner:
+            self.assertEqual(research({"keywords": ["solar"]}, 3), ["https://solar.example"])
+            scanner.assert_called_once_with("https://solar.example", max_bytes=16 * 1024)
+        self.assertEqual(provider_status()["search_provider"], "searxng")
+        self.assertEqual(provider_status()["scrape_provider"], "native_crawler")
+        self.assertNotIn("nous-secret", json.dumps(provider_status()))
+
+    def test_self_hosted_unsafe_endpoint_fails_closed(self):
+        self.configure_nous(); os.environ.update({"SEARXNG_BASE_URL": "http://127.0.0.1:8080", "SEARXNG_ALLOWED_HOSTS": "searxng"})
+        self.assertEqual(provider_status()["status"], "unsafe_provider")
+
 
 if __name__ == "__main__":
     unittest.main()

apps/web/index.html

@@ -121,7 +121,7 @@
           <div class="crm-two-col"><article class="panel"><div class="panel-heading"><div><p class="eyebrow">ADD RULE</p><h3>Add suppression</h3></div><span class="small-label">Explicit confirmation required</span></div><form id="suppressionForm" class="crm-form"><label>Kind<select name="kind"><option value="email">Email</option><option value="domain">Domain</option><option value="phone">Phone</option></select></label><label>Value<input name="value" required placeholder="person@example.com"></label><label>Reason <span class="optional">optional</span><input name="reason" placeholder="Customer request / policy"></label><p id="suppressionMessage" class="form-message" role="status"></p><button class="button danger" type="submit">Add suppression</button></form></article><article class="panel"><div class="panel-heading"><div><p class="eyebrow">REGISTER</p><h3>Current suppressions</h3></div><div class="suppression-bulk-actions"><label class="checkbox-label"><input id="selectAllSuppressions" type="checkbox"> Select all</label><button class="button ghost compact" id="bulkReviewSuppressionsBtn" type="button" disabled>Review selected</button></div></div><div id="suppressionState" class="detail-loading">Sign in to load suppressions.</div></article></div>
         </section>
         <section class="crm-section outreach-settings-section" id="outreachSettings" aria-labelledby="outreachSettingsTitle" data-smoke="outreach-provider-policy"><div class="crm-header panel"><div><p class="eyebrow">GOVERNANCE</p><h2 id="outreachSettingsTitle">Outreach provider policy</h2><p class="muted">View approved provider status without exposing credentials or secrets.</p></div><button class="button ghost" id="outreachPolicyRefreshBtn" type="button">↻ Refresh policy</button></div><div class="outreach-settings-safety" role="note"><strong>Sending is disabled by default.</strong> This panel is status-only. Provider configuration never creates a send trigger, and no credentials are displayed.</div><div id="providerPolicyPanel" class="provider-policy-panel" aria-live="polite"><div class="detail-loading">Sign in to load provider policy.</div></div></section>
-        <section class="crm-section ai-provider-settings-section" id="aiProviderSettings" aria-labelledby="aiProviderSettingsTitle" data-smoke="ai-provider-settings"><div class="crm-header panel"><div><p class="eyebrow">SYSTEM CONFIGURATION</p><h2 id="aiProviderSettingsTitle">AI Provider Settings</h2><p class="muted">Configure the approved Nous Portal and Firecrawl services used by workspace AI research.</p></div><button class="button ghost" id="aiProviderRefreshBtn" type="button">↻ Refresh status</button></div><div class="ai-provider-safety" role="note"><strong>Admin-only and write-only credentials.</strong> Keys are sent only over the authenticated API, are never displayed or stored in this browser, and updating settings does not start discovery or AI work.</div><div id="aiProviderAdminState" class="provider-state" role="status" hidden></div><div class="ai-provider-grid"><article class="panel ai-provider-form-panel"><div class="panel-heading"><div><p class="eyebrow">PROVIDER CONFIGURATION</p><h3>Service connection</h3></div><span class="small-label">Approved providers only</span></div><form id="aiProviderForm"><div class="form-grid"><label>AI provider<select id="aiProvider" name="provider"><option value="nous_portal">Nous Portal</option></select></label><label>Nous Portal model<input id="nousModel" name="nous_model" required maxlength="160" placeholder="Hermes 4 405B" autocomplete="off"></label><label>Nous Portal base URL<input id="nousBaseUrl" name="nous_base_url" type="url" required placeholder="https://inference-api.nousresearch.com/v1" autocomplete="off"></label><label>Nous Portal API key <span class="optional">write-only · leave blank to keep</span><input id="nousApiKey" name="nous_api_key" type="password" maxlength="512" placeholder="Enter a new key to rotate" autocomplete="new-password"></label><label>Firecrawl base URL<input id="firecrawlBaseUrl" name="firecrawl_base_url" type="url" required placeholder="https://api.firecrawl.dev/v1" autocomplete="off"></label><label>Firecrawl API key <span class="optional">write-only · leave blank to keep</span><input id="firecrawlApiKey" name="firecrawl_api_key" type="password" maxlength="512" placeholder="Enter a new key to rotate" autocomplete="new-password"></label></div><div class="form-footer"><p id="aiProviderMessage" class="form-message" role="status" aria-live="polite"></p><button class="button primary" id="saveAiProviderBtn" type="submit">Save provider settings</button></div></form></article><aside class="panel ai-provider-status-panel"><div class="panel-heading"><div><p class="eyebrow">SAFE STATUS</p><h3>Connection status</h3></div><button class="button ghost compact" id="testAiProviderBtn" type="button">Test connection</button></div><div id="aiProviderStatus" aria-live="polite"><div class="detail-loading">Sign in to load provider status.</div></div></aside></div></section>
+        <section class="crm-section ai-provider-settings-section" id="aiProviderSettings" aria-labelledby="aiProviderSettingsTitle" data-smoke="ai-provider-settings"><div class="crm-header panel"><div><p class="eyebrow">SYSTEM CONFIGURATION</p><h2 id="aiProviderSettingsTitle">AI Provider Settings</h2><p class="muted">Configure the approved Nous Portal service and the internal SearXNG service used by workspace AI research.</p></div><button class="button ghost" id="aiProviderRefreshBtn" type="button">↻ Refresh status</button></div><div class="ai-provider-safety" role="note"><strong>Admin-only and write-only credentials.</strong> Keys are sent only over the authenticated API, are never displayed or stored in this browser, and updating settings does not start discovery or AI work.</div><div id="aiProviderAdminState" class="provider-state" role="status" hidden></div><div class="ai-provider-grid"><article class="panel ai-provider-form-panel"><div class="panel-heading"><div><p class="eyebrow">PROVIDER CONFIGURATION</p><h3>Service connection</h3></div><span class="small-label">Approved providers only</span></div><form id="aiProviderForm"><div class="form-grid"><label>AI provider<select id="aiProvider" name="provider"><option value="nous_portal">Nous Portal</option></select></label><label>Nous Portal model<input id="nousModel" name="nous_model" required maxlength="160" placeholder="Hermes 4 405B" autocomplete="off"></label><label>Nous Portal base URL<input id="nousBaseUrl" name="nous_base_url" type="url" required placeholder="https://inference-api.nousresearch.com/v1" autocomplete="off"></label><label>Nous Portal API key <span class="optional">write-only · leave blank to keep</span><input id="nousApiKey" name="nous_api_key" type="password" maxlength="512" placeholder="Enter a new key to rotate" autocomplete="new-password"></label><label>SearXNG base URL<input id="firecrawlBaseUrl" name="firecrawl_base_url" type="url" required placeholder="https://api.firecrawl.dev/v1" autocomplete="off"></label><label>Firecrawl API key <span class="optional">write-only · leave blank to keep</span><input id="firecrawlApiKey" name="firecrawl_api_key" type="password" maxlength="512" placeholder="Enter a new key to rotate" autocomplete="new-password"></label></div><div class="form-footer"><p id="aiProviderMessage" class="form-message" role="status" aria-live="polite"></p><button class="button primary" id="saveAiProviderBtn" type="submit">Save provider settings</button></div></form></article><aside class="panel ai-provider-status-panel"><div class="panel-heading"><div><p class="eyebrow">SAFE STATUS</p><h3>Connection status</h3></div><button class="button ghost compact" id="testAiProviderBtn" type="button">Test connection</button></div><div id="aiProviderStatus" aria-live="polite"><div class="detail-loading">Sign in to load provider status.</div></div></aside></div></section>
         <footer><span>ProspectOS</span><span>Evidence-led prospecting · <a href="#explorer">Review queue</a></span></footer>
       </div>
     </main>

docker-compose.yml

@@ -25,6 +25,8 @@ services:
       NOUS_MODEL: ${NOUS_MODEL:-Hermes-4-405B}
       NOUS_BASE_URL: ${NOUS_BASE_URL:-https://inference-api.nousresearch.com/v1}
       NOUS_ALLOWED_HOSTS: ${NOUS_ALLOWED_HOSTS:-inference-api.nousresearch.com}
+      SEARXNG_BASE_URL: ${SEARXNG_BASE_URL:-http://searxng:8080}
+      SEARXNG_ALLOWED_HOSTS: ${SEARXNG_ALLOWED_HOSTS:-searxng}
       FIRECRAWL_API_KEY: ${FIRECRAWL_API_KEY:-}
       FIRECRAWL_BASE_URL: ${FIRECRAWL_BASE_URL:-https://api.firecrawl.dev/v2}
       FIRECRAWL_ALLOWED_HOSTS: ${FIRECRAWL_ALLOWED_HOSTS:-api.firecrawl.dev}
@@ -52,6 +54,8 @@ services:
       retries: 5
       start_period: 5s
     restart: unless-stopped
+    networks:
+      - prospect_internal
 
   web:
     build:
@@ -80,7 +84,34 @@ services:
       retries: 5
       start_period: 3s
     restart: unless-stopped
+    networks:
+      - prospect_internal
+
+  searxng:
+    image: searxng/searxng:latest
+    environment:
+      SEARXNG_SECRET_KEY: ${SEARXNG_SECRET_KEY:-change-me-in-production}
+    volumes:
+      - ./searxng/settings.yml:/etc/searxng/settings.yml:ro
+    read_only: true
+    tmpfs:
+      - /tmp
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL
+    expose:
+      - "8080"
+    networks:
+      - prospect_internal
+      - searxng_egress
+    restart: unless-stopped
 
 volumes:
   prospect_api_data:
     name: prospect-platform-api-data
+
+networks:
+  prospect_internal:
+    internal: true
+  searxng_egress:

docs/DEPLOYMENT.md

@@ -17,8 +17,7 @@ After the first successful login, remove both values from `.env`, restart the AP
 
 Copy `.env.example` to an untracked deployment environment file. Production requires a secret-manager supplied `SESSION_SECRET` of at least 32 characters and refuses `AUTOMATED_OUTREACH_ENABLED=true`. Keep bootstrap credentials one-time only; remove and rotate them after provisioning. Never place secrets in images, Compose YAML, logs, backups, or public web roots.
 
-For optional native Nous Portal Chat Completions tool-calling discovery, set these
+For criteria-first AI web research without a paid scraper, use the self-hosted mode:
-server-side variables:
 
 ```dotenv
 AI_RESEARCH_PROVIDER=nous_portal
@@ -26,20 +25,24 @@ NOUS_API_KEY=<Nous Portal API key>
 NOUS_MODEL=Hermes-4-405B
 NOUS_BASE_URL=https://inference-api.nousresearch.com/v1
 NOUS_ALLOWED_HOSTS=inference-api.nousresearch.com
-FIRECRAWL_API_KEY=<Firecrawl API key>
+SEARXNG_BASE_URL=http://searxng:8080
-FIRECRAWL_BASE_URL=https://api.firecrawl.dev/v2
+SEARXNG_ALLOWED_HOSTS=searxng
-FIRECRAWL_ALLOWED_HOSTS=api.firecrawl.dev
+SEARXNG_SECRET_KEY=<random secret>
 ```
 
-The adapter calls Nous at `/chat/completions` with strict `web_search` and
+Compose starts SearXNG without publishing a host port. The API reaches it only
-`scrape_website` function tools. Tool calls are executed only against the
+on the internal Compose network; SearXNG alone has a separate egress network for
-allowlisted Firecrawl-compatible API, capped at 4 calls and 16 KiB per tool
+its upstream search engines. `web_search` calls SearXNG's JSON endpoint and
-result. Prompt-injection-shaped criteria are rejected and tool/page content is
+`scrape_website` uses the API's bounded SSRF-safe scanner. No Firecrawl key is
-untrusted data. The final model response is parsed only as structured JSON
+required. Firecrawl variables remain an optional legacy compatibility fallback.
-HTTPS targets; the existing crawler performs SSRF validation and persists
+
-fetched-page evidence. Missing either key, unavailable providers, unsafe base
+The Nous adapter calls `/chat/completions` with strict `web_search` and
-URLs, malformed tool calls, oversized responses, and exhausted budgets fail
+`scrape_website` function tools. Tool calls are capped at 4 and each result at
-closed. Status metadata never includes secrets.
+16 KiB; criteria, model output, page text, redirects, and candidate URLs remain
+bounded and untrusted. Only structured HTTPS targets are accepted and the
+existing crawler fetches/persists evidence. Missing/unsafe configuration,
+unavailable providers, malformed calls, oversized responses, SSRF targets, and
+exhausted budgets fail closed. Status metadata never includes secrets.
 
 The prior OpenAI Responses and generic provider variables remain supported only
 as compatibility adapters.

searxng/settings.yml

@@ -0,0 +1,26 @@
+use_default_settings: true
+server:
+  bind_address: 0.0.0.0
+  port: 8080
+  secret_key: "${SEARXNG_SECRET_KEY}"
+  limiter: true
+  image_proxy: false
+  public_instance: false
+search:
+  formats:
+    - html
+    - json
+ui:
+  static_use_hash: true
+outgoing:
+  request_timeout: 8.0
+  max_request_timeout: 10.0
+  retries: 0
+categories_as_tabs:
+  general:
+    engines:
+      - google
+      - bing
+      - duckduckgo
+      - brave
+      - wikipedia