diff --git a/.env.example b/.env.example index 1911579..0d57368 100644 --- a/.env.example +++ b/.env.example @@ -20,7 +20,7 @@ AUTOMATED_OUTREACH_ENABLED=false # they are never copied into API responses. Connectivity tests use bounded GET # requests only and report outbound_calls=false (no outreach is implemented). # The following variables are legacy/bootstrap fallback values only. -AI_RESEARCH_PROVIDER= +# AI_RESEARCH_PROVIDER is set below for the primary self-hosted mode. # NOUS_MODEL=Hermes-4-405B # NOUS_BASE_URL=https://inference-api.nousresearch.com/v1 # NOUS_ALLOWED_HOSTS=inference-api.nousresearch.com @@ -28,11 +28,17 @@ AI_RESEARCH_PROVIDER= # FIRECRAWL_BASE_URL=https://api.firecrawl.dev/v2 # FIRECRAWL_ALLOWED_HOSTS=api.firecrawl.dev # FIRECRAWL_API_KEY= -AI_RESEARCH_PROVIDER= +# Primary no-paid-scraper mode: Nous orchestrates, internal SearXNG searches, +# and the API's SSRF-safe crawler reads pages. No Firecrawl key is required. +AI_RESEARCH_PROVIDER=nous_portal NOUS_API_KEY= NOUS_MODEL=Hermes-4-405B NOUS_BASE_URL=https://inference-api.nousresearch.com/v1 NOUS_ALLOWED_HOSTS=inference-api.nousresearch.com +SEARXNG_BASE_URL=http://searxng:8080 +SEARXNG_ALLOWED_HOSTS=searxng +SEARXNG_SECRET_KEY= +# Optional legacy fallback only; never required by self-hosted mode. FIRECRAWL_API_KEY= FIRECRAWL_BASE_URL=https://api.firecrawl.dev/v2 FIRECRAWL_ALLOWED_HOSTS=api.firecrawl.dev diff --git a/apps/api/README.md b/apps/api/README.md index b2222e0..efa4987 100644 --- a/apps/api/README.md +++ b/apps/api/README.md @@ -31,13 +31,12 @@ The provider status is available at authenticated `GET `/api/v1/discovery/provider-status` alias is retained). Configure only on the server. The native Nous adapter uses OpenAI-compatible Chat Completions at `https://inference-api.nousresearch.com/v1/chat/completions` and strict -`web_search`/`scrape_website` tools backed by an allowlisted Firecrawl-compatible -API. Configure server-side `NOUS_API_KEY`, `NOUS_MODEL`, `NOUS_BASE_URL`, -`NOUS_ALLOWED_HOSTS`, `FIRECRAWL_API_KEY`, `FIRECRAWL_BASE_URL`, and -`FIRECRAWL_ALLOWED_HOSTS` with `AI_RESEARCH_PROVIDER=nous_portal`. Tool calls, -responses, criteria, and results are bounded; page text is untrusted; only -structured HTTPS targets are accepted and the existing SSRF-safe crawler fetches -and persists evidence. Status is fail-closed and never returns secrets. +`web_search`/`scrape_website` tools. In the primary no-paid-scraper mode, +`web_search` uses internal SearXNG (`SEARXNG_BASE_URL`, normally +`http://searxng:8080`) and `scrape_website` uses the existing SSRF-safe scanner; +configure server-side `NOUS_API_KEY`, `NOUS_MODEL`, `NOUS_BASE_URL`, +`NOUS_ALLOWED_HOSTS`, `SEARXNG_ALLOWED_HOSTS`, and `SEARXNG_SECRET_KEY`. +Firecrawl settings are optional legacy compatibility only. All protected endpoints require the server-side session cookie. Every query is constrained by the authenticated user's `organization_id`; IDs from another tenant behave as not found and must not disclose whether a record exists. diff --git a/apps/api/app/ai_research.py b/apps/api/app/ai_research.py index 78142c2..10fe4cb 100644 --- a/apps/api/app/ai_research.py +++ b/apps/api/app/ai_research.py @@ -1,9 +1,8 @@ """Fail-closed AI web-research providers for criteria-first discovery. -The native Nous adapter is a locator only. It may ask an approved Firecrawl- -compatible service for bounded search/scrape observations, but only structured -HTTPS targets returned by the model are handed to discovery.py. The existing -crawler performs the final SSRF validation and persists the evidence. +Nous is the model/orchestrator. Search is provided by an internal-only SearXNG +service and page reads use ProspectOS's own SSRF-safe website scanner. Firecrawl +remains a backwards-compatible legacy path when explicitly configured. """ from __future__ import annotations @@ -11,10 +10,11 @@ import json import os import re import sqlite3 +from html import unescape from urllib.parse import urlparse from urllib.request import Request, urlopen -from .website_scanner import validate_url +from .website_scanner import scan_website, validate_url MAX_CANDIDATES = 50 MAX_CRITERIA_BYTES = 8192 @@ -61,7 +61,7 @@ def _config(): if row: from .provider_config import decrypt credentials = json.loads(decrypt(row["credentials_ciphertext"])) if row["credentials_ciphertext"] else {} - return {"provider": row["provider"], "model": row["model"], "nous_url": row["nous_base_url"], "nous_allowed": {urlparse(row["nous_base_url"]).hostname}, "nous_key": credentials.get("nous_api_key", ""), "firecrawl_url": row["firecrawl_base_url"], "firecrawl_allowed": {urlparse(row["firecrawl_base_url"]).hostname}, "firecrawl_key": credentials.get("firecrawl_api_key", "")} + return {"provider": row["provider"], "model": row["model"], "nous_url": row["nous_base_url"], "nous_allowed": {urlparse(row["nous_base_url"]).hostname}, "nous_key": credentials.get("nous_api_key", ""), "searxng_url": os.environ.get("SEARXNG_BASE_URL", "").strip(), "searxng_allowed": _hosts("SEARXNG_ALLOWED_HOSTS", "searxng"), "firecrawl_url": row["firecrawl_base_url"], "firecrawl_allowed": {urlparse(row["firecrawl_base_url"]).hostname}, "firecrawl_key": credentials.get("firecrawl_api_key", "")} except Exception: return {"provider": "", "model": "", "endpoint": "", "allowed": set(), "api_key": ""} provider = os.environ.get("AI_RESEARCH_PROVIDER", "").strip().lower() @@ -73,7 +73,8 @@ def _config(): return {"provider": provider, "model": os.environ.get("NOUS_MODEL", "Hermes-4-405B").strip(), "nous_url": os.environ.get("NOUS_BASE_URL", "https://inference-api.nousresearch.com/v1").strip(), "nous_allowed": _hosts("NOUS_ALLOWED_HOSTS", "inference-api.nousresearch.com"), - "nous_key": nous_key, "firecrawl_url": os.environ.get("FIRECRAWL_BASE_URL", "https://api.firecrawl.dev/v2").strip(), + "nous_key": nous_key, "searxng_url": os.environ.get("SEARXNG_BASE_URL", "").strip(), + "searxng_allowed": _hosts("SEARXNG_ALLOWED_HOSTS", "searxng"), "firecrawl_url": os.environ.get("FIRECRAWL_BASE_URL", "https://api.firecrawl.dev/v2").strip(), "firecrawl_allowed": _hosts("FIRECRAWL_ALLOWED_HOSTS", "api.firecrawl.dev"), "firecrawl_key": firecrawl_key} api_key = generic_key if provider == "openai_web_search": api_key = api_key or os.environ.get("OPENAI_API_KEY", "").strip() @@ -85,9 +86,17 @@ def _config(): def _endpoint(): cfg = _config() if cfg["provider"] in NOUS_PROVIDER_IDS: - if not cfg["model"] or not cfg["nous_key"] or not cfg["firecrawl_key"]: + if not cfg["model"] or not cfg["nous_key"]: raise AIResearchConfigError("not_configured") - return cfg, _safe_endpoint(cfg["nous_url"], cfg["nous_allowed"]), _safe_endpoint(cfg["firecrawl_url"], cfg["firecrawl_allowed"]) + nous = _safe_endpoint(cfg["nous_url"], cfg["nous_allowed"]) + if cfg.get("searxng_url"): + parsed = urlparse(cfg["searxng_url"]); host = (parsed.hostname or "").lower().rstrip(".") + if parsed.scheme != "http" or host not in cfg["searxng_allowed"] or parsed.username or parsed.password or parsed.query or parsed.fragment: + raise AIResearchConfigError("unsafe_provider") + return cfg, nous, cfg["searxng_url"].rstrip("/") + if not cfg["firecrawl_key"]: + raise AIResearchConfigError("not_configured") + return cfg, nous, _safe_endpoint(cfg["firecrawl_url"], cfg["firecrawl_allowed"]) if not cfg["provider"] or not cfg["endpoint"] or not cfg["model"]: raise AIResearchConfigError("not_configured") if cfg["provider"] not in APPROVED_PROVIDER_IDS: raise AIResearchConfigError("unapproved_provider") parsed = urlparse(cfg["endpoint"]); host = (parsed.hostname or "").lower().rstrip(".") @@ -99,10 +108,13 @@ def _endpoint(): def provider_status() -> dict[str, object]: cfg = _config() if cfg["provider"] in NOUS_PROVIDER_IDS: - try: _, nous_url, firecrawl_url = _endpoint() + try: _, nous_url, tool_url = _endpoint() except AIResearchConfigError as exc: return {"provider": cfg["provider"], "status": str(exc), "configured": False, "network_enabled": False, "outbound_calls": False} - return {"provider": cfg["provider"], "model": cfg["model"], "nous_host": urlparse(nous_url).hostname, "firecrawl_host": urlparse(firecrawl_url).hostname, "status": "ready", "configured": True, "network_enabled": True, "outbound_calls": True, "max_candidates": MAX_CANDIDATES, "max_tool_calls": MAX_TOOL_CALLS} + result = {"provider": cfg["provider"], "model": cfg["model"], "nous_host": urlparse(nous_url).hostname, "status": "ready", "configured": True, "network_enabled": True, "outbound_calls": True, "max_candidates": MAX_CANDIDATES, "max_tool_calls": MAX_TOOL_CALLS} + if cfg.get("searxng_url"): result.update({"search_provider": "searxng", "searxng_host": urlparse(tool_url).hostname, "scrape_provider": "native_crawler"}) + else: result.update({"search_provider": "firecrawl_legacy", "firecrawl_host": urlparse(tool_url).hostname, "scrape_provider": "firecrawl_legacy"}) + return result if not cfg["provider"] and not cfg["endpoint"]: return {"provider": "", "status": "not_configured", "configured": False, "network_enabled": False, "outbound_calls": False} if cfg["provider"] and cfg["provider"] not in APPROVED_PROVIDER_IDS: return {"provider": cfg["provider"], "status": "unapproved_provider", "configured": False, "network_enabled": False, "outbound_calls": False} try: _, endpoint, _ = _endpoint() @@ -147,17 +159,35 @@ def _post(url: str, key: str, body_obj: dict, *, limit: int = MAX_RESPONSE_BYTES return payload +def _page_text(html: str) -> str: + """Return small, non-executable page text for the model.""" + text = re.sub(r"(?is)<(script|style|noscript).*?>.*?", " ", html or "") + text = re.sub(r"(?s)<[^>]*>", " ", text) + return re.sub(r"\s+", " ", unescape(text)).strip()[:MAX_TOOL_RESULT_BYTES] + + def _tool_result(cfg, name: str, arguments: str, remaining: int) -> dict: if remaining < 0: raise AIResearchConfigError("tool_budget_exhausted") try: args = json.loads(arguments or "{}") except json.JSONDecodeError as exc: raise AIResearchConfigError("invalid_tool_arguments") from exc if not isinstance(args, dict): raise AIResearchConfigError("invalid_tool_arguments") - base = _safe_endpoint(cfg["firecrawl_url"], cfg["firecrawl_allowed"]) if name == "web_search": query = args.get("query") try: requested_limit = int(args.get("limit", MAX_SEARCH_RESULTS)) except (TypeError, ValueError) as exc: raise AIResearchConfigError("invalid_tool_arguments") from exc if not isinstance(query, str) or not query.strip() or len(query.encode()) > 1000 or not 1 <= requested_limit <= MAX_SEARCH_RESULTS: raise AIResearchConfigError("invalid_tool_arguments") + if cfg.get("searxng_url"): + request = Request(cfg["searxng_url"].rstrip("/") + "/search" + "?q=" + __import__("urllib.parse", fromlist=["quote"]).quote(query.strip()) + "&format=json", headers={"Accept": "application/json"}, method="GET") + try: + with urlopen(request, timeout=TIMEOUT_SECONDS) as response: raw = response.read(MAX_TOOL_RESULT_BYTES + 1) + except Exception as exc: raise AIResearchConfigError("provider_unavailable") from exc + if len(raw) > MAX_TOOL_RESULT_BYTES: raise AIResearchConfigError("provider_response_too_large") + try: payload = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: raise AIResearchConfigError("invalid_provider_response") from exc + if not isinstance(payload, dict): raise AIResearchConfigError("invalid_provider_response") + results = [{"title": x.get("title", "")[:300], "url": x.get("url", ""), "snippet": x.get("content", "")[:500]} for x in payload.get("results", [])[:requested_limit] if isinstance(x, dict) and isinstance(x.get("url"), str)] + return {"type": "web_search_result", "data": results} + base = _safe_endpoint(cfg["firecrawl_url"], cfg["firecrawl_allowed"]) payload = _post(base + "/search", cfg["firecrawl_key"], {"query": query.strip(), "limit": requested_limit}, limit=MAX_TOOL_RESULT_BYTES) return {"type": "web_search_result", "data": payload.get("data", payload.get("results", []))} if name == "scrape_website": @@ -165,6 +195,11 @@ def _tool_result(cfg, name: str, arguments: str, remaining: int) -> dict: if not isinstance(target, str) or urlparse(target).scheme != "https": raise AIResearchConfigError("invalid_tool_arguments") try: safe = validate_url(target) except (TypeError, ValueError) as exc: raise AIResearchConfigError("unsafe_target_url") from exc + if cfg.get("searxng_url"): + scanned = scan_website(safe, max_bytes=MAX_TOOL_RESULT_BYTES) + if scanned.get("error_code"): raise AIResearchConfigError("scrape_" + str(scanned["error_code"])) + return {"type": "scrape_result", "url": scanned.get("final_url") or safe, "data": {"status": scanned.get("status"), "title": scanned.get("title", ""), "description": scanned.get("meta_description", ""), "headings": scanned.get("headings", [])[:20], "content": _page_text(scanned.get("html", ""))}} + base = _safe_endpoint(cfg["firecrawl_url"], cfg["firecrawl_allowed"]) payload = _post(base + "/scrape", cfg["firecrawl_key"], {"url": safe, "formats": ["markdown"], "onlyMainContent": True}, limit=MAX_TOOL_RESULT_BYTES) return {"type": "scrape_result", "url": safe, "data": payload.get("data", payload)} raise AIResearchConfigError("unknown_tool") diff --git a/apps/api/app/config.py b/apps/api/app/config.py index 7ff7186..ee4089b 100644 --- a/apps/api/app/config.py +++ b/apps/api/app/config.py @@ -2,6 +2,7 @@ from dataclasses import dataclass import os from pathlib import Path +from urllib.parse import urlparse class ConfigError(ValueError): @@ -17,6 +18,8 @@ class Config: log_level: str ai_research_provider: str ai_research_model: str + nous_base_url: str = "" + searxng_base_url: str = "" def _env(values, key, default=""): @@ -42,6 +45,15 @@ def load_config(values=None): log_level = _env(values, "LOG_LEVEL", "INFO").upper() if log_level not in {"QUIET", "ERROR", "WARNING", "INFO", "DEBUG"}: raise ConfigError("LOG_LEVEL is invalid") - return Config(app_env, data_dir, secret, False, log_level, - _env(values, "AI_RESEARCH_PROVIDER").lower(), - _env(values, "AI_RESEARCH_PROVIDER_MODEL")) + provider = _env(values, "AI_RESEARCH_PROVIDER").lower() + nous_url = _env(values, "NOUS_BASE_URL", "https://inference-api.nousresearch.com/v1") + searx_url = _env(values, "SEARXNG_BASE_URL", "http://searxng:8080") + if provider in {"nous_portal", "nous_portal_web_research"}: + parsed = urlparse(nous_url); allowed = {x.strip().lower() for x in _env(values, "NOUS_ALLOWED_HOSTS", "inference-api.nousresearch.com").split(",") if x.strip()} + if parsed.scheme != "https" or parsed.hostname not in allowed or parsed.username or parsed.password or parsed.query or parsed.fragment: + raise ConfigError("NOUS_BASE_URL is unsafe") + parsed = urlparse(searx_url); allowed = {x.strip().lower() for x in _env(values, "SEARXNG_ALLOWED_HOSTS", "searxng").split(",") if x.strip()} + if parsed.scheme != "http" or parsed.hostname not in allowed or parsed.username or parsed.password or parsed.query or parsed.fragment: + raise ConfigError("SEARXNG_BASE_URL is unsafe") + return Config(app_env, data_dir, secret, False, log_level, provider, + _env(values, "AI_RESEARCH_PROVIDER_MODEL") or _env(values, "NOUS_MODEL", "Hermes-4-405B"), nous_url, searx_url) diff --git a/apps/api/app/main.py b/apps/api/app/main.py index a1f77ee..7d3b725 100644 --- a/apps/api/app/main.py +++ b/apps/api/app/main.py @@ -496,7 +496,7 @@ class ApiHandler(BaseHTTPRequestHandler): except Exception: old = {} for name in ("nous_api_key", "firecrawl_api_key"): if name not in credentials and name in old: credentials[name] = old[name] - if config["enabled"] and any(name not in credentials for name in ("nous_api_key", "firecrawl_api_key")): + if config["enabled"] and ("nous_api_key" not in credentials or ("SEARXNG_BASE_URL" not in os.environ and "firecrawl_api_key" not in credentials)): return self.send_json(400, {"error": "provider_credentials_required"}) ciphertext = encrypt_provider_secret(json.dumps(credentials, sort_keys=True)) if credentials else "" fingerprint = hashlib.sha256(json.dumps(credentials, sort_keys=True).encode()).hexdigest() if credentials else "" diff --git a/apps/api/app/provider_config.py b/apps/api/app/provider_config.py index c562c36..128a31d 100644 --- a/apps/api/app/provider_config.py +++ b/apps/api/app/provider_config.py @@ -86,7 +86,7 @@ def validate_payload(payload: dict) -> dict: for field, default in urls.items(): value = str(payload.get(field, default)).strip().rstrip("/") parsed = urlparse(value) - if parsed.scheme != "https" or not parsed.hostname or parsed.username or parsed.password or parsed.fragment or parsed.query: + if (parsed.scheme != "https" and not (field == "firecrawl_base_url" and parsed.scheme == "http" and parsed.hostname == "searxng")) or not parsed.hostname or parsed.username or parsed.password or parsed.fragment or parsed.query: raise ValueError("unsafe_provider_url") urls[field] = value credentials = payload.get("credentials", {}) diff --git a/apps/api/tests/test_ai_research.py b/apps/api/tests/test_ai_research.py index 796c05a..6cb5ab8 100644 --- a/apps/api/tests/test_ai_research.py +++ b/apps/api/tests/test_ai_research.py @@ -11,7 +11,7 @@ class AIResearchTests(unittest.TestCase): "AI_RESEARCH_PROVIDER", "AI_RESEARCH_PROVIDER_MODEL", "AI_RESEARCH_PROVIDER_URL", "AI_RESEARCH_PROVIDER_ALLOWED_HOSTS", "AI_RESEARCH_PROVIDER_API_KEY", "OPENAI_API_KEY", "NOUS_API_KEY", "NOUS_MODEL", "NOUS_BASE_URL", "NOUS_ALLOWED_HOSTS", - "FIRECRAWL_API_KEY", "FIRECRAWL_BASE_URL", "FIRECRAWL_ALLOWED_HOSTS", + "FIRECRAWL_API_KEY", "FIRECRAWL_BASE_URL", "FIRECRAWL_ALLOWED_HOSTS", "SEARXNG_BASE_URL", "SEARXNG_ALLOWED_HOSTS", ) def tearDown(self): @@ -164,6 +164,27 @@ class AIResearchTests(unittest.TestCase): with self.assertRaisesRegex(AIResearchConfigError, "not_configured"): research({"keywords": ["solar"]}, 5) + def test_self_hosted_searxng_search_and_native_scrape_are_bounded(self): + self.configure_nous(); os.environ.pop("FIRECRAWL_API_KEY", None) + os.environ.update({"SEARXNG_BASE_URL": "http://searxng:8080", "SEARXNG_ALLOWED_HOSTS": "searxng"}) + responses = [ + self.response({"choices": [{"message": {"tool_calls": [{"id": "s", "type": "function", "function": {"name": "web_search", "arguments": '{"query":"solar","limit":1}'}}]}}]}), + self.response({"results": [{"title": "Solar", "url": "https://solar.example", "content": "snippet"}]}), + self.response({"choices": [{"message": {"tool_calls": [{"id": "p", "type": "function", "function": {"name": "scrape_website", "arguments": '{"url":"https://solar.example"}'}}]}}]}), + self.response({"choices": [{"message": {"content": '{"targets":[{"url":"https://solar.example"}]}'}}]}), + ] + scan = {"status": 200, "final_url": "https://solar.example", "title": "Solar", "meta_description": "", "headings": [], "html": "

Solar

Public page

", "error_code": None} + with patch("app.ai_research.urlopen", side_effect=responses), patch("app.ai_research.validate_url", side_effect=lambda url, **_: url), patch("app.ai_research.scan_website", return_value=scan) as scanner: + self.assertEqual(research({"keywords": ["solar"]}, 3), ["https://solar.example"]) + scanner.assert_called_once_with("https://solar.example", max_bytes=16 * 1024) + self.assertEqual(provider_status()["search_provider"], "searxng") + self.assertEqual(provider_status()["scrape_provider"], "native_crawler") + self.assertNotIn("nous-secret", json.dumps(provider_status())) + + def test_self_hosted_unsafe_endpoint_fails_closed(self): + self.configure_nous(); os.environ.update({"SEARXNG_BASE_URL": "http://127.0.0.1:8080", "SEARXNG_ALLOWED_HOSTS": "searxng"}) + self.assertEqual(provider_status()["status"], "unsafe_provider") + if __name__ == "__main__": unittest.main() diff --git a/apps/web/index.html b/apps/web/index.html index 41693f3..0538891 100644 --- a/apps/web/index.html +++ b/apps/web/index.html @@ -121,7 +121,7 @@

ADD RULE

Add suppression

Explicit confirmation required

REGISTER

Current suppressions

Sign in to load suppressions.

GOVERNANCE

Outreach provider policy

View approved provider status without exposing credentials or secrets.

Sending is disabled by default. This panel is status-only. Provider configuration never creates a send trigger, and no credentials are displayed.
Sign in to load provider policy.
-

SYSTEM CONFIGURATION

AI Provider Settings

Configure the approved Nous Portal and Firecrawl services used by workspace AI research.

Admin-only and write-only credentials. Keys are sent only over the authenticated API, are never displayed or stored in this browser, and updating settings does not start discovery or AI work.

PROVIDER CONFIGURATION

Service connection

Approved providers only

+

SYSTEM CONFIGURATION

AI Provider Settings

Configure the approved Nous Portal service and the internal SearXNG service used by workspace AI research.

Admin-only and write-only credentials. Keys are sent only over the authenticated API, are never displayed or stored in this browser, and updating settings does not start discovery or AI work.

PROVIDER CONFIGURATION

Service connection

Approved providers only

diff --git a/docker-compose.yml b/docker-compose.yml index 946457a..59f5399 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,6 +25,8 @@ services: NOUS_MODEL: ${NOUS_MODEL:-Hermes-4-405B} NOUS_BASE_URL: ${NOUS_BASE_URL:-https://inference-api.nousresearch.com/v1} NOUS_ALLOWED_HOSTS: ${NOUS_ALLOWED_HOSTS:-inference-api.nousresearch.com} + SEARXNG_BASE_URL: ${SEARXNG_BASE_URL:-http://searxng:8080} + SEARXNG_ALLOWED_HOSTS: ${SEARXNG_ALLOWED_HOSTS:-searxng} FIRECRAWL_API_KEY: ${FIRECRAWL_API_KEY:-} FIRECRAWL_BASE_URL: ${FIRECRAWL_BASE_URL:-https://api.firecrawl.dev/v2} FIRECRAWL_ALLOWED_HOSTS: ${FIRECRAWL_ALLOWED_HOSTS:-api.firecrawl.dev} @@ -52,6 +54,8 @@ services: retries: 5 start_period: 5s restart: unless-stopped + networks: + - prospect_internal web: build: @@ -80,7 +84,34 @@ services: retries: 5 start_period: 3s restart: unless-stopped + networks: + - prospect_internal + + searxng: + image: searxng/searxng:latest + environment: + SEARXNG_SECRET_KEY: ${SEARXNG_SECRET_KEY:-change-me-in-production} + volumes: + - ./searxng/settings.yml:/etc/searxng/settings.yml:ro + read_only: true + tmpfs: + - /tmp + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + expose: + - "8080" + networks: + - prospect_internal + - searxng_egress + restart: unless-stopped volumes: prospect_api_data: name: prospect-platform-api-data + +networks: + prospect_internal: + internal: true + searxng_egress: diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 95c12a2..de9614c 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -17,8 +17,7 @@ After the first successful login, remove both values from `.env`, restart the AP Copy `.env.example` to an untracked deployment environment file. Production requires a secret-manager supplied `SESSION_SECRET` of at least 32 characters and refuses `AUTOMATED_OUTREACH_ENABLED=true`. Keep bootstrap credentials one-time only; remove and rotate them after provisioning. Never place secrets in images, Compose YAML, logs, backups, or public web roots. -For optional native Nous Portal Chat Completions tool-calling discovery, set these -server-side variables: +For criteria-first AI web research without a paid scraper, use the self-hosted mode: ```dotenv AI_RESEARCH_PROVIDER=nous_portal @@ -26,20 +25,24 @@ NOUS_API_KEY= NOUS_MODEL=Hermes-4-405B NOUS_BASE_URL=https://inference-api.nousresearch.com/v1 NOUS_ALLOWED_HOSTS=inference-api.nousresearch.com -FIRECRAWL_API_KEY= -FIRECRAWL_BASE_URL=https://api.firecrawl.dev/v2 -FIRECRAWL_ALLOWED_HOSTS=api.firecrawl.dev +SEARXNG_BASE_URL=http://searxng:8080 +SEARXNG_ALLOWED_HOSTS=searxng +SEARXNG_SECRET_KEY= ``` -The adapter calls Nous at `/chat/completions` with strict `web_search` and -`scrape_website` function tools. Tool calls are executed only against the -allowlisted Firecrawl-compatible API, capped at 4 calls and 16 KiB per tool -result. Prompt-injection-shaped criteria are rejected and tool/page content is -untrusted data. The final model response is parsed only as structured JSON -HTTPS targets; the existing crawler performs SSRF validation and persists -fetched-page evidence. Missing either key, unavailable providers, unsafe base -URLs, malformed tool calls, oversized responses, and exhausted budgets fail -closed. Status metadata never includes secrets. +Compose starts SearXNG without publishing a host port. The API reaches it only +on the internal Compose network; SearXNG alone has a separate egress network for +its upstream search engines. `web_search` calls SearXNG's JSON endpoint and +`scrape_website` uses the API's bounded SSRF-safe scanner. No Firecrawl key is +required. Firecrawl variables remain an optional legacy compatibility fallback. + +The Nous adapter calls `/chat/completions` with strict `web_search` and +`scrape_website` function tools. Tool calls are capped at 4 and each result at +16 KiB; criteria, model output, page text, redirects, and candidate URLs remain +bounded and untrusted. Only structured HTTPS targets are accepted and the +existing crawler fetches/persists evidence. Missing/unsafe configuration, +unavailable providers, malformed calls, oversized responses, SSRF targets, and +exhausted budgets fail closed. Status metadata never includes secrets. The prior OpenAI Responses and generic provider variables remain supported only as compatibility adapters. diff --git a/searxng/settings.yml b/searxng/settings.yml new file mode 100644 index 0000000..9c1ddfc --- /dev/null +++ b/searxng/settings.yml @@ -0,0 +1,26 @@ +use_default_settings: true +server: + bind_address: 0.0.0.0 + port: 8080 + secret_key: "${SEARXNG_SECRET_KEY}" + limiter: true + image_proxy: false + public_instance: false +search: + formats: + - html + - json +ui: + static_use_hash: true +outgoing: + request_timeout: 8.0 + max_request_timeout: 10.0 + retries: 0 +categories_as_tabs: + general: + engines: + - google + - bing + - duckduckgo + - brave + - wikipedia