Add secure admin AI provider configuration

apps/api/app/ai_research.py

@@ -10,6 +10,7 @@ from __future__ import annotations
 import json
 import os
 import re
+import sqlite3
 from urllib.parse import urlparse
 from urllib.request import Request, urlopen
 
@@ -43,7 +44,26 @@ def _safe_endpoint(value: str, allowed: set[str]) -> str:
     return value.rstrip("/")
 
 
+_DB_PATH = ""
+_DB_ORG = "demo-tenant"
+
+
+def configure_db(path: str, organization_id: str = "demo-tenant") -> None:
+    global _DB_PATH, _DB_ORG
+    _DB_PATH, _DB_ORG = path, organization_id
+
+
 def _config():
+    if _DB_PATH:
+        try:
+            db = sqlite3.connect(_DB_PATH); db.row_factory = sqlite3.Row
+            row = db.execute("SELECT * FROM ai_remote_provider_configs WHERE organization_id=?", (_DB_ORG,)).fetchone(); db.close()
+            if row:
+                from .provider_config import decrypt
+                credentials = json.loads(decrypt(row["credentials_ciphertext"])) if row["credentials_ciphertext"] else {}
+                return {"provider": row["provider"], "model": row["model"], "nous_url": row["nous_base_url"], "nous_allowed": {urlparse(row["nous_base_url"]).hostname}, "nous_key": credentials.get("nous_api_key", ""), "firecrawl_url": row["firecrawl_base_url"], "firecrawl_allowed": {urlparse(row["firecrawl_base_url"]).hostname}, "firecrawl_key": credentials.get("firecrawl_api_key", "")}
+        except Exception:
+            return {"provider": "", "model": "", "endpoint": "", "allowed": set(), "api_key": ""}
     provider = os.environ.get("AI_RESEARCH_PROVIDER", "").strip().lower()
     # Nous uses its conventional key directly; no gateway or key translation is needed.
     nous_key = os.environ.get("NOUS_API_KEY", "").strip()

apps/api/app/main.py

@@ -15,9 +15,10 @@ if __package__ in (None, ""):
     from app.scoring import DEFAULT_RULES, signals_for_business, evaluate_score, SCORE_VERSION
     from app.ai_assistance import generate as generate_ai, input_fingerprint, provider_status, MAX_INPUT_ITEMS, MAX_OUTPUT_CHARS
     from app.discovery import discover as scoped_discover
-    from app.ai_research import provider_status as ai_research_provider_status, validate_criteria as validate_ai_research_criteria, AIResearchConfigError
+    from app.ai_research import provider_status as ai_research_provider_status, configure_db as configure_ai_research_db, validate_criteria as validate_ai_research_criteria, AIResearchConfigError
     from app.search_provider import provider_status as search_provider_status
     from app.config import load_config
+    from app.provider_config import validate_payload as validate_remote_provider, encrypt as encrypt_provider_secret, decrypt as decrypt_provider_secret, safe_status as remote_provider_status, test_connectivity as test_remote_connectivity
 else:
     from .domain import deduplication_key, deduplicate_businesses, is_suppressed, normalize_business, score_business, normalize_domain, normalize_phone, match_businesses
     from .sources import adapter_for, contains_secret
@@ -27,9 +28,10 @@ else:
     from .scoring import DEFAULT_RULES, signals_for_business, evaluate_score, SCORE_VERSION
     from .ai_assistance import generate as generate_ai, input_fingerprint, provider_status, MAX_INPUT_ITEMS, MAX_OUTPUT_CHARS
     from .discovery import discover as scoped_discover
-    from .ai_research import provider_status as ai_research_provider_status, validate_criteria as validate_ai_research_criteria, AIResearchConfigError
+    from .ai_research import provider_status as ai_research_provider_status, configure_db as configure_ai_research_db, validate_criteria as validate_ai_research_criteria, AIResearchConfigError
     from .search_provider import provider_status as search_provider_status
 from .config import load_config
+from .provider_config import validate_payload as validate_remote_provider, encrypt as encrypt_provider_secret, decrypt as decrypt_provider_secret, safe_status as remote_provider_status, test_connectivity as test_remote_connectivity
 ORGANIZATION_ID = "demo-tenant"
 SCHEMA = Path(__file__).resolve().parents[1] / "schema.sql"
 SESSION_DAYS = 7
@@ -476,7 +478,39 @@ class ApiHandler(BaseHTTPRequestHandler):
         reasons.append("network_send_disabled"); self.audit(db, user, "outreach_draft.send_blocked", f"{did}:network_send_disabled"); db.commit()
         return self.send_json(409, {"status": "blocked", "blocked_reasons": reasons, "network_send": False, "id": did})
 
+    def remote_ai_provider_config(self, db, user, payload=None, connectivity=False):
+        org = user["organization_id"]
+        row = db.execute("SELECT * FROM ai_remote_provider_configs WHERE organization_id=?", (org,)).fetchone()
+        if connectivity:
+            if user["role"] not in {"owner", "admin"}:
+                return self.send_json(403, {"error": "forbidden"})
+            return self.send_json(200, test_remote_connectivity(row))
+        if payload is not None:
+            if user["role"] not in {"owner", "admin"}:
+                return self.send_json(403, {"error": "forbidden"})
+            try:
+                config = validate_remote_provider(payload)
+                credentials = dict(config["credentials"])
+                if row:
+                    try: old = json.loads(decrypt_provider_secret(row["credentials_ciphertext"])) if row["credentials_ciphertext"] else {}
+                    except Exception: old = {}
+                    for name in ("nous_api_key", "firecrawl_api_key"):
+                        if name not in credentials and name in old: credentials[name] = old[name]
+                if config["enabled"] and any(name not in credentials for name in ("nous_api_key", "firecrawl_api_key")):
+                    return self.send_json(400, {"error": "provider_credentials_required"})
+                ciphertext = encrypt_provider_secret(json.dumps(credentials, sort_keys=True)) if credentials else ""
+                fingerprint = hashlib.sha256(json.dumps(credentials, sort_keys=True).encode()).hexdigest() if credentials else ""
+            except (ValueError, TypeError, RuntimeError) as exc:
+                return self.send_json(400, {"error": str(exc)})
+            db.execute("INSERT INTO ai_remote_provider_configs(organization_id,provider,model,enabled,nous_base_url,firecrawl_base_url,credentials_ciphertext,credentials_fingerprint) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(organization_id) DO UPDATE SET provider=excluded.provider,model=excluded.model,enabled=excluded.enabled,nous_base_url=excluded.nous_base_url,firecrawl_base_url=excluded.firecrawl_base_url,credentials_ciphertext=excluded.credentials_ciphertext,credentials_fingerprint=excluded.credentials_fingerprint,updated_at=CURRENT_TIMESTAMP", (org, config["provider"], config["model"], int(config["enabled"]), config["nous_base_url"], config["firecrawl_base_url"], ciphertext, fingerprint))
+            self.audit(db, user, "ai.remote_provider.updated", config["provider"]); db.commit()
+            row = db.execute("SELECT * FROM ai_remote_provider_configs WHERE organization_id=?", (org,)).fetchone()
+        return self.send_json(200, dict(remote_provider_status(row), organization_id=org))
+
     def ai_provider_config(self, db, user, payload=None):
+        remote = db.execute("SELECT * FROM ai_remote_provider_configs WHERE organization_id=?", (user["organization_id"],)).fetchone()
+        if remote or (payload and str(payload.get("provider", "")).lower() in {"nous_portal", "nous_portal_web_research"}):
+            return self.remote_ai_provider_config(db, user, payload)
         org = user["organization_id"]
         row = db.execute("SELECT * FROM ai_provider_configs WHERE organization_id=?", (org,)).fetchone()
         if payload is not None:
@@ -613,6 +647,7 @@ class ApiHandler(BaseHTTPRequestHandler):
             if path=="/api/v1/suppressions": return self.list_suppressions(db,org,parse_qs(parsed.query))
             if path=="/api/v1/ai-runs": return self.list_ai_runs(db,user,parse_qs(parsed.query))
             if path=="/api/v1/ai/provider-config": return self.ai_provider_config(db,user)
+            if path=="/api/v1/admin/ai-provider-config": return self.remote_ai_provider_config(db,user)
             if path=="/api/v1/outreach/drafts": return self.list_outreach_drafts(db,user,parse_qs(parsed.query))
             if path=="/api/v1/outreach/provider-config": return self.provider_config(db,user)
             if path in ("/api/v1/reports/pipeline","/api/v1/reports/outcomes","/api/v1/reports/activity"): return self.report(db,org,path.rsplit('/',1)[1],parse_qs(parsed.query))
@@ -1029,6 +1064,8 @@ class ApiHandler(BaseHTTPRequestHandler):
             if path=="/api/v1/saved-filters": return self.save_filter(payload,db,user)
             if path=="/api/v1/outreach/provider-config": return self.provider_config(db,user,payload)
             if path=="/api/v1/ai/provider-config": return self.ai_provider_config(db,user,payload)
+            if path=="/api/v1/admin/ai-provider-config": return self.remote_ai_provider_config(db,user,payload)
+            if path=="/api/v1/admin/ai-provider-config/test": return self.remote_ai_provider_config(db,user,connectivity=True)
             if path=="/api/v1/businesses/bulk-review": return self.bulk_review(payload,db,user)
             bits_ai=path.split("/")
             if len(bits_ai)==7 and bits_ai[:4]==["","api","v1","businesses"] and bits_ai[5]=="ai" and bits_ai[6]=="suggest": return self.suggest_ai(int(bits_ai[4]) if bits_ai[4].isdigit() else -1,payload,db,user)
@@ -1440,6 +1477,7 @@ def _job_worker(server):
 
 def create_server(host="127.0.0.1",port=8000,db_path="prospects.db"):
     load_config()
+    configure_ai_research_db(db_path)
     server=ThreadingHTTPServer((host,port),ApiHandler);server.db_path=db_path;connect(db_path).close();server.job_stop=threading.Event();server.job_wakeup=threading.Event();server.job_thread=threading.Thread(target=_job_worker,args=(server,),daemon=True);server.job_thread.start()
     original_close=server.server_close
     def close():

apps/api/app/provider_config.py

@@ -0,0 +1,129 @@
+"""Encrypted, tenant-scoped AI provider configuration storage.
+
+The key is generated under the private data volume and is never stored in SQLite.
+The small authenticated stream construction here uses HMAC-SHA256 for the
+keystream and integrity tag; ciphertext is prefixed with ``pc1`` and never
+returned by the API.
+"""
+from __future__ import annotations
+
+import base64
+import hashlib
+import hmac
+import json
+import os
+import secrets
+from pathlib import Path
+from urllib.parse import urlparse
+from urllib.request import Request, urlopen
+
+MAX_SECRET = 4096
+MAX_MODEL = 160
+ALLOWED_PROVIDERS = {"nous_portal", "nous_portal_web_research"}
+DEFAULT_NOUS_URL = "https://inference-api.nousresearch.com/v1"
+DEFAULT_FIRECRAWL_URL = "https://api.firecrawl.dev/v1"
+
+
+def key_path() -> Path:
+    return Path(os.environ.get("PROVIDER_CONFIG_KEY_FILE", "/data/provider-config.key")).expanduser()
+
+
+def _key() -> bytes:
+    path = key_path()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    if path.exists():
+        key = path.read_bytes()
+        if len(key) != 32:
+            raise RuntimeError("invalid_provider_config_key")
+        return key
+    key = secrets.token_bytes(32)
+    flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY
+    fd = os.open(path, flags, 0o600)
+    try:
+        os.write(fd, key)
+    finally:
+        os.close(fd)
+    os.chmod(path, 0o600)
+    return key
+
+
+def _stream(key: bytes, nonce: bytes, size: int) -> bytes:
+    return b"".join(hmac.new(key, nonce + i.to_bytes(8, "big"), hashlib.sha256).digest() for i in range((size + 31) // 32))[:size]
+
+
+def encrypt(value: str) -> str:
+    raw = value.encode("utf-8")
+    nonce = secrets.token_bytes(16)
+    cipher = bytes(a ^ b for a, b in zip(raw, _stream(_key(), nonce, len(raw))))
+    tag = hmac.new(_key(), nonce + cipher, hashlib.sha256).digest()
+    return "pc1:" + base64.urlsafe_b64encode(nonce + tag + cipher).decode("ascii")
+
+
+def decrypt(value: str) -> str:
+    if not isinstance(value, str) or not value.startswith("pc1:"):
+        raise ValueError("invalid_ciphertext")
+    raw = base64.urlsafe_b64decode(value[4:].encode("ascii"))
+    nonce, tag, cipher = raw[:16], raw[16:48], raw[48:]
+    key = _key()
+    if not hmac.compare_digest(tag, hmac.new(key, nonce + cipher, hashlib.sha256).digest()):
+        raise ValueError("invalid_ciphertext")
+    return bytes(a ^ b for a, b in zip(cipher, _stream(key, nonce, len(cipher)))).decode("utf-8")
+
+
+def validate_payload(payload: dict) -> dict:
+    if not isinstance(payload, dict):
+        raise ValueError("invalid_config")
+    provider = str(payload.get("provider", "nous_portal")).strip().lower()
+    if provider not in ALLOWED_PROVIDERS:
+        raise ValueError("invalid_provider")
+    model = str(payload.get("model", "Hermes-4-405B")).strip()
+    if not model or len(model) > MAX_MODEL:
+        raise ValueError("invalid_model")
+    enabled = payload.get("enabled", True)
+    if not isinstance(enabled, bool):
+        raise ValueError("invalid_enabled")
+    urls = {"nous_base_url": DEFAULT_NOUS_URL, "firecrawl_base_url": DEFAULT_FIRECRAWL_URL}
+    for field, default in urls.items():
+        value = str(payload.get(field, default)).strip().rstrip("/")
+        parsed = urlparse(value)
+        if parsed.scheme != "https" or not parsed.hostname or parsed.username or parsed.password or parsed.fragment or parsed.query:
+            raise ValueError("unsafe_provider_url")
+        urls[field] = value
+    credentials = payload.get("credentials", {})
+    if not isinstance(credentials, dict):
+        raise ValueError("invalid_credentials")
+    result = {"provider": provider, "model": model, "enabled": enabled, **urls, "credentials": {}}
+    for name in ("nous_api_key", "firecrawl_api_key"):
+        if name in credentials:
+            value = credentials[name]
+            if not isinstance(value, str) or not value or len(value) > MAX_SECRET:
+                raise ValueError("invalid_secret")
+            result["credentials"][name] = value
+    return result
+
+
+def safe_status(row, *, env_bootstrap=False) -> dict:
+    if not row:
+        return {"provider": "", "status": "not_configured", "configured": False, "enabled": False, "network_enabled": False, "outbound_calls": False, "source": "env_bootstrap" if env_bootstrap else "none"}
+    return {"provider": row["provider"], "model": row["model"], "enabled": bool(row["enabled"]), "configured": bool(row["credentials_ciphertext"]), "status": "ready" if row["enabled"] and row["credentials_ciphertext"] else "disabled", "network_enabled": bool(row["enabled"] and row["credentials_ciphertext"]), "outbound_calls": bool(row["enabled"] and row["credentials_ciphertext"]), "source": "database", "nous_host": urlparse(row["nous_base_url"]).hostname, "firecrawl_host": urlparse(row["firecrawl_base_url"]).hostname, "updated_at": row["updated_at"]}
+
+
+def test_connectivity(row) -> dict:
+    """Make only bounded GET requests to fixed configured HTTPS hosts."""
+    if not row or not row["enabled"] or not row["credentials_ciphertext"]:
+        return {"status": "not_configured", "network_calls": 0, "outbound_calls": False}
+    credentials = json.loads(decrypt(row["credentials_ciphertext"]))
+    checks = []
+    for label, url, key_name in (("nous", row["nous_base_url"] + "/models", "nous_api_key"), ("firecrawl", row["firecrawl_base_url"], "firecrawl_api_key")):
+        key = credentials.get(key_name)
+        if not isinstance(key, str) or not key:
+            checks.append({"provider": label, "ok": False})
+            continue
+        request = Request(url, headers={"Accept": "application/json", "Authorization": "Bearer " + key}, method="GET")
+        try:
+            with urlopen(request, timeout=3) as response:
+                response.read(8193)
+                checks.append({"provider": label, "ok": 200 <= response.status < 500})
+        except Exception:
+            checks.append({"provider": label, "ok": False})
+    return {"status": "ready" if all(x["ok"] for x in checks) else "unavailable", "network_calls": len(checks), "outbound_calls": False, "checks": checks}