diff --git a/.env.example b/.env.example index 303b117..7973690 100644 --- a/.env.example +++ b/.env.example @@ -12,11 +12,15 @@ BOOTSTRAP_ADMIN_EMAIL= BOOTSTRAP_ADMIN_PASSWORD= # Hard safety default; this release has no delivery capability. AUTOMATED_OUTREACH_ENABLED=false -# Optional criteria-first Nous Portal AI web research. The model uses strict -# web_search/scrape_website tools through an approved Firecrawl-compatible API. -# Only structured HTTPS targets are returned; the server crawler validates and -# persists evidence. Both credentials are server-side only. -# AI_RESEARCH_PROVIDER=nous_portal +# Runtime provider configuration is managed in the authenticated admin API: +# POST /api/v1/admin/ai-provider-config. It is stored per organization with +# encrypted credentials in SQLite and a generated 0600 key at /data/provider-config.key. +# On startup, a database row takes precedence over every provider environment +# variable. Environment values are bootstrap fallback only when no DB row exists; +# they are never copied into API responses. Connectivity tests use bounded GET +# requests only and report outbound_calls=false (no outreach is implemented). +# The following variables are legacy/bootstrap fallback values only. +AI_RESEARCH_PROVIDER= # NOUS_MODEL=Hermes-4-405B # NOUS_BASE_URL=https://inference-api.nousresearch.com/v1 # NOUS_ALLOWED_HOSTS=inference-api.nousresearch.com diff --git a/apps/api/app/ai_research.py b/apps/api/app/ai_research.py index d333dd2..1f25362 100644 --- a/apps/api/app/ai_research.py +++ b/apps/api/app/ai_research.py @@ -10,6 +10,7 @@ from __future__ import annotations import json import os import re +import sqlite3 from urllib.parse import urlparse from urllib.request import Request, urlopen @@ -43,7 +44,26 @@ def _safe_endpoint(value: str, allowed: set[str]) -> str: return value.rstrip("/") +_DB_PATH = "" +_DB_ORG = "demo-tenant" + + +def configure_db(path: str, organization_id: str = "demo-tenant") -> None: + global _DB_PATH, _DB_ORG + _DB_PATH, _DB_ORG = path, organization_id + + def _config(): + if _DB_PATH: + try: + db = sqlite3.connect(_DB_PATH); db.row_factory = sqlite3.Row + row = db.execute("SELECT * FROM ai_remote_provider_configs WHERE organization_id=?", (_DB_ORG,)).fetchone(); db.close() + if row: + from .provider_config import decrypt + credentials = json.loads(decrypt(row["credentials_ciphertext"])) if row["credentials_ciphertext"] else {} + return {"provider": row["provider"], "model": row["model"], "nous_url": row["nous_base_url"], "nous_allowed": {urlparse(row["nous_base_url"]).hostname}, "nous_key": credentials.get("nous_api_key", ""), "firecrawl_url": row["firecrawl_base_url"], "firecrawl_allowed": {urlparse(row["firecrawl_base_url"]).hostname}, "firecrawl_key": credentials.get("firecrawl_api_key", "")} + except Exception: + return {"provider": "", "model": "", "endpoint": "", "allowed": set(), "api_key": ""} provider = os.environ.get("AI_RESEARCH_PROVIDER", "").strip().lower() # Nous uses its conventional key directly; no gateway or key translation is needed. nous_key = os.environ.get("NOUS_API_KEY", "").strip() diff --git a/apps/api/app/main.py b/apps/api/app/main.py index d595670..a1f77ee 100644 --- a/apps/api/app/main.py +++ b/apps/api/app/main.py @@ -15,9 +15,10 @@ if __package__ in (None, ""): from app.scoring import DEFAULT_RULES, signals_for_business, evaluate_score, SCORE_VERSION from app.ai_assistance import generate as generate_ai, input_fingerprint, provider_status, MAX_INPUT_ITEMS, MAX_OUTPUT_CHARS from app.discovery import discover as scoped_discover - from app.ai_research import provider_status as ai_research_provider_status, validate_criteria as validate_ai_research_criteria, AIResearchConfigError + from app.ai_research import provider_status as ai_research_provider_status, configure_db as configure_ai_research_db, validate_criteria as validate_ai_research_criteria, AIResearchConfigError from app.search_provider import provider_status as search_provider_status from app.config import load_config + from app.provider_config import validate_payload as validate_remote_provider, encrypt as encrypt_provider_secret, decrypt as decrypt_provider_secret, safe_status as remote_provider_status, test_connectivity as test_remote_connectivity else: from .domain import deduplication_key, deduplicate_businesses, is_suppressed, normalize_business, score_business, normalize_domain, normalize_phone, match_businesses from .sources import adapter_for, contains_secret @@ -27,9 +28,10 @@ else: from .scoring import DEFAULT_RULES, signals_for_business, evaluate_score, SCORE_VERSION from .ai_assistance import generate as generate_ai, input_fingerprint, provider_status, MAX_INPUT_ITEMS, MAX_OUTPUT_CHARS from .discovery import discover as scoped_discover - from .ai_research import provider_status as ai_research_provider_status, validate_criteria as validate_ai_research_criteria, AIResearchConfigError + from .ai_research import provider_status as ai_research_provider_status, configure_db as configure_ai_research_db, validate_criteria as validate_ai_research_criteria, AIResearchConfigError from .search_provider import provider_status as search_provider_status from .config import load_config +from .provider_config import validate_payload as validate_remote_provider, encrypt as encrypt_provider_secret, decrypt as decrypt_provider_secret, safe_status as remote_provider_status, test_connectivity as test_remote_connectivity ORGANIZATION_ID = "demo-tenant" SCHEMA = Path(__file__).resolve().parents[1] / "schema.sql" SESSION_DAYS = 7 @@ -476,7 +478,39 @@ class ApiHandler(BaseHTTPRequestHandler): reasons.append("network_send_disabled"); self.audit(db, user, "outreach_draft.send_blocked", f"{did}:network_send_disabled"); db.commit() return self.send_json(409, {"status": "blocked", "blocked_reasons": reasons, "network_send": False, "id": did}) + def remote_ai_provider_config(self, db, user, payload=None, connectivity=False): + org = user["organization_id"] + row = db.execute("SELECT * FROM ai_remote_provider_configs WHERE organization_id=?", (org,)).fetchone() + if connectivity: + if user["role"] not in {"owner", "admin"}: + return self.send_json(403, {"error": "forbidden"}) + return self.send_json(200, test_remote_connectivity(row)) + if payload is not None: + if user["role"] not in {"owner", "admin"}: + return self.send_json(403, {"error": "forbidden"}) + try: + config = validate_remote_provider(payload) + credentials = dict(config["credentials"]) + if row: + try: old = json.loads(decrypt_provider_secret(row["credentials_ciphertext"])) if row["credentials_ciphertext"] else {} + except Exception: old = {} + for name in ("nous_api_key", "firecrawl_api_key"): + if name not in credentials and name in old: credentials[name] = old[name] + if config["enabled"] and any(name not in credentials for name in ("nous_api_key", "firecrawl_api_key")): + return self.send_json(400, {"error": "provider_credentials_required"}) + ciphertext = encrypt_provider_secret(json.dumps(credentials, sort_keys=True)) if credentials else "" + fingerprint = hashlib.sha256(json.dumps(credentials, sort_keys=True).encode()).hexdigest() if credentials else "" + except (ValueError, TypeError, RuntimeError) as exc: + return self.send_json(400, {"error": str(exc)}) + db.execute("INSERT INTO ai_remote_provider_configs(organization_id,provider,model,enabled,nous_base_url,firecrawl_base_url,credentials_ciphertext,credentials_fingerprint) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(organization_id) DO UPDATE SET provider=excluded.provider,model=excluded.model,enabled=excluded.enabled,nous_base_url=excluded.nous_base_url,firecrawl_base_url=excluded.firecrawl_base_url,credentials_ciphertext=excluded.credentials_ciphertext,credentials_fingerprint=excluded.credentials_fingerprint,updated_at=CURRENT_TIMESTAMP", (org, config["provider"], config["model"], int(config["enabled"]), config["nous_base_url"], config["firecrawl_base_url"], ciphertext, fingerprint)) + self.audit(db, user, "ai.remote_provider.updated", config["provider"]); db.commit() + row = db.execute("SELECT * FROM ai_remote_provider_configs WHERE organization_id=?", (org,)).fetchone() + return self.send_json(200, dict(remote_provider_status(row), organization_id=org)) + def ai_provider_config(self, db, user, payload=None): + remote = db.execute("SELECT * FROM ai_remote_provider_configs WHERE organization_id=?", (user["organization_id"],)).fetchone() + if remote or (payload and str(payload.get("provider", "")).lower() in {"nous_portal", "nous_portal_web_research"}): + return self.remote_ai_provider_config(db, user, payload) org = user["organization_id"] row = db.execute("SELECT * FROM ai_provider_configs WHERE organization_id=?", (org,)).fetchone() if payload is not None: @@ -613,6 +647,7 @@ class ApiHandler(BaseHTTPRequestHandler): if path=="/api/v1/suppressions": return self.list_suppressions(db,org,parse_qs(parsed.query)) if path=="/api/v1/ai-runs": return self.list_ai_runs(db,user,parse_qs(parsed.query)) if path=="/api/v1/ai/provider-config": return self.ai_provider_config(db,user) + if path=="/api/v1/admin/ai-provider-config": return self.remote_ai_provider_config(db,user) if path=="/api/v1/outreach/drafts": return self.list_outreach_drafts(db,user,parse_qs(parsed.query)) if path=="/api/v1/outreach/provider-config": return self.provider_config(db,user) if path in ("/api/v1/reports/pipeline","/api/v1/reports/outcomes","/api/v1/reports/activity"): return self.report(db,org,path.rsplit('/',1)[1],parse_qs(parsed.query)) @@ -1029,6 +1064,8 @@ class ApiHandler(BaseHTTPRequestHandler): if path=="/api/v1/saved-filters": return self.save_filter(payload,db,user) if path=="/api/v1/outreach/provider-config": return self.provider_config(db,user,payload) if path=="/api/v1/ai/provider-config": return self.ai_provider_config(db,user,payload) + if path=="/api/v1/admin/ai-provider-config": return self.remote_ai_provider_config(db,user,payload) + if path=="/api/v1/admin/ai-provider-config/test": return self.remote_ai_provider_config(db,user,connectivity=True) if path=="/api/v1/businesses/bulk-review": return self.bulk_review(payload,db,user) bits_ai=path.split("/") if len(bits_ai)==7 and bits_ai[:4]==["","api","v1","businesses"] and bits_ai[5]=="ai" and bits_ai[6]=="suggest": return self.suggest_ai(int(bits_ai[4]) if bits_ai[4].isdigit() else -1,payload,db,user) @@ -1440,6 +1477,7 @@ def _job_worker(server): def create_server(host="127.0.0.1",port=8000,db_path="prospects.db"): load_config() + configure_ai_research_db(db_path) server=ThreadingHTTPServer((host,port),ApiHandler);server.db_path=db_path;connect(db_path).close();server.job_stop=threading.Event();server.job_wakeup=threading.Event();server.job_thread=threading.Thread(target=_job_worker,args=(server,),daemon=True);server.job_thread.start() original_close=server.server_close def close(): diff --git a/apps/api/app/provider_config.py b/apps/api/app/provider_config.py new file mode 100644 index 0000000..7e0a89f --- /dev/null +++ b/apps/api/app/provider_config.py @@ -0,0 +1,129 @@ +"""Encrypted, tenant-scoped AI provider configuration storage. + +The key is generated under the private data volume and is never stored in SQLite. +The small authenticated stream construction here uses HMAC-SHA256 for the +keystream and integrity tag; ciphertext is prefixed with ``pc1`` and never +returned by the API. +""" +from __future__ import annotations + +import base64 +import hashlib +import hmac +import json +import os +import secrets +from pathlib import Path +from urllib.parse import urlparse +from urllib.request import Request, urlopen + +MAX_SECRET = 4096 +MAX_MODEL = 160 +ALLOWED_PROVIDERS = {"nous_portal", "nous_portal_web_research"} +DEFAULT_NOUS_URL = "https://inference-api.nousresearch.com/v1" +DEFAULT_FIRECRAWL_URL = "https://api.firecrawl.dev/v1" + + +def key_path() -> Path: + return Path(os.environ.get("PROVIDER_CONFIG_KEY_FILE", "/data/provider-config.key")).expanduser() + + +def _key() -> bytes: + path = key_path() + path.parent.mkdir(parents=True, exist_ok=True) + if path.exists(): + key = path.read_bytes() + if len(key) != 32: + raise RuntimeError("invalid_provider_config_key") + return key + key = secrets.token_bytes(32) + flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY + fd = os.open(path, flags, 0o600) + try: + os.write(fd, key) + finally: + os.close(fd) + os.chmod(path, 0o600) + return key + + +def _stream(key: bytes, nonce: bytes, size: int) -> bytes: + return b"".join(hmac.new(key, nonce + i.to_bytes(8, "big"), hashlib.sha256).digest() for i in range((size + 31) // 32))[:size] + + +def encrypt(value: str) -> str: + raw = value.encode("utf-8") + nonce = secrets.token_bytes(16) + cipher = bytes(a ^ b for a, b in zip(raw, _stream(_key(), nonce, len(raw)))) + tag = hmac.new(_key(), nonce + cipher, hashlib.sha256).digest() + return "pc1:" + base64.urlsafe_b64encode(nonce + tag + cipher).decode("ascii") + + +def decrypt(value: str) -> str: + if not isinstance(value, str) or not value.startswith("pc1:"): + raise ValueError("invalid_ciphertext") + raw = base64.urlsafe_b64decode(value[4:].encode("ascii")) + nonce, tag, cipher = raw[:16], raw[16:48], raw[48:] + key = _key() + if not hmac.compare_digest(tag, hmac.new(key, nonce + cipher, hashlib.sha256).digest()): + raise ValueError("invalid_ciphertext") + return bytes(a ^ b for a, b in zip(cipher, _stream(key, nonce, len(cipher)))).decode("utf-8") + + +def validate_payload(payload: dict) -> dict: + if not isinstance(payload, dict): + raise ValueError("invalid_config") + provider = str(payload.get("provider", "nous_portal")).strip().lower() + if provider not in ALLOWED_PROVIDERS: + raise ValueError("invalid_provider") + model = str(payload.get("model", "Hermes-4-405B")).strip() + if not model or len(model) > MAX_MODEL: + raise ValueError("invalid_model") + enabled = payload.get("enabled", True) + if not isinstance(enabled, bool): + raise ValueError("invalid_enabled") + urls = {"nous_base_url": DEFAULT_NOUS_URL, "firecrawl_base_url": DEFAULT_FIRECRAWL_URL} + for field, default in urls.items(): + value = str(payload.get(field, default)).strip().rstrip("/") + parsed = urlparse(value) + if parsed.scheme != "https" or not parsed.hostname or parsed.username or parsed.password or parsed.fragment or parsed.query: + raise ValueError("unsafe_provider_url") + urls[field] = value + credentials = payload.get("credentials", {}) + if not isinstance(credentials, dict): + raise ValueError("invalid_credentials") + result = {"provider": provider, "model": model, "enabled": enabled, **urls, "credentials": {}} + for name in ("nous_api_key", "firecrawl_api_key"): + if name in credentials: + value = credentials[name] + if not isinstance(value, str) or not value or len(value) > MAX_SECRET: + raise ValueError("invalid_secret") + result["credentials"][name] = value + return result + + +def safe_status(row, *, env_bootstrap=False) -> dict: + if not row: + return {"provider": "", "status": "not_configured", "configured": False, "enabled": False, "network_enabled": False, "outbound_calls": False, "source": "env_bootstrap" if env_bootstrap else "none"} + return {"provider": row["provider"], "model": row["model"], "enabled": bool(row["enabled"]), "configured": bool(row["credentials_ciphertext"]), "status": "ready" if row["enabled"] and row["credentials_ciphertext"] else "disabled", "network_enabled": bool(row["enabled"] and row["credentials_ciphertext"]), "outbound_calls": bool(row["enabled"] and row["credentials_ciphertext"]), "source": "database", "nous_host": urlparse(row["nous_base_url"]).hostname, "firecrawl_host": urlparse(row["firecrawl_base_url"]).hostname, "updated_at": row["updated_at"]} + + +def test_connectivity(row) -> dict: + """Make only bounded GET requests to fixed configured HTTPS hosts.""" + if not row or not row["enabled"] or not row["credentials_ciphertext"]: + return {"status": "not_configured", "network_calls": 0, "outbound_calls": False} + credentials = json.loads(decrypt(row["credentials_ciphertext"])) + checks = [] + for label, url, key_name in (("nous", row["nous_base_url"] + "/models", "nous_api_key"), ("firecrawl", row["firecrawl_base_url"], "firecrawl_api_key")): + key = credentials.get(key_name) + if not isinstance(key, str) or not key: + checks.append({"provider": label, "ok": False}) + continue + request = Request(url, headers={"Accept": "application/json", "Authorization": "Bearer " + key}, method="GET") + try: + with urlopen(request, timeout=3) as response: + response.read(8193) + checks.append({"provider": label, "ok": 200 <= response.status < 500}) + except Exception: + checks.append({"provider": label, "ok": False}) + return {"status": "ready" if all(x["ok"] for x in checks) else "unavailable", "network_calls": len(checks), "outbound_calls": False, "checks": checks} diff --git a/apps/api/schema.sql b/apps/api/schema.sql index c1913ca..f920014 100644 --- a/apps/api/schema.sql +++ b/apps/api/schema.sql @@ -292,6 +292,19 @@ CREATE TABLE IF NOT EXISTS ai_provider_configs ( UNIQUE(organization_id) ); +-- Tenant-scoped remote AI credentials. Ciphertext is encrypted with the private +-- key in /data; this table never stores plaintext credentials. +CREATE TABLE IF NOT EXISTS ai_remote_provider_configs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + organization_id TEXT NOT NULL REFERENCES organizations(id) ON DELETE CASCADE, + provider TEXT NOT NULL, model TEXT NOT NULL, enabled INTEGER NOT NULL DEFAULT 0, + nous_base_url TEXT NOT NULL, firecrawl_base_url TEXT NOT NULL, + credentials_ciphertext TEXT NOT NULL DEFAULT '', credentials_fingerprint TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE(organization_id) +); +CREATE INDEX IF NOT EXISTS idx_ai_remote_provider_org ON ai_remote_provider_configs(organization_id); + -- Phase 14 outreach preparation. Provider configuration is metadata plus a -- one-way secret fingerprint; outbound transport is intentionally disabled. CREATE TABLE IF NOT EXISTS outreach_provider_configs ( diff --git a/apps/api/tests/test_provider_config.py b/apps/api/tests/test_provider_config.py new file mode 100644 index 0000000..161c0f8 --- /dev/null +++ b/apps/api/tests/test_provider_config.py @@ -0,0 +1,43 @@ +import json, os, sqlite3, stat, threading, unittest +from http.client import HTTPConnection +from tempfile import TemporaryDirectory +from unittest.mock import patch +from app.main import create_server, hash_password + +class ProviderConfigTests(unittest.TestCase): + def setUp(self): + self.tmp=TemporaryDirectory(); self.old_env={k:os.environ.get(k) for k in ('BOOTSTRAP_ADMIN_EMAIL','BOOTSTRAP_ADMIN_PASSWORD','PROVIDER_CONFIG_KEY_FILE','AI_RESEARCH_PROVIDER')}; os.environ['BOOTSTRAP_ADMIN_EMAIL']='pc-owner@test'; os.environ['BOOTSTRAP_ADMIN_PASSWORD']='pw'; os.environ['PROVIDER_CONFIG_KEY_FILE']=self.tmp.name+'/provider.key' + self.db=self.tmp.name+'/db.sqlite'; self.server=create_server('127.0.0.1',0,self.db); self.thread=threading.Thread(target=self.server.serve_forever,daemon=True); self.thread.start(); self.conn=HTTPConnection('127.0.0.1',self.server.server_port); self.cookie=None; self.req('POST','/api/v1/auth/login',{'email':'pc-owner@test','password':'pw'}) + def tearDown(self): + self.server.shutdown(); self.server.server_close(); self.thread.join(2) + for k,v in self.old_env.items(): + if v is None: os.environ.pop(k,None) + else: os.environ[k]=v + self.tmp.cleanup() + def req(self,m,p,b=None): + body=json.dumps(b).encode() if b is not None else None; h={'Content-Type':'application/json'} + if self.cookie:h['Cookie']=self.cookie + self.conn.request(m,p,body,h); r=self.conn.getresponse(); c=r.getheader('Set-Cookie'); + if c:self.cookie=c.split(';',1)[0] + return r.status,json.loads(r.read() or b'{}') + def test_admin_can_store_write_only_encrypted_config_and_read_status(self): + payload={'provider':'nous_portal','model':'Hermes-test','enabled':True,'credentials':{'nous_api_key':'nous-secret','firecrawl_api_key':'fire-secret'}} + status,out=self.req('POST','/api/v1/admin/ai-provider-config',payload); self.assertEqual(status,200); self.assertNotIn('secret',json.dumps(out)); self.assertEqual(out['source'],'database'); self.assertTrue(out['configured']) + with open(self.db,'rb') as handle: raw=handle.read() + self.assertNotIn(b'nous-secret',raw); self.assertNotIn(b'fire-secret',raw) + mode=stat.S_IMODE(os.stat(os.environ['PROVIDER_CONFIG_KEY_FILE']).st_mode); self.assertEqual(mode,0o600) + status,out=self.req('GET','/api/v1/admin/ai-provider-config'); self.assertNotIn('credentials',out); self.assertEqual(out['status'],'ready') + def test_viewer_cannot_mutate_but_can_read_safe_status(self): + ph,s=hash_password('viewer'); db=sqlite3.connect(self.db); db.execute("INSERT INTO users(organization_id,email,password_hash,password_salt,role) VALUES(?,?,?,?,?)",('demo-tenant','pc-viewer@test',ph,s,'viewer')); db.commit(); db.close(); self.cookie=None; self.req('POST','/api/v1/auth/login',{'email':'pc-viewer@test','password':'viewer'}) + self.assertEqual(self.req('GET','/api/v1/admin/ai-provider-config')[0],200); self.assertEqual(self.req('POST','/api/v1/admin/ai-provider-config',{'provider':'nous_portal'})[0],403) + def test_invalid_config_and_connectivity_never_sends_outreach(self): + self.assertEqual(self.req('POST','/api/v1/admin/ai-provider-config',{'provider':'nous_portal','enabled':True,'credentials':{'nous_api_key':'x'}})[0],400) + self.req('POST','/api/v1/admin/ai-provider-config',{'provider':'nous_portal','enabled':True,'credentials':{'nous_api_key':'x','firecrawl_api_key':'y'}}) + with patch('app.provider_config.urlopen') as opened: + status,out=self.req('POST','/api/v1/admin/ai-provider-config/test',{}); self.assertEqual(status,200); self.assertFalse(out['outbound_calls']); self.assertEqual(out['network_calls'],2); self.assertEqual(opened.call_count,2) + for call in opened.call_args_list: self.assertEqual(call.args[0].method,'GET') + def test_persists_after_server_restart_and_db_config_beats_env(self): + self.req('POST','/api/v1/admin/ai-provider-config',{'provider':'nous_portal','enabled':True,'credentials':{'nous_api_key':'x','firecrawl_api_key':'y'}}); self.server.shutdown(); self.server.server_close(); self.thread.join(2) + os.environ['AI_RESEARCH_PROVIDER']='untrusted'; self.server=create_server('127.0.0.1',0,self.db); self.thread=threading.Thread(target=self.server.serve_forever,daemon=True); self.thread.start(); self.conn=HTTPConnection('127.0.0.1',self.server.server_port); self.cookie=None; self.req('POST','/api/v1/auth/login',{'email':'pc-owner@test','password':'pw'}); status,out=self.req('GET','/api/v1/admin/ai-provider-config'); self.assertEqual(status,200); self.assertEqual(out['provider'],'nous_portal') + +if __name__=='__main__': unittest.main()