add policy-aware source adapter framework
This commit is contained in:
Marco0300
+20
-2
@@ -1,6 +1,6 @@
|
||||
# Prospect Platform API — Phase 4 boundary
|
||||
# Prospect Platform API — Phase 5 boundary
|
||||
|
||||
Dependency-light JSON API for tenant-scoped, **manual** prospect workflows. Core domain rules use Python's standard library and persistence is SQLite. The API stores businesses plus child intelligence/evidence records, pipeline state, notes, and audit context, with the Phase 4 job/live-log contract described below. It never performs automated discovery, DNS/website scanning, or outreach.
|
||||
Dependency-light JSON API for tenant-scoped, **manual** prospect workflows and the Phase 5 source-ingestion contract. Core domain rules use Python's standard library and persistence is SQLite. The API stores businesses plus child intelligence/evidence records, pipeline state, notes, and audit context. Source queries and adapter results must remain auditable and fail closed; the current runtime does not perform network discovery, DNS/website scanning, or outreach.
|
||||
|
||||
## Run
|
||||
|
||||
@@ -34,6 +34,24 @@ The MVP job routes are `POST /api/v1/jobs`, `GET /api/v1/jobs`, `GET /api/v1/job
|
||||
|
||||
The current runtime has no durable job queue and no Redis/Celery integration. Its in-process/SQLite worker is single-instance, non-durable across process loss, and unsuitable for horizontal scaling or guaranteed execution; it is pilot-only.
|
||||
|
||||
### Phase 5 source adapter contract
|
||||
|
||||
Phase 5 treats a source as a registered, reviewable capability rather than an arbitrary URL or scraper. Every adapter contract must identify:
|
||||
|
||||
- a stable adapter/source ID and version, owner, permitted purpose, and terms/robots contact;
|
||||
- accepted discovery-query fields, result schema, provenance fields, and validation/error behavior;
|
||||
- request and concurrency rate limits, retry/backoff rules, timeout/size bounds, and a retention class for raw source records;
|
||||
- health signals and circuit-breaker states (`closed`, `open`, `half-open`), including fail-closed behavior when unhealthy; and
|
||||
- an explicit `dry_run` mode that validates and plans work without contacting a source or writing prospect facts.
|
||||
|
||||
The initial safe adapters are `csv` and `manual_reference`. CSV input may be parsed and previewed; a manual reference records operator-supplied source identity, citation/reference, observed value, and timestamp. Neither adapter independently verifies a source or authorizes outreach. Raw source records should be retained immutably enough to reproduce the normalized result, with tenant/source/query identifiers, capture time, adapter version, and redaction/retention metadata; never retain secrets or unnecessary personal data.
|
||||
|
||||
A source registry entry must include its terms owner, approval status/expiry, allowed tenants or scopes, rate-limit policy, retention class, and health/circuit policy. A discovery query is bounded and tenant-scoped, and its execution mode must be explicit (`dry_run` by default). A live network source is not implemented and must be rejected unless product, legal, and security approval is recorded and operations explicitly enables the registered adapter. No query, job acceptance, or successful parse may be described as network discovery.
|
||||
|
||||
### Phase 5 source controls (contract, not current live routes)
|
||||
|
||||
Implementations should expose source/query/job state without leaking raw payloads across tenants, including source approval, terms, rate-limit, retention, health, and circuit-open reason. On rate-limit, terms, approval, or circuit failure, return a safe non-live outcome and preserve an audit event; do not silently retry against another source. Dry-run must be side-effect-free with respect to external sources and prospect facts. These are Phase 5 design requirements; the current MVP has no network adapter or discovery endpoint.
|
||||
|
||||
### Health and workspace
|
||||
|
||||
- `GET /api/v1/health/live` — unauthenticated liveness check.
|
||||
|
||||
+80
-2
@@ -8,14 +8,16 @@ from urllib.parse import parse_qs, urlparse
|
||||
if __package__ in (None, ""):
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from app.domain import deduplication_key, deduplicate_businesses, is_suppressed, normalize_business, score_business, normalize_domain, normalize_phone
|
||||
from app.sources import adapter_for, contains_secret
|
||||
else:
|
||||
from .domain import deduplication_key, deduplicate_businesses, is_suppressed, normalize_business, score_business, normalize_domain, normalize_phone
|
||||
from .sources import adapter_for, contains_secret
|
||||
ORGANIZATION_ID = "demo-tenant"
|
||||
SCHEMA = Path(__file__).resolve().parents[1] / "schema.sql"
|
||||
SESSION_DAYS = 7
|
||||
PBKDF2_ITERATIONS = 300_000
|
||||
MUTATING_ROLES = {"owner", "admin", "researcher"}
|
||||
JOB_TYPES = {"noop", "prospect_recalculate"}
|
||||
JOB_TYPES = {"noop", "prospect_recalculate", "source_discovery"}
|
||||
JOB_PAGE_SIZE = 100
|
||||
SECRET_KEYS = {"password", "passwd", "secret", "token", "api_key", "apikey", "authorization", "credential", "private_key"}
|
||||
CHILD_TABLES = {"contacts": ("name", "email", "phone", "title", "do_not_contact"), "domains": ("domain", "kind"), "websites": ("url", "website_class"), "evidence": ("kind", "url", "claim"), "notes": ("body",)}
|
||||
@@ -110,6 +112,9 @@ class ApiHandler(BaseHTTPRequestHandler):
|
||||
if path=="/api/v1/dashboard/summary":
|
||||
row=db.execute("SELECT COUNT(*) businesses,COALESCE(AVG(score),0) average_score FROM businesses WHERE organization_id=?",(org,)).fetchone(); return self.send_json(200,{"organization_id":org,"businesses":row["businesses"],"average_score":round(row["average_score"],2),"suppressed":db.execute("SELECT COUNT(*) FROM suppressions WHERE organization_id=?",(org,)).fetchone()[0]})
|
||||
if path=="/api/v1/businesses": return self.list_businesses(db,org,parse_qs(parsed.query))
|
||||
if path=="/api/v1/sources": return self.list_sources(db,org)
|
||||
if path=="/api/v1/discovery-queries": return self.list_queries(db,org)
|
||||
if path=="/api/v1/source-records": return self.list_source_records(db,org,parse_qs(parsed.query))
|
||||
if path=="/api/v1/jobs": return self.list_jobs(db,org,parse_qs(parsed.query))
|
||||
if path.startswith("/api/v1/jobs/"): return self.get_job_route(db,org,path,parse_qs(parsed.query))
|
||||
if path.startswith("/api/v1/businesses/"):
|
||||
@@ -153,7 +158,7 @@ class ApiHandler(BaseHTTPRequestHandler):
|
||||
try:
|
||||
cur=db.execute("INSERT INTO jobs(organization_id,idempotency_key,type,payload,max_attempts) VALUES(?,?,?,?,?)",(user["organization_id"],key,kind,safe,max_attempts)); jid=cur.lastrowid
|
||||
self.add_job_event(db,jid,user["organization_id"],"queued","Job queued",0); self.audit(db,user,"job.created",str(jid)); db.commit(); getattr(self.server,"job_wakeup",threading.Event()).set()
|
||||
return self.send_json(201,job_json(db.execute("SELECT * FROM jobs WHERE id=?",(jid,)).fetchone()))
|
||||
return self.send_json(202 if payload.get("_accepted") else 201,job_json(db.execute("SELECT * FROM jobs WHERE id=?",(jid,)).fetchone()))
|
||||
except sqlite3.IntegrityError:
|
||||
row=db.execute("SELECT * FROM jobs WHERE organization_id=? AND idempotency_key=?",(user["organization_id"],key)).fetchone(); return self.send_json(200,job_json(row))
|
||||
|
||||
@@ -210,9 +215,14 @@ class ApiHandler(BaseHTTPRequestHandler):
|
||||
if path.startswith("/api/v1/jobs/"):
|
||||
return self.job_action(db,user,path)
|
||||
if path=="/api/v1/businesses":return self.create_business(payload,db,user)
|
||||
if path=="/api/v1/sources":return self.create_source(payload,db,user)
|
||||
if path=="/api/v1/discovery-queries":return self.create_query(payload,db,user)
|
||||
if path=="/api/v1/suppressions":return self.create_suppression(payload,db,user)
|
||||
if path=="/api/v1/imports/preview":return self.preview_import(payload,db,org)
|
||||
bits=path.split("/")
|
||||
if len(bits)==6 and bits[3] == "sources" and bits[4].isdigit() and bits[5] in {"test","ingest"}:
|
||||
return self.test_source(int(bits[4]),db,user) if bits[5]=="test" else self.ingest_source(int(bits[4]),payload,db,user)
|
||||
if len(bits)==6 and bits[3] == "discovery-queries" and bits[4].isdigit() and bits[5]=="run": return self.run_query(int(bits[4]),db,user)
|
||||
if len(bits)==7 and bits[:4]==["","api","v1","businesses"] and bits[5] in CHILD_TABLES and bits[6]=="": pass
|
||||
if len(bits)==6 and bits[:4]==["","api","v1","businesses"] and bits[5] in CHILD_TABLES:return self.create_child(int(bits[4]) if bits[4].isdigit() else -1,bits[5],payload,db,user)
|
||||
if len(bits)==6 and bits[:4]==["","api","v1","businesses"] and bits[5]=="verify":return self.verify_business(int(bits[4]) if bits[4].isdigit() else -1,payload,db,user)
|
||||
@@ -225,6 +235,7 @@ class ApiHandler(BaseHTTPRequestHandler):
|
||||
if not user:return
|
||||
if user["role"] not in MUTATING_ROLES:return self.send_json(403,{"error":"forbidden"})
|
||||
bits=path.split("/")
|
||||
if len(bits)==5 and bits[:4]==["","api","v1","sources"] and bits[4].isdigit(): return self.update_source(int(bits[4]),self.read_json(),db,user)
|
||||
if len(bits)==6 and bits[:4]==["","api","v1","businesses"] and bits[5]=="pipeline":return self.update_pipeline(int(bits[4]) if bits[4].isdigit() else -1,self.read_json(),db,user)
|
||||
return self.send_json(404,{"error":"not_found"})
|
||||
finally:db.close()
|
||||
@@ -286,6 +297,73 @@ class ApiHandler(BaseHTTPRequestHandler):
|
||||
elif key in existing_keys or key in seen:continue
|
||||
else:seen.add(key);accepted.append(b)
|
||||
return self.send_json(200,{"accepted":len(accepted),"duplicates":len(rows)-len(normalized)+len(normalized)-len(accepted)-suppressed,"suppressed":suppressed,"rows":accepted})
|
||||
def list_sources(self,db,org):
|
||||
cols='id,organization_id,name,kind,enabled,health_status,consecutive_failures,circuit_open,last_success_at,last_failure_at,last_error,created_at,updated_at'
|
||||
return self.send_json(200,{"organization_id":org,"items":[row_json(r) for r in db.execute(f"SELECT {cols} FROM sources WHERE organization_id=? ORDER BY id",(org,))]})
|
||||
def list_queries(self,db,org):
|
||||
return self.send_json(200,{"organization_id":org,"items":[row_json(r) for r in db.execute("SELECT * FROM discovery_queries WHERE organization_id=? ORDER BY id",(org,))]})
|
||||
def list_source_records(self,db,org,q):
|
||||
try:
|
||||
limit=int(q.get('page_size',[50])[0]); offset=max(0,int(q.get('offset',[0])[0]))
|
||||
if limit<1 or limit>100: raise ValueError
|
||||
except (ValueError,TypeError): return self.send_json(400,{"error":"invalid_pagination"})
|
||||
rows=db.execute("SELECT * FROM source_records WHERE organization_id=? ORDER BY id DESC LIMIT ? OFFSET ?",(org,limit+1,offset)).fetchall(); out=[]
|
||||
for r in rows[:limit]:
|
||||
x=row_json(r)
|
||||
for k in ('raw_json','normalized_json','query_context_json','cursor_json','rate_policy_json'):
|
||||
try:x[k]=json.loads(x[k])
|
||||
except (ValueError,TypeError):pass
|
||||
out.append(x)
|
||||
return self.send_json(200,{"organization_id":org,"items":out,"limit":limit,"offset":offset,"has_more":len(rows)>limit})
|
||||
def create_source(self,payload,db,user):
|
||||
name=str(payload.get('name','')).strip(); kind=str(payload.get('kind','')).strip().lower(); config=payload.get('config',{})
|
||||
if not name or kind not in ('csv','manual') or not isinstance(config,dict):return self.send_json(400,{"error":"invalid_source"})
|
||||
if contains_secret(config):return self.send_json(400,{"error":"secret_not_permitted"})
|
||||
try:
|
||||
validation=adapter_for(kind).validate(config)
|
||||
if config and not validation.valid:return self.send_json(400,{"error":"invalid_source_config","details":validation.errors})
|
||||
cur=db.execute("INSERT INTO sources(organization_id,name,kind,enabled,config_json) VALUES(?,?,?,?,?)",(user['organization_id'],name,kind,int(bool(payload.get('enabled',False))),json.dumps(config,sort_keys=True)))
|
||||
except sqlite3.IntegrityError:return self.send_json(409,{"error":"duplicate_source"})
|
||||
self.audit(db,user,'source.created',str(cur.lastrowid));db.commit();return self.send_json(201,row_json(db.execute("SELECT id,organization_id,name,kind,enabled,health_status,consecutive_failures,circuit_open,last_success_at,last_failure_at,last_error,created_at,updated_at FROM sources WHERE id=?",(cur.lastrowid,)).fetchone()))
|
||||
def update_source(self,sid,payload,db,user):
|
||||
if not db.execute("SELECT id FROM sources WHERE id=? AND organization_id=?",(sid,user['organization_id'])).fetchone():return self.send_json(404,{"error":"not_found"})
|
||||
if 'enabled' not in payload:return self.send_json(400,{"error":"enabled_required"})
|
||||
value=int(bool(payload['enabled']));db.execute("UPDATE sources SET enabled=?,updated_at=CURRENT_TIMESTAMP WHERE id=?",(value,sid));self.audit(db,user,'source.enabled' if value else 'source.disabled',str(sid));db.commit();return self.send_json(200,row_json(db.execute("SELECT * FROM sources WHERE id=?",(sid,)).fetchone()))
|
||||
def create_query(self,payload,db,user):
|
||||
sid=payload.get('source_id');name=str(payload.get('name','')).strip();query=payload.get('query',{})
|
||||
if not isinstance(sid,int) or not name or not isinstance(query,dict) or contains_secret(query):return self.send_json(400,{"error":"invalid_query"})
|
||||
if not db.execute("SELECT id FROM sources WHERE id=? AND organization_id=?",(sid,user['organization_id'])).fetchone():return self.send_json(404,{"error":"not_found"})
|
||||
try:cur=db.execute("INSERT INTO discovery_queries(organization_id,source_id,name,query_json) VALUES(?,?,?,?)",(user['organization_id'],sid,name,json.dumps(query,sort_keys=True)))
|
||||
except sqlite3.IntegrityError:return self.send_json(409,{"error":"duplicate_query"})
|
||||
self.audit(db,user,'discovery_query.created',str(cur.lastrowid));db.commit();return self.send_json(201,row_json(db.execute("SELECT * FROM discovery_queries WHERE id=?",(cur.lastrowid,)).fetchone()))
|
||||
def run_query(self,qid,db,user):
|
||||
if not db.execute("SELECT id FROM discovery_queries WHERE id=? AND organization_id=?",(qid,user['organization_id'])).fetchone():return self.send_json(404,{"error":"not_found"})
|
||||
return self.create_job({"type":"source_discovery","_accepted":True,"payload":{"discovery_query_id":qid},"idempotency_key":f"discovery-query-{qid}-{int(time.time())}"},db,user)
|
||||
def test_source(self,sid,db,user):
|
||||
source=db.execute("SELECT * FROM sources WHERE id=? AND organization_id=?",(sid,user['organization_id'])).fetchone()
|
||||
if not source:return self.send_json(404,{"error":"not_found"})
|
||||
try: result=adapter_for(source['kind']).validate(json.loads(source['config_json'])); ok=result.valid; error='; '.join(result.errors) if not ok else None
|
||||
except Exception as exc:ok=False;error=str(exc)[:300]
|
||||
if ok:db.execute("UPDATE sources SET health_status='healthy',consecutive_failures=0,circuit_open=0,last_success_at=CURRENT_TIMESTAMP,last_error=NULL WHERE id=?",(sid,));action='source.test.succeeded'
|
||||
else:db.execute("UPDATE sources SET health_status='unhealthy',consecutive_failures=consecutive_failures+1,circuit_open=CASE WHEN consecutive_failures+1>=3 THEN 1 ELSE circuit_open END,last_failure_at=CURRENT_TIMESTAMP,last_error=? WHERE id=?",(error,sid));action='source.test.failed'
|
||||
self.audit(db,user,action,str(sid));db.commit();return self.send_json(200,{"ok":ok,"errors":[] if ok else [error]})
|
||||
def ingest_source(self,sid,payload,db,user):
|
||||
source=db.execute("SELECT * FROM sources WHERE id=? AND organization_id=?",(sid,user['organization_id'])).fetchone()
|
||||
if not source:return self.send_json(404,{"error":"not_found"})
|
||||
if not source['enabled'] or source['circuit_open']:return self.send_json(409,{"error":"source_disabled"})
|
||||
if contains_secret(payload):return self.send_json(400,{"error":"secret_not_permitted"})
|
||||
config={k:v for k,v in payload.items() if k not in ('source_url','query_context','cursor','rate_policy')}
|
||||
if len(json.dumps(config).encode())>5*1024*1024:return self.send_json(400,{"error":"ingest_limits"})
|
||||
if isinstance(config.get('rows'),list) and (len(config['rows'])>1000 or any(not isinstance(r,dict) or len(r)>50 or any(len(str(v))>10000 for v in r.values()) for r in config['rows'])):return self.send_json(400,{"error":"ingest_limits"})
|
||||
if isinstance(config.get('csv'),str) and config['csv'].count('\n')>1001:return self.send_json(400,{"error":"ingest_limits"})
|
||||
try:page=adapter_for(source['kind']).discover(config)
|
||||
except (ValueError,KeyError) as exc:return self.send_json(400,{"error":"invalid_ingest","detail":str(exc)})
|
||||
inserted=0
|
||||
for record in page.records[:1000]:
|
||||
raw=json.dumps(record,sort_keys=True,separators=(',',':'));digest=hashlib.sha256(raw.encode()).hexdigest()
|
||||
try:db.execute("INSERT INTO source_records(organization_id,source_id,content_hash,raw_json,normalized_json,source_url,query_context_json,cursor_json,rate_policy_json) VALUES(?,?,?,?,?,?,?,?,?)",(user['organization_id'],sid,digest,raw,raw,str(payload.get('source_url','')),json.dumps(payload.get('query_context',{}),sort_keys=True),json.dumps(payload.get('cursor',{}),sort_keys=True),json.dumps(payload.get('rate_policy',{}),sort_keys=True)));inserted+=1
|
||||
except sqlite3.IntegrityError:pass
|
||||
db.execute("UPDATE sources SET health_status='healthy',consecutive_failures=0,last_success_at=CURRENT_TIMESTAMP,last_error=NULL WHERE id=?",(sid,));self.audit(db,user,'source.ingested',f'{sid}:{inserted}');db.commit();return self.send_json(201 if inserted else 200,{"inserted":inserted,"records":len(page.records)})
|
||||
def log_message(self,*_):pass
|
||||
|
||||
def _job_worker(server):
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Deterministic, network-free discovery source contracts and adapters."""
|
||||
from __future__ import annotations
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Mapping, Protocol, Sequence
|
||||
import csv, io, re
|
||||
|
||||
SECRET_KEYS = {"password", "passwd", "secret", "token", "api_key", "apikey", "authorization", "private_key", "credential"}
|
||||
|
||||
def contains_secret(value: Any, path: str = "") -> str | None:
|
||||
if isinstance(value, Mapping):
|
||||
for key, child in value.items():
|
||||
key_text = str(key).lower()
|
||||
if key_text in SECRET_KEYS or any(x in key_text for x in ("password", "token", "secret", "api_key", "private_key")):
|
||||
return path + str(key)
|
||||
found = contains_secret(child, path + str(key) + ".")
|
||||
if found: return found
|
||||
elif isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
|
||||
for index, child in enumerate(value):
|
||||
found = contains_secret(child, path + str(index) + ".")
|
||||
if found: return found
|
||||
return None
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ValidationResult:
|
||||
valid: bool
|
||||
errors: list[str] = field(default_factory=list)
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DiscoveryPage:
|
||||
records: list[dict[str, str]]
|
||||
next_cursor: str | None = None
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SourceHealth:
|
||||
status: str = "unknown"
|
||||
consecutive_failures: int = 0
|
||||
circuit_open: bool = False
|
||||
last_error: str | None = None
|
||||
|
||||
class DiscoverySource(Protocol):
|
||||
kind: str
|
||||
def validate(self, config: Mapping[str, Any]) -> ValidationResult: ...
|
||||
def discover(self, config: Mapping[str, Any], cursor: str | None = None) -> DiscoveryPage: ...
|
||||
|
||||
_FIELDS = ("name", "website", "email", "phone", "description")
|
||||
|
||||
def normalize_record(row: Mapping[str, Any]) -> dict[str, str]:
|
||||
result = {field: str(row.get(field, "")).strip() for field in _FIELDS}
|
||||
# Accept common CSV spellings without retaining arbitrary sensitive fields.
|
||||
aliases = {"company": "name", "url": "website", "domain": "website"}
|
||||
for key, target in aliases.items():
|
||||
if not result[target] and row.get(key) is not None: result[target] = str(row[key]).strip()
|
||||
return result
|
||||
|
||||
class ManualSource:
|
||||
kind = "manual"
|
||||
def validate(self, config: Mapping[str, Any]) -> ValidationResult:
|
||||
if not isinstance(config, Mapping): return ValidationResult(False, ["config must be an object"])
|
||||
found = contains_secret(config)
|
||||
if found: return ValidationResult(False, [f"secret field is not permitted: {found}"])
|
||||
rows = config.get("rows")
|
||||
if not isinstance(rows, list): return ValidationResult(False, ["rows must be a list"])
|
||||
return ValidationResult(True)
|
||||
def discover(self, config: Mapping[str, Any], cursor: str | None = None) -> DiscoveryPage:
|
||||
validation = self.validate(config)
|
||||
if not validation.valid: raise ValueError(validation.errors[0])
|
||||
rows = [normalize_record(row) for row in config["rows"] if isinstance(row, Mapping)]
|
||||
return DiscoveryPage(rows, None, {"adapter": self.kind})
|
||||
|
||||
class CsvSource:
|
||||
kind = "csv"
|
||||
def validate(self, config: Mapping[str, Any]) -> ValidationResult:
|
||||
if not isinstance(config, Mapping): return ValidationResult(False, ["config must be an object"])
|
||||
found = contains_secret(config)
|
||||
if found: return ValidationResult(False, [f"secret field is not permitted: {found}"])
|
||||
if not isinstance(config.get("csv"), str): return ValidationResult(False, ["csv must be text"])
|
||||
try:
|
||||
reader = csv.DictReader(io.StringIO(config["csv"]));
|
||||
if not reader.fieldnames: return ValidationResult(False, ["CSV header is required"])
|
||||
except csv.Error as exc: return ValidationResult(False, [f"invalid CSV: {exc}"])
|
||||
return ValidationResult(True)
|
||||
def discover(self, config: Mapping[str, Any], cursor: str | None = None) -> DiscoveryPage:
|
||||
validation = self.validate(config)
|
||||
if not validation.valid: raise ValueError(validation.errors[0])
|
||||
reader = csv.DictReader(io.StringIO(config["csv"].replace("\r\n", "\n")))
|
||||
records = [normalize_record({str(k).strip().lower(): v for k, v in row.items()}) for row in reader]
|
||||
return DiscoveryPage(records, None, {"adapter": self.kind, "columns": reader.fieldnames or []})
|
||||
|
||||
ADAPTERS = {"manual": ManualSource, "csv": CsvSource}
|
||||
|
||||
def adapter_for(kind: str) -> DiscoverySource:
|
||||
try: return ADAPTERS[kind]()
|
||||
except KeyError: raise ValueError("unsupported source kind")
|
||||
@@ -114,3 +114,34 @@ CREATE TABLE IF NOT EXISTS job_events (
|
||||
UNIQUE(job_id,sequence)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_job_events_job_sequence ON job_events(job_id,sequence);
|
||||
|
||||
-- Phase 5 source framework (additive-safe; credentials contain metadata only).
|
||||
CREATE TABLE IF NOT EXISTS sources (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT, organization_id TEXT NOT NULL REFERENCES organizations(id),
|
||||
name TEXT NOT NULL, kind TEXT NOT NULL CHECK(kind IN ('csv','manual')), enabled INTEGER NOT NULL DEFAULT 0,
|
||||
config_json TEXT NOT NULL DEFAULT '{}', health_status TEXT NOT NULL DEFAULT 'unknown',
|
||||
consecutive_failures INTEGER NOT NULL DEFAULT 0, circuit_open INTEGER NOT NULL DEFAULT 0,
|
||||
last_success_at TEXT, last_failure_at TEXT, last_error TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, UNIQUE(organization_id,name)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_sources_org ON sources(organization_id,id);
|
||||
CREATE TABLE IF NOT EXISTS source_credentials (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT, source_id INTEGER NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
|
||||
organization_id TEXT NOT NULL REFERENCES organizations(id), provider TEXT NOT NULL, key_name TEXT NOT NULL,
|
||||
secret_ref TEXT NOT NULL DEFAULT '', metadata_json TEXT NOT NULL DEFAULT '{}', created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(source_id,provider,key_name)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS discovery_queries (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT, organization_id TEXT NOT NULL REFERENCES organizations(id), source_id INTEGER NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
|
||||
name TEXT NOT NULL, query_json TEXT NOT NULL DEFAULT '{}', enabled INTEGER NOT NULL DEFAULT 1, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, UNIQUE(organization_id,name)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_discovery_queries_org ON discovery_queries(organization_id,id);
|
||||
CREATE TABLE IF NOT EXISTS source_records (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT, organization_id TEXT NOT NULL REFERENCES organizations(id), source_id INTEGER NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
|
||||
discovery_query_id INTEGER REFERENCES discovery_queries(id) ON DELETE SET NULL, content_hash TEXT NOT NULL, raw_json TEXT NOT NULL,
|
||||
normalized_json TEXT NOT NULL, source_url TEXT NOT NULL DEFAULT '', query_context_json TEXT NOT NULL DEFAULT '{}', processing_status TEXT NOT NULL DEFAULT 'raw',
|
||||
cursor_json TEXT NOT NULL DEFAULT '{}', rate_policy_json TEXT NOT NULL DEFAULT '{}', created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(organization_id,source_id,content_hash)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_source_records_org ON source_records(organization_id,id DESC);
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
import json, os, sqlite3, threading, unittest
|
||||
from http.client import HTTPConnection
|
||||
from tempfile import TemporaryDirectory
|
||||
from app.main import create_server
|
||||
from app.sources import CsvSource, ManualSource
|
||||
|
||||
class SourceAdapterTests(unittest.TestCase):
|
||||
def test_csv_adapter_is_deterministic_and_normalizes(self):
|
||||
src = CsvSource()
|
||||
a = src.discover({'csv': 'Name,Website,Email\n Acme ,https://acme.test,a@acme.test\n'})
|
||||
b = src.discover({'csv': 'Name,Website,Email\n Acme ,https://acme.test,a@acme.test\n'})
|
||||
self.assertEqual(a.records, b.records)
|
||||
self.assertEqual(a.records[0]['name'], 'Acme')
|
||||
self.assertEqual(a.records[0]['email'], 'a@acme.test')
|
||||
|
||||
def test_manual_validation_rejects_secret_fields(self):
|
||||
result = ManualSource().validate({'rows': [{'name': 'x', 'api_key': 'secret'}]})
|
||||
self.assertFalse(result.valid)
|
||||
self.assertIn('secret', result.errors[0].lower())
|
||||
|
||||
class SourceApiTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp=TemporaryDirectory(); os.environ['BOOTSTRAP_ADMIN_EMAIL']='owner@example.test'; os.environ['BOOTSTRAP_ADMIN_PASSWORD']='development-password'
|
||||
self.server=create_server('127.0.0.1',0,self.tmp.name+'/x.db'); self.thread=threading.Thread(target=self.server.serve_forever,daemon=True); self.thread.start(); self.c=HTTPConnection('127.0.0.1',self.server.server_port); self.cookie=None
|
||||
self.req('POST','/api/v1/auth/login',{'email':'owner@example.test','password':'development-password'})
|
||||
def tearDown(self): self.server.shutdown(); self.server.server_close(); self.thread.join(2); self.tmp.cleanup()
|
||||
def req(self,m,p,x=None,cookie=True):
|
||||
body=json.dumps(x).encode() if x is not None else None; h={'Content-Type':'application/json'} if body else {};
|
||||
if cookie and self.cookie:h['Cookie']=self.cookie
|
||||
self.c.request(m,p,body,h); r=self.c.getresponse(); sc=r.getheader('Set-Cookie');
|
||||
if sc:self.cookie=sc.split(';',1)[0]
|
||||
raw=r.read(); return r.status,json.loads(raw or b'{}')
|
||||
def test_source_lifecycle_ingestion_idempotency_health_and_disabled(self):
|
||||
s, source=self.req('POST','/api/v1/sources',{'name':'Import','kind':'manual','config':{}}); self.assertEqual(s,201)
|
||||
sid=source['id']; self.assertFalse(source['enabled'])
|
||||
self.assertEqual(self.req('PATCH',f'/api/v1/sources/{sid}',{'enabled':True})[0],200)
|
||||
payload={'rows':[{'name':'Acme','website':'https://acme.test'}], 'source_url':'file://import.csv','query_context':{'q':'test'}}
|
||||
self.assertEqual(self.req('POST',f'/api/v1/sources/{sid}/ingest',payload)[0],201)
|
||||
self.assertEqual(self.req('POST',f'/api/v1/sources/{sid}/ingest',payload)[0],200)
|
||||
self.assertEqual(len(self.req('GET','/api/v1/source-records')[1]['items']),1)
|
||||
self.assertEqual(self.req('POST',f'/api/v1/sources/{sid}/test')[0],200)
|
||||
self.assertEqual(self.req('PATCH',f'/api/v1/sources/{sid}',{'enabled':False})[0],200)
|
||||
self.assertEqual(self.req('POST',f'/api/v1/sources/{sid}/ingest',payload)[0],409)
|
||||
db=sqlite3.connect(self.tmp.name+'/x.db'); self.assertTrue(db.execute("select 1 from audit_log where action='source.disabled'").fetchone()); db.close()
|
||||
def test_queries_enqueue_and_records_are_tenant_scoped(self):
|
||||
_,source=self.req('POST','/api/v1/sources',{'name':'CSV','kind':'csv','enabled':True})
|
||||
_,q=self.req('POST','/api/v1/discovery-queries',{'source_id':source['id'],'name':'q','query':{'csv':'name\nA'}})
|
||||
status,job=self.req('POST',f"/api/v1/discovery-queries/{q['id']}/run",{})
|
||||
self.assertEqual(status,202); self.assertEqual(job['type'],'source_discovery')
|
||||
self.assertEqual(self.req('GET','/api/v1/source-records?page_size=101')[0],400)
|
||||
def test_sources_require_auth(self):
|
||||
self.cookie=None; self.assertEqual(self.req('GET','/api/v1/sources',cookie=False)[0],401)
|
||||
|
||||
if __name__=='__main__': unittest.main()
|
||||
Reference in New Issue
Block a user