add governed source adapter discovery framework

apps/api/schema.sql

@@ -136,8 +136,8 @@ CREATE INDEX IF NOT EXISTS idx_job_events_job_sequence ON job_events(job_id,sequ
 -- Phase 5 source framework (additive-safe; credentials contain metadata only).
 CREATE TABLE IF NOT EXISTS sources (
  id INTEGER PRIMARY KEY AUTOINCREMENT, organization_id TEXT NOT NULL REFERENCES organizations(id),
- name TEXT NOT NULL, kind TEXT NOT NULL CHECK(kind IN ('csv','manual')), enabled INTEGER NOT NULL DEFAULT 0,
- config_json TEXT NOT NULL DEFAULT '{}', health_status TEXT NOT NULL DEFAULT 'unknown',
+ name TEXT NOT NULL, kind TEXT NOT NULL CHECK(kind IN ('csv','manual','google_places','bing_local','approved_directory','public_website','permitted_social','ct_logs','dns','rdap')), source_code TEXT NOT NULL DEFAULT '', display_name TEXT NOT NULL DEFAULT '', enabled INTEGER NOT NULL DEFAULT 0, approved INTEGER NOT NULL DEFAULT 0,
+ config_json TEXT NOT NULL DEFAULT '{}', policy_json TEXT NOT NULL DEFAULT '{}', quota_json TEXT NOT NULL DEFAULT '{}', health_status TEXT NOT NULL DEFAULT 'unknown',
  consecutive_failures INTEGER NOT NULL DEFAULT 0, circuit_open INTEGER NOT NULL DEFAULT 0,
  last_success_at TEXT, last_failure_at TEXT, last_error TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
  updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, UNIQUE(organization_id,name)
@@ -151,18 +151,25 @@ CREATE TABLE IF NOT EXISTS source_credentials (
 );
 CREATE TABLE IF NOT EXISTS discovery_queries (
  id INTEGER PRIMARY KEY AUTOINCREMENT, organization_id TEXT NOT NULL REFERENCES organizations(id), source_id INTEGER NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
- name TEXT NOT NULL, query_json TEXT NOT NULL DEFAULT '{}', enabled INTEGER NOT NULL DEFAULT 1, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ name TEXT NOT NULL, query_json TEXT NOT NULL DEFAULT '{}', enabled INTEGER NOT NULL DEFAULT 1, selected_adapters_json TEXT NOT NULL DEFAULT '[]', location TEXT NOT NULL DEFAULT '', category TEXT NOT NULL DEFAULT '', max_records INTEGER NOT NULL DEFAULT 100, daily_limit INTEGER NOT NULL DEFAULT 1000, schedule TEXT NOT NULL DEFAULT '', dry_run INTEGER NOT NULL DEFAULT 0, lifecycle TEXT NOT NULL DEFAULT 'draft', created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
  updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, UNIQUE(organization_id,name)
 );
 CREATE INDEX IF NOT EXISTS idx_discovery_queries_org ON discovery_queries(organization_id,id);
 CREATE TABLE IF NOT EXISTS source_records (
  id INTEGER PRIMARY KEY AUTOINCREMENT, organization_id TEXT NOT NULL REFERENCES organizations(id), source_id INTEGER NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
- discovery_query_id INTEGER REFERENCES discovery_queries(id) ON DELETE SET NULL, content_hash TEXT NOT NULL, raw_json TEXT NOT NULL,
- normalized_json TEXT NOT NULL, source_url TEXT NOT NULL DEFAULT '', query_context_json TEXT NOT NULL DEFAULT '{}', processing_status TEXT NOT NULL DEFAULT 'raw',
+ discovery_query_id INTEGER REFERENCES discovery_queries(id) ON DELETE SET NULL, discovery_run_id INTEGER REFERENCES discovery_runs(id) ON DELETE SET NULL, content_hash TEXT NOT NULL, raw_json TEXT NOT NULL,
+ normalized_json TEXT NOT NULL, normalized_key TEXT NOT NULL DEFAULT '', source_url TEXT NOT NULL DEFAULT '', provenance_json TEXT NOT NULL DEFAULT '{}', query_context_json TEXT NOT NULL DEFAULT '{}', processing_status TEXT NOT NULL DEFAULT 'raw',
  cursor_json TEXT NOT NULL DEFAULT '{}', rate_policy_json TEXT NOT NULL DEFAULT '{}', created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
  UNIQUE(organization_id,source_id,content_hash)
 );
 CREATE INDEX IF NOT EXISTS idx_source_records_org ON source_records(organization_id,id DESC);
+CREATE TABLE IF NOT EXISTS enrichment_queue (
+ id INTEGER PRIMARY KEY AUTOINCREMENT, organization_id TEXT NOT NULL REFERENCES organizations(id) ON DELETE CASCADE, source_record_id INTEGER NOT NULL REFERENCES source_records(id) ON DELETE CASCADE, status TEXT NOT NULL DEFAULT 'queued', attempts INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, UNIQUE(organization_id,source_record_id)
+);
+CREATE TABLE IF NOT EXISTS source_events (
+ id INTEGER PRIMARY KEY AUTOINCREMENT, organization_id TEXT NOT NULL REFERENCES organizations(id) ON DELETE CASCADE, source_id INTEGER REFERENCES sources(id) ON DELETE SET NULL, discovery_run_id INTEGER REFERENCES discovery_runs(id) ON DELETE SET NULL, event_type TEXT NOT NULL, message TEXT NOT NULL DEFAULT '', metadata_json TEXT NOT NULL DEFAULT '{}', created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+CREATE INDEX IF NOT EXISTS idx_source_events_org ON source_events(organization_id,created_at DESC,id DESC);
 
 -- Phase 7 domain intelligence (additive-safe; DNS results are explicitly stateful).
 CREATE TABLE IF NOT EXISTS domain_checks (
@@ -341,9 +348,8 @@ CREATE INDEX IF NOT EXISTS idx_outreach_drafts_business ON outreach_drafts(organ
 -- Built-in scoped discovery runs. Results are reviewable business records only;
 -- this feature never sends outreach.
 CREATE TABLE IF NOT EXISTS discovery_runs (
- id INTEGER PRIMARY KEY AUTOINCREMENT,
- organization_id TEXT NOT NULL REFERENCES organizations(id) ON DELETE CASCADE,
- job_id INTEGER NOT NULL REFERENCES jobs(id) ON DELETE CASCADE,
+ id INTEGER PRIMARY KEY AUTOINCREMENT, organization_id TEXT NOT NULL REFERENCES organizations(id), job_id INTEGER NOT NULL REFERENCES jobs(id) ON DELETE CASCADE,
+ selected_adapters_json TEXT NOT NULL DEFAULT '[]', location TEXT NOT NULL DEFAULT '', category TEXT NOT NULL DEFAULT '', max_records INTEGER NOT NULL DEFAULT 100, daily_limit INTEGER NOT NULL DEFAULT 1000, schedule TEXT NOT NULL DEFAULT '', dry_run INTEGER NOT NULL DEFAULT 0, lifecycle TEXT NOT NULL DEFAULT 'draft', paused_at TEXT,
  criteria_json TEXT NOT NULL DEFAULT '{}', seed_urls_json TEXT NOT NULL DEFAULT '[]',
  result_json TEXT NOT NULL DEFAULT '{}', result_count INTEGER NOT NULL DEFAULT 0,
  created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,