root/MarketingTool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

complete source discovery processing lifecycle

Browse Source
This commit is contained in:
Marco0300
2026-09-03 23:01:25 +02:00
parent a9213c0282
commit 58c4d93208
4 changed files with 157 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/schema.sql
+2 -1
View File
@@ -158,11 +158,12 @@ CREATE INDEX IF NOT EXISTS idx_discovery_queries_org ON discovery_queries(organi
CREATE TABLE IF NOT EXISTS source_records (
id INTEGER PRIMARY KEY AUTOINCREMENT, organization_id TEXT NOT NULL REFERENCES organizations(id), source_id INTEGER NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
discovery_query_id INTEGER REFERENCES discovery_queries(id) ON DELETE SET NULL, discovery_run_id INTEGER REFERENCES discovery_runs(id) ON DELETE SET NULL, content_hash TEXT NOT NULL, raw_json TEXT NOT NULL,
normalized_json TEXT NOT NULL, normalized_key TEXT NOT NULL DEFAULT '', source_url TEXT NOT NULL DEFAULT '', provenance_json TEXT NOT NULL DEFAULT '{}', query_context_json TEXT NOT NULL DEFAULT '{}', processing_status TEXT NOT NULL DEFAULT 'raw',
normalized_json TEXT NOT NULL, normalized_key TEXT NOT NULL DEFAULT '', source_url TEXT NOT NULL DEFAULT '', provenance_json TEXT NOT NULL DEFAULT '{}', query_context_json TEXT NOT NULL DEFAULT '{}', response_metadata_json TEXT NOT NULL DEFAULT '{}', processing_status TEXT NOT NULL DEFAULT 'raw' CHECK(processing_status IN ('raw','processed','matched','failed','skipped')),
cursor_json TEXT NOT NULL DEFAULT '{}', rate_policy_json TEXT NOT NULL DEFAULT '{}', created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
UNIQUE(organization_id,source_id,content_hash)
);
CREATE INDEX IF NOT EXISTS idx_source_records_org ON source_records(organization_id,id DESC);
CREATE INDEX IF NOT EXISTS idx_source_records_processing ON source_records(organization_id,processing_status,created_at DESC);
CREATE TABLE IF NOT EXISTS enrichment_queue (
id INTEGER PRIMARY KEY AUTOINCREMENT, organization_id TEXT NOT NULL REFERENCES organizations(id) ON DELETE CASCADE, source_record_id INTEGER NOT NULL REFERENCES source_records(id) ON DELETE CASCADE, status TEXT NOT NULL DEFAULT 'queued', attempts INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, UNIQUE(organization_id,source_record_id)
);