From 1e4c3a1e15519ead828454e46a10241a84fbbed7 Mon Sep 17 00:00:00 2001 From: Marco0300 Date: Fri, 4 Sep 2026 15:51:59 +0200 Subject: [PATCH] migrate legacy source kind constraint --- apps/api/app/main.py | 22 ++++++++++++++++++++++ apps/api/tests/test_sources_phase5.py | 13 ++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/apps/api/app/main.py b/apps/api/app/main.py index f361270..ad6601a 100644 --- a/apps/api/app/main.py +++ b/apps/api/app/main.py @@ -84,6 +84,28 @@ def connect(db_path: str) -> sqlite3.Connection: existing = {r[1] for r in db.execute(f"PRAGMA table_info({table})")} for col, definition in additions: if col not in existing: db.execute(f"ALTER TABLE {table} ADD COLUMN {col} {definition}") + db.execute("INSERT OR IGNORE INTO organizations (id,name) VALUES (?,?)", (ORGANIZATION_ID, "Demo organization")) + source_sql_row=db.execute("SELECT sql FROM sqlite_master WHERE type='table' AND name='sources'").fetchone() + source_sql=(source_sql_row[0] or '') if source_sql_row else '' + if "CHECK(kind IN ('csv','manual'))" in source_sql: + # SQLite cannot change foreign-key enforcement during a transaction. + db.commit(); db.execute("PRAGMA foreign_keys=OFF") + db.executescript(""" + CREATE TABLE sources_rebuilt ( + id INTEGER PRIMARY KEY AUTOINCREMENT, organization_id TEXT NOT NULL REFERENCES organizations(id), + name TEXT NOT NULL, kind TEXT NOT NULL CHECK(kind IN ('csv','manual','google_places','bing_local','approved_directory','public_website','permitted_social','ct_logs','dns','rdap')), source_code TEXT NOT NULL DEFAULT '', display_name TEXT NOT NULL DEFAULT '', enabled INTEGER NOT NULL DEFAULT 0, approved INTEGER NOT NULL DEFAULT 0, + config_json TEXT NOT NULL DEFAULT '{}', policy_json TEXT NOT NULL DEFAULT '{}', quota_json TEXT NOT NULL DEFAULT '{}', health_status TEXT NOT NULL DEFAULT 'unknown', + consecutive_failures INTEGER NOT NULL DEFAULT 0, circuit_open INTEGER NOT NULL DEFAULT 0, + last_success_at TEXT, last_failure_at TEXT, last_error TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, UNIQUE(organization_id,name) + ); + INSERT INTO sources_rebuilt(id,organization_id,name,kind,source_code,display_name,enabled,approved,config_json,policy_json,quota_json,health_status,consecutive_failures,circuit_open,last_success_at,last_failure_at,last_error,created_at,updated_at) + SELECT id,organization_id,name,kind,source_code,display_name,enabled,approved,config_json,policy_json,quota_json,health_status,consecutive_failures,circuit_open,last_success_at,last_failure_at,last_error,created_at,updated_at FROM sources; + DROP TABLE sources; + ALTER TABLE sources_rebuilt RENAME TO sources; + CREATE INDEX IF NOT EXISTS idx_sources_org ON sources(organization_id,id); + """) + db.execute("PRAGMA foreign_keys=ON") db.execute("CREATE UNIQUE INDEX IF NOT EXISTS uq_pipeline_idempotency ON pipeline_entries(organization_id,idempotency_key) WHERE idempotency_key IS NOT NULL AND idempotency_key <> ''") db.execute("CREATE UNIQUE INDEX IF NOT EXISTS uq_interaction_idempotency ON interactions(organization_id,idempotency_key) WHERE idempotency_key IS NOT NULL AND idempotency_key <> ''") db.execute("INSERT OR IGNORE INTO organizations (id,name) VALUES (?,?)", (ORGANIZATION_ID, "Demo organization")) diff --git a/apps/api/tests/test_sources_phase5.py b/apps/api/tests/test_sources_phase5.py index 1101319..497ce5a 100644 --- a/apps/api/tests/test_sources_phase5.py +++ b/apps/api/tests/test_sources_phase5.py @@ -1,7 +1,7 @@ import json, os, sqlite3, threading, unittest from http.client import HTTPConnection from tempfile import TemporaryDirectory -from app.main import create_server +from app.main import ORGANIZATION_ID, connect, create_server from app.sources import CsvSource, ManualSource, available_adapters class SourceAdapterTests(unittest.TestCase): @@ -31,6 +31,17 @@ class SourceAdapterTests(unittest.TestCase): self.assertFalse(result.valid) self.assertIn('secret', result.errors[0].lower()) + def test_legacy_source_kind_constraint_is_migrated(self): + with TemporaryDirectory() as tmp: + path=os.path.join(tmp,'legacy.db') + legacy=sqlite3.connect(path) + legacy.execute("CREATE TABLE sources (id INTEGER PRIMARY KEY AUTOINCREMENT, organization_id TEXT NOT NULL, name TEXT NOT NULL, kind TEXT NOT NULL CHECK(kind IN ('csv','manual')), enabled INTEGER NOT NULL DEFAULT 0, config_json TEXT NOT NULL DEFAULT '{}', health_status TEXT NOT NULL DEFAULT 'unknown', consecutive_failures INTEGER NOT NULL DEFAULT 0, circuit_open INTEGER NOT NULL DEFAULT 0, last_success_at TEXT, last_failure_at TEXT, last_error TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, source_code TEXT NOT NULL DEFAULT '', display_name TEXT NOT NULL DEFAULT '', approved INTEGER NOT NULL DEFAULT 0, policy_json TEXT NOT NULL DEFAULT '{}', quota_json TEXT NOT NULL DEFAULT '{}', UNIQUE(organization_id,name))") + legacy.execute("INSERT INTO sources(organization_id,name,kind) VALUES(?, 'Existing manual','manual')",(ORGANIZATION_ID,)); legacy.commit(); legacy.close() + db=connect(path) + self.assertEqual(db.execute("SELECT kind FROM sources WHERE name='Existing manual'").fetchone()[0],'manual') + db.execute("INSERT INTO sources(organization_id,name,kind,source_code) VALUES(?,?,?,?)",(ORGANIZATION_ID,'OpenStreetMap / Overpass ยท plumbers','approved_directory','openstreetmap')) + db.commit(); self.assertEqual(db.execute("SELECT kind FROM sources WHERE source_code='openstreetmap'").fetchone()[0],'approved_directory'); db.close() + class SourceApiTests(unittest.TestCase): def setUp(self): self.tmp=TemporaryDirectory(); os.environ['BOOTSTRAP_ADMIN_EMAIL']='owner@example.test'; os.environ['BOOTSTRAP_ADMIN_PASSWORD']='development-password'