rebuild source discovery workflows

2026-09-04 20:52:37 +02:00
parent 3a9b553440
commit 6aefbdc1f3
18 changed files with 853 additions and 106 deletions
@@ -37,13 +37,14 @@ class SourceAdapterTests(unittest.TestCase):
            def read(self, _): return b'{"elements":[{"tags":{"name":"Cape Plumber","craft":"plumber"}}]}'
            def __enter__(self): return self
            def __exit__(self, *_): return False
-        config={'provider':'openstreetmap','query':'plumbers','location':'Cape Town','approved':True,'public_access':True,'terms_accepted':True,'rate_limit':1}
+        config={'provider':'openstreetmap','approved':True,'public_access':True,'terms_accepted':True,'rate_limit':1}
+        criteria={'keywords':['plumbers'],'city':'Cape Town'}
        with patch('app.sources.urlopen',return_value=Response()) as request:
-            page=ApprovedDirectorySource().discover(config)
+            result=ApprovedDirectorySource().discover(config,criteria=criteria,limits={'max_records':10})
        query=request.call_args.args[0].data.decode()
        self.assertIn('"craft"~"plumber|plumbers",i]',query)
        self.assertIn('"shop"~"plumber|plumbers",i]',query)
-        self.assertEqual(page.records[0]['name'],'Cape Plumber')
+        self.assertEqual(result.records[0]['name'],'Cape Plumber')

    def test_legacy_source_kind_constraint_is_migrated(self):
        with TemporaryDirectory() as tmp:
@@ -54,7 +55,8 @@ class SourceAdapterTests(unittest.TestCase):
            db=connect(path)
            self.assertEqual(db.execute("SELECT kind FROM sources WHERE name='Existing manual'").fetchone()[0],'manual')
            db.execute("INSERT INTO sources(organization_id,name,kind,source_code) VALUES(?,?,?,?)",(ORGANIZATION_ID,'OpenStreetMap / Overpass · plumbers','approved_directory','openstreetmap'))
-            db.commit(); self.assertEqual(db.execute("SELECT kind FROM sources WHERE source_code='openstreetmap'").fetchone()[0],'approved_directory'); db.close()
+            db.execute("INSERT INTO sources(organization_id,name,kind,source_code) VALUES(?,?,?,?)",(ORGANIZATION_ID,'Experimental Google','google_browser_search','google_browser_search'))
+            db.commit(); self.assertEqual(db.execute("SELECT kind FROM sources WHERE source_code='openstreetmap'").fetchone()[0],'approved_directory'); self.assertEqual(db.execute("SELECT kind FROM sources WHERE source_code='google_browser_search'").fetchone()[0],'google_browser_search'); db.close()

 class SourceApiTests(unittest.TestCase):
    def setUp(self):
@@ -177,6 +179,52 @@ class SourceApiTests(unittest.TestCase):
            threading.Event().wait(.01)
        self.assertEqual(current['status'],'failed'); self.assertEqual(current['error_code'],'SOURCE_DISABLED')

+    def test_google_browser_source_cannot_enable_without_runtime_feature_flag(self):
+        with patch.dict(os.environ, {'GOOGLE_BROWSER_SEARCH_ENABLED': 'false'}):
+            status, source = self.req('POST', '/api/v1/sources', {
+                'name': 'Disabled Experimental Google', 'kind': 'google_browser_search',
+                'config': {'approved': True, 'public_access': True, 'terms_accepted': True, 'rate_limit': 6},
+            })
+            self.assertEqual(status, 201)
+            status, body = self.req('PATCH', f"/api/v1/sources/{source['id']}", {'enabled': True})
+        self.assertEqual(status, 409)
+        self.assertEqual(body['error'], 'source_feature_disabled')
+        with patch.dict(os.environ, {'GOOGLE_BROWSER_SEARCH_ENABLED': 'false'}):
+            status, body = self.req('POST', '/api/v1/sources', {
+                'name': 'Still Disabled Experimental Google', 'kind': 'google_browser_search', 'enabled': True,
+                'config': {'approved': True, 'public_access': True, 'terms_accepted': True, 'rate_limit': 6},
+            })
+        self.assertEqual(status, 409)
+        self.assertEqual(body['error'], 'source_feature_disabled')
+
+    def test_google_browser_block_is_structured_and_stops_source_job(self):
+        class Response:
+            def read(self, _limit): return b"<html>Our systems have detected unusual traffic from your computer network.</html>"
+            def __enter__(self): return self
+            def __exit__(self, *_): return False
+        with patch.dict(os.environ, {'GOOGLE_BROWSER_SEARCH_ENABLED': 'true'}):
+            status, source = self.req('POST', '/api/v1/sources', {
+                'name': 'Experimental Google', 'kind': 'google_browser_search', 'enabled': True,
+                'config': {'approved': True, 'public_access': True, 'terms_accepted': True, 'rate_limit': 6},
+            })
+            self.assertEqual(status, 201)
+            status, job = self.req('POST', '/api/v1/discovery', {
+                'criteria': {'keywords': ['solar'], 'city': 'Cape Town'},
+                'selected_adapters': ['google_browser_search'], 'idempotency_key': 'google-blocked', 'max_records': 2,
+            })
+            self.assertEqual(status, 202)
+            with patch('app.sources.urlopen', return_value=Response()) as network:
+                for _ in range(100):
+                    _, current = self.req('GET', f"/api/v1/jobs/{job['id']}")
+                    if current['status'] in ('succeeded', 'failed'):
+                        break
+                    threading.Event().wait(.01)
+            self.assertEqual(current['status'], 'failed')
+            self.assertEqual(current['error_code'], 'GOOGLE_BROWSER_BLOCKED')
+            self.assertEqual(network.call_count, 1)
+            events = self.req('GET', f"/api/v1/jobs/{job['id']}/events")[1]['items']
+            self.assertIn('GOOGLE_BROWSER_BLOCKED', [event.get('error_code') for event in events])
+
    def test_source_discovery_persists_pipeline_and_is_idempotent(self):
        status, source = self.req('POST', '/api/v1/sources', {
            'name': 'Manual leads', 'kind': 'manual', 'enabled': True,