separate reusable source config from discovery criteria

apps/api/tests/test_sources_phase5.py

@@ -81,8 +81,8 @@ class SourceApiTests(unittest.TestCase):
         self.assertEqual(self.req('POST',f'/api/v1/sources/{sid}/ingest',payload)[0],409)
         db=sqlite3.connect(self.tmp.name+'/x.db'); self.assertTrue(db.execute("select 1 from audit_log where action='source.disabled'").fetchone()); db.close()
     def test_queries_enqueue_and_records_are_tenant_scoped(self):
-        _,source=self.req('POST','/api/v1/sources',{'name':'CSV','kind':'csv','enabled':True})
-        _,q=self.req('POST','/api/v1/discovery-queries',{'source_id':source['id'],'name':'q','query':{'csv':'name\nA'}})
+        _,source=self.req('POST','/api/v1/sources',{'name':'CSV','kind':'csv','enabled':True,'config':{'csv':'name\nA'}})
+        _,q=self.req('POST','/api/v1/discovery-queries',{'source_id':source['id'],'name':'q','query':{}})
         status,job=self.req('POST',f"/api/v1/discovery-queries/{q['id']}/run",{})
         self.assertEqual(status,202); self.assertEqual(job['type'],'source_discovery')
         for _ in range(100):
@@ -105,12 +105,52 @@ class SourceApiTests(unittest.TestCase):
         self.assertEqual(self.req('PATCH', f"/api/v1/sources/{website['id']}", {'enabled': True})[0], 409)
 
     def test_duplicate_source_registration_is_idempotent(self):
-        payload={'name':'OpenStreetMap / Overpass · plumbers','kind':'openstreetmap','config':{'provider':'openstreetmap','query':'plumbers','location':'Cape Town','approved':True,'public_access':True,'terms_accepted':True,'rate_limit':1}}
+        payload={'name':'OpenStreetMap / Overpass · plumbers','kind':'openstreetmap','config':{'provider':'openstreetmap','approved':True,'public_access':True,'terms_accepted':True,'rate_limit':1}}
         status, created=self.req('POST','/api/v1/sources',payload); self.assertEqual(status,201); self.assertTrue(created['created'])
         status, reused=self.req('POST','/api/v1/sources',payload); self.assertEqual(status,200); self.assertFalse(reused['created']); self.assertEqual(reused['id'],created['id'])
-        second={**payload,'name':'OpenStreetMap / Overpass · plumbers · Durban','config':{**payload['config'],'location':'Durban'}}
+        second={**payload,'name':'OpenStreetMap / Overpass · plumbers · Durban'}
         status, other=self.req('POST','/api/v1/sources',second); self.assertEqual(status,201); self.assertTrue(other['created']); self.assertNotEqual(other['id'],created['id'])
 
+    def test_source_configuration_rejects_discovery_criteria(self):
+        forbidden = {'query': 'plumbers', 'category': 'trades', 'city': 'Cape Town', 'location': 'Western Cape'}
+        for field, value in forbidden.items():
+            with self.subTest(field=field):
+                status, body = self.req('POST', '/api/v1/sources', {
+                    'name': 'Manual ' + field, 'kind': 'manual',
+                    'config': {'rows': [], field: value},
+                })
+                self.assertEqual(status, 400)
+                self.assertEqual(body['error'], 'source_configuration_contains_criteria')
+
+    def test_source_worker_passes_query_criteria_and_effective_limits_to_connector(self):
+        status, source = self.req('POST', '/api/v1/sources', {
+            'name': 'Criteria manual', 'kind': 'manual', 'enabled': True,
+            'config': {'rows': [{'name': 'Criteria Acme'}]},
+        })
+        self.assertEqual(status, 201)
+        status, query = self.req('POST', '/api/v1/discovery-queries', {
+            'source_id': source['id'], 'name': 'Cape solar',
+            'query': {'keywords': ['solar'], 'city': 'Cape Town'},
+            'max_records': 7, 'daily_limit': 9,
+        })
+        self.assertEqual(status, 201)
+        observed = []
+        from app.sources import ManualSource
+        original = ManualSource.discover
+        def spy(adapter, config, cursor=None, criteria=None, limits=None):
+            observed.append((criteria, limits))
+            return original(adapter, config, cursor, criteria=criteria, limits=limits)
+        with patch('app.sources.ManualSource.discover', new=spy):
+            status, job = self.req('POST', f"/api/v1/discovery-queries/{query['id']}/run", {})
+            self.assertEqual(status, 202)
+            for _ in range(100):
+                _, current = self.req('GET', f"/api/v1/jobs/{job['id']}")
+                if current['status'] in ('succeeded', 'failed'):
+                    break
+                threading.Event().wait(.01)
+        self.assertEqual(current['status'], 'succeeded')
+        self.assertEqual(observed, [({'keywords': ['solar'], 'city': 'Cape Town'}, {'max_records': 7, 'daily_limit': 9, 'per_run_limit': 7})])
+
     def test_source_configuration_can_be_saved_before_enablement(self):
         status, source = self.req('POST', '/api/v1/sources', {'name': 'DNS', 'kind': 'dns', 'config': {}})
         self.assertEqual(status, 201)