add criteria-first discovery search provider

apps/api/tests/test_scoped_discovery.py

@@ -14,6 +14,8 @@ from app.main import create_server, hash_password
 class ScopedDiscoveryApiTests(unittest.TestCase):
     def setUp(self):
         self.tmp = TemporaryDirectory()
+        for key in ('SEARCH_PROVIDER_URL', 'SEARCH_PROVIDER_ALLOWED_HOSTS', 'SEARCH_PROVIDER_API_KEY'):
+            os.environ.pop(key, None)
         os.environ['BOOTSTRAP_ADMIN_EMAIL'] = 'discover-owner@example.test'
         os.environ['BOOTSTRAP_ADMIN_PASSWORD'] = 'password'
         self.server = create_server('127.0.0.1', 0, self.tmp.name + '/db.sqlite')
@@ -60,10 +62,28 @@ class ScopedDiscoveryApiTests(unittest.TestCase):
         self.assertFalse(detail.get('outreach_enabled', False))
 
     def test_requires_bounded_seed_allowlist_and_rejects_ssrf(self):
-        self.assertEqual(self.request('POST', '/api/v1/discovery', {'criteria': {'keywords': ['x']}})[0], 400)
+        status, body = self.request('POST', '/api/v1/discovery', {'criteria': {'keywords': ['x']}})
+        self.assertEqual(status, 503); self.assertEqual(body['error'], 'not_configured')
         status, body = self.request('POST', '/api/v1/discovery', {'criteria': {}, 'seed_urls': ['http://127.0.0.1/'], 'idempotency_key': 'bad'})
         self.assertEqual(status, 400); self.assertEqual(body['error'], 'unsafe_seed_url')
 
+    def test_criteria_first_search_results_flow_through_existing_job_persistence(self):
+        os.environ['SEARCH_PROVIDER_URL'] = 'https://search.example.test/query'
+        os.environ['SEARCH_PROVIDER_ALLOWED_HOSTS'] = 'search.example.test'
+        pages = {'https://acme.test/': {'status': 200, 'final_url': 'https://acme.test/', 'content_type': 'text/html', 'body': b'<title>Acme Solar</title><h1>Acme Solar</h1><p>solar</p>'}}
+        def fetch(url, **_):
+            value = pages[url]; return dict(value, redirect_chain=[], elapsed_ms=1, tls=True, certificate_status='valid')
+        with patch('app.discovery.search_provider', return_value=['https://acme.test/']) as provider, patch('app.discovery._fetch', side_effect=fetch), patch('app.discovery.validate_url', side_effect=lambda url, **_: url), patch('app.main.validate_url', side_effect=lambda url, **_: url):
+            status, created = self.request('POST', '/api/v1/discovery', {'criteria': {'keywords': ['solar']}, 'max_candidates': 1, 'idempotency_key': 'criteria-1'})
+            self.assertEqual(status, 202)
+            for _ in range(50):
+                _, job = self.request('GET', '/api/v1/jobs/' + str(created['id']))
+                if job['status'] in ('succeeded', 'failed'): break
+                time.sleep(.02)
+        self.assertEqual(job['status'], 'succeeded'); provider.assert_called_once_with({'keywords': ['solar']}, 1)
+        run = self.request('GET', '/api/v1/discovery-runs')[1]['items'][0]
+        self.assertEqual(run['seed_urls'], []); self.assertEqual(run['result']['candidates'][0]['provenance']['mechanism'], 'criteria_search_provider')
+
     def test_results_are_tenant_isolated(self):
         ph, salt = hash_password('other-password')
         db = sqlite3.connect(self.server.db_path); db.execute("INSERT INTO organizations VALUES ('other-tenant','Other',CURRENT_TIMESTAMP)"); db.execute("INSERT INTO users (organization_id,email,password_hash,password_salt,role) VALUES (?,?,?,?,?)", ('other-tenant','other@example.test',ph,salt,'owner')); db.commit(); db.close()

apps/api/tests/test_search_provider.py

@@ -0,0 +1,44 @@
+import os
+import unittest
+from unittest.mock import patch
+
+from app.search_provider import ProviderConfigError, provider_status, search
+
+
+class SearchProviderTests(unittest.TestCase):
+    def tearDown(self):
+        for key in ("SEARCH_PROVIDER_URL", "SEARCH_PROVIDER_ALLOWED_HOSTS", "SEARCH_PROVIDER_API_KEY"):
+            os.environ.pop(key, None)
+
+    def test_not_configured_is_fail_closed(self):
+        self.assertEqual(provider_status()["status"], "not_configured")
+        with self.assertRaisesRegex(ProviderConfigError, "not_configured"):
+            search({"keywords": ["solar"]}, 5)
+
+    def test_unsafe_provider_is_rejected(self):
+        os.environ["SEARCH_PROVIDER_URL"] = "http://search.example.test/query"
+        os.environ["SEARCH_PROVIDER_ALLOWED_HOSTS"] = "search.example.test"
+        self.assertEqual(provider_status()["status"], "unsafe_configured")
+        with self.assertRaisesRegex(ProviderConfigError, "unsafe_provider"):
+            search({}, 5)
+
+    def test_successful_mocked_search_returns_bounded_https_urls(self):
+        os.environ["SEARCH_PROVIDER_URL"] = "https://search.example.test/query"
+        os.environ["SEARCH_PROVIDER_ALLOWED_HOSTS"] = "search.example.test"
+        response = type("Response", (), {"__enter__": lambda self: self, "__exit__": lambda self, *args: None, "read": lambda self, *_: b'{"results":[{"url":"https://acme.test"},{"url":"http://bad.test"},{"url":"https://acme.test"}]}'})()
+        with patch("app.search_provider.urlopen", return_value=response), patch("app.search_provider.validate_url", side_effect=lambda url, **_: url):
+            self.assertEqual(search({"keywords": ["solar"]}, 5), ["https://acme.test"])
+
+    def test_limit_is_bounded_and_sent_to_provider(self):
+        os.environ["SEARCH_PROVIDER_URL"] = "https://search.example.test/query"
+        os.environ["SEARCH_PROVIDER_ALLOWED_HOSTS"] = "search.example.test"
+        response = type("Response", (), {"__enter__": lambda self: self, "__exit__": lambda self, *args: None, "read": lambda self, *_: b'{"results":[]}'})()
+        with patch("app.search_provider.urlopen", return_value=response) as opened:
+            self.assertEqual(search({}, 500), [])
+            self.assertEqual(opened.call_args.kwargs["timeout"], 8)
+            request = opened.call_args.args[0]
+            self.assertIn(b'"limit":50', request.data)
+
+
+if __name__ == "__main__":
+    unittest.main()