rebuild source discovery workflows

apps/api/tests/test_ai_opportunity.py

@@ -0,0 +1,170 @@
+import json
+import os
+import sqlite3
+import threading
+import unittest
+from http.client import HTTPConnection
+from tempfile import TemporaryDirectory
+from unittest.mock import Mock
+
+from app.ai_opportunity import normalize_assessment
+from app.ai_research import configure_db as configure_ai_research_db
+from app.main import create_server, hash_password
+
+
+class OpportunityNormalizationTests(unittest.TestCase):
+    def test_normalizes_scores_enums_and_known_evidence_only(self):
+        provider = Mock(return_value={
+            "opportunity_score": 81,
+            "confidence_score": 0.82,
+            "recommendation": "contact",
+            "priority": "high",
+            "reasons": ["Two independent public listings corroborate the business."],
+            "missing_evidence": [],
+            "website_assessment": {"status": "healthy", "broken": False, "outdated": False, "mobile_issue": False, "https_issue": False, "performance_issue": False},
+            "domain_assessment": {"status": "registered"},
+            "contactability": {"public_business_contact_found": True, "contact_type": "general_business"},
+            "recommended_services": ["seo"],
+            "evidence_references": [2, 1, 2],
+            "human_review_required": False,
+        })
+        result = normalize_assessment(provider(), {1, 2})
+        self.assertEqual(result["opportunity_score"], 81)
+        self.assertEqual(result["confidence_score"], 82)
+        self.assertEqual(result["evidence_references"], [1, 2])
+        self.assertFalse(result["human_review_required"])
+        self.assertEqual(result["recommendation"], "contact")
+        self.assertEqual(result["reasons"], ["Two independent public listings corroborate the business."])
+        self.assertEqual(result["missing_evidence"], [])
+        self.assertEqual(result["website_assessment"]["status"], "healthy")
+        self.assertEqual(result["domain_assessment"]["status"], "registered")
+        self.assertEqual(result["contactability"]["contact_type"], "general_business")
+        self.assertEqual(result["recommended_services"], ["seo"])
+
+    def test_contact_recommendation_is_an_internal_no_send_recommendation(self):
+        payload = {"recommendation": "contact", "evidence_references": [1, 2], "confidence_score": 90}
+        self.assertEqual(normalize_assessment(payload, {1, 2})["recommendation"], "contact")
+
+    def test_unknown_values_default_and_weak_evidence_requires_human_review(self):
+        result = normalize_assessment({
+            "opportunity_score": "not-a-number",
+            "confidence_score": 20,
+            "recommendation": "email_them_now",
+            "priority": "urgent",
+            "evidence_references": [],
+            "human_review_required": False,
+        }, set())
+        self.assertEqual(result["opportunity_score"], 0)
+        self.assertEqual(result["confidence_score"], 20)
+        self.assertEqual(result["recommendation"], "insufficient_evidence")
+        self.assertEqual(result["priority"], "low")
+        self.assertTrue(result["human_review_required"])
+
+    def test_unknown_evidence_reference_is_rejected(self):
+        with self.assertRaisesRegex(ValueError, "unknown_evidence_reference"):
+            normalize_assessment({"evidence_references": [99]}, {1})
+
+    def test_suppression_overrides_provider_recommendation(self):
+        result = normalize_assessment({
+            "opportunity_score": 99,
+            "confidence_score": 99,
+            "recommendation": "contact",
+            "priority": "high",
+            "evidence_references": [1, 2],
+            "human_review_required": False,
+        }, {1, 2}, suppressed=True)
+        self.assertEqual(result["recommendation"], "do_not_contact")
+        self.assertTrue(result["human_review_required"])
+
+
+class OpportunityAssessmentApiTests(unittest.TestCase):
+    def setUp(self):
+        self.tmp = TemporaryDirectory()
+        self.old = {key: os.environ.get(key) for key in ("AI_PROVIDER", "BOOTSTRAP_ADMIN_EMAIL", "BOOTSTRAP_ADMIN_PASSWORD")}
+        os.environ.update({"AI_PROVIDER": "local", "BOOTSTRAP_ADMIN_EMAIL": "opportunity-owner@example.test", "BOOTSTRAP_ADMIN_PASSWORD": "password"})
+        self.server = create_server("127.0.0.1", 0, self.tmp.name + "/opportunity.db")
+        self.thread = threading.Thread(target=self.server.serve_forever, daemon=True); self.thread.start()
+        self.conn = HTTPConnection("127.0.0.1", self.server.server_port, timeout=3); self.cookie = None
+        self.request("POST", "/api/v1/auth/login", {"email": "opportunity-owner@example.test", "password": "password"})
+
+    def tearDown(self):
+        self.server.shutdown(); self.server.server_close(); self.thread.join(timeout=2)
+        configure_ai_research_db("")
+        self.tmp.cleanup()
+        for key, value in self.old.items():
+            if value is None: os.environ.pop(key, None)
+            else: os.environ[key] = value
+
+    def request(self, method, path, payload=None):
+        headers = {"Content-Type": "application/json"}
+        if self.cookie: headers["Cookie"] = self.cookie
+        self.conn.request(method, path, json.dumps(payload).encode() if payload is not None else None, headers)
+        response = self.conn.getresponse(); cookie = response.getheader("Set-Cookie")
+        if cookie: self.cookie = cookie.split(";", 1)[0]
+        return response.status, json.loads(response.read() or b"{}")
+
+    def selected_business_at_threshold(self):
+        status, business = self.request("POST", "/api/v1/businesses", {"name": "Selected Co"})
+        self.assertEqual(status, 201)
+        db = sqlite3.connect(self.tmp.name + "/opportunity.db")
+        db.execute("UPDATE businesses SET score=70 WHERE id=?", (business["id"],)); db.commit(); db.close()
+        return business["id"]
+
+    def test_manual_selected_business_after_threshold_returns_grounded_review_only_assessment(self):
+        bid = self.selected_business_at_threshold()
+        self.request("POST", f"/api/v1/businesses/{bid}/evidence", {"kind": "source", "url": "https://source.test", "claim": "Needs a modern website"})
+        status, result = self.request("POST", f"/api/v1/businesses/{bid}/ai/opportunity-assessment", {})
+        self.assertEqual(status, 201)
+        self.assertEqual(result["business_id"], bid)
+        self.assertEqual(result["assessment"]["evidence_references"], [1])
+        self.assertIn("opportunity_score", result["assessment"])
+        self.assertIn("confidence_score", result["assessment"])
+        self.assertTrue(result["assessment"]["human_review_required"])
+        self.assertFalse(result["network_send"])
+        self.assertFalse(result["automatic_outreach"])
+        status, runs = self.request("GET", "/api/v1/ai-runs")
+        self.assertEqual(status, 200)
+        self.assertEqual(runs["items"][0]["output"]["assessment"], result["assessment"])
+        self.assertEqual(len(runs["items"][0]["input_evidence_hashes"]), 1)
+
+    def test_manual_assessment_requires_deterministic_threshold(self):
+        status, business = self.request("POST", "/api/v1/businesses", {"name": "Below threshold"})
+        self.assertEqual(status, 201)
+        status, result = self.request("POST", f"/api/v1/businesses/{business['id']}/ai/opportunity-assessment", {})
+        self.assertEqual(status, 409)
+        self.assertEqual(result["error"], "deterministic_threshold_not_met")
+
+    def test_unconfigured_provider_fails_closed_without_assessment_output(self):
+        bid = self.selected_business_at_threshold()
+        with unittest.mock.patch("app.main.provider_status", return_value={"status": "not_configured", "provider": ""}):
+            status, result = self.request("POST", f"/api/v1/businesses/{bid}/ai/opportunity-assessment", {})
+        self.assertEqual(status, 409)
+        self.assertEqual(result["error"], "ai_provider_not_configured")
+        self.assertFalse(result["network_send"])
+        self.assertNotIn("assessment", result)
+
+    def test_suppressed_selected_business_assessment_is_do_not_contact(self):
+        bid = self.selected_business_at_threshold()
+        self.request("POST", f"/api/v1/businesses/{bid}/evidence", {"kind": "source", "url": "https://source.test", "claim": "Evidence"})
+        self.request("POST", "/api/v1/suppressions", {"kind": "domain", "value": "selected.test"})
+        db = sqlite3.connect(self.tmp.name + "/opportunity.db")
+        db.execute("UPDATE businesses SET website_domain='selected.test' WHERE id=?", (bid,)); db.commit(); db.close()
+        status, result = self.request("POST", f"/api/v1/businesses/{bid}/ai/opportunity-assessment", {})
+        self.assertEqual(status, 201)
+        self.assertEqual(result["assessment"]["recommendation"], "do_not_contact")
+        self.assertTrue(result["assessment"]["human_review_required"])
+
+    def test_manual_assessment_is_tenant_scoped(self):
+        bid = self.selected_business_at_threshold()
+        password_hash, password_salt = hash_password("other-password")
+        db = sqlite3.connect(self.tmp.name + "/opportunity.db")
+        db.execute("INSERT INTO organizations(id,name) VALUES(?,?)", ("other-tenant", "Other"))
+        db.execute("INSERT INTO users(organization_id,email,password_hash,password_salt,role) VALUES(?,?,?,?,?)", ("other-tenant", "other-opportunity@example.test", password_hash, password_salt, "owner"))
+        db.commit(); db.close()
+        self.cookie = None
+        self.assertEqual(self.request("POST", "/api/v1/auth/login", {"email": "other-opportunity@example.test", "password": "other-password"})[0], 200)
+        self.assertEqual(self.request("POST", f"/api/v1/businesses/{bid}/ai/opportunity-assessment", {})[0], 404)
+
+
+if __name__ == "__main__":
+    unittest.main()

apps/api/tests/test_google_browser_search.py

@@ -0,0 +1,59 @@
+import os
+import unittest
+from unittest.mock import patch
+
+from app.sources import GoogleBrowserSearchBlocked, GoogleBrowserSearchSource
+
+
+class GoogleBrowserSearchSourceTests(unittest.TestCase):
+    def setUp(self):
+        self.old_enabled = os.environ.pop("GOOGLE_BROWSER_SEARCH_ENABLED", None)
+
+    def tearDown(self):
+        GoogleBrowserSearchSource._last_request_at = None
+        if self.old_enabled is None:
+            os.environ.pop("GOOGLE_BROWSER_SEARCH_ENABLED", None)
+        else:
+            os.environ["GOOGLE_BROWSER_SEARCH_ENABLED"] = self.old_enabled
+
+    def test_disabled_feature_blocks_without_network_io(self):
+        source = GoogleBrowserSearchSource()
+        with patch("app.sources.urlopen") as network:
+            with self.assertRaises(GoogleBrowserSearchBlocked) as raised:
+                source.discover(
+                    {"approved": True, "public_access": True, "terms_accepted": True, "rate_limit": 6},
+                    criteria={"keywords": ["solar installers"], "city": "Cape Town"},
+                    limits={"max_records": 5},
+                )
+        self.assertEqual(raised.exception.code, "GOOGLE_BROWSER_BLOCKED")
+        self.assertEqual(raised.exception.reason, "feature_disabled")
+        network.assert_not_called()
+
+    def test_enabled_fetch_builds_query_from_criteria_and_parses_visible_result_links(self):
+        os.environ["GOOGLE_BROWSER_SEARCH_ENABLED"] = "true"
+
+        class Response:
+            def read(self, _limit):
+                return b'<html><body><a href="https://acme.example/about"><h3>Acme Solar</h3></a><a href="https://www.google.com/preferences"><h3>Settings</h3></a></body></html>'
+
+            def __enter__(self):
+                return self
+
+            def __exit__(self, *_):
+                return False
+
+        with patch("app.sources.urlopen", return_value=Response()) as network:
+            page = GoogleBrowserSearchSource().discover(
+                {"approved": True, "public_access": True, "terms_accepted": True, "rate_limit": 6},
+                criteria={"keywords": ["solar"], "city": "Cape Town"},
+                limits={"max_records": 4},
+            )
+        self.assertEqual(page.records, [{"name": "Acme Solar", "website": "https://acme.example/about", "email": "", "phone": "", "description": "Public Google search result", "location": ""}])
+        request = network.call_args.args[0]
+        self.assertIn("q=solar+Cape+Town", request.full_url)
+        self.assertNotIn("query", request.full_url)
+        self.assertTrue(page.metadata["public_html_only"])
+
+
+if __name__ == "__main__":
+    unittest.main()

apps/api/tests/test_scoped_discovery.py

@@ -84,6 +84,46 @@ class ScopedDiscoveryApiTests(unittest.TestCase):
         run = self.request('GET', '/api/v1/discovery-runs')[1]['items'][0]
         self.assertEqual(run['seed_urls'], []); self.assertEqual(run['result']['candidates'][0]['provenance']['mechanism'], 'criteria_search_provider')
 
+    def test_selected_disabled_source_is_rejected_before_job_creation(self):
+        status, source = self.request('POST', '/api/v1/sources', {
+            'name': 'Disabled manual', 'kind': 'manual',
+            'config': {'rows': [{'name': 'Not runnable'}]},
+        })
+        self.assertEqual(status, 201)
+        status, body = self.request('POST', '/api/v1/discovery', {
+            'criteria': {'category': 'plumbers'}, 'source_ids': [source['id']],
+            'idempotency_key': 'disabled-source',
+        })
+        self.assertEqual(status, 409)
+        self.assertEqual(body['error'], 'selected_source_not_ready')
+        self.assertEqual(self.request('GET', '/api/v1/discovery-runs')[1]['items'], [])
+
+    def test_source_dry_run_validates_enabled_source_without_persisting_candidates(self):
+        status, source = self.request('POST', '/api/v1/sources', {
+            'name': 'Preview manual', 'kind': 'manual', 'enabled': True,
+            'config': {'rows': [{'name': 'Preview Plumbing', 'website': 'https://preview.example.test'}]},
+        })
+        self.assertEqual(status, 201)
+        with patch('app.main.enrich_source_business') as enrich:
+            status, job = self.request('POST', '/api/v1/discovery', {
+                'criteria': {'category': 'plumbers', 'city': 'Cape Town'},
+                'source_ids': [source['id']], 'dry_run': True,
+                'idempotency_key': 'source-dry-run', 'max_candidates': 5,
+            })
+            self.assertEqual(status, 202)
+            for _ in range(100):
+                _, current = self.request('GET', '/api/v1/jobs/' + str(job['id']))
+                if current['status'] in ('succeeded', 'failed'):
+                    break
+                time.sleep(.02)
+        self.assertEqual(current['status'], 'succeeded')
+        self.assertFalse(enrich.called)
+        self.assertEqual(self.request('GET', '/api/v1/businesses')[1]['items'], [])
+        self.assertEqual(self.request('GET', '/api/v1/source-records')[1]['items'], [])
+        run = self.request('GET', '/api/v1/discovery-runs')[1]['items'][0]
+        self.assertTrue(run['dry_run'])
+        self.assertEqual(run['result_count'], 1)
+
     def test_results_are_tenant_isolated(self):
         ph, salt = hash_password('other-password')
         db = sqlite3.connect(self.server.db_path); db.execute("INSERT INTO organizations VALUES ('other-tenant','Other',CURRENT_TIMESTAMP)"); db.execute("INSERT INTO users (organization_id,email,password_hash,password_salt,role) VALUES (?,?,?,?,?)", ('other-tenant','other@example.test',ph,salt,'owner')); db.commit(); db.close()

apps/api/tests/test_sources_phase5.py

@@ -37,13 +37,14 @@ class SourceAdapterTests(unittest.TestCase):
             def read(self, _): return b'{"elements":[{"tags":{"name":"Cape Plumber","craft":"plumber"}}]}'
             def __enter__(self): return self
             def __exit__(self, *_): return False
-        config={'provider':'openstreetmap','query':'plumbers','location':'Cape Town','approved':True,'public_access':True,'terms_accepted':True,'rate_limit':1}
+        config={'provider':'openstreetmap','approved':True,'public_access':True,'terms_accepted':True,'rate_limit':1}
+        criteria={'keywords':['plumbers'],'city':'Cape Town'}
         with patch('app.sources.urlopen',return_value=Response()) as request:
-            page=ApprovedDirectorySource().discover(config)
+            result=ApprovedDirectorySource().discover(config,criteria=criteria,limits={'max_records':10})
         query=request.call_args.args[0].data.decode()
         self.assertIn('"craft"~"plumber|plumbers",i]',query)
         self.assertIn('"shop"~"plumber|plumbers",i]',query)
-        self.assertEqual(page.records[0]['name'],'Cape Plumber')
+        self.assertEqual(result.records[0]['name'],'Cape Plumber')
 
     def test_legacy_source_kind_constraint_is_migrated(self):
         with TemporaryDirectory() as tmp:
@@ -54,7 +55,8 @@ class SourceAdapterTests(unittest.TestCase):
             db=connect(path)
             self.assertEqual(db.execute("SELECT kind FROM sources WHERE name='Existing manual'").fetchone()[0],'manual')
             db.execute("INSERT INTO sources(organization_id,name,kind,source_code) VALUES(?,?,?,?)",(ORGANIZATION_ID,'OpenStreetMap / Overpass · plumbers','approved_directory','openstreetmap'))
-            db.commit(); self.assertEqual(db.execute("SELECT kind FROM sources WHERE source_code='openstreetmap'").fetchone()[0],'approved_directory'); db.close()
+            db.execute("INSERT INTO sources(organization_id,name,kind,source_code) VALUES(?,?,?,?)",(ORGANIZATION_ID,'Experimental Google','google_browser_search','google_browser_search'))
+            db.commit(); self.assertEqual(db.execute("SELECT kind FROM sources WHERE source_code='openstreetmap'").fetchone()[0],'approved_directory'); self.assertEqual(db.execute("SELECT kind FROM sources WHERE source_code='google_browser_search'").fetchone()[0],'google_browser_search'); db.close()
 
 class SourceApiTests(unittest.TestCase):
     def setUp(self):
@@ -177,6 +179,52 @@ class SourceApiTests(unittest.TestCase):
             threading.Event().wait(.01)
         self.assertEqual(current['status'],'failed'); self.assertEqual(current['error_code'],'SOURCE_DISABLED')
 
+    def test_google_browser_source_cannot_enable_without_runtime_feature_flag(self):
+        with patch.dict(os.environ, {'GOOGLE_BROWSER_SEARCH_ENABLED': 'false'}):
+            status, source = self.req('POST', '/api/v1/sources', {
+                'name': 'Disabled Experimental Google', 'kind': 'google_browser_search',
+                'config': {'approved': True, 'public_access': True, 'terms_accepted': True, 'rate_limit': 6},
+            })
+            self.assertEqual(status, 201)
+            status, body = self.req('PATCH', f"/api/v1/sources/{source['id']}", {'enabled': True})
+        self.assertEqual(status, 409)
+        self.assertEqual(body['error'], 'source_feature_disabled')
+        with patch.dict(os.environ, {'GOOGLE_BROWSER_SEARCH_ENABLED': 'false'}):
+            status, body = self.req('POST', '/api/v1/sources', {
+                'name': 'Still Disabled Experimental Google', 'kind': 'google_browser_search', 'enabled': True,
+                'config': {'approved': True, 'public_access': True, 'terms_accepted': True, 'rate_limit': 6},
+            })
+        self.assertEqual(status, 409)
+        self.assertEqual(body['error'], 'source_feature_disabled')
+
+    def test_google_browser_block_is_structured_and_stops_source_job(self):
+        class Response:
+            def read(self, _limit): return b"<html>Our systems have detected unusual traffic from your computer network.</html>"
+            def __enter__(self): return self
+            def __exit__(self, *_): return False
+        with patch.dict(os.environ, {'GOOGLE_BROWSER_SEARCH_ENABLED': 'true'}):
+            status, source = self.req('POST', '/api/v1/sources', {
+                'name': 'Experimental Google', 'kind': 'google_browser_search', 'enabled': True,
+                'config': {'approved': True, 'public_access': True, 'terms_accepted': True, 'rate_limit': 6},
+            })
+            self.assertEqual(status, 201)
+            status, job = self.req('POST', '/api/v1/discovery', {
+                'criteria': {'keywords': ['solar'], 'city': 'Cape Town'},
+                'selected_adapters': ['google_browser_search'], 'idempotency_key': 'google-blocked', 'max_records': 2,
+            })
+            self.assertEqual(status, 202)
+            with patch('app.sources.urlopen', return_value=Response()) as network:
+                for _ in range(100):
+                    _, current = self.req('GET', f"/api/v1/jobs/{job['id']}")
+                    if current['status'] in ('succeeded', 'failed'):
+                        break
+                    threading.Event().wait(.01)
+            self.assertEqual(current['status'], 'failed')
+            self.assertEqual(current['error_code'], 'GOOGLE_BROWSER_BLOCKED')
+            self.assertEqual(network.call_count, 1)
+            events = self.req('GET', f"/api/v1/jobs/{job['id']}/events")[1]['items']
+            self.assertIn('GOOGLE_BROWSER_BLOCKED', [event.get('error_code') for event in events])
+
     def test_source_discovery_persists_pipeline_and_is_idempotent(self):
         status, source = self.req('POST', '/api/v1/sources', {
             'name': 'Manual leads', 'kind': 'manual', 'enabled': True,