add deterministic pilot benchmarks

2026-09-03 12:50:00 +02:00
parent 6b41d5b9ee
commit 9622f76977
11 changed files with 763 additions and 0 deletions
@@ -0,0 +1,82 @@
+{
+  "seed": 1601,
+  "limitations": [
+    "Synthetic, small, English-heavy cases are not representative of production traffic.",
+    "Contact labels cover parser false positives but do not establish consent, identity, or deliverability.",
+    "Website fixtures classify supplied HTML only; they do not measure network, DNS, redirects, or adversarial pages.",
+    "Latency and cache measurements are local pilot signals and vary by host load."
+  ],
+  "normalization": [
+    {"name": "SA contact fields", "input": {"name": "  Acme   Solar ", "website": "HTTPS://WWW.ACME.TEST/path", "phone": "082 555 1234", "email": " SALES@ACME.TEST "}, "expected": {"name": "Acme Solar", "website_domain": "acme.test", "email": "sales@acme.test"}},
+    {"name": "accented location", "input": {"name": "Cafe", "province": "  Western Cape ", "city": "CAPE TOWN", "suburb": "Sea Point"}, "expected": {"province": "western cape", "city": "cape town", "suburb": "sea point"}}
+  ],
+  "matching": [
+    {
+      "name": "shared exact domain is a positive",
+      "source": {"name": "Acme Solar", "website": "https://www.acme.test", "city": "Johannesburg"},
+      "candidates": [
+        {"id": 1, "name": "Acme Solar (Pty) Ltd", "website": "http://acme.test/", "city": "Johannesburg"},
+        {"id": 2, "name": "Acme Supplies", "website": "https://supplies.test", "city": "Johannesburg"}
+      ],
+      "threshold": 0.72,
+      "expected_ids": [1],
+      "forbidden_ids": [2]
+    },
+    {
+      "name": "similar name without corroboration is not a positive",
+      "source": {"name": "Brightline Consulting", "city": "Cape Town"},
+      "candidates": [
+        {"id": 3, "name": "Brightline Consulting", "city": "Durban"},
+        {"id": 4, "name": "Brightline Consulting", "city": "Cape Town"}
+      ],
+      "threshold": 0.72,
+      "expected_ids": [4],
+      "forbidden_ids": [3]
+    },
+    {
+      "name": "unrelated businesses are negatives",
+      "source": {"name": "Green Oak Architects", "website": "green-oak.test"},
+      "candidates": [
+        {"id": 5, "name": "Green Oak Accounting", "website": "accounting.test"},
+        {"id": 6, "name": "Red River Bakery", "website": "redriver.test"}
+      ],
+      "threshold": 0.72,
+      "expected_ids": [],
+      "forbidden_ids": [5, 6]
+    }
+  ],
+  "contacts": [
+    {
+      "name": "public contacts and ignored markup",
+      "source_url": "https://acme.test/contact",
+      "html": "<p>Sales: sales@acme.test</p><p>Call +27 (12) 345-6789</p><script>secret@acme.test</script><img src=\"https://tracker.test/pixel?email=tracker@tracker.test\"><p>API key: token@acme.test</p>",
+      "expected": [["email", "sales@acme.test"], ["phone", "+27123456789"]],
+      "forbidden": [["email", "secret@acme.test"], ["email", "tracker@tracker.test"], ["email", "token@acme.test"]]
+    },
+    {
+      "name": "forms are provenance not email destinations",
+      "source_url": "https://bright.test/contact",
+      "html": "<form><input name=\"email\" placeholder=\"Your email\"></form><p>hello [at] bright.test</p>",
+      "expected": [["email", "hello@bright.test"]],
+      "forbidden": [["email", "email@bright.test"]]
+    }
+  ],
+  "websites": [
+    {"name": "healthy", "status": 200, "url": "https://acme.test", "body": "<html><h1>Acme Solar</h1><p>Welcome to our business.</p></html>", "expected": "healthy"},
+    {"name": "parked", "status": 200, "url": "https://parked.test", "body": "Domain for sale - buy this domain", "expected": "parked"},
+    {"name": "construction", "status": 200, "url": "https://new.test", "body": "Website coming soon", "expected": "under_construction"},
+    {"name": "broken", "status": 404, "url": "https://gone.test", "body": "", "expected": "broken"},
+    {"name": "blocked", "status": null, "url": "https://private.test", "body": "", "error": "unsafe_address", "expected": "blocked"},
+    {"name": "unknown is not healthy", "status": null, "url": "https://unknown.test", "body": "", "expected": "unknown"}
+  ],
+  "scoring": [
+    {"name": "complete active business", "signals": {"business": {"name": "Acme", "email": "a@acme.test", "phone": "+27123456789"}, "website": {"classification": "healthy"}, "contacts": {"public_count": 1}, "domain": {"status": "resolved"}, "state": {"verified": true, "suppressed": false, "merge_status": "active"}}},
+    {"name": "suppressed remains ineligible", "signals": {"business": {"name": "Suppressed"}, "state": {"suppressed": true, "merge_status": "active"}}, "must_be_ineligible": true},
+    {"name": "stale evidence does not create positive", "signals": {"business": {"name": "Stale"}, "website": {"classification": "healthy", "stale": true}, "state": {"suppressed": false, "merge_status": "active"}}}
+  ],
+  "tenant_isolation": {
+    "tenant_a": {"id": 101, "business_ids": [1, 2]},
+    "tenant_b": {"id": 202, "business_ids": [3, 4]},
+    "cross_tenant_ids": [3, 4]
+  }
+}
@@ -0,0 +1,61 @@
+import json
+import sys
+import unittest
+from pathlib import Path
+
+from app.domain import match_businesses
+from app.contact_extractor import extract_contacts
+from app.website_scanner import classify_website
+from app.scoring import DEFAULT_RULES, evaluate_score
+
+ROOT = Path(__file__).resolve().parents[1]
+FIXTURE_PATH = ROOT / "fixtures" / "phase16.json"
+sys.path.insert(0, str(ROOT.parent.parent / "scripts"))
+from benchmark_phase16 import run_benchmark
+
+
+class Phase16BenchmarkRegressionTests(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.fixtures = json.loads(FIXTURE_PATH.read_text())
+
+    def test_fixture_is_stable_and_benchmark_cases_are_deterministic(self):
+        self.assertEqual(self.fixtures["seed"], 1601)
+        first = json.dumps(self.fixtures, sort_keys=True, separators=(",", ":"))
+        second = json.dumps(json.loads(FIXTURE_PATH.read_text()), sort_keys=True, separators=(",", ":"))
+        self.assertEqual(first, second)
+        case = self.fixtures["matching"][0]
+        self.assertEqual(
+            match_businesses(case["source"], case["candidates"], threshold=case["threshold"]),
+            match_businesses(case["source"], case["candidates"], threshold=case["threshold"]),
+        )
+
+    def test_benchmark_semantic_report_is_deterministic_and_passes_thresholds(self):
+        first = run_benchmark(measure_latency=False)
+        second = run_benchmark(measure_latency=False)
+        self.assertEqual(first, second)
+        self.assertTrue(first["passed"])
+        self.assertEqual(first["results"]["matching"]["false_positive"], 0)
+        self.assertEqual(first["results"]["contacts"]["false_positive"], 0)
+
+    def test_labeled_false_positives_stay_suppressed(self):
+        for case in self.fixtures["matching"]:
+            predicted = {item["id"] for item in match_businesses(case["source"], case["candidates"], threshold=case["threshold"])}
+            self.assertTrue(predicted.isdisjoint(set(case["forbidden_ids"])), case["name"])
+
+        for case in self.fixtures["contacts"]:
+            values = {(item["kind"], item["value"]) for item in extract_contacts(case["html"], case["source_url"])}
+            self.assertTrue(values.isdisjoint({tuple(item) for item in case["forbidden"]}), case["name"])
+
+    def test_website_labels_and_scores_have_no_unsafe_positive(self):
+        for case in self.fixtures["websites"]:
+            self.assertEqual(classify_website(case["status"], case["url"], case["body"], error=case.get("error")), case["expected"], case["name"])
+        for case in self.fixtures["scoring"]:
+            result = evaluate_score(case["signals"], DEFAULT_RULES)
+            self.assertEqual(result, evaluate_score(case["signals"], DEFAULT_RULES))
+            if case.get("must_be_ineligible"):
+                self.assertFalse(result["eligible"])
+
+
+if __name__ == "__main__":
+    unittest.main()