add deterministic pilot benchmarks

2026-09-03 12:50:00 +02:00
parent 6b41d5b9ee
commit 9622f76977
11 changed files with 763 additions and 0 deletions
@@ -0,0 +1,61 @@
+import json
+import sys
+import unittest
+from pathlib import Path
+
+from app.domain import match_businesses
+from app.contact_extractor import extract_contacts
+from app.website_scanner import classify_website
+from app.scoring import DEFAULT_RULES, evaluate_score
+
+ROOT = Path(__file__).resolve().parents[1]
+FIXTURE_PATH = ROOT / "fixtures" / "phase16.json"
+sys.path.insert(0, str(ROOT.parent.parent / "scripts"))
+from benchmark_phase16 import run_benchmark
+
+
+class Phase16BenchmarkRegressionTests(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.fixtures = json.loads(FIXTURE_PATH.read_text())
+
+    def test_fixture_is_stable_and_benchmark_cases_are_deterministic(self):
+        self.assertEqual(self.fixtures["seed"], 1601)
+        first = json.dumps(self.fixtures, sort_keys=True, separators=(",", ":"))
+        second = json.dumps(json.loads(FIXTURE_PATH.read_text()), sort_keys=True, separators=(",", ":"))
+        self.assertEqual(first, second)
+        case = self.fixtures["matching"][0]
+        self.assertEqual(
+            match_businesses(case["source"], case["candidates"], threshold=case["threshold"]),
+            match_businesses(case["source"], case["candidates"], threshold=case["threshold"]),
+        )
+
+    def test_benchmark_semantic_report_is_deterministic_and_passes_thresholds(self):
+        first = run_benchmark(measure_latency=False)
+        second = run_benchmark(measure_latency=False)
+        self.assertEqual(first, second)
+        self.assertTrue(first["passed"])
+        self.assertEqual(first["results"]["matching"]["false_positive"], 0)
+        self.assertEqual(first["results"]["contacts"]["false_positive"], 0)
+
+    def test_labeled_false_positives_stay_suppressed(self):
+        for case in self.fixtures["matching"]:
+            predicted = {item["id"] for item in match_businesses(case["source"], case["candidates"], threshold=case["threshold"])}
+            self.assertTrue(predicted.isdisjoint(set(case["forbidden_ids"])), case["name"])
+
+        for case in self.fixtures["contacts"]:
+            values = {(item["kind"], item["value"]) for item in extract_contacts(case["html"], case["source_url"])}
+            self.assertTrue(values.isdisjoint({tuple(item) for item in case["forbidden"]}), case["name"])
+
+    def test_website_labels_and_scores_have_no_unsafe_positive(self):
+        for case in self.fixtures["websites"]:
+            self.assertEqual(classify_website(case["status"], case["url"], case["body"], error=case.get("error")), case["expected"], case["name"])
+        for case in self.fixtures["scoring"]:
+            result = evaluate_score(case["signals"], DEFAULT_RULES)
+            self.assertEqual(result, evaluate_score(case["signals"], DEFAULT_RULES))
+            if case.get("must_be_ineligible"):
+                self.assertFalse(result["eligible"])
+
+
+if __name__ == "__main__":
+    unittest.main()