add deterministic pilot benchmarks

2026-09-03 12:50:00 +02:00
parent 6b41d5b9ee
commit 9622f76977
11 changed files with 763 additions and 0 deletions
@@ -0,0 +1,136 @@
+{
+  "acceptance_thresholds": {
+    "contact_precision": 0.85,
+    "contact_recall": 0.85,
+    "matching_precision": 0.9,
+    "matching_recall": 0.9,
+    "normalization_accuracy": 1.0,
+    "score_reproducible": true,
+    "tenant_leakage": 0,
+    "website_accuracy": 0.9
+  },
+  "benchmark": "phase16",
+  "checks": {
+    "contact_precision": true,
+    "contact_recall": true,
+    "matching_precision": true,
+    "matching_recall": true,
+    "normalization_accuracy": true,
+    "score_reproducible": true,
+    "tenant_leakage": true,
+    "website_accuracy": true
+  },
+  "fixture": "apps/api/fixtures/phase16.json",
+  "limitations": [
+    "Synthetic, small, English-heavy cases are not representative of production traffic.",
+    "Contact labels cover parser false positives but do not establish consent, identity, or deliverability.",
+    "Website fixtures classify supplied HTML only; they do not measure network, DNS, redirects, or adversarial pages.",
+    "Latency and cache measurements are local pilot signals and vary by host load."
+  ],
+  "offline": true,
+  "passed": true,
+  "results": {
+    "cache": {
+      "enabled": true,
+      "timing_omitted": true
+    },
+    "contacts": {
+      "cases": [
+        {
+          "expected_count": 2,
+          "false_positives": [],
+          "name": "public contacts and ignored markup",
+          "predicted_count": 2
+        },
+        {
+          "expected_count": 1,
+          "false_positives": [],
+          "name": "forms are provenance not email destinations",
+          "predicted_count": 1
+        }
+      ],
+      "false_negative": 0,
+      "false_positive": 0,
+      "precision": 1.0,
+      "recall": 1.0,
+      "true_positive": 3
+    },
+    "latency": {},
+    "matching": {
+      "cases": [
+        {
+          "expected_ids": [
+            1
+          ],
+          "name": "shared exact domain is a positive",
+          "predicted_ids": [
+            1
+          ]
+        },
+        {
+          "expected_ids": [
+            4
+          ],
+          "name": "similar name without corroboration is not a positive",
+          "predicted_ids": [
+            4
+          ]
+        },
+        {
+          "expected_ids": [],
+          "name": "unrelated businesses are negatives",
+          "predicted_ids": []
+        }
+      ],
+      "false_negative": 0,
+      "false_positive": 0,
+      "false_positive_cases": 0,
+      "precision": 1.0,
+      "recall": 1.0,
+      "true_positive": 2
+    },
+    "normalization": {
+      "accuracy": 1.0,
+      "correct": 2,
+      "total": 2
+    },
+    "scoring": {
+      "fixture_scores": [
+        {
+          "eligible": true,
+          "name": "complete active business",
+          "score": 80
+        },
+        {
+          "eligible": false,
+          "name": "suppressed remains ineligible",
+          "score": 15
+        },
+        {
+          "eligible": true,
+          "name": "stale evidence does not create positive",
+          "score": 15
+        }
+      ],
+      "reproducible": true
+    },
+    "tenant_isolation": {
+      "isolated": true,
+      "leakage": 0,
+      "tenant_a_visible_ids": [
+        1,
+        2
+      ],
+      "tenant_b_ids": [
+        3,
+        4
+      ]
+    },
+    "website_classification": {
+      "accuracy": 1.0,
+      "correct": 6,
+      "total": 6
+    }
+  },
+  "version": 1
+}