add deterministic pilot benchmarks

2026-09-03 12:50:00 +02:00
parent 6b41d5b9ee
commit 9622f76977
11 changed files with 763 additions and 0 deletions
@@ -0,0 +1,82 @@
+{
+  "seed": 1601,
+  "limitations": [
+    "Synthetic, small, English-heavy cases are not representative of production traffic.",
+    "Contact labels cover parser false positives but do not establish consent, identity, or deliverability.",
+    "Website fixtures classify supplied HTML only; they do not measure network, DNS, redirects, or adversarial pages.",
+    "Latency and cache measurements are local pilot signals and vary by host load."
+  ],
+  "normalization": [
+    {"name": "SA contact fields", "input": {"name": "  Acme   Solar ", "website": "HTTPS://WWW.ACME.TEST/path", "phone": "082 555 1234", "email": " SALES@ACME.TEST "}, "expected": {"name": "Acme Solar", "website_domain": "acme.test", "email": "sales@acme.test"}},
+    {"name": "accented location", "input": {"name": "Cafe", "province": "  Western Cape ", "city": "CAPE TOWN", "suburb": "Sea Point"}, "expected": {"province": "western cape", "city": "cape town", "suburb": "sea point"}}
+  ],
+  "matching": [
+    {
+      "name": "shared exact domain is a positive",
+      "source": {"name": "Acme Solar", "website": "https://www.acme.test", "city": "Johannesburg"},
+      "candidates": [
+        {"id": 1, "name": "Acme Solar (Pty) Ltd", "website": "http://acme.test/", "city": "Johannesburg"},
+        {"id": 2, "name": "Acme Supplies", "website": "https://supplies.test", "city": "Johannesburg"}
+      ],
+      "threshold": 0.72,
+      "expected_ids": [1],
+      "forbidden_ids": [2]
+    },
+    {
+      "name": "similar name without corroboration is not a positive",
+      "source": {"name": "Brightline Consulting", "city": "Cape Town"},
+      "candidates": [
+        {"id": 3, "name": "Brightline Consulting", "city": "Durban"},
+        {"id": 4, "name": "Brightline Consulting", "city": "Cape Town"}
+      ],
+      "threshold": 0.72,
+      "expected_ids": [4],
+      "forbidden_ids": [3]
+    },
+    {
+      "name": "unrelated businesses are negatives",
+      "source": {"name": "Green Oak Architects", "website": "green-oak.test"},
+      "candidates": [
+        {"id": 5, "name": "Green Oak Accounting", "website": "accounting.test"},
+        {"id": 6, "name": "Red River Bakery", "website": "redriver.test"}
+      ],
+      "threshold": 0.72,
+      "expected_ids": [],
+      "forbidden_ids": [5, 6]
+    }
+  ],
+  "contacts": [
+    {
+      "name": "public contacts and ignored markup",
+      "source_url": "https://acme.test/contact",
+      "html": "<p>Sales: sales@acme.test</p><p>Call +27 (12) 345-6789</p><script>secret@acme.test</script><img src=\"https://tracker.test/pixel?email=tracker@tracker.test\"><p>API key: token@acme.test</p>",
+      "expected": [["email", "sales@acme.test"], ["phone", "+27123456789"]],
+      "forbidden": [["email", "secret@acme.test"], ["email", "tracker@tracker.test"], ["email", "token@acme.test"]]
+    },
+    {
+      "name": "forms are provenance not email destinations",
+      "source_url": "https://bright.test/contact",
+      "html": "<form><input name=\"email\" placeholder=\"Your email\"></form><p>hello [at] bright.test</p>",
+      "expected": [["email", "hello@bright.test"]],
+      "forbidden": [["email", "email@bright.test"]]
+    }
+  ],
+  "websites": [
+    {"name": "healthy", "status": 200, "url": "https://acme.test", "body": "<html><h1>Acme Solar</h1><p>Welcome to our business.</p></html>", "expected": "healthy"},
+    {"name": "parked", "status": 200, "url": "https://parked.test", "body": "Domain for sale - buy this domain", "expected": "parked"},
+    {"name": "construction", "status": 200, "url": "https://new.test", "body": "Website coming soon", "expected": "under_construction"},
+    {"name": "broken", "status": 404, "url": "https://gone.test", "body": "", "expected": "broken"},
+    {"name": "blocked", "status": null, "url": "https://private.test", "body": "", "error": "unsafe_address", "expected": "blocked"},
+    {"name": "unknown is not healthy", "status": null, "url": "https://unknown.test", "body": "", "expected": "unknown"}
+  ],
+  "scoring": [
+    {"name": "complete active business", "signals": {"business": {"name": "Acme", "email": "a@acme.test", "phone": "+27123456789"}, "website": {"classification": "healthy"}, "contacts": {"public_count": 1}, "domain": {"status": "resolved"}, "state": {"verified": true, "suppressed": false, "merge_status": "active"}}},
+    {"name": "suppressed remains ineligible", "signals": {"business": {"name": "Suppressed"}, "state": {"suppressed": true, "merge_status": "active"}}, "must_be_ineligible": true},
+    {"name": "stale evidence does not create positive", "signals": {"business": {"name": "Stale"}, "website": {"classification": "healthy", "stale": true}, "state": {"suppressed": false, "merge_status": "active"}}}
+  ],
+  "tenant_isolation": {
+    "tenant_a": {"id": 101, "business_ids": [1, 2]},
+    "tenant_b": {"id": 202, "business_ids": [3, 4]},
+    "cross_tenant_ids": [3, 4]
+  }
+}