add deterministic pilot benchmarks

2026-09-03 12:50:00 +02:00
parent 6b41d5b9ee
commit 9622f76977
11 changed files with 763 additions and 0 deletions
@@ -0,0 +1,82 @@
+{
+  "seed": 1601,
+  "limitations": [
+    "Synthetic, small, English-heavy cases are not representative of production traffic.",
+    "Contact labels cover parser false positives but do not establish consent, identity, or deliverability.",
+    "Website fixtures classify supplied HTML only; they do not measure network, DNS, redirects, or adversarial pages.",
+    "Latency and cache measurements are local pilot signals and vary by host load."
+  ],
+  "normalization": [
+    {"name": "SA contact fields", "input": {"name": "  Acme   Solar ", "website": "HTTPS://WWW.ACME.TEST/path", "phone": "082 555 1234", "email": " SALES@ACME.TEST "}, "expected": {"name": "Acme Solar", "website_domain": "acme.test", "email": "sales@acme.test"}},
+    {"name": "accented location", "input": {"name": "Cafe", "province": "  Western Cape ", "city": "CAPE TOWN", "suburb": "Sea Point"}, "expected": {"province": "western cape", "city": "cape town", "suburb": "sea point"}}
+  ],
+  "matching": [
+    {
+      "name": "shared exact domain is a positive",
+      "source": {"name": "Acme Solar", "website": "https://www.acme.test", "city": "Johannesburg"},
+      "candidates": [
+        {"id": 1, "name": "Acme Solar (Pty) Ltd", "website": "http://acme.test/", "city": "Johannesburg"},
+        {"id": 2, "name": "Acme Supplies", "website": "https://supplies.test", "city": "Johannesburg"}
+      ],
+      "threshold": 0.72,
+      "expected_ids": [1],
+      "forbidden_ids": [2]
+    },
+    {
+      "name": "similar name without corroboration is not a positive",
+      "source": {"name": "Brightline Consulting", "city": "Cape Town"},
+      "candidates": [
+        {"id": 3, "name": "Brightline Consulting", "city": "Durban"},
+        {"id": 4, "name": "Brightline Consulting", "city": "Cape Town"}
+      ],
+      "threshold": 0.72,
+      "expected_ids": [4],
+      "forbidden_ids": [3]
+    },
+    {
+      "name": "unrelated businesses are negatives",
+      "source": {"name": "Green Oak Architects", "website": "green-oak.test"},
+      "candidates": [
+        {"id": 5, "name": "Green Oak Accounting", "website": "accounting.test"},
+        {"id": 6, "name": "Red River Bakery", "website": "redriver.test"}
+      ],
+      "threshold": 0.72,
+      "expected_ids": [],
+      "forbidden_ids": [5, 6]
+    }
+  ],
+  "contacts": [
+    {
+      "name": "public contacts and ignored markup",
+      "source_url": "https://acme.test/contact",
+      "html": "<p>Sales: sales@acme.test</p><p>Call +27 (12) 345-6789</p><script>secret@acme.test</script><img src=\"https://tracker.test/pixel?email=tracker@tracker.test\"><p>API key: token@acme.test</p>",
+      "expected": [["email", "sales@acme.test"], ["phone", "+27123456789"]],
+      "forbidden": [["email", "secret@acme.test"], ["email", "tracker@tracker.test"], ["email", "token@acme.test"]]
+    },
+    {
+      "name": "forms are provenance not email destinations",
+      "source_url": "https://bright.test/contact",
+      "html": "<form><input name=\"email\" placeholder=\"Your email\"></form><p>hello [at] bright.test</p>",
+      "expected": [["email", "hello@bright.test"]],
+      "forbidden": [["email", "email@bright.test"]]
+    }
+  ],
+  "websites": [
+    {"name": "healthy", "status": 200, "url": "https://acme.test", "body": "<html><h1>Acme Solar</h1><p>Welcome to our business.</p></html>", "expected": "healthy"},
+    {"name": "parked", "status": 200, "url": "https://parked.test", "body": "Domain for sale - buy this domain", "expected": "parked"},
+    {"name": "construction", "status": 200, "url": "https://new.test", "body": "Website coming soon", "expected": "under_construction"},
+    {"name": "broken", "status": 404, "url": "https://gone.test", "body": "", "expected": "broken"},
+    {"name": "blocked", "status": null, "url": "https://private.test", "body": "", "error": "unsafe_address", "expected": "blocked"},
+    {"name": "unknown is not healthy", "status": null, "url": "https://unknown.test", "body": "", "expected": "unknown"}
+  ],
+  "scoring": [
+    {"name": "complete active business", "signals": {"business": {"name": "Acme", "email": "a@acme.test", "phone": "+27123456789"}, "website": {"classification": "healthy"}, "contacts": {"public_count": 1}, "domain": {"status": "resolved"}, "state": {"verified": true, "suppressed": false, "merge_status": "active"}}},
+    {"name": "suppressed remains ineligible", "signals": {"business": {"name": "Suppressed"}, "state": {"suppressed": true, "merge_status": "active"}}, "must_be_ineligible": true},
+    {"name": "stale evidence does not create positive", "signals": {"business": {"name": "Stale"}, "website": {"classification": "healthy", "stale": true}, "state": {"suppressed": false, "merge_status": "active"}}}
+  ],
+  "tenant_isolation": {
+    "tenant_a": {"id": 101, "business_ids": [1, 2]},
+    "tenant_b": {"id": 202, "business_ids": [3, 4]},
+    "cross_tenant_ids": [3, 4]
+  }
+}
@@ -0,0 +1,61 @@
+import json
+import sys
+import unittest
+from pathlib import Path
+
+from app.domain import match_businesses
+from app.contact_extractor import extract_contacts
+from app.website_scanner import classify_website
+from app.scoring import DEFAULT_RULES, evaluate_score
+
+ROOT = Path(__file__).resolve().parents[1]
+FIXTURE_PATH = ROOT / "fixtures" / "phase16.json"
+sys.path.insert(0, str(ROOT.parent.parent / "scripts"))
+from benchmark_phase16 import run_benchmark
+
+
+class Phase16BenchmarkRegressionTests(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.fixtures = json.loads(FIXTURE_PATH.read_text())
+
+    def test_fixture_is_stable_and_benchmark_cases_are_deterministic(self):
+        self.assertEqual(self.fixtures["seed"], 1601)
+        first = json.dumps(self.fixtures, sort_keys=True, separators=(",", ":"))
+        second = json.dumps(json.loads(FIXTURE_PATH.read_text()), sort_keys=True, separators=(",", ":"))
+        self.assertEqual(first, second)
+        case = self.fixtures["matching"][0]
+        self.assertEqual(
+            match_businesses(case["source"], case["candidates"], threshold=case["threshold"]),
+            match_businesses(case["source"], case["candidates"], threshold=case["threshold"]),
+        )
+
+    def test_benchmark_semantic_report_is_deterministic_and_passes_thresholds(self):
+        first = run_benchmark(measure_latency=False)
+        second = run_benchmark(measure_latency=False)
+        self.assertEqual(first, second)
+        self.assertTrue(first["passed"])
+        self.assertEqual(first["results"]["matching"]["false_positive"], 0)
+        self.assertEqual(first["results"]["contacts"]["false_positive"], 0)
+
+    def test_labeled_false_positives_stay_suppressed(self):
+        for case in self.fixtures["matching"]:
+            predicted = {item["id"] for item in match_businesses(case["source"], case["candidates"], threshold=case["threshold"])}
+            self.assertTrue(predicted.isdisjoint(set(case["forbidden_ids"])), case["name"])
+
+        for case in self.fixtures["contacts"]:
+            values = {(item["kind"], item["value"]) for item in extract_contacts(case["html"], case["source_url"])}
+            self.assertTrue(values.isdisjoint({tuple(item) for item in case["forbidden"]}), case["name"])
+
+    def test_website_labels_and_scores_have_no_unsafe_positive(self):
+        for case in self.fixtures["websites"]:
+            self.assertEqual(classify_website(case["status"], case["url"], case["body"], error=case.get("error")), case["expected"], case["name"])
+        for case in self.fixtures["scoring"]:
+            result = evaluate_score(case["signals"], DEFAULT_RULES)
+            self.assertEqual(result, evaluate_score(case["signals"], DEFAULT_RULES))
+            if case.get("must_be_ineligible"):
+                self.assertFalse(result["eligible"])
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -0,0 +1,136 @@
+#!/usr/bin/env node
+/**
+ * Deterministic, Chromium-free frontend pilot smoke harness.
+ *
+ * Usage:
+ *   node scripts/smoke-frontend.mjs http://127.0.0.1:8080
+ *
+ * The final stdout value is JSON so CI can consume it directly. Exit code is
+ * non-zero when any check fails. The checks intentionally inspect source HTML,
+ * JavaScript, and CSS instead of executing the application or making API calls.
+ */
+import { readFile } from 'node:fs/promises';
+import { dirname, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const webRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const baseUrl = (process.argv[2] || 'http://127.0.0.1:8080').replace(/\/$/, '');
+const files = Object.fromEntries(await Promise.all(
+  ['index.html', 'app.js', 'styles.css', 'config.js', 'health.html', 'error.html', 'healthz', 'smoke-test.html']
+    .map(async name => [name, await readFile(resolve(webRoot, name), 'utf8')])
+));
+
+const results = [];
+function check(id, description, pass, details = '') {
+  results.push({ id, description, pass: Boolean(pass), ...(details ? { details } : {}) });
+}
+function has(text, needles) {
+  return needles.every(needle => text.includes(needle));
+}
+function ids(html) {
+  return [...html.matchAll(/\bid=["']([^"']+)["']/g)].map(match => match[1]);
+}
+function smokeMarkers(html) {
+  return [...html.matchAll(/data-smoke=["']([^"']+)["']/g)].map(match => match[1]);
+}
+
+const html = files['index.html'];
+const js = files['app.js'];
+const css = files['styles.css'];
+const htmlIds = new Set(ids(html));
+const markers = new Set(smokeMarkers(html));
+
+check('dom.operator-markers', 'operator-critical DOM markers are present', has(html, [
+  'id="loginScreen"', 'id="dashboardShell"', 'id="explorer"', 'id="detailPanel"',
+  'id="reviewQueueCount"', 'id="jobsList"', 'id="sourcesList"', 'id="pipelineBoard"',
+  'id="interactionState"', 'id="pipelineReport"', 'id="suppressionState"',
+  'id="providerPolicyPanel"', 'id="scoreRulesPanel"', 'id="scoreDistributionPanel"'
+]));
+check('dom.safety-markers', 'safety and approval sections have stable smoke markers', [
+  'saved-views', 'review-queue', 'crm-pipeline', 'crm-interactions', 'crm-reports',
+  'suppression-center', 'outreach-provider-policy', 'score-rules', 'score-distribution'
+].every(marker => markers.has(marker)));
+check('dom.required-controls', 'operator controls have stable IDs', [
+  'loginEmail', 'loginPassword', 'logoutBtn', 'nextPageBtn', 'bulkVerifyBtn', 'bulkRejectBtn',
+  'savedFilterForm', 'pipelineViewToggle', 'interactionForm', 'suppressionForm', 'outreachPolicyRefreshBtn'
+].every(id => htmlIds.has(id)) && has(js, [
+  'scanWebsiteBtn', 'extractContactsBtn', 'recalculateScoreBtn', 'generateAiSuggestionBtn', 'createOutreachDraftBtn'
+]));
+
+check('css.responsive', 'responsive CSS is present for mobile operator layouts',
+  /@media\s*\(max-width\s*:\s*700px\)/.test(css) &&
+  has(css, ['.saved-view-controls', '.website-scan-grid', '.crm-two-col', '.reports-grid', '.outreach-panel']));
+check('css.layout-contracts', 'critical layout selectors are defined', has(css, [
+  '.app-shell', '.sidebar', '.workspace-grid', '.table-scroll', '.pipeline-board',
+  '.jobs-grid', '.sources-grid', '.score-config-row', '.provider-policy-row'
+]));
+
+check('safety.no-send-copy', 'persistent no-send and approval safety copy is present', has(`${html}\n${js}`, [
+  'No outreach from this workspace.', 'Suppression always wins.', 'Sending is disabled by default.',
+  'Approval does not send a message.', 'no outreach will be sent', 'send:false',
+  'autonomous_action:false'
+]));
+check('safety.no-send-controls', 'static HTML exposes no send or delivery button',
+  ![...html.matchAll(/<button\b[^>]*>([\s\S]*?)<\/button>/gi)]
+    .some(match => /\b(send|deliver|campaign)\b/i.test(match[1])));
+check('safety.no-secrets', 'frontend source has no obvious hardcoded secrets',
+  !/(api[_-]?key|secret|token|password|private[_-]?key|authorization)\s*[:=]\s*["'][^"']+/i.test(
+    `${html}\n${js}\n${files['config.js']}`
+  ));
+check('safety.approval-gated', 'approval is explicitly human-confirmed and non-delivering',
+  has(js, ['window.confirm', 'human_approval:true', 'send:false', 'Approval does not send a message.']));
+
+const routes = [
+  '/api/v1/auth/me', '/api/v1/auth/login', '/api/v1/auth/logout', '/api/v1/businesses',
+  '/api/v1/review-queue', '/api/v1/saved-filters', '/api/v1/businesses/bulk-review',
+  '/api/v1/jobs', '/api/v1/sources', '/api/v1/source-records', '/api/v1/discovery-queries',
+  '/api/v1/merge-history', '/api/v1/scoring/summary', '/api/v1/score-rules',
+  '/api/v1/pipeline-entries', '/api/v1/interactions', '/api/v1/reports/pipeline',
+  '/api/v1/reports/outcomes', '/api/v1/reports/activity', '/api/v1/suppressions',
+  '/api/v1/ai-runs', '/api/v1/outreach/drafts', '/api/v1/outreach/provider-config'
+];
+check('routes.contracts', 'operator API route contracts are referenced by the client',
+  routes.every(route => js.includes(route)), routes.filter(route => !js.includes(route)).join(', '));
+check('routes.authenticated', 'API requests use cookie credentials',
+  js.includes("credentials:'include'") && js.includes('jsonRequest'));
+check('routes.no-target-fetch', 'client does not directly fetch arbitrary target URLs',
+  !/fetch\s*\(\s*(?:url|target|website|href)\b/i.test(js));
+
+const linkedAssets = new Set(['index.html', 'config.js', 'app.js', 'styles.css', 'health.html', 'error.html', 'healthz', 'smoke-test.html']);
+for (const match of html.matchAll(/(?:src|href)=["']([^"'#?]+)(?:\?[^"']*)?["']/gi)) {
+  const asset = match[1];
+  if (!asset.startsWith('http') && !asset.startsWith('data:')) linkedAssets.add(asset.replace(/^\.\//, ''));
+}
+check('assets.local', 'all linked static assets exist and are non-empty', [...linkedAssets].every(asset => files[asset]?.length > 0),
+  [...linkedAssets].filter(asset => !files[asset]?.length).join(', '));
+
+async function verifyHttp() {
+  for (const asset of [...linkedAssets].sort()) {
+    const url = `${baseUrl}/${asset}`;
+    try {
+      const response = await fetch(url);
+      const body = await response.text();
+      const contentType = response.headers.get('content-type') || '';
+      const expectedType = asset.endsWith('.html') ? 'text/html' : asset.endsWith('.css') ? 'text/css' : asset.endsWith('.js') ? 'javascript' : null;
+      const pass = response.ok && body.length > 0 && (!expectedType || contentType.includes(expectedType));
+      check(`http.${asset}`, `HTTP delivery: ${asset}`, pass,
+        pass ? `${response.status} ${contentType}` : `${response.status} ${contentType || 'missing content-type'}`);
+    } catch (error) {
+      check(`http.${asset}`, `HTTP delivery: ${asset}`, false, error.message);
+    }
+  }
+}
+await verifyHttp();
+
+const failed = results.filter(result => !result.pass);
+const report = {
+  schema: 1,
+  harness: 'prospectos-frontend-pilot-smoke',
+  base_url: baseUrl,
+  chromium_required: false,
+  pass: failed.length === 0,
+  totals: { checks: results.length, passed: results.length - failed.length, failed: failed.length },
+  checks: results
+};
+console.log(JSON.stringify(report, null, 2));
+if (failed.length) process.exitCode = 1;