add post-discovery website/domain/contact enrichment

2026-09-04 21:36:16 +02:00
parent 4ec9dc608b
commit 73ababe90c
4 changed files with 419 additions and 0 deletions
@@ -0,0 +1,103 @@
+"""Tests for the bounded post-discovery enrichment pipeline."""
+import os
+import unittest
+from unittest.mock import patch
+
+from app.enrichment import enrich_contacts, enrich_domain, enrich_website
+
+
+def _page(status=200, html=b"<html><head><title>Acme</title></head><body><h1>Acme</h1></body></html>", redirects=None, elapsed=12, tls=True):
+    return {
+        "status": status,
+        "final_url": "https://acme.test/",
+        "redirect_chain": redirects or [],
+        "body": html,
+        "content_type": "text/html",
+        "elapsed_ms": elapsed,
+        "tls": tls,
+        "certificate_status": "valid" if tls else "not_applicable",
+    }
+
+
+class EnrichmentTests(unittest.TestCase):
+    def test_enrich_website_uses_existing_scanner_fetcher(self):
+        fetch = lambda url, timeout=5.0, max_bytes=262144: _page()
+        with patch("app.enrichment.validate_url", side_effect=lambda url: url), \
+             patch("app.enrichment.normalize_registrable_domain", return_value="acme.test"):
+            result = enrich_website("https://acme.test/", fetch=fetch)
+        self.assertEqual(result["status"], "working")
+        self.assertTrue(result["has_working_website"])
+        self.assertEqual(result["domain"], "acme.test")
+        self.assertIsNotNone(result["response_time_ms"])
+
+    def test_enrich_website_marks_broken_on_4xx(self):
+        fetch = lambda url, **_: _page(status=404)
+        with patch("app.enrichment.validate_url", side_effect=lambda url: url):
+            result = enrich_website("https://acme.test/", fetch=fetch)
+        self.assertEqual(result["status"], "broken")
+        self.assertFalse(result["has_working_website"])
+
+    def test_enrich_website_detects_mobile_viewport(self):
+        html = b'<html><head><meta name="viewport" content="width=device-width"></head><body><h1>Acme</h1></body></html>'
+        fetch = lambda url, **_: _page(html=html)
+        with patch("app.enrichment.validate_url", side_effect=lambda url: url):
+            result = enrich_website("https://acme.test/", fetch=fetch)
+        self.assertTrue(result["mobile_viewport"])
+
+    def test_enrich_website_extracts_phone_and_email(self):
+        html = b'<html><body><h1>Acme</h1><p>+27 12 345 6789</p><a href="mailto:hello@acme.test">hello@acme.test</a></body></html>'
+        fetch = lambda url, **_: _page(html=html)
+        with patch("app.enrichment.validate_url", side_effect=lambda url: url):
+            result = enrich_website("https://acme.test/", fetch=fetch)
+        self.assertEqual(result["visible_phone"], "+27 12 345 6789")
+        self.assertEqual(result["visible_email"], "hello@acme.test")
+
+    def test_enrich_website_no_evidence_is_no_claim(self):
+        fetch = lambda url, **_: _page(status=404)
+        with patch("app.enrichment.validate_url", side_effect=lambda url: url):
+            result = enrich_website("https://acme.test/", fetch=fetch)
+        self.assertIsNone(result["visible_phone"])
+        self.assertIsNone(result["visible_email"])
+
+    def test_enrich_website_handles_fetch_failure(self):
+        def fetch(url, **_):
+            raise ConnectionError("timed out")
+        with patch("app.enrichment.validate_url", side_effect=lambda url: url):
+            result = enrich_website("https://acme.test/", fetch=fetch)
+        self.assertIn("error", result)
+        self.assertIsNone(result["has_working_website"])
+
+    def test_enrich_https_flag(self):
+        fetch = lambda url, **_: _page(tls=True)
+        with patch("app.enrichment.validate_url", side_effect=lambda url: url):
+            result = enrich_website("https://acme.test/", fetch=fetch)
+        self.assertTrue(result["https"])
+        self.assertTrue(result["ssl_valid"])
+
+    def test_enrich_contacts_delegates_to_extractor(self):
+        html = '<html><body><a href="mailto:hello@acme.test">Email</a><span>+27 12 345 6789</span></body></html>'
+        contacts = enrich_contacts(html, "https://acme.test/", max_results=10)
+        self.assertTrue(any(c["value"] == "hello@acme.test" for c in contacts))
+        self.assertTrue(any(c["kind"] == "phone" for c in contacts))
+        self.assertTrue(all(c["source_url"] == "https://acme.test/" for c in contacts))
+
+    def test_enrich_domain_unknown_for_unsupported_suffix(self):
+        result = enrich_domain("localhost")
+        self.assertEqual(result["status"], "unknown")
+
+    def test_enrich_domain_uses_resolution_evidence(self):
+        with patch("app.enrichment.normalize_registrable_domain", return_value="acme.co.za"), \
+             patch("app.enrichment.resolve_domain", return_value={"status": "ok", "addresses": ["1.2.3.4"]}):
+            result = enrich_domain("acme.co.za")
+            self.assertTrue(result["resolves"])
+            self.assertEqual(result["status"], "registered")
+
+    def test_enrich_domain_fails_closed(self):
+        with patch("app.enrichment.normalize_registrable_domain", return_value="acme.co.za"), \
+             patch("app.enrichment.resolve_domain", return_value={"status": "nxdomain"}):
+            result = enrich_domain("acme.co.za")
+            self.assertFalse(result["resolves"])
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -0,0 +1,66 @@
+"""Tests for the enrich business API endpoint."""
+import json
+import os
+import sqlite3
+import threading
+import unittest
+from http.client import HTTPConnection
+from tempfile import TemporaryDirectory
+from unittest.mock import patch
+
+from app.main import create_server, hash_password
+
+
+class EnrichmentApiTests(unittest.TestCase):
+    def setUp(self):
+        self.tmp = TemporaryDirectory()
+        os.environ["BOOTSTRAP_ADMIN_EMAIL"] = "enrich-owner@example.test"
+        os.environ["BOOTSTRAP_ADMIN_PASSWORD"] = "password"
+        self.server = create_server("127.0.0.1", 0, self.tmp.name + "/enrich.db")
+        self.thread = threading.Thread(target=self.server.serve_forever, daemon=True); self.thread.start()
+        self.conn = HTTPConnection("127.0.0.1", self.server.server_port, timeout=3); self.cookie = None
+        self.request("POST", "/api/v1/auth/login", {"email": "enrich-owner@example.test", "password": "password"})
+
+    def tearDown(self):
+        self.server.shutdown(); self.server.server_close(); self.thread.join(2); self.tmp.cleanup()
+        for key in ("BOOTSTRAP_ADMIN_EMAIL", "BOOTSTRAP_ADMIN_PASSWORD"):
+            os.environ.pop(key, None)
+
+    def request(self, method, path, payload=None):
+        headers = {"Content-Type": "application/json"}
+        if self.cookie: headers["Cookie"] = self.cookie
+        self.conn.request(method, path, json.dumps(payload).encode() if payload is not None else None, headers)
+        response = self.conn.getresponse(); cookie = response.getheader("Set-Cookie")
+        if cookie: self.cookie = cookie.split(";", 1)[0]
+        return response.status, json.loads(response.read() or b"{}")
+
+    def test_enrich_business_returns_website_domain_contacts(self):
+        status, business = self.request("POST", "/api/v1/businesses", {"name": "Enrich Co", "website": "https://enrich.example.test"})
+        self.assertEqual(status, 201)
+        scan = {"status": 200, "final_url": "https://enrich.example.test/", "redirect_chain": [], "body": b"<html><head><title>Enrich</title></head><body><h1>Enrich Co</h1><a href=\"mailto:hello@enrich.example.test\">hello@enrich.example.test</a></body></html>", "html": "<html><head><title>Enrich</title></head><body><h1>Enrich Co</h1><a href=\"mailto:hello@enrich.example.test\">hello@enrich.example.test</a></body></html>", "content_type": "text/html", "elapsed_ms": 15, "tls": True, "certificate_status": "valid"}
+        with patch("app.main.scan_website", return_value=scan), \
+             patch("app.enrichment.validate_url", side_effect=lambda url: url), \
+             patch("app.enrichment.normalize_registrable_domain", return_value="enrich.example.test"), \
+             patch("app.main.enrich_domain", return_value={"domain": "enrich.example.test", "status": "registered", "resolves": True}):
+            status, result = self.request("POST", f"/api/v1/businesses/{business['id']}/enrichment", {})
+        self.assertEqual(status, 200)
+        self.assertEqual(result["business_id"], business["id"])
+        self.assertEqual(result["website"]["status"], "working")
+        self.assertEqual(result["domain"]["status"], "registered")
+        self.assertTrue(any(c["value"] == "hello@enrich.example.test" for c in result["contacts"]))
+
+    def test_enrich_business_tenant_isolated(self):
+        status, business = self.request("POST", "/api/v1/businesses", {"name": "Tenant Co"})
+        self.assertEqual(status, 201)
+        other_hash, other_salt = hash_password("other-password")
+        db = sqlite3.connect(self.tmp.name + "/enrich.db")
+        db.execute("INSERT INTO organizations(id,name) VALUES(?,?)", ("other-tenant", "Other"))
+        db.execute("INSERT INTO users(organization_id,email,password_hash,password_salt,role) VALUES(?,?,?,?,?)", ("other-tenant", "other@example.test", other_hash, other_salt, "owner"))
+        db.commit(); db.close()
+        self.cookie = None
+        self.assertEqual(self.request("POST", "/api/v1/auth/login", {"email": "other@example.test", "password": "other-password"})[0], 200)
+        self.assertEqual(self.request("POST", f"/api/v1/businesses/{business['id']}/enrichment", {})[0], 404)
+
+
+if __name__ == "__main__":
+    unittest.main()