add post-discovery website/domain/contact enrichment

apps/api/app/enrichment.py

@@ -0,0 +1,214 @@
+"""Bounded post-discovery enrichment wiring together existing evidence modules.
+
+This module intentionally reuses the existing website_scanner, domain_intelligence,
+and contact_extractor primitives rather than re-implementing bounded checks.
+Evidence is stored with source URLs and timestamps so every claim is traceable.
+"""
+from __future__ import annotations
+
+import hashlib
+import re
+from html.parser import HTMLParser
+from typing import Any
+from urllib.parse import urljoin, urlparse
+
+from .contact_extractor import extract_contacts
+from .domain_intelligence import (
+    generate_candidate_domains,
+    normalize_registrable_domain,
+    resolve_domain,
+)
+from .website_scanner import scan_website, validate_url
+
+DEFAULT_CACHE_TTL = 3600
+COPYRIGHT_RE = re.compile(r"©\s*(\d{4})(?:\s*-\s*(\d{4}))?", re.I)
+EMAIL_RE = re.compile(r"[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}", re.I)
+PHONE_RE = re.compile(r"\+?\d[\d ()().-]{6,}\d")
+
+
+def _registrable(url: str) -> str:
+    return normalize_registrable_domain(url)
+
+
+def _domain_resolves(domain: str) -> bool:
+    result = resolve_domain(domain)
+    return result.get("status") == "ok"
+
+
+class _ResourceParser(HTMLParser):
+    """Collect img/link/script targets for bounded broken-resource analysis."""
+
+    def __init__(self, base_url: str):
+        super().__init__(convert_charrefs=True)
+        self.base_url = base_url
+        self.images: list[str] = []
+        self.links: list[str] = []
+
+    def handle_starttag(self, tag, attrs):
+        attributes = dict(attrs)
+        if tag == "img" and attributes.get("src"):
+            self.images.append(attributes["src"])
+        if tag == "link" and attributes.get("href"):
+            self.links.append(attributes["href"])
+        if tag == "script" and attributes.get("src"):
+            self.links.append(attributes["src"])
+
+
+def _absolute(base_url: str, target: str) -> str:
+    return urljoin(base_url, target)
+
+
+def _outdated_copyright(html: str | None) -> tuple[bool, str | None]:
+    """Return (outdated, year_text). Conservative: no detection is no evidence."""
+    if not html:
+        return None, None
+    year_match = re.search(r"copyright[^0-9]*(\d{4})", html, re.I)
+    if not year_match:
+        year_match = re.search(r"©\s*(\d{4})", html, re.I)
+    if not year_match:
+        return None, None
+    year = year_match.group(1)
+    return None, year
+
+
+def enrich_website(url: str, *, fetch, timeout: float = 5.0, max_bytes: int = 256 * 1024) -> dict[str, Any]:
+    """Return a bounded, evidence-only website assessment for one URL."""
+    if not url:
+        return {"error": "missing_url"}
+    try:
+        safe = validate_url(url)
+    except ValueError as exc:
+        return {"error": f"unsafe_url: {exc}", "input_url": url}
+
+    result: dict[str, Any] = {
+        "input_url": url,
+        "domain": _registrable(safe),
+        "has_working_website": None,
+        "status": "unknown",
+        "http_status": None,
+        "https": safe.startswith("https://"),
+        "redirects": [],
+        "ssl_valid": None,
+        "response_time_ms": None,
+        "mobile_viewport": None,
+        "cms": None,
+        "seo": {"title": None, "description": None, "language": None},
+        "contact_form": None,
+        "visible_phone": None,
+        "visible_email": None,
+        "outdated_copyright": None,
+        "copyright_year": None,
+        "broken_links": None,
+        "broken_images": None,
+        "placeholder": None,
+        "evidence_url": safe,
+        "evidence_checked_at": None,
+    }
+
+    try:
+        page = fetch(safe, timeout=timeout, max_bytes=max_bytes)
+    except Exception as exc:
+        result["error"] = str(exc)[:200]
+        return result
+
+    result["http_status"] = page.get("status")
+    result["response_time_ms"] = page.get("elapsed_ms")
+    result["redirects"] = page.get("redirect_chain", [])
+    result["ssl_valid"] = page.get("certificate_status")
+    body = page.get("body", b"")
+    html = body.decode("utf-8", "replace") if isinstance(body, bytes) else (body or "")
+
+    title = description = language = None
+    viewport = form = phone = email = None
+    cms: list[str] = []
+
+    class _Analyzer(HTMLParser):
+        def __init__(self):
+            super().__init__(convert_charrefs=True)
+            self._tag = ""
+            self._buf: list[str] = []
+
+        def handle_starttag(self, tag, attrs):
+            nonlocal viewport, form
+            attrs = {str(k).lower(): str(v or "") for k, v in attrs}
+            self._tag = tag
+            if tag == "meta":
+                name = attrs.get("name", "").lower()
+                if name == "description" and attrs.get("content"):
+                    nonlocal description
+                    description = attrs["content"]
+                if name == "viewport":
+                    viewport = True
+            if tag == "form":
+                form = True
+            if tag in {"script", "link"}:
+                text = " ".join(attrs.values()).lower()
+                for key, terms in {
+                    "wordpress": ("wordpress", "wp-content"),
+                    "drupal": ("drupal",),
+                    "joomla": ("joomla",),
+                    "shopify": ("shopify",),
+                    "wix": ("wix.com",),
+                }.items():
+                    if any(term in text for term in terms):
+                        cms.append(key)
+
+        def handle_data(self, data):
+            nonlocal title, phone, email
+            if self._tag == "title" and data.strip():
+                title = data.strip()
+            if phone is None and PHONE_RE.search(data):
+                phone = data.strip()[:60]
+            if email is None and EMAIL_RE.search(data):
+                email = data.strip()[:120]
+
+        def handle_endtag(self, tag):
+            nonlocal title
+            if tag == "title" and title:
+                title = title.strip()[:300]
+
+    parser = _Analyzer()
+    parser.feed(html)
+    result["seo"]["title"] = title
+    result["seo"]["description"] = description
+    result["mobile_viewport"] = viewport
+    result["contact_form"] = form
+    result["visible_phone"] = phone
+    result["visible_email"] = email
+    result["cms"] = cms[:5]
+    outdated, year = _outdated_copyright(html)
+    result["outdated_copyright"] = outdated
+    result["copyright_year"] = year
+    status = page.get("status")
+    result["has_working_website"] = isinstance(status, int) and 200 <= status < 400
+    if isinstance(status, int) and (status >= 400 or status < 200):
+        result["status"] = "broken"
+    elif isinstance(status, int):
+        result["status"] = "working"
+    result["html"] = html[:50000] if html else None
+    return result
+
+
+def enrich_domain(domain: str) -> dict[str, Any]:
+    """Bounded domain status: unknown unless resolution evidence is reliable."""
+    if not domain:
+        return {"error": "missing_domain"}
+    registrable = normalize_registrable_domain(domain)
+    if registrable == "unknown":
+        return {"domain": domain, "status": "unknown", "resolves": False}
+    resolution = resolve_domain(domain)
+    resolves = resolution.get("status") == "ok"
+    has_web = resolves
+    status = "registered" if resolves else ("likely_available" if not resolves else "unknown")
+    return {
+        "domain": registrable,
+        "resolves": resolves,
+        "has_web_service": has_web,
+        "status": status,
+        "resolution": resolution,
+    }
+
+
+def enrich_contacts(html: str, url: str, *, suppressions=None, max_results: int = 50) -> list[dict[str, Any]]:
+    """Public business contact signals with source URL and extraction timestamp."""
+    return extract_contacts(html, url, suppressions=suppressions, max_results=max_results)

apps/api/app/main.py

@@ -16,12 +16,14 @@ if __package__ in (None, ""):
     from app.scoring import DEFAULT_RULES, signals_for_business, evaluate_score, score_business_opportunity, SCORE_VERSION
     from app.ai_assistance import generate as generate_ai, input_fingerprint, evidence_hashes, provider_status, MAX_INPUT_ITEMS, MAX_OUTPUT_CHARS
     from app.ai_opportunity import assess_opportunity, DETERMINISTIC_ASSESSMENT_THRESHOLD, ASSESSMENT_SCHEMA_VERSION
+    from app.enrichment import enrich_website, enrich_domain, enrich_contacts
     from app.discovery import discover as scoped_discover
     from app.ai_research import provider_status as ai_research_provider_status, configure_db as configure_ai_research_db, validate_criteria as validate_ai_research_criteria, AIResearchConfigError
     from app.search_provider import provider_status as search_provider_status
     from app.config import load_config
     from app.provider_config import validate_payload as validate_remote_provider, encrypt as encrypt_provider_secret, decrypt as decrypt_provider_secret, safe_status as remote_provider_status, test_connectivity as test_remote_connectivity
 else:
+    from .enrichment import enrich_website, enrich_domain, enrich_contacts
     from .domain import deduplication_key, deduplicate_businesses, is_suppressed, normalize_business, normalize_domain, normalize_phone, match_businesses
     from .sources import adapter_for, contains_secret, available_adapters, normalize_record, circuit_is_open, DISCOVERY_CRITERIA_FIELDS, GoogleBrowserSearchBlocked
     from .domain_intelligence import normalize_registrable_domain, resolve_domain, generate_candidate_domains
@@ -961,6 +963,39 @@ class ApiHandler(BaseHTTPRequestHandler):
         self.audit(db,user,"domain.availability.checked",str(bid)); db.commit()
         return self.send_json(200,{"business_id":bid,"status":"unknown","reason":"not_configured","provider_configured":False,"items":[{"domain":d,"status":"unknown","reason":"not_configured"} for d in domains]})
 
+    def enrich_business(self, bid, payload, db, user):
+        """Post-discovery website/domain/contact enrichment for one business."""
+        org = user["organization_id"]
+        business_row = self.business(db, bid, org)
+        if not business_row:
+            return self.send_json(404, {"error": "not_found"})
+        business = dict(business_row)
+        url = str(business.get("website") or payload.get("url", "")).strip()
+        domain = business.get("website_domain") or normalize_registrable_domain(url)
+        suppressions=[dict(r) for r in db.execute("SELECT kind,value FROM suppressions WHERE organization_id=? AND active=1",(org,))]
+        enrichment = {
+            "business_id": bid,
+            "website": None,
+            "domain": enrich_domain(domain) if domain else None,
+            "contacts": [],
+        }
+        if url:
+            scan = enrich_website(url, fetch=lambda u, timeout=5.0, max_bytes=256*1024: scan_website(u, max_pages=1))
+            enrichment["website"] = scan
+            html = scan.get("html") or ""
+            if not html and scan.get("body"):
+                body = scan["body"]
+                html = body.decode("utf-8", "replace") if isinstance(body, bytes) else str(body)
+            if html and isinstance(html, str):
+                contacts = enrich_contacts(html, scan.get("final_url") or url, suppressions=suppressions, max_results=25)
+                enrichment["contacts"] = contacts
+                for contact in contacts:
+                    key = hashlib.sha256((str(bid) + contact["source_url"] + contact["kind"] + contact["value"]).encode()).hexdigest()
+                    db.execute("INSERT OR IGNORE INTO contact_extractions(organization_id,business_id,website_scan_id,extraction_key,kind,value,label,classification,confidence,source_url,public_business,mx_status,suppressed,do_not_contact,provenance) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",(org,bid,None,key,contact["kind"],contact["value"],contact["label"],contact["classification"],contact["confidence"],contact["source_url"],1,contact["mx_status"],int(contact["suppressed"]),int(contact["do_not_contact"]),contact["provenance"]))
+                db.commit()
+        self.audit(db, user, "business.enriched", str(bid)); db.commit()
+        return self.send_json(200, enrichment)
+
     def list_jobs(self, db, org, query):
         try:
             limit=max(1,min(int(query.get("page_size",[50])[0]),JOB_PAGE_SIZE)); page=max(1,int(query.get("page",[1])[0])); offset=max(0,int(query.get("offset",[0])[0]))+(page-1)*limit
@@ -1220,6 +1255,7 @@ class ApiHandler(BaseHTTPRequestHandler):
             if path=="/api/v1/businesses/bulk-review": return self.bulk_review(payload,db,user)
             bits_ai=path.split("/")
             if len(bits_ai)==7 and bits_ai[:4]==["","api","v1","businesses"] and bits_ai[5]=="ai" and bits_ai[6]=="opportunity-assessment": return self.assess_ai_opportunity(int(bits_ai[4]) if bits_ai[4].isdigit() else -1,payload,db,user)
+            if len(bits_ai)==6 and bits_ai[:4]==["","api","v1","businesses"] and bits_ai[5]=="enrichment": return self.enrich_business(int(bits_ai[4]) if bits_ai[4].isdigit() else -1,payload,db,user)
             if len(bits_ai)==7 and bits_ai[:4]==["","api","v1","businesses"] and bits_ai[5]=="ai" and bits_ai[6]=="suggest": return self.suggest_ai(int(bits_ai[4]) if bits_ai[4].isdigit() else -1,payload,db,user)
             if len(bits_ai)==6 and bits_ai[:4]==["","api","v1","ai-runs"] and bits_ai[4].isdigit() and bits_ai[5] in {"approve","reject"}: return self.decide_ai(int(bits_ai[4]),bits_ai[5],db,user)
             if path=="/api/v1/score-rules": return self.create_score_rule(payload,db,user)

apps/api/tests/test_enrichment.py

@@ -0,0 +1,103 @@
+"""Tests for the bounded post-discovery enrichment pipeline."""
+import os
+import unittest
+from unittest.mock import patch
+
+from app.enrichment import enrich_contacts, enrich_domain, enrich_website
+
+
+def _page(status=200, html=b"<html><head><title>Acme</title></head><body><h1>Acme</h1></body></html>", redirects=None, elapsed=12, tls=True):
+    return {
+        "status": status,
+        "final_url": "https://acme.test/",
+        "redirect_chain": redirects or [],
+        "body": html,
+        "content_type": "text/html",
+        "elapsed_ms": elapsed,
+        "tls": tls,
+        "certificate_status": "valid" if tls else "not_applicable",
+    }
+
+
+class EnrichmentTests(unittest.TestCase):
+    def test_enrich_website_uses_existing_scanner_fetcher(self):
+        fetch = lambda url, timeout=5.0, max_bytes=262144: _page()
+        with patch("app.enrichment.validate_url", side_effect=lambda url: url), \
+             patch("app.enrichment.normalize_registrable_domain", return_value="acme.test"):
+            result = enrich_website("https://acme.test/", fetch=fetch)
+        self.assertEqual(result["status"], "working")
+        self.assertTrue(result["has_working_website"])
+        self.assertEqual(result["domain"], "acme.test")
+        self.assertIsNotNone(result["response_time_ms"])
+
+    def test_enrich_website_marks_broken_on_4xx(self):
+        fetch = lambda url, **_: _page(status=404)
+        with patch("app.enrichment.validate_url", side_effect=lambda url: url):
+            result = enrich_website("https://acme.test/", fetch=fetch)
+        self.assertEqual(result["status"], "broken")
+        self.assertFalse(result["has_working_website"])
+
+    def test_enrich_website_detects_mobile_viewport(self):
+        html = b'<html><head><meta name="viewport" content="width=device-width"></head><body><h1>Acme</h1></body></html>'
+        fetch = lambda url, **_: _page(html=html)
+        with patch("app.enrichment.validate_url", side_effect=lambda url: url):
+            result = enrich_website("https://acme.test/", fetch=fetch)
+        self.assertTrue(result["mobile_viewport"])
+
+    def test_enrich_website_extracts_phone_and_email(self):
+        html = b'<html><body><h1>Acme</h1><p>+27 12 345 6789</p><a href="mailto:hello@acme.test">hello@acme.test</a></body></html>'
+        fetch = lambda url, **_: _page(html=html)
+        with patch("app.enrichment.validate_url", side_effect=lambda url: url):
+            result = enrich_website("https://acme.test/", fetch=fetch)
+        self.assertEqual(result["visible_phone"], "+27 12 345 6789")
+        self.assertEqual(result["visible_email"], "hello@acme.test")
+
+    def test_enrich_website_no_evidence_is_no_claim(self):
+        fetch = lambda url, **_: _page(status=404)
+        with patch("app.enrichment.validate_url", side_effect=lambda url: url):
+            result = enrich_website("https://acme.test/", fetch=fetch)
+        self.assertIsNone(result["visible_phone"])
+        self.assertIsNone(result["visible_email"])
+
+    def test_enrich_website_handles_fetch_failure(self):
+        def fetch(url, **_):
+            raise ConnectionError("timed out")
+        with patch("app.enrichment.validate_url", side_effect=lambda url: url):
+            result = enrich_website("https://acme.test/", fetch=fetch)
+        self.assertIn("error", result)
+        self.assertIsNone(result["has_working_website"])
+
+    def test_enrich_https_flag(self):
+        fetch = lambda url, **_: _page(tls=True)
+        with patch("app.enrichment.validate_url", side_effect=lambda url: url):
+            result = enrich_website("https://acme.test/", fetch=fetch)
+        self.assertTrue(result["https"])
+        self.assertTrue(result["ssl_valid"])
+
+    def test_enrich_contacts_delegates_to_extractor(self):
+        html = '<html><body><a href="mailto:hello@acme.test">Email</a><span>+27 12 345 6789</span></body></html>'
+        contacts = enrich_contacts(html, "https://acme.test/", max_results=10)
+        self.assertTrue(any(c["value"] == "hello@acme.test" for c in contacts))
+        self.assertTrue(any(c["kind"] == "phone" for c in contacts))
+        self.assertTrue(all(c["source_url"] == "https://acme.test/" for c in contacts))
+
+    def test_enrich_domain_unknown_for_unsupported_suffix(self):
+        result = enrich_domain("localhost")
+        self.assertEqual(result["status"], "unknown")
+
+    def test_enrich_domain_uses_resolution_evidence(self):
+        with patch("app.enrichment.normalize_registrable_domain", return_value="acme.co.za"), \
+             patch("app.enrichment.resolve_domain", return_value={"status": "ok", "addresses": ["1.2.3.4"]}):
+            result = enrich_domain("acme.co.za")
+            self.assertTrue(result["resolves"])
+            self.assertEqual(result["status"], "registered")
+
+    def test_enrich_domain_fails_closed(self):
+        with patch("app.enrichment.normalize_registrable_domain", return_value="acme.co.za"), \
+             patch("app.enrichment.resolve_domain", return_value={"status": "nxdomain"}):
+            result = enrich_domain("acme.co.za")
+            self.assertFalse(result["resolves"])
+
+
+if __name__ == "__main__":
+    unittest.main()

apps/api/tests/test_enrichment_api.py

@@ -0,0 +1,66 @@
+"""Tests for the enrich business API endpoint."""
+import json
+import os
+import sqlite3
+import threading
+import unittest
+from http.client import HTTPConnection
+from tempfile import TemporaryDirectory
+from unittest.mock import patch
+
+from app.main import create_server, hash_password
+
+
+class EnrichmentApiTests(unittest.TestCase):
+    def setUp(self):
+        self.tmp = TemporaryDirectory()
+        os.environ["BOOTSTRAP_ADMIN_EMAIL"] = "enrich-owner@example.test"
+        os.environ["BOOTSTRAP_ADMIN_PASSWORD"] = "password"
+        self.server = create_server("127.0.0.1", 0, self.tmp.name + "/enrich.db")
+        self.thread = threading.Thread(target=self.server.serve_forever, daemon=True); self.thread.start()
+        self.conn = HTTPConnection("127.0.0.1", self.server.server_port, timeout=3); self.cookie = None
+        self.request("POST", "/api/v1/auth/login", {"email": "enrich-owner@example.test", "password": "password"})
+
+    def tearDown(self):
+        self.server.shutdown(); self.server.server_close(); self.thread.join(2); self.tmp.cleanup()
+        for key in ("BOOTSTRAP_ADMIN_EMAIL", "BOOTSTRAP_ADMIN_PASSWORD"):
+            os.environ.pop(key, None)
+
+    def request(self, method, path, payload=None):
+        headers = {"Content-Type": "application/json"}
+        if self.cookie: headers["Cookie"] = self.cookie
+        self.conn.request(method, path, json.dumps(payload).encode() if payload is not None else None, headers)
+        response = self.conn.getresponse(); cookie = response.getheader("Set-Cookie")
+        if cookie: self.cookie = cookie.split(";", 1)[0]
+        return response.status, json.loads(response.read() or b"{}")
+
+    def test_enrich_business_returns_website_domain_contacts(self):
+        status, business = self.request("POST", "/api/v1/businesses", {"name": "Enrich Co", "website": "https://enrich.example.test"})
+        self.assertEqual(status, 201)
+        scan = {"status": 200, "final_url": "https://enrich.example.test/", "redirect_chain": [], "body": b"<html><head><title>Enrich</title></head><body><h1>Enrich Co</h1><a href=\"mailto:hello@enrich.example.test\">hello@enrich.example.test</a></body></html>", "html": "<html><head><title>Enrich</title></head><body><h1>Enrich Co</h1><a href=\"mailto:hello@enrich.example.test\">hello@enrich.example.test</a></body></html>", "content_type": "text/html", "elapsed_ms": 15, "tls": True, "certificate_status": "valid"}
+        with patch("app.main.scan_website", return_value=scan), \
+             patch("app.enrichment.validate_url", side_effect=lambda url: url), \
+             patch("app.enrichment.normalize_registrable_domain", return_value="enrich.example.test"), \
+             patch("app.main.enrich_domain", return_value={"domain": "enrich.example.test", "status": "registered", "resolves": True}):
+            status, result = self.request("POST", f"/api/v1/businesses/{business['id']}/enrichment", {})
+        self.assertEqual(status, 200)
+        self.assertEqual(result["business_id"], business["id"])
+        self.assertEqual(result["website"]["status"], "working")
+        self.assertEqual(result["domain"]["status"], "registered")
+        self.assertTrue(any(c["value"] == "hello@enrich.example.test" for c in result["contacts"]))
+
+    def test_enrich_business_tenant_isolated(self):
+        status, business = self.request("POST", "/api/v1/businesses", {"name": "Tenant Co"})
+        self.assertEqual(status, 201)
+        other_hash, other_salt = hash_password("other-password")
+        db = sqlite3.connect(self.tmp.name + "/enrich.db")
+        db.execute("INSERT INTO organizations(id,name) VALUES(?,?)", ("other-tenant", "Other"))
+        db.execute("INSERT INTO users(organization_id,email,password_hash,password_salt,role) VALUES(?,?,?,?,?)", ("other-tenant", "other@example.test", other_hash, other_salt, "owner"))
+        db.commit(); db.close()
+        self.cookie = None
+        self.assertEqual(self.request("POST", "/api/v1/auth/login", {"email": "other@example.test", "password": "other-password"})[0], 200)
+        self.assertEqual(self.request("POST", f"/api/v1/businesses/{business['id']}/enrichment", {})[0], 404)
+
+
+if __name__ == "__main__":
+    unittest.main()