add public contact extraction

2026-09-03 11:15:54 +02:00
parent fb89a28f2c
commit 89eb7e07e6
14 changed files with 402 additions and 7 deletions
@@ -169,6 +169,8 @@ def scan_website(url: str, *, timeout: float = DEFAULT_TIMEOUT, max_bytes: int =
        if not 0 < int(max_redirects) <= MAX_REDIRECTS or not 0 < int(max_pages) <= MAX_PAGES: raise ValueError("invalid_limits")
        fetched = _fetch(url, timeout=max(0.1, min(float(timeout), 10.0)), max_bytes=max(1, min(int(max_bytes), MAX_BYTES)), max_redirects=int(max_redirects))
        result.update({k: fetched[k] for k in ("status", "final_url", "redirect_chain", "elapsed_ms", "tls", "certificate_status")}); result["size_bytes"] = len(fetched["body"])
+        if fetched["content_type"] in {"text/html", "application/xhtml+xml"}:
+            result["html"] = fetched["body"].decode("utf-8", "replace")
        if fetched["content_type"] not in {"text/html", "application/xhtml+xml"}:
            result["classification"] = classify_website(fetched["status"], fetched["final_url"], ""); return result
        parser = _PageParser(); parser.feed(fetched["body"].decode("utf-8", "replace"))