183 lines
10 KiB
Python
183 lines
10 KiB
Python
"""Bounded, passive and SSRF-safe website analysis using only the stdlib."""
|
|
from __future__ import annotations
|
|
|
|
import html
|
|
import ipaddress
|
|
import re
|
|
import socket
|
|
import ssl
|
|
import time
|
|
from html.parser import HTMLParser
|
|
from urllib.parse import urljoin, urlparse
|
|
from urllib.error import HTTPError
|
|
from urllib.request import HTTPRedirectHandler, Request, build_opener
|
|
|
|
MAX_BYTES = 512 * 1024
|
|
MAX_REDIRECTS = 5
|
|
MAX_PAGES = 1
|
|
DEFAULT_TIMEOUT = 5.0
|
|
_METADATA_IPS = {ipaddress.ip_address("169.254.169.254"), ipaddress.ip_address("100.100.100.200")}
|
|
|
|
|
|
def _resolved_addresses(host: str, timeout: float) -> list[str]:
|
|
try:
|
|
socket.setdefaulttimeout(timeout)
|
|
records = socket.getaddrinfo(host, None, type=socket.SOCK_STREAM)
|
|
except (OSError, socket.gaierror) as exc:
|
|
raise ValueError("dns_failure") from exc
|
|
addresses = sorted({str(r[4][0]) for r in records if len(r) > 4})
|
|
if not addresses:
|
|
raise ValueError("dns_failure")
|
|
for value in addresses:
|
|
try:
|
|
ip = ipaddress.ip_address(value)
|
|
except ValueError as exc:
|
|
raise ValueError("unsafe_address") from exc
|
|
if ip in _METADATA_IPS or not ip.is_global or ip.is_loopback or ip.is_private or ip.is_link_local or ip.is_multicast or ip.is_reserved or ip.is_unspecified:
|
|
raise ValueError("unsafe_address")
|
|
return addresses
|
|
|
|
|
|
def validate_url(value: str, *, timeout: float = DEFAULT_TIMEOUT) -> str:
|
|
raw = str(value or "").strip()
|
|
parsed = urlparse(raw)
|
|
if parsed.scheme.lower() not in {"http", "https"} or not parsed.hostname or parsed.username or parsed.password:
|
|
raise ValueError("invalid_url")
|
|
if parsed.fragment:
|
|
raw = raw.split("#", 1)[0]
|
|
parsed = urlparse(raw)
|
|
host = (parsed.hostname or "").rstrip(".").lower()
|
|
if len(raw) > 2048 or len(host) > 253:
|
|
raise ValueError("invalid_url")
|
|
try:
|
|
ipaddress.ip_address(host)
|
|
_resolved_addresses(host, timeout)
|
|
except ValueError:
|
|
_resolved_addresses(host, timeout)
|
|
return parsed.geturl()
|
|
|
|
|
|
class _Redirects(HTTPRedirectHandler):
|
|
def __init__(self, timeout: float, max_redirects: int):
|
|
self.timeout = timeout
|
|
self.max_redirects = max_redirects
|
|
self.chain: list[str] = []
|
|
|
|
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
if len(self.chain) >= self.max_redirects:
|
|
raise ValueError("redirect_limit")
|
|
target = validate_url(urljoin(req.full_url, newurl), timeout=self.timeout)
|
|
self.chain.append(target)
|
|
return Request(target, headers={"User-Agent": "ProspectPlatformWebsiteScanner/1.0", "Accept": "text/html,application/xhtml+xml"}, method="GET")
|
|
|
|
|
|
def _fetch(url: str, *, timeout: float = DEFAULT_TIMEOUT, max_bytes: int = MAX_BYTES, max_redirects: int = MAX_REDIRECTS) -> dict:
|
|
safe_url = validate_url(url, timeout=timeout)
|
|
redirects = _Redirects(timeout, max_redirects)
|
|
opener = build_opener(redirects)
|
|
request = Request(safe_url, headers={"User-Agent": "ProspectPlatformWebsiteScanner/1.0", "Accept": "text/html,application/xhtml+xml"}, method="GET")
|
|
started = time.monotonic()
|
|
try:
|
|
with opener.open(request, timeout=timeout) as response:
|
|
chunks, total = [], 0
|
|
while True:
|
|
chunk = response.read(min(65536, max_bytes - total + 1))
|
|
if not chunk:
|
|
break
|
|
total += len(chunk)
|
|
if total > max_bytes:
|
|
raise ValueError("response_too_large")
|
|
chunks.append(chunk)
|
|
final_url = validate_url(response.geturl(), timeout=timeout)
|
|
return {"status": int(response.status), "final_url": final_url, "redirect_chain": redirects.chain, "body": b"".join(chunks), "content_type": response.headers.get_content_type(), "elapsed_ms": round((time.monotonic() - started) * 1000, 2), "tls": final_url.startswith("https://"), "certificate_status": "valid" if final_url.startswith("https://") else "not_applicable"}
|
|
except HTTPError as exc:
|
|
# HTTP errors are still useful website observations; read only the bounded body.
|
|
body = exc.read(max_bytes + 1)
|
|
if len(body) > max_bytes: raise ValueError("response_too_large") from exc
|
|
return {"status": int(exc.code), "final_url": validate_url(exc.geturl(), timeout=timeout), "redirect_chain": redirects.chain, "body": body, "content_type": exc.headers.get_content_type() if exc.headers else "text/html", "elapsed_ms": round((time.monotonic() - started) * 1000, 2), "tls": str(exc.geturl()).startswith("https://"), "certificate_status": "valid" if str(exc.geturl()).startswith("https://") else "not_applicable"}
|
|
except ssl.SSLCertVerificationError as exc:
|
|
raise ValueError("certificate_invalid") from exc
|
|
except ValueError:
|
|
raise
|
|
except TimeoutError as exc:
|
|
raise ValueError("timeout") from exc
|
|
except OSError as exc:
|
|
raise ValueError("connection_failed") from exc
|
|
|
|
|
|
class _PageParser(HTMLParser):
|
|
def __init__(self):
|
|
super().__init__(convert_charrefs=True)
|
|
self.title = ""; self.meta_description = ""; self.language = ""; self.headings: list[str] = []
|
|
self.viewport = False; self.cms_hints: set[str] = set(); self.contact_page = False; self.form = False
|
|
self.mail = False; self.phone = False; self.whatsapp = False; self.social = False; self._tag = ""; self._buf: list[str] = []
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
attrs = {str(k).lower(): str(v or "") for k, v in attrs}; tag = tag.lower()
|
|
self._tag = tag
|
|
if tag == "html": self.language = attrs.get("lang", "")[:20]
|
|
if tag in {"h1", "h2", "h3", "h4", "h5", "h6"}: self._buf = []
|
|
if tag == "title": self._buf = []
|
|
if tag == "meta":
|
|
name = attrs.get("name", "").lower()
|
|
if name == "description": self.meta_description = attrs.get("content", "")[:1000]
|
|
if name == "viewport": self.viewport = True
|
|
generator = attrs.get("content", "").lower()
|
|
if name == "generator": self._cms(generator)
|
|
if tag == "form": self.form = True
|
|
if tag == "a":
|
|
href = attrs.get("href", "").lower()
|
|
self.contact_page |= any(x in href for x in ("contact", "get-in-touch", "reach-us"))
|
|
self.mail |= href.startswith("mailto:"); self.whatsapp |= "wa.me" in href or "whatsapp" in href
|
|
self.social |= any(x in href for x in ("facebook.com", "instagram.com", "linkedin.com", "twitter.com", "x.com", "youtube.com", "tiktok.com"))
|
|
if tag in {"script", "link"}:
|
|
text = " ".join(attrs.values()).lower(); self._cms(text)
|
|
|
|
def _cms(self, text):
|
|
for key, terms in {"wordpress": ("wordpress", "wp-content"), "drupal": ("drupal",), "joomla": ("joomla",), "shopify": ("shopify",), "wix": ("wix.com",)}.items():
|
|
if any(term in text for term in terms): self.cms_hints.add(key)
|
|
|
|
def handle_data(self, data):
|
|
if self._tag in {"title", "h1", "h2", "h3", "h4", "h5", "h6"}: self._buf.append(data)
|
|
if re.search(r"(?:tel:|\+?\d[\d ()-]{6,})", data): self.phone = True
|
|
|
|
def handle_endtag(self, tag):
|
|
tag = tag.lower()
|
|
if tag == "title" and self._buf: self.title = " ".join("".join(self._buf).split())[:500]
|
|
if tag in {"h1", "h2", "h3", "h4", "h5", "h6"} and self._buf:
|
|
self.headings.append(" ".join("".join(self._buf).split())[:300])
|
|
self._tag = ""
|
|
|
|
|
|
def classify_website(status, final_url, body, *, error=None) -> str:
|
|
if error:
|
|
return "blocked" if error in {"timeout", "connection_failed", "dns_failure", "unsafe_address", "certificate_invalid", "redirect_limit", "response_too_large"} else "unknown"
|
|
if status is None: return "unknown"
|
|
if 400 <= status or status < 200: return "broken"
|
|
text = re.sub(r"<[^>]+>", " ", body if isinstance(body, str) else body.decode("utf-8", "replace")).lower()
|
|
if status in {301, 302, 303, 307, 308} and not text.strip(): return "redirect_only"
|
|
if re.search(r"domain (is )?for sale|buy this domain|parking page|parked free", text): return "parked"
|
|
if re.search(r"under construction|coming soon|website coming", text): return "under_construction"
|
|
if re.search(r"placeholder|lorem ipsum|sample page|default web page", text): return "placeholder"
|
|
if status < 300 and len(re.sub(r"\s+", "", text)) >= 8: return "healthy"
|
|
return "unknown"
|
|
|
|
|
|
def scan_website(url: str, *, timeout: float = DEFAULT_TIMEOUT, max_bytes: int = MAX_BYTES, max_redirects: int = MAX_REDIRECTS, max_pages: int = MAX_PAGES) -> dict:
|
|
result = {"input_url": str(url), "status": None, "final_url": None, "redirect_chain": [], "title": "", "meta_description": "", "language": "", "headings": [], "responsive_signal": None, "cms_hints": [], "contact_page_signal": None, "form_signal": None, "mail_signal": None, "phone_signal": None, "whatsapp_signal": None, "social_signal": None, "elapsed_ms": None, "size_bytes": 0, "tls": None, "certificate_status": "unknown", "error_code": None}
|
|
try:
|
|
if not 0 < int(max_redirects) <= MAX_REDIRECTS or not 0 < int(max_pages) <= MAX_PAGES: raise ValueError("invalid_limits")
|
|
fetched = _fetch(url, timeout=max(0.1, min(float(timeout), 10.0)), max_bytes=max(1, min(int(max_bytes), MAX_BYTES)), max_redirects=int(max_redirects))
|
|
result.update({k: fetched[k] for k in ("status", "final_url", "redirect_chain", "elapsed_ms", "tls", "certificate_status")}); result["size_bytes"] = len(fetched["body"])
|
|
if fetched["content_type"] in {"text/html", "application/xhtml+xml"}:
|
|
result["html"] = fetched["body"].decode("utf-8", "replace")
|
|
if fetched["content_type"] not in {"text/html", "application/xhtml+xml"}:
|
|
result["classification"] = classify_website(fetched["status"], fetched["final_url"], ""); return result
|
|
parser = _PageParser(); parser.feed(fetched["body"].decode("utf-8", "replace"))
|
|
for key in ("title", "meta_description", "language", "headings", "viewport", "cms_hints", "contact_page", "form", "mail", "phone", "whatsapp", "social"):
|
|
result[{"viewport":"responsive_signal","cms_hints":"cms_hints","contact_page":"contact_page_signal","form":"form_signal","mail":"mail_signal","phone":"phone_signal","whatsapp":"whatsapp_signal","social":"social_signal"}.get(key,key)] = sorted(parser.cms_hints) if key == "cms_hints" else getattr(parser, key)
|
|
result["classification"] = classify_website(result["status"], result["final_url"], fetched["body"])
|
|
except ValueError as exc:
|
|
result["error_code"] = str(exc); result["classification"] = classify_website(None, url, "", error=str(exc))
|
|
return result
|