add ssrf-safe website analysis
This commit is contained in:
Marco0300
@@ -10,10 +10,12 @@ if __package__ in (None, ""):
|
||||
from app.domain import deduplication_key, deduplicate_businesses, is_suppressed, normalize_business, score_business, normalize_domain, normalize_phone, match_businesses
|
||||
from app.sources import adapter_for, contains_secret
|
||||
from app.domain_intelligence import normalize_registrable_domain, resolve_domain, generate_candidate_domains
|
||||
from app.website_scanner import scan_website, validate_url
|
||||
else:
|
||||
from .domain import deduplication_key, deduplicate_businesses, is_suppressed, normalize_business, score_business, normalize_domain, normalize_phone, match_businesses
|
||||
from .sources import adapter_for, contains_secret
|
||||
from .domain_intelligence import normalize_registrable_domain, resolve_domain, generate_candidate_domains
|
||||
from .website_scanner import scan_website, validate_url
|
||||
ORGANIZATION_ID = "demo-tenant"
|
||||
SCHEMA = Path(__file__).resolve().parents[1] / "schema.sql"
|
||||
SESSION_DAYS = 7
|
||||
@@ -21,6 +23,8 @@ PBKDF2_ITERATIONS = 300_000
|
||||
MUTATING_ROLES = {"owner", "admin", "researcher"}
|
||||
JOB_TYPES = {"noop", "prospect_recalculate", "source_discovery", "domain_check"}
|
||||
JOB_PAGE_SIZE = 100
|
||||
WEBSITE_SCAN_PAGE_SIZE = 100
|
||||
WEBSITE_SCAN_CACHE_SECONDS = 3600
|
||||
SECRET_KEYS = {"password", "passwd", "secret", "token", "api_key", "apikey", "authorization", "credential", "private_key"}
|
||||
CHILD_TABLES = {"contacts": ("name", "email", "phone", "title", "do_not_contact"), "domains": ("domain", "kind"), "websites": ("url", "website_class"), "evidence": ("kind", "url", "claim"), "notes": ("body",)}
|
||||
|
||||
@@ -120,12 +124,14 @@ class ApiHandler(BaseHTTPRequestHandler):
|
||||
if path=="/api/v1/source-records": return self.list_source_records(db,org,parse_qs(parsed.query))
|
||||
if path=="/api/v1/jobs": return self.list_jobs(db,org,parse_qs(parsed.query))
|
||||
if path=="/api/v1/domain-checks": return self.list_domain_checks(db,org,parse_qs(parsed.query))
|
||||
if path=="/api/v1/website-scans": return self.list_website_scans(db,org,parse_qs(parsed.query))
|
||||
if path.startswith("/api/v1/jobs/"): return self.get_job_route(db,org,path,parse_qs(parsed.query))
|
||||
if path.startswith("/api/v1/businesses/"):
|
||||
bits=path.split("/"); ident=bits[4] if len(bits)>4 else ""
|
||||
if not ident.isdigit(): return self.send_json(404,{"error":"not_found"})
|
||||
row=self.business(db,int(ident),org)
|
||||
if not row:return self.send_json(404,{"error":"not_found"})
|
||||
if len(bits)==7 and bits[5:]==["websites","scan"]: return self.get_latest_website_scan(int(ident),db,user)
|
||||
if len(bits)==7 and bits[5:]==["domains","check"]: return self.get_domain_check(int(ident),db,user,parse_qs(parsed.query))
|
||||
if len(bits)==6 and bits[5]=="domain-candidates": return self.list_domain_candidates(int(ident),db,org)
|
||||
if len(bits)==6 and bits[5]=="matches": return self.matches(int(ident),db,org)
|
||||
@@ -173,6 +179,48 @@ class ApiHandler(BaseHTTPRequestHandler):
|
||||
rows=db.execute("SELECT * FROM domain_checks WHERE organization_id=? ORDER BY id DESC LIMIT ? OFFSET ?",(org,limit+1,offset)).fetchall()
|
||||
return self.send_json(200,{"organization_id":org,"items":[self._domain_result(r) for r in rows[:limit]],"limit":limit,"offset":offset,"has_more":len(rows)>limit})
|
||||
|
||||
def _website_scan_result(self, row, cache_hit=False):
|
||||
result = json.loads(row["result_json"] or "{}")
|
||||
result.update({"id": row["id"], "business_id": row["business_id"], "website_id": row["website_id"], "input_url": row["input_url"], "classification": row["classification"], "scanned_at": row["scanned_at"], "cache_expires_at": row["cache_expires_at"], "cache_hit": cache_hit})
|
||||
return result
|
||||
|
||||
def list_website_scans(self, db, org, query):
|
||||
try:
|
||||
limit = int((query.get("page_size") or [50])[0]); offset = max(0, int((query.get("offset") or [0])[0]))
|
||||
if limit < 1 or limit > WEBSITE_SCAN_PAGE_SIZE: raise ValueError
|
||||
except (ValueError, TypeError): return self.send_json(400, {"error": "invalid_pagination"})
|
||||
params = [org]; where = ["organization_id=?"]
|
||||
if (query.get("business_id") or [""])[0].isdigit(): where.append("business_id=?"); params.append(int(query["business_id"][0]))
|
||||
rows = db.execute("SELECT * FROM website_scans WHERE " + " AND ".join(where) + " ORDER BY id DESC LIMIT ? OFFSET ?", params + [limit + 1, offset]).fetchall()
|
||||
return self.send_json(200, {"organization_id": org, "items": [self._website_scan_result(r) for r in rows[:limit]], "limit": limit, "offset": offset, "has_more": len(rows) > limit})
|
||||
|
||||
def get_latest_website_scan(self, bid, db, user):
|
||||
row = db.execute("SELECT * FROM website_scans WHERE business_id=? AND organization_id=? ORDER BY id DESC LIMIT 1", (bid, user["organization_id"])).fetchone()
|
||||
if not row: return self.send_json(404, {"error": "scan_not_found"})
|
||||
return self.send_json(200, self._website_scan_result(row, False))
|
||||
|
||||
def scan_business_website(self, bid, payload, db, user):
|
||||
org = user["organization_id"]; business = self.business(db, bid, org)
|
||||
if not business: return self.send_json(404, {"error": "not_found"})
|
||||
requested = str(payload.get("url", "")).strip() if isinstance(payload, dict) else ""
|
||||
if not requested:
|
||||
child = db.execute("SELECT * FROM websites WHERE business_id=? AND organization_id=? ORDER BY id LIMIT 1", (bid, org)).fetchone()
|
||||
requested = (child["url"] if child else business["website"]) or ""
|
||||
try: safe_url = validate_url(requested)
|
||||
except ValueError as exc:
|
||||
self.audit(db, user, "website.scan.rejected", f"{bid}:{str(exc)}"); db.commit()
|
||||
return self.send_json(400, {"error": "unsafe_url", "reason": str(exc)})
|
||||
website = db.execute("SELECT id FROM websites WHERE business_id=? AND organization_id=? AND url=? ORDER BY id LIMIT 1", (bid, org, safe_url)).fetchone()
|
||||
cache_key = hashlib.sha256(safe_url.encode()).hexdigest(); now = datetime.now(timezone.utc).replace(microsecond=0); expires = now + timedelta(seconds=WEBSITE_SCAN_CACHE_SECONDS)
|
||||
cached = db.execute("SELECT * FROM website_scans WHERE organization_id=? AND business_id=? AND cache_key=? AND cache_expires_at>? ORDER BY id DESC LIMIT 1", (org, bid, cache_key, now.isoformat())).fetchone()
|
||||
if cached:
|
||||
self.audit(db, user, "website.scan.cache_hit", f"{bid}:{safe_url}"); db.commit()
|
||||
return self.send_json(200, self._website_scan_result(cached, True))
|
||||
result = scan_website(safe_url); result["business_id"] = bid
|
||||
cur = db.execute("INSERT INTO website_scans(organization_id,business_id,website_id,input_url,classification,result_json,cache_key,scanned_at,cache_expires_at) VALUES(?,?,?,?,?,?,?,?,?)", (org, bid, website["id"] if website else None, safe_url, result["classification"], json.dumps(result, sort_keys=True), cache_key, now.isoformat(), expires.isoformat()))
|
||||
self.audit(db, user, "website.scanned", f"{bid}:{result['classification']}"); db.commit()
|
||||
return self.send_json(201, self._website_scan_result(db.execute("SELECT * FROM website_scans WHERE id=?", (cur.lastrowid,)).fetchone()))
|
||||
|
||||
def list_domain_candidates(self,bid,db,org):
|
||||
business=self.business(db,bid,org)
|
||||
if not business:return self.send_json(404,{"error":"not_found"})
|
||||
@@ -286,6 +334,7 @@ class ApiHandler(BaseHTTPRequestHandler):
|
||||
if path=="/api/v1/discovery-queries":return self.create_query(payload,db,user)
|
||||
if path=="/api/v1/suppressions":return self.create_suppression(payload,db,user)
|
||||
if path=="/api/v1/imports/preview":return self.preview_import(payload,db,org)
|
||||
if len(path.split("/"))==7 and path.split("/")[3:6]==["businesses",path.split("/")[4],"websites"] and path.split("/")[6]=="scan": return self.scan_business_website(int(path.split("/")[4]) if path.split("/")[4].isdigit() else -1,payload,db,user)
|
||||
if len(path.split("/"))==7 and path.split("/")[3:6]==["businesses",path.split("/")[4],"domains"] and path.split("/")[6]=="check": return self.post_domain_check(int(path.split("/")[4]) if path.split("/")[4].isdigit() else -1,payload,db,user)
|
||||
if len(path.split("/"))==7 and path.split("/")[3:6]==["businesses",path.split("/")[4],"domain-candidates"] and path.split("/")[6]=="check-availability": return self.check_availability(int(path.split("/")[4]) if path.split("/")[4].isdigit() else -1,payload,db,user)
|
||||
if path.startswith("/api/v1/merge-history/") and path.endswith("/reverse"):
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
"""Bounded, passive and SSRF-safe website analysis using only the stdlib."""
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import ipaddress
|
||||
import re
|
||||
import socket
|
||||
import ssl
|
||||
import time
|
||||
from html.parser import HTMLParser
|
||||
from urllib.parse import urljoin, urlparse
|
||||
from urllib.error import HTTPError
|
||||
from urllib.request import HTTPRedirectHandler, Request, build_opener
|
||||
|
||||
MAX_BYTES = 512 * 1024
|
||||
MAX_REDIRECTS = 5
|
||||
MAX_PAGES = 1
|
||||
DEFAULT_TIMEOUT = 5.0
|
||||
_METADATA_IPS = {ipaddress.ip_address("169.254.169.254"), ipaddress.ip_address("100.100.100.200")}
|
||||
|
||||
|
||||
def _resolved_addresses(host: str, timeout: float) -> list[str]:
|
||||
try:
|
||||
socket.setdefaulttimeout(timeout)
|
||||
records = socket.getaddrinfo(host, None, type=socket.SOCK_STREAM)
|
||||
except (OSError, socket.gaierror) as exc:
|
||||
raise ValueError("dns_failure") from exc
|
||||
addresses = sorted({str(r[4][0]) for r in records if len(r) > 4})
|
||||
if not addresses:
|
||||
raise ValueError("dns_failure")
|
||||
for value in addresses:
|
||||
try:
|
||||
ip = ipaddress.ip_address(value)
|
||||
except ValueError as exc:
|
||||
raise ValueError("unsafe_address") from exc
|
||||
if ip in _METADATA_IPS or not ip.is_global or ip.is_loopback or ip.is_private or ip.is_link_local or ip.is_multicast or ip.is_reserved or ip.is_unspecified:
|
||||
raise ValueError("unsafe_address")
|
||||
return addresses
|
||||
|
||||
|
||||
def validate_url(value: str, *, timeout: float = DEFAULT_TIMEOUT) -> str:
|
||||
raw = str(value or "").strip()
|
||||
parsed = urlparse(raw)
|
||||
if parsed.scheme.lower() not in {"http", "https"} or not parsed.hostname or parsed.username or parsed.password:
|
||||
raise ValueError("invalid_url")
|
||||
if parsed.fragment:
|
||||
raw = raw.split("#", 1)[0]
|
||||
parsed = urlparse(raw)
|
||||
host = (parsed.hostname or "").rstrip(".").lower()
|
||||
if len(raw) > 2048 or len(host) > 253:
|
||||
raise ValueError("invalid_url")
|
||||
try:
|
||||
ipaddress.ip_address(host)
|
||||
_resolved_addresses(host, timeout)
|
||||
except ValueError:
|
||||
_resolved_addresses(host, timeout)
|
||||
return parsed.geturl()
|
||||
|
||||
|
||||
class _Redirects(HTTPRedirectHandler):
|
||||
def __init__(self, timeout: float, max_redirects: int):
|
||||
self.timeout = timeout
|
||||
self.max_redirects = max_redirects
|
||||
self.chain: list[str] = []
|
||||
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
if len(self.chain) >= self.max_redirects:
|
||||
raise ValueError("redirect_limit")
|
||||
target = validate_url(urljoin(req.full_url, newurl), timeout=self.timeout)
|
||||
self.chain.append(target)
|
||||
return Request(target, headers={"User-Agent": "ProspectPlatformWebsiteScanner/1.0", "Accept": "text/html,application/xhtml+xml"}, method="GET")
|
||||
|
||||
|
||||
def _fetch(url: str, *, timeout: float = DEFAULT_TIMEOUT, max_bytes: int = MAX_BYTES, max_redirects: int = MAX_REDIRECTS) -> dict:
|
||||
safe_url = validate_url(url, timeout=timeout)
|
||||
redirects = _Redirects(timeout, max_redirects)
|
||||
opener = build_opener(redirects)
|
||||
request = Request(safe_url, headers={"User-Agent": "ProspectPlatformWebsiteScanner/1.0", "Accept": "text/html,application/xhtml+xml"}, method="GET")
|
||||
started = time.monotonic()
|
||||
try:
|
||||
with opener.open(request, timeout=timeout) as response:
|
||||
chunks, total = [], 0
|
||||
while True:
|
||||
chunk = response.read(min(65536, max_bytes - total + 1))
|
||||
if not chunk:
|
||||
break
|
||||
total += len(chunk)
|
||||
if total > max_bytes:
|
||||
raise ValueError("response_too_large")
|
||||
chunks.append(chunk)
|
||||
final_url = validate_url(response.geturl(), timeout=timeout)
|
||||
return {"status": int(response.status), "final_url": final_url, "redirect_chain": redirects.chain, "body": b"".join(chunks), "content_type": response.headers.get_content_type(), "elapsed_ms": round((time.monotonic() - started) * 1000, 2), "tls": final_url.startswith("https://"), "certificate_status": "valid" if final_url.startswith("https://") else "not_applicable"}
|
||||
except HTTPError as exc:
|
||||
# HTTP errors are still useful website observations; read only the bounded body.
|
||||
body = exc.read(max_bytes + 1)
|
||||
if len(body) > max_bytes: raise ValueError("response_too_large") from exc
|
||||
return {"status": int(exc.code), "final_url": validate_url(exc.geturl(), timeout=timeout), "redirect_chain": redirects.chain, "body": body, "content_type": exc.headers.get_content_type() if exc.headers else "text/html", "elapsed_ms": round((time.monotonic() - started) * 1000, 2), "tls": str(exc.geturl()).startswith("https://"), "certificate_status": "valid" if str(exc.geturl()).startswith("https://") else "not_applicable"}
|
||||
except ssl.SSLCertVerificationError as exc:
|
||||
raise ValueError("certificate_invalid") from exc
|
||||
except ValueError:
|
||||
raise
|
||||
except TimeoutError as exc:
|
||||
raise ValueError("timeout") from exc
|
||||
except OSError as exc:
|
||||
raise ValueError("connection_failed") from exc
|
||||
|
||||
|
||||
class _PageParser(HTMLParser):
|
||||
def __init__(self):
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.title = ""; self.meta_description = ""; self.language = ""; self.headings: list[str] = []
|
||||
self.viewport = False; self.cms_hints: set[str] = set(); self.contact_page = False; self.form = False
|
||||
self.mail = False; self.phone = False; self.whatsapp = False; self.social = False; self._tag = ""; self._buf: list[str] = []
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
attrs = {str(k).lower(): str(v or "") for k, v in attrs}; tag = tag.lower()
|
||||
self._tag = tag
|
||||
if tag == "html": self.language = attrs.get("lang", "")[:20]
|
||||
if tag in {"h1", "h2", "h3", "h4", "h5", "h6"}: self._buf = []
|
||||
if tag == "title": self._buf = []
|
||||
if tag == "meta":
|
||||
name = attrs.get("name", "").lower()
|
||||
if name == "description": self.meta_description = attrs.get("content", "")[:1000]
|
||||
if name == "viewport": self.viewport = True
|
||||
generator = attrs.get("content", "").lower()
|
||||
if name == "generator": self._cms(generator)
|
||||
if tag == "form": self.form = True
|
||||
if tag == "a":
|
||||
href = attrs.get("href", "").lower()
|
||||
self.contact_page |= any(x in href for x in ("contact", "get-in-touch", "reach-us"))
|
||||
self.mail |= href.startswith("mailto:"); self.whatsapp |= "wa.me" in href or "whatsapp" in href
|
||||
self.social |= any(x in href for x in ("facebook.com", "instagram.com", "linkedin.com", "twitter.com", "x.com", "youtube.com", "tiktok.com"))
|
||||
if tag in {"script", "link"}:
|
||||
text = " ".join(attrs.values()).lower(); self._cms(text)
|
||||
|
||||
def _cms(self, text):
|
||||
for key, terms in {"wordpress": ("wordpress", "wp-content"), "drupal": ("drupal",), "joomla": ("joomla",), "shopify": ("shopify",), "wix": ("wix.com",)}.items():
|
||||
if any(term in text for term in terms): self.cms_hints.add(key)
|
||||
|
||||
def handle_data(self, data):
|
||||
if self._tag in {"title", "h1", "h2", "h3", "h4", "h5", "h6"}: self._buf.append(data)
|
||||
if re.search(r"(?:tel:|\+?\d[\d ()-]{6,})", data): self.phone = True
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
tag = tag.lower()
|
||||
if tag == "title" and self._buf: self.title = " ".join("".join(self._buf).split())[:500]
|
||||
if tag in {"h1", "h2", "h3", "h4", "h5", "h6"} and self._buf:
|
||||
self.headings.append(" ".join("".join(self._buf).split())[:300])
|
||||
self._tag = ""
|
||||
|
||||
|
||||
def classify_website(status, final_url, body, *, error=None) -> str:
|
||||
if error:
|
||||
return "blocked" if error in {"timeout", "connection_failed", "dns_failure", "unsafe_address", "certificate_invalid", "redirect_limit", "response_too_large"} else "unknown"
|
||||
if status is None: return "unknown"
|
||||
if 400 <= status or status < 200: return "broken"
|
||||
text = re.sub(r"<[^>]+>", " ", body if isinstance(body, str) else body.decode("utf-8", "replace")).lower()
|
||||
if status in {301, 302, 303, 307, 308} and not text.strip(): return "redirect_only"
|
||||
if re.search(r"domain (is )?for sale|buy this domain|parking page|parked free", text): return "parked"
|
||||
if re.search(r"under construction|coming soon|website coming", text): return "under_construction"
|
||||
if re.search(r"placeholder|lorem ipsum|sample page|default web page", text): return "placeholder"
|
||||
if status < 300 and len(re.sub(r"\s+", "", text)) >= 8: return "healthy"
|
||||
return "unknown"
|
||||
|
||||
|
||||
def scan_website(url: str, *, timeout: float = DEFAULT_TIMEOUT, max_bytes: int = MAX_BYTES, max_redirects: int = MAX_REDIRECTS, max_pages: int = MAX_PAGES) -> dict:
|
||||
result = {"input_url": str(url), "status": None, "final_url": None, "redirect_chain": [], "title": "", "meta_description": "", "language": "", "headings": [], "responsive_signal": None, "cms_hints": [], "contact_page_signal": None, "form_signal": None, "mail_signal": None, "phone_signal": None, "whatsapp_signal": None, "social_signal": None, "elapsed_ms": None, "size_bytes": 0, "tls": None, "certificate_status": "unknown", "error_code": None}
|
||||
try:
|
||||
if not 0 < int(max_redirects) <= MAX_REDIRECTS or not 0 < int(max_pages) <= MAX_PAGES: raise ValueError("invalid_limits")
|
||||
fetched = _fetch(url, timeout=max(0.1, min(float(timeout), 10.0)), max_bytes=max(1, min(int(max_bytes), MAX_BYTES)), max_redirects=int(max_redirects))
|
||||
result.update({k: fetched[k] for k in ("status", "final_url", "redirect_chain", "elapsed_ms", "tls", "certificate_status")}); result["size_bytes"] = len(fetched["body"])
|
||||
if fetched["content_type"] not in {"text/html", "application/xhtml+xml"}:
|
||||
result["classification"] = classify_website(fetched["status"], fetched["final_url"], ""); return result
|
||||
parser = _PageParser(); parser.feed(fetched["body"].decode("utf-8", "replace"))
|
||||
for key in ("title", "meta_description", "language", "headings", "viewport", "cms_hints", "contact_page", "form", "mail", "phone", "whatsapp", "social"):
|
||||
result[{"viewport":"responsive_signal","cms_hints":"cms_hints","contact_page":"contact_page_signal","form":"form_signal","mail":"mail_signal","phone":"phone_signal","whatsapp":"whatsapp_signal","social":"social_signal"}.get(key,key)] = sorted(parser.cms_hints) if key == "cms_hints" else getattr(parser, key)
|
||||
result["classification"] = classify_website(result["status"], result["final_url"], fetched["body"])
|
||||
except ValueError as exc:
|
||||
result["error_code"] = str(exc); result["classification"] = classify_website(None, url, "", error=str(exc))
|
||||
return result
|
||||
Reference in New Issue
Block a user