root/MarketingTool
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

add ssrf-safe website analysis

Browse Source
This commit is contained in:
Marco0300
2026-09-03 11:07:34 +02:00
parent f1efe39de4
commit fb89a28f2c
13 changed files with 454 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/app/main.py
+49
View File
@@ -10,10 +10,12 @@ if __package__ in (None, ""):
from app.domain import deduplication_key, deduplicate_businesses, is_suppressed, normalize_business, score_business, normalize_domain, normalize_phone, match_businesses
from app.sources import adapter_for, contains_secret
from app.domain_intelligence import normalize_registrable_domain, resolve_domain, generate_candidate_domains
from app.website_scanner import scan_website, validate_url
else:
from .domain import deduplication_key, deduplicate_businesses, is_suppressed, normalize_business, score_business, normalize_domain, normalize_phone, match_businesses
from .sources import adapter_for, contains_secret
from .domain_intelligence import normalize_registrable_domain, resolve_domain, generate_candidate_domains
from .website_scanner import scan_website, validate_url
ORGANIZATION_ID = "demo-tenant"
SCHEMA = Path(__file__).resolve().parents[1] / "schema.sql"
SESSION_DAYS = 7
@@ -21,6 +23,8 @@ PBKDF2_ITERATIONS = 300_000
MUTATING_ROLES = {"owner", "admin", "researcher"}
JOB_TYPES = {"noop", "prospect_recalculate", "source_discovery", "domain_check"}
JOB_PAGE_SIZE = 100
WEBSITE_SCAN_PAGE_SIZE = 100
WEBSITE_SCAN_CACHE_SECONDS = 3600
SECRET_KEYS = {"password", "passwd", "secret", "token", "api_key", "apikey", "authorization", "credential", "private_key"}
CHILD_TABLES = {"contacts": ("name", "email", "phone", "title", "do_not_contact"), "domains": ("domain", "kind"), "websites": ("url", "website_class"), "evidence": ("kind", "url", "claim"), "notes": ("body",)}
@@ -120,12 +124,14 @@ class ApiHandler(BaseHTTPRequestHandler):
if path=="/api/v1/source-records": return self.list_source_records(db,org,parse_qs(parsed.query))
if path=="/api/v1/jobs": return self.list_jobs(db,org,parse_qs(parsed.query))
if path=="/api/v1/domain-checks": return self.list_domain_checks(db,org,parse_qs(parsed.query))
if path=="/api/v1/website-scans": return self.list_website_scans(db,org,parse_qs(parsed.query))
if path.startswith("/api/v1/jobs/"): return self.get_job_route(db,org,path,parse_qs(parsed.query))
if path.startswith("/api/v1/businesses/"):
bits=path.split("/"); ident=bits[4] if len(bits)>4 else ""
if not ident.isdigit(): return self.send_json(404,{"error":"not_found"})
row=self.business(db,int(ident),org)
if not row:return self.send_json(404,{"error":"not_found"})
if len(bits)==7 and bits[5:]==["websites","scan"]: return self.get_latest_website_scan(int(ident),db,user)
if len(bits)==7 and bits[5:]==["domains","check"]: return self.get_domain_check(int(ident),db,user,parse_qs(parsed.query))
if len(bits)==6 and bits[5]=="domain-candidates": return self.list_domain_candidates(int(ident),db,org)
if len(bits)==6 and bits[5]=="matches": return self.matches(int(ident),db,org)
@@ -173,6 +179,48 @@ class ApiHandler(BaseHTTPRequestHandler):
rows=db.execute("SELECT * FROM domain_checks WHERE organization_id=? ORDER BY id DESC LIMIT ? OFFSET ?",(org,limit+1,offset)).fetchall()
return self.send_json(200,{"organization_id":org,"items":[self._domain_result(r) for r in rows[:limit]],"limit":limit,"offset":offset,"has_more":len(rows)>limit})
def _website_scan_result(self, row, cache_hit=False):
result = json.loads(row["result_json"] or "{}")
result.update({"id": row["id"], "business_id": row["business_id"], "website_id": row["website_id"], "input_url": row["input_url"], "classification": row["classification"], "scanned_at": row["scanned_at"], "cache_expires_at": row["cache_expires_at"], "cache_hit": cache_hit})
return result
def list_website_scans(self, db, org, query):
try:
limit = int((query.get("page_size") or [50])[0]); offset = max(0, int((query.get("offset") or [0])[0]))
if limit < 1 or limit > WEBSITE_SCAN_PAGE_SIZE: raise ValueError
except (ValueError, TypeError): return self.send_json(400, {"error": "invalid_pagination"})
params = [org]; where = ["organization_id=?"]
if (query.get("business_id") or [""])[0].isdigit(): where.append("business_id=?"); params.append(int(query["business_id"][0]))
rows = db.execute("SELECT * FROM website_scans WHERE " + " AND ".join(where) + " ORDER BY id DESC LIMIT ? OFFSET ?", params + [limit + 1, offset]).fetchall()
return self.send_json(200, {"organization_id": org, "items": [self._website_scan_result(r) for r in rows[:limit]], "limit": limit, "offset": offset, "has_more": len(rows) > limit})
def get_latest_website_scan(self, bid, db, user):
row = db.execute("SELECT * FROM website_scans WHERE business_id=? AND organization_id=? ORDER BY id DESC LIMIT 1", (bid, user["organization_id"])).fetchone()
if not row: return self.send_json(404, {"error": "scan_not_found"})
return self.send_json(200, self._website_scan_result(row, False))
def scan_business_website(self, bid, payload, db, user):
org = user["organization_id"]; business = self.business(db, bid, org)
if not business: return self.send_json(404, {"error": "not_found"})
requested = str(payload.get("url", "")).strip() if isinstance(payload, dict) else ""
if not requested:
child = db.execute("SELECT * FROM websites WHERE business_id=? AND organization_id=? ORDER BY id LIMIT 1", (bid, org)).fetchone()
requested = (child["url"] if child else business["website"]) or ""
try: safe_url = validate_url(requested)
except ValueError as exc:
self.audit(db, user, "website.scan.rejected", f"{bid}:{str(exc)}"); db.commit()
return self.send_json(400, {"error": "unsafe_url", "reason": str(exc)})
website = db.execute("SELECT id FROM websites WHERE business_id=? AND organization_id=? AND url=? ORDER BY id LIMIT 1", (bid, org, safe_url)).fetchone()
cache_key = hashlib.sha256(safe_url.encode()).hexdigest(); now = datetime.now(timezone.utc).replace(microsecond=0); expires = now + timedelta(seconds=WEBSITE_SCAN_CACHE_SECONDS)
cached = db.execute("SELECT * FROM website_scans WHERE organization_id=? AND business_id=? AND cache_key=? AND cache_expires_at>? ORDER BY id DESC LIMIT 1", (org, bid, cache_key, now.isoformat())).fetchone()
if cached:
self.audit(db, user, "website.scan.cache_hit", f"{bid}:{safe_url}"); db.commit()
return self.send_json(200, self._website_scan_result(cached, True))
result = scan_website(safe_url); result["business_id"] = bid
cur = db.execute("INSERT INTO website_scans(organization_id,business_id,website_id,input_url,classification,result_json,cache_key,scanned_at,cache_expires_at) VALUES(?,?,?,?,?,?,?,?,?)", (org, bid, website["id"] if website else None, safe_url, result["classification"], json.dumps(result, sort_keys=True), cache_key, now.isoformat(), expires.isoformat()))
self.audit(db, user, "website.scanned", f"{bid}:{result['classification']}"); db.commit()
return self.send_json(201, self._website_scan_result(db.execute("SELECT * FROM website_scans WHERE id=?", (cur.lastrowid,)).fetchone()))
def list_domain_candidates(self,bid,db,org):
business=self.business(db,bid,org)
if not business:return self.send_json(404,{"error":"not_found"})
@@ -286,6 +334,7 @@ class ApiHandler(BaseHTTPRequestHandler):
if path=="/api/v1/discovery-queries":return self.create_query(payload,db,user)
if path=="/api/v1/suppressions":return self.create_suppression(payload,db,user)
if path=="/api/v1/imports/preview":return self.preview_import(payload,db,org)
if len(path.split("/"))==7 and path.split("/")[3:6]==["businesses",path.split("/")[4],"websites"] and path.split("/")[6]=="scan": return self.scan_business_website(int(path.split("/")[4]) if path.split("/")[4].isdigit() else -1,payload,db,user)
if len(path.split("/"))==7 and path.split("/")[3:6]==["businesses",path.split("/")[4],"domains"] and path.split("/")[6]=="check": return self.post_domain_check(int(path.split("/")[4]) if path.split("/")[4].isdigit() else -1,payload,db,user)
if len(path.split("/"))==7 and path.split("/")[3:6]==["businesses",path.split("/")[4],"domain-candidates"] and path.split("/")[6]=="check-availability": return self.check_availability(int(path.split("/")[4]) if path.split("/")[4].isdigit() else -1,payload,db,user)
if path.startswith("/api/v1/merge-history/") and path.endswith("/reverse"):