root/MarketingTool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add public contact extraction

Browse Source
This commit is contained in:
Marco0300
2026-09-03 11:15:54 +02:00
parent fb89a28f2c
commit 89eb7e07e6
14 changed files with 402 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/app/main.py
+65
View File
@@ -11,11 +11,13 @@ if __package__ in (None, ""):
from app.sources import adapter_for, contains_secret
from app.domain_intelligence import normalize_registrable_domain, resolve_domain, generate_candidate_domains
from app.website_scanner import scan_website, validate_url
from app.contact_extractor import extract_contacts, MAX_HTML_BYTES, MAX_RESULTS
else:
from .domain import deduplication_key, deduplicate_businesses, is_suppressed, normalize_business, score_business, normalize_domain, normalize_phone, match_businesses
from .sources import adapter_for, contains_secret
from .domain_intelligence import normalize_registrable_domain, resolve_domain, generate_candidate_domains
from .website_scanner import scan_website, validate_url
from .contact_extractor import extract_contacts, MAX_HTML_BYTES, MAX_RESULTS
ORGANIZATION_ID = "demo-tenant"
SCHEMA = Path(__file__).resolve().parents[1] / "schema.sql"
SESSION_DAYS = 7
@@ -25,6 +27,7 @@ JOB_TYPES = {"noop", "prospect_recalculate", "source_discovery", "domain_check"}
JOB_PAGE_SIZE = 100
WEBSITE_SCAN_PAGE_SIZE = 100
WEBSITE_SCAN_CACHE_SECONDS = 3600
CONTACT_EXTRACTION_PAGE_SIZE = 100
SECRET_KEYS = {"password", "passwd", "secret", "token", "api_key", "apikey", "authorization", "credential", "private_key"}
CHILD_TABLES = {"contacts": ("name", "email", "phone", "title", "do_not_contact"), "domains": ("domain", "kind"), "websites": ("url", "website_class"), "evidence": ("kind", "url", "claim"), "notes": ("body",)}
@@ -125,6 +128,7 @@ class ApiHandler(BaseHTTPRequestHandler):
if path=="/api/v1/jobs": return self.list_jobs(db,org,parse_qs(parsed.query))
if path=="/api/v1/domain-checks": return self.list_domain_checks(db,org,parse_qs(parsed.query))
if path=="/api/v1/website-scans": return self.list_website_scans(db,org,parse_qs(parsed.query))
if path=="/api/v1/contact-extractions": return self.list_contact_extractions(db,org,parse_qs(parsed.query))
if path.startswith("/api/v1/jobs/"): return self.get_job_route(db,org,path,parse_qs(parsed.query))
if path.startswith("/api/v1/businesses/"):
bits=path.split("/"); ident=bits[4] if len(bits)>4 else ""
@@ -133,6 +137,7 @@ class ApiHandler(BaseHTTPRequestHandler):
if not row:return self.send_json(404,{"error":"not_found"})
if len(bits)==7 and bits[5:]==["websites","scan"]: return self.get_latest_website_scan(int(ident),db,user)
if len(bits)==7 and bits[5:]==["domains","check"]: return self.get_domain_check(int(ident),db,user,parse_qs(parsed.query))
if len(bits)==7 and bits[5:]==["contacts","extract"]: return self.send_json(405,{"error":"method_not_allowed"})
if len(bits)==6 and bits[5]=="domain-candidates": return self.list_domain_candidates(int(ident),db,org)
if len(bits)==6 and bits[5]=="matches": return self.matches(int(ident),db,org)
payload=row_json(row); payload.update(self.nested(db,int(ident),org)); return self.send_json(200,payload)
@@ -194,6 +199,64 @@ class ApiHandler(BaseHTTPRequestHandler):
rows = db.execute("SELECT * FROM website_scans WHERE " + " AND ".join(where) + " ORDER BY id DESC LIMIT ? OFFSET ?", params + [limit + 1, offset]).fetchall()
return self.send_json(200, {"organization_id": org, "items": [self._website_scan_result(r) for r in rows[:limit]], "limit": limit, "offset": offset, "has_more": len(rows) > limit})
def _contact_json(self, row):
result = row_json(row)
for key in ("public_business", "suppressed", "do_not_contact"):
if key in result: result[key] = bool(result[key])
return result
def list_contact_extractions(self, db, org, query):
try:
limit = int((query.get("page_size") or [50])[0]); offset = max(0, int((query.get("offset") or [0])[0]))
if limit < 1 or limit > CONTACT_EXTRACTION_PAGE_SIZE: raise ValueError
except (ValueError, TypeError): return self.send_json(400, {"error": "invalid_pagination"})
params = [org]; where = ["organization_id=?"]
business_id = (query.get("business_id") or [""])[0]
if business_id.isdigit(): where.append("business_id=?"); params.append(int(business_id))
rows = db.execute("SELECT * FROM contact_extractions WHERE " + " AND ".join(where) + " ORDER BY id DESC LIMIT ? OFFSET ?", params + [limit + 1, offset]).fetchall()
return self.send_json(200, {"organization_id": org, "items": [self._contact_json(r) for r in rows[:limit]], "limit": limit, "offset": offset, "has_more": len(rows) > limit})
def extract_business_contacts(self, bid, payload, db, user):
org = user["organization_id"]; business = self.business(db, bid, org)
if not business: return self.send_json(404, {"error": "not_found"})
scan_id = payload.get("website_scan_id", payload.get("scan_id"))
scan = None
if scan_id is not None:
if not isinstance(scan_id, int): return self.send_json(400, {"error": "invalid_scan"})
scan = db.execute("SELECT * FROM website_scans WHERE id=? AND business_id=? AND organization_id=?", (scan_id, bid, org)).fetchone()
if not scan: return self.send_json(404, {"error": "scan_not_found"})
try: stored = json.loads(scan["result_json"] or "{}")
except (TypeError, ValueError): stored = {}
source_url = str(payload.get("source_url") or scan["input_url"]).strip()
source_html = payload.get("html") if isinstance(payload.get("html"), str) else stored.get("html")
if source_html is None: return self.send_json(409, {"error": "scan_html_unavailable"})
if source_url != scan["input_url"] and source_url != (stored.get("final_url") or ""): return self.send_json(400, {"error": "source_not_approved"})
else:
source_url = str(payload.get("source_url") or "").strip()
source_html = payload.get("html")
if not source_url or not isinstance(source_html, str): return self.send_json(400, {"error": "approved_scan_required"})
parsed = urlparse(source_url)
official_hosts = {str(business["website_domain"]).lower().strip(".")}
if business["website"]:
official_hosts.add((urlparse(business["website"]).hostname or "").lower().strip("."))
if parsed.scheme not in {"http", "https"} or not parsed.hostname or parsed.hostname.lower().strip(".") not in official_hosts:
return self.send_json(400, {"error": "source_not_approved"})
if len(source_html.encode("utf-8")) > MAX_HTML_BYTES: return self.send_json(413, {"error": "html_too_large"})
try: requested_limit = int(payload.get("limit", MAX_RESULTS))
except (ValueError, TypeError): return self.send_json(400, {"error": "invalid_limits"})
if requested_limit < 1 or requested_limit > MAX_RESULTS: return self.send_json(400, {"error": "invalid_limits"})
key = str(payload.get("idempotency_key") or hashlib.sha256((str(scan_id or "") + source_url + source_html).encode()).hexdigest())[:200]
existing = db.execute("SELECT * FROM contact_extractions WHERE organization_id=? AND business_id=? AND extraction_key=? ORDER BY id", (org, bid, key)).fetchall()
if existing: return self.send_json(200, {"business_id": bid, "extraction_key": key, "items": [self._contact_json(r) for r in existing], "idempotent": True})
suppressions = [dict(r) for r in db.execute("SELECT kind,value FROM suppressions WHERE organization_id=?", (org,))]
try: found = extract_contacts(source_html, source_url, suppressions=suppressions, max_results=requested_limit)
except ValueError as exc: return self.send_json(400, {"error": str(exc)})
for item in found:
db.execute("INSERT INTO contact_extractions(organization_id,business_id,website_scan_id,extraction_key,kind,value,label,classification,confidence,source_url,public_business,mx_status,suppressed,do_not_contact,provenance) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", (org,bid,scan["id"] if scan else None,key,item["kind"],item["value"],item["label"],item["classification"],item["confidence"],item["source_url"],int(item["public_business"]),item["mx_status"],int(item["suppressed"]),int(item["do_not_contact"]),item["provenance"]))
self.audit(db, user, "contacts.extracted", f"{bid}:{len(found)}:{key}"); db.commit()
rows = db.execute("SELECT * FROM contact_extractions WHERE organization_id=? AND business_id=? AND extraction_key=? ORDER BY id", (org,bid,key)).fetchall()
return self.send_json(201, {"business_id": bid, "extraction_key": key, "items": [self._contact_json(r) for r in rows], "idempotent": False})
def get_latest_website_scan(self, bid, db, user):
row = db.execute("SELECT * FROM website_scans WHERE business_id=? AND organization_id=? ORDER BY id DESC LIMIT 1", (bid, user["organization_id"])).fetchone()
if not row: return self.send_json(404, {"error": "scan_not_found"})
@@ -333,8 +396,10 @@ class ApiHandler(BaseHTTPRequestHandler):
if path=="/api/v1/sources":return self.create_source(payload,db,user)
if path=="/api/v1/discovery-queries":return self.create_query(payload,db,user)
if path=="/api/v1/suppressions":return self.create_suppression(payload,db,user)
if path=="/api/v1/contact-extractions": return self.send_json(405,{"error":"method_not_allowed"})
if path=="/api/v1/imports/preview":return self.preview_import(payload,db,org)
if len(path.split("/"))==7 and path.split("/")[3:6]==["businesses",path.split("/")[4],"websites"] and path.split("/")[6]=="scan": return self.scan_business_website(int(path.split("/")[4]) if path.split("/")[4].isdigit() else -1,payload,db,user)
if len(path.split("/"))==7 and path.split("/")[3:6]==["businesses",path.split("/")[4],"contacts"] and path.split("/")[6]=="extract": return self.extract_business_contacts(int(path.split("/")[4]) if path.split("/")[4].isdigit() else -1,payload,db,user)
if len(path.split("/"))==7 and path.split("/")[3:6]==["businesses",path.split("/")[4],"domains"] and path.split("/")[6]=="check": return self.post_domain_check(int(path.split("/")[4]) if path.split("/")[4].isdigit() else -1,payload,db,user)
if len(path.split("/"))==7 and path.split("/")[3:6]==["businesses",path.split("/")[4],"domain-candidates"] and path.split("/")[6]=="check-availability": return self.check_availability(int(path.split("/")[4]) if path.split("/")[4].isdigit() else -1,payload,db,user)
if path.startswith("/api/v1/merge-history/") and path.endswith("/reverse"):