root/MarketingTool
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

add scoped authenticated discovery and scraping

Browse Source
This commit is contained in:
Marco0300
2026-09-03 19:47:57 +02:00
parent 6b8ad8f419
commit 6269cdd6f1
4 changed files with 306 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/app/main.py
+90 -4
View File
@@ -14,6 +14,7 @@ if __package__ in (None, ""):
from app.contact_extractor import extract_contacts, MAX_HTML_BYTES, MAX_RESULTS
from app.scoring import DEFAULT_RULES, signals_for_business, evaluate_score, SCORE_VERSION
from app.ai_assistance import generate as generate_ai, MAX_INPUT_ITEMS, MAX_OUTPUT_CHARS
from app.discovery import discover as scoped_discover
from app.config import load_config
else:
from .domain import deduplication_key, deduplicate_businesses, is_suppressed, normalize_business, score_business, normalize_domain, normalize_phone, match_businesses
@@ -23,13 +24,14 @@ else:
from .contact_extractor import extract_contacts, MAX_HTML_BYTES, MAX_RESULTS
from .scoring import DEFAULT_RULES, signals_for_business, evaluate_score, SCORE_VERSION
from .ai_assistance import generate as generate_ai, MAX_INPUT_ITEMS, MAX_OUTPUT_CHARS
from .discovery import discover as scoped_discover
from .config import load_config
ORGANIZATION_ID = "demo-tenant"
SCHEMA = Path(__file__).resolve().parents[1] / "schema.sql"
SESSION_DAYS = 7
PBKDF2_ITERATIONS = 300_000
MUTATING_ROLES = {"owner", "admin", "researcher"}
JOB_TYPES = {"noop", "prospect_recalculate", "source_discovery", "domain_check"}
JOB_TYPES = {"noop", "prospect_recalculate", "source_discovery", "domain_check", "scoped_discovery"}
JOB_PAGE_SIZE = 100
WEBSITE_SCAN_PAGE_SIZE = 100
WEBSITE_SCAN_CACHE_SECONDS = 3600
@@ -121,8 +123,8 @@ class ApiHandler(BaseHTTPRequestHandler):
db.execute("INSERT INTO audit_log (organization_id,user_id,action,details) VALUES (?,?,?,?)",(user["organization_id"],user["id"],action,details))
def business(self, db, ident, org): return db.execute("SELECT * FROM businesses WHERE id=? AND organization_id=?",(ident,org)).fetchone()
def nested(self, db, bid, org):
result={"contacts":[],"domains":[],"websites":[],"evidence":[],"pipeline":[],"interactions":[],"notes":[]}
tables={"contacts":"contacts","domains":"domains","websites":"websites","evidence":"evidence","pipeline":"pipeline_entries","interactions":"interactions","notes":"notes"}
result={"contacts":[],"contact_extractions":[],"domains":[],"websites":[],"evidence":[],"pipeline":[],"interactions":[],"notes":[]}
tables={"contacts":"contacts","contact_extractions":"contact_extractions","domains":"domains","websites":"websites","evidence":"evidence","pipeline":"pipeline_entries","interactions":"interactions","notes":"notes"}
for key, table in tables.items():
result[key]=[row_json(r) for r in db.execute(f"SELECT * FROM {table} WHERE business_id=? AND organization_id=? ORDER BY id",(bid,org))]
return result
@@ -552,6 +554,7 @@ class ApiHandler(BaseHTTPRequestHandler):
if path=="/api/v1/merge-history": return self.list_merge_history(db,org)
if path=="/api/v1/sources": return self.list_sources(db,org)
if path=="/api/v1/discovery-queries": return self.list_queries(db,org)
if path=="/api/v1/discovery-runs": return self.list_discovery_runs(db,org,parse_qs(parsed.query))
if path=="/api/v1/source-records": return self.list_source_records(db,org,parse_qs(parsed.query))
if path=="/api/v1/jobs": return self.list_jobs(db,org,parse_qs(parsed.query))
if path=="/api/v1/domain-checks": return self.list_domain_checks(db,org,parse_qs(parsed.query))
@@ -746,6 +749,45 @@ class ApiHandler(BaseHTTPRequestHandler):
rows=db.execute("SELECT * FROM jobs WHERE organization_id=? ORDER BY id DESC LIMIT ? OFFSET ?",(org,limit+1,offset)).fetchall(); more=len(rows)>limit
return self.send_json(200,{"organization_id":org,"items":[job_json(r) for r in rows[:limit]],"limit":limit,"offset":offset,"has_more":more})
def _discovery_run_json(self, row):
item = row_json(row)
for field, default in (("criteria_json", {}), ("seed_urls_json", []), ("result_json", {})):
try: item[field[:-5]] = json.loads(item.pop(field) or json.dumps(default))
except (TypeError, ValueError): item[field[:-5]] = default
return item
def list_discovery_runs(self, db, org, query):
try: limit = max(1, min(int((query.get("page_size") or [50])[0]), 100)); offset = max(0, int((query.get("offset") or [0])[0]))
except (ValueError, TypeError): return self.send_json(400, {"error": "invalid_pagination"})
rows = db.execute("SELECT * FROM discovery_runs WHERE organization_id=? ORDER BY id DESC LIMIT ? OFFSET ?", (org, limit + 1, offset)).fetchall()
return self.send_json(200, {"organization_id": org, "items": [self._discovery_run_json(r) for r in rows[:limit]], "limit": limit, "offset": offset, "has_more": len(rows) > limit})
def create_scoped_discovery(self, payload, db, user):
criteria = payload.get("criteria", {}); seeds = payload.get("seed_urls")
if not isinstance(criteria, dict) or not isinstance(seeds, list) or not seeds: return self.send_json(400, {"error": "seed_urls_required"})
try:
if len(seeds) > 5 or len(json.dumps(criteria).encode()) > 8192: raise ValueError("invalid_criteria")
if not isinstance(criteria.get("keywords", criteria.get("keyword", [])), (list, str)): raise ValueError("invalid_criteria")
max_pages = int(payload.get("max_pages", 20)); max_candidates = int(payload.get("max_candidates", 50))
if not 1 <= max_pages <= 20 or not 1 <= max_candidates <= 50: raise ValueError("invalid_limits")
except (ValueError, TypeError) as exc: return self.send_json(400, {"error": str(exc) or "invalid_criteria"})
try:
for url in seeds: validate_url(url)
except (ValueError, TypeError):
return self.send_json(400, {"error": "unsafe_seed_url"})
key = str(payload.get("idempotency_key", "")).strip()
if not key or len(key) > 200: return self.send_json(400, {"error": "invalid_idempotency_key"})
job_payload = {"criteria": criteria, "seed_urls": seeds, "max_pages": max_pages, "max_candidates": max_candidates}
result = self.create_job({"type": "scoped_discovery", "payload": job_payload, "idempotency_key": key, "_accepted": True, "_defer_wakeup": True}, db, user)
# create_job has already committed; read its id from the response is not available,
# so resolve by the tenant-scoped idempotency key.
job = db.execute("SELECT * FROM jobs WHERE organization_id=? AND idempotency_key=?", (user["organization_id"], key)).fetchone()
if not db.execute("SELECT id FROM discovery_runs WHERE organization_id=? AND job_id=?", (user["organization_id"], job["id"])).fetchone():
db.execute("INSERT INTO discovery_runs(organization_id,job_id,criteria_json,seed_urls_json) VALUES(?,?,?,?)", (user["organization_id"], job["id"], json.dumps(criteria, sort_keys=True), json.dumps(seeds)))
self.audit(db, user, "discovery.created", str(job["id"])); db.commit()
getattr(self.server, "job_wakeup", threading.Event()).set()
return result
def get_job_route(self, db, org, path, query):
bits=path.split("/")
if len(bits)<5 or not bits[4].isdigit(): return self.send_json(404,{"error":"not_found"})
@@ -772,7 +814,8 @@ class ApiHandler(BaseHTTPRequestHandler):
safe=json.dumps(redact(data),sort_keys=True,separators=(",",":")); max_attempts=max(1,min(int(payload.get("max_attempts",3)),5)) if str(payload.get("max_attempts",3)).isdigit() else 3
try:
cur=db.execute("INSERT INTO jobs(organization_id,idempotency_key,type,payload,max_attempts) VALUES(?,?,?,?,?)",(user["organization_id"],key,kind,safe,max_attempts)); jid=cur.lastrowid
self.add_job_event(db,jid,user["organization_id"],"queued","Job queued",0); self.audit(db,user,"job.created",str(jid)); db.commit(); getattr(self.server,"job_wakeup",threading.Event()).set()
self.add_job_event(db,jid,user["organization_id"],"queued","Job queued",0); self.audit(db,user,"job.created",str(jid)); db.commit()
if not payload.get("_defer_wakeup"): getattr(self.server,"job_wakeup",threading.Event()).set()
return self.send_json(202 if payload.get("_accepted") else 201,job_json(db.execute("SELECT * FROM jobs WHERE id=?",(jid,)).fetchone()))
except sqlite3.IntegrityError:
row=db.execute("SELECT * FROM jobs WHERE organization_id=? AND idempotency_key=?",(user["organization_id"],key)).fetchone(); return self.send_json(200,job_json(row))
@@ -938,6 +981,7 @@ class ApiHandler(BaseHTTPRequestHandler):
bits_outreach=path.split("/")
if len(bits_outreach)==7 and bits_outreach[:4]==["","api","v1","outreach"] and bits_outreach[4]=="drafts" and bits_outreach[5].isdigit() and bits_outreach[6] in {"approve","send"}: return self.approve_outreach_draft(int(bits_outreach[5]),db,user) if bits_outreach[6]=="approve" else self.send_outreach_draft(int(bits_outreach[5]),db,user)
if path=="/api/v1/sources":return self.create_source(payload,db,user)
if path=="/api/v1/discovery":return self.create_scoped_discovery(payload,db,user)
if path=="/api/v1/discovery-queries":return self.create_query(payload,db,user)
if path=="/api/v1/suppressions":return self.create_suppression(payload,db,user)
if path=="/api/v1/suppressions/import":return self.import_suppressions(payload,db,user)
@@ -1262,6 +1306,41 @@ class ApiHandler(BaseHTTPRequestHandler):
def log_message(self,*_):pass
def _run_scoped_discovery(db, job, handler):
payload = json.loads(job["payload"] or "{}")
result = scoped_discover(payload.get("criteria", {}), payload.get("seed_urls", []), max_pages=payload.get("max_pages", 20), max_candidates=payload.get("max_candidates", 50))
org = job["organization_id"]; persisted = []
for candidate in result["candidates"]:
b = normalize_business(candidate)
suppressed = is_suppressed(b, [dict(r) for r in db.execute("SELECT kind,value FROM suppressions WHERE organization_id=? AND active=1", (org,))])
if suppressed or not b["website_domain"]: continue
existing = db.execute("SELECT id FROM businesses WHERE organization_id=? AND website_domain=?", (org, b["website_domain"])).fetchone()
if existing: bid = existing["id"]
else:
scored = score_business(b)
cur = db.execute("INSERT INTO businesses(organization_id,name,website,website_domain,email,phone,description,province,city,suburb,score,score_version,score_factors,website_class) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)", (org,b["name"],b["website"],b["website_domain"],b["email"],b["phone"],b.get("description", ""),b["province"],b["city"],b["suburb"],scored["score"],scored["score_version"],json.dumps(scored["factors"]),scored["website_class"]))
bid = cur.lastrowid
priority = "high" if scored["score"] >= 70 else "medium" if scored["score"] >= 40 else "low"
db.execute("INSERT INTO score_history(organization_id,business_id,score,eligible,priority_band,score_version,explanations_json,signals_json) VALUES(?,?,?,?,?,?,?,?)", (org,bid,scored["score"],1,priority,scored["score_version"],json.dumps(scored["factors"]),json.dumps({"source": "scoped_discovery"}, sort_keys=True)))
scan_ids = {}
for page in candidate.get("pages", []):
scan_key = hashlib.sha256((org + ":" + page["url"]).encode()).hexdigest()
scan = db.execute("SELECT id FROM website_scans WHERE organization_id=? AND business_id=? AND cache_key=? ORDER BY id DESC LIMIT 1", (org, bid, scan_key)).fetchone()
if scan: scan_ids[page["url"]] = scan["id"]; continue
scan_result = {"input_url": page["url"], "final_url": page.get("final_url"), "status": page.get("status"), "title": page.get("title", ""), "headings": page.get("headings", []), "html": page.get("html", ""), "provenance": "scoped_discovery"}
cur_scan = db.execute("INSERT INTO website_scans(organization_id,business_id,website_id,input_url,classification,result_json,cache_key,scanned_at,cache_expires_at) VALUES(?,?,?,?,?,?,?,?,?)", (org,bid,None,page["url"],"healthy",json.dumps(scan_result, sort_keys=True),scan_key,datetime.now(timezone.utc).replace(microsecond=0).isoformat(),None))
scan_ids[page["url"]] = cur_scan.lastrowid
for page in candidate["evidence"]:
db.execute("INSERT INTO evidence(business_id,organization_id,kind,url,claim) VALUES(?,?,?,?,?)", (bid, org, page["kind"], page["url"], page["claim"]))
for contact in candidate["contacts"]:
key = hashlib.sha256((str(job["id"]) + contact["source_url"] + contact["kind"] + contact["value"]).encode()).hexdigest()
db.execute("INSERT OR IGNORE INTO contact_extractions(organization_id,business_id,website_scan_id,extraction_key,kind,value,label,classification,confidence,source_url,public_business,mx_status,suppressed,do_not_contact,provenance) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", (org,bid,scan_ids.get(contact["source_url"]),key,contact["kind"],contact["value"],contact["label"],contact["classification"],contact["confidence"],contact["source_url"],1,"unknown",int(contact["suppressed"]),int(contact["do_not_contact"]),contact["provenance"]))
persisted.append({"business_id": bid, "domain": b["website_domain"], "provenance": candidate["provenance"]})
safe_result = dict(result); safe_result["candidates"] = persisted
db.execute("UPDATE discovery_runs SET result_json=?,result_count=?,updated_at=CURRENT_TIMESTAMP WHERE organization_id=? AND job_id=?", (json.dumps(safe_result, sort_keys=True), len(persisted), org, job["id"]))
handler.add_job_event(db, job["id"], org, "discovery.completed", f"Persisted {len(persisted)} candidates", 100)
def _job_worker(server):
while not server.job_stop.is_set():
db=connect(server.db_path)
@@ -1275,6 +1354,13 @@ def _job_worker(server):
server_handler.add_job_event(db,jid,org,"started","Job started",0); db.commit()
try: payload=json.loads(job["payload"] or "{}")
except ValueError: payload={}
if job["type"] == "scoped_discovery":
try:
_run_scoped_discovery(db, job, server_handler)
db.execute("UPDATE jobs SET status='succeeded',progress=100,completed_at=CURRENT_TIMESTAMP,updated_at=CURRENT_TIMESTAMP WHERE id=?", (jid,)); db.commit()
except Exception as exc:
db.execute("UPDATE jobs SET status='failed',error_code=?,completed_at=CURRENT_TIMESTAMP,updated_at=CURRENT_TIMESTAMP WHERE id=?", (str(exc)[:80] or "DISCOVERY_FAILED", jid)); server_handler.add_job_event(db,jid,org,"failed","Discovery failed",job["progress"],str(exc)[:80]); db.commit()
continue
try: steps=1 if job["type"]=="noop" else max(1,min(int(payload.get("steps",5)),20))
except (ValueError,TypeError): steps=5
cancelled=False