581 lines
54 KiB
Python
581 lines
54 KiB
Python
from __future__ import annotations
|
|
import argparse, hashlib, json, os, re, secrets, sqlite3, sys, threading, time
|
|
from datetime import datetime, timedelta, timezone
|
|
from http.cookies import SimpleCookie
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
from pathlib import Path
|
|
from urllib.parse import parse_qs, urlparse
|
|
if __package__ in (None, ""):
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
from app.domain import deduplication_key, deduplicate_businesses, is_suppressed, normalize_business, score_business, normalize_domain, normalize_phone, match_businesses
|
|
from app.sources import adapter_for, contains_secret
|
|
from app.domain_intelligence import normalize_registrable_domain, resolve_domain, generate_candidate_domains
|
|
from app.website_scanner import scan_website, validate_url
|
|
else:
|
|
from .domain import deduplication_key, deduplicate_businesses, is_suppressed, normalize_business, score_business, normalize_domain, normalize_phone, match_businesses
|
|
from .sources import adapter_for, contains_secret
|
|
from .domain_intelligence import normalize_registrable_domain, resolve_domain, generate_candidate_domains
|
|
from .website_scanner import scan_website, validate_url
|
|
ORGANIZATION_ID = "demo-tenant"
|
|
SCHEMA = Path(__file__).resolve().parents[1] / "schema.sql"
|
|
SESSION_DAYS = 7
|
|
PBKDF2_ITERATIONS = 300_000
|
|
MUTATING_ROLES = {"owner", "admin", "researcher"}
|
|
JOB_TYPES = {"noop", "prospect_recalculate", "source_discovery", "domain_check"}
|
|
JOB_PAGE_SIZE = 100
|
|
WEBSITE_SCAN_PAGE_SIZE = 100
|
|
WEBSITE_SCAN_CACHE_SECONDS = 3600
|
|
SECRET_KEYS = {"password", "passwd", "secret", "token", "api_key", "apikey", "authorization", "credential", "private_key"}
|
|
CHILD_TABLES = {"contacts": ("name", "email", "phone", "title", "do_not_contact"), "domains": ("domain", "kind"), "websites": ("url", "website_class"), "evidence": ("kind", "url", "claim"), "notes": ("body",)}
|
|
|
|
def redact(value):
|
|
if isinstance(value, dict): return {k: ("[REDACTED]" if str(k).lower() in SECRET_KEYS or any(s in str(k).lower() for s in ("password", "token", "secret", "api_key")) else redact(v)) for k,v in value.items()}
|
|
if isinstance(value, list): return [redact(v) for v in value[:100]]
|
|
if isinstance(value, str): return value[:2000]
|
|
return value
|
|
|
|
def job_json(row):
|
|
result = row_json(row)
|
|
try: result["payload"] = json.loads(result.get("payload") or "{}")
|
|
except (ValueError, TypeError): result["payload"] = {}
|
|
return result
|
|
|
|
def hash_password(password: str, salt: bytes | None = None) -> tuple[str, str]:
|
|
salt = salt or secrets.token_bytes(16); return hashlib.pbkdf2_hmac("sha256", password.encode(), salt, PBKDF2_ITERATIONS).hex(), salt.hex()
|
|
def verify_password(password, encoded_hash, encoded_salt):
|
|
try: return secrets.compare_digest(hashlib.pbkdf2_hmac("sha256", password.encode(), bytes.fromhex(encoded_salt), PBKDF2_ITERATIONS).hex(), encoded_hash)
|
|
except (TypeError, ValueError): return False
|
|
|
|
def connect(db_path: str) -> sqlite3.Connection:
|
|
db = sqlite3.connect(db_path); db.row_factory = sqlite3.Row; db.execute("PRAGMA foreign_keys = ON"); db.executescript(SCHEMA.read_text())
|
|
# Upgrade databases created by Phase 1/2 without destroying data.
|
|
cols = {r[1] for r in db.execute("PRAGMA table_info(businesses)")}
|
|
for col, definition in (("verified", "INTEGER NOT NULL DEFAULT 0"), ("verified_at", "TEXT"), ("updated_at", "TEXT"), ("province", "TEXT NOT NULL DEFAULT ''"), ("city", "TEXT NOT NULL DEFAULT ''"), ("suburb", "TEXT NOT NULL DEFAULT ''"), ("merge_status", "TEXT NOT NULL DEFAULT 'active'"), ("merged_into_id", "INTEGER")):
|
|
if col not in cols: db.execute(f"ALTER TABLE businesses ADD COLUMN {col} {definition}")
|
|
db.execute("UPDATE businesses SET updated_at=COALESCE(updated_at,created_at) WHERE updated_at IS NULL")
|
|
db.execute("INSERT OR IGNORE INTO organizations (id,name) VALUES (?,?)", (ORGANIZATION_ID, "Demo organization"))
|
|
email, password = os.environ.get("BOOTSTRAP_ADMIN_EMAIL"), os.environ.get("BOOTSTRAP_ADMIN_PASSWORD")
|
|
if email and password and not db.execute("SELECT id FROM users WHERE email=?", (email.strip().lower(),)).fetchone():
|
|
ph, salt = hash_password(password); db.execute("INSERT INTO users (organization_id,email,password_hash,password_salt,role) VALUES (?,?,?,?,?)", (ORGANIZATION_ID,email.strip().lower(),ph,salt,"owner"))
|
|
db.commit(); return db
|
|
|
|
def safe_value(value):
|
|
if isinstance(value, bytes): return value.decode("utf-8", "replace")
|
|
return value
|
|
|
|
def row_json(row):
|
|
result = {k: safe_value(v) for k, v in dict(row).items()}
|
|
if "score_factors" in result:
|
|
try: result["score_factors"] = json.loads(result["score_factors"] or "[]")
|
|
except (TypeError, ValueError): result["score_factors"] = []
|
|
for key in ("verified", "do_not_contact"):
|
|
if key in result: result[key] = bool(result[key])
|
|
return result
|
|
|
|
class ApiHandler(BaseHTTPRequestHandler):
|
|
server_version = "ProspectPlatform/0.1"
|
|
def send_json(self, status, payload, extra_headers=None):
|
|
body = json.dumps(payload, sort_keys=True, default=str).encode(); self.send_response(status); self.send_header("Content-Type","application/json; charset=utf-8"); self.send_header("Access-Control-Allow-Origin",os.environ.get("CORS_ORIGINS","http://localhost:8080")); self.send_header("Access-Control-Allow-Credentials","true"); self.send_header("Access-Control-Allow-Methods","GET, POST, PATCH, OPTIONS"); self.send_header("Access-Control-Allow-Headers","Content-Type")
|
|
for k,v in (extra_headers or {}).items(): self.send_header(k,v)
|
|
self.send_header("Content-Length",str(len(body))); self.end_headers(); self.wfile.write(body)
|
|
def read_json(self):
|
|
try:
|
|
value=json.loads(self.rfile.read(int(self.headers.get("Content-Length","0"))) or b"{}"); return value if isinstance(value,dict) else {}
|
|
except (ValueError,json.JSONDecodeError): return {}
|
|
def db(self): return connect(getattr(self.server,"db_path"))
|
|
def do_OPTIONS(self): self.send_response(204); self.send_header("Access-Control-Allow-Methods","GET, POST, PATCH, OPTIONS"); self.end_headers()
|
|
def session_user(self, db):
|
|
cookie=SimpleCookie(); cookie.load(self.headers.get("Cookie","")); token=cookie.get("session")
|
|
if not token: return None
|
|
now=datetime.now(timezone.utc).replace(microsecond=0).isoformat(); h=hashlib.sha256(token.value.encode()).hexdigest()
|
|
return db.execute("SELECT u.id,u.email,u.role,u.organization_id FROM sessions s JOIN users u ON u.id=s.user_id WHERE s.token_hash=? AND s.expires_at>?",(h,now)).fetchone()
|
|
def require_auth(self,db):
|
|
user=self.session_user(db)
|
|
if not user: self.send_json(401,{"error":"unauthorized"}); return None
|
|
return user
|
|
def auth_cookie(self,token,max_age): return f"session={token}; Max-Age={max_age}; Path=/; HttpOnly; SameSite=Lax"
|
|
def audit(self, db, user, action, details=""):
|
|
db.execute("INSERT INTO audit_log (organization_id,user_id,action,details) VALUES (?,?,?,?)",(user["organization_id"],user["id"],action,details))
|
|
def business(self, db, ident, org): return db.execute("SELECT * FROM businesses WHERE id=? AND organization_id=?",(ident,org)).fetchone()
|
|
def nested(self, db, bid, org):
|
|
result={"contacts":[],"domains":[],"websites":[],"evidence":[],"pipeline":[],"notes":[]}
|
|
tables={"contacts":"contacts","domains":"domains","websites":"websites","evidence":"evidence","pipeline":"pipeline_entries","notes":"notes"}
|
|
for key, table in tables.items():
|
|
result[key]=[row_json(r) for r in db.execute(f"SELECT * FROM {table} WHERE business_id=? AND organization_id=? ORDER BY id",(bid,org))]
|
|
return result
|
|
def do_GET(self):
|
|
parsed=urlparse(self.path); path=parsed.path.rstrip("/")
|
|
if path=="/api/v1/health/live": return self.send_json(200,{"status":"ok","organization_id":ORGANIZATION_ID})
|
|
db=self.db()
|
|
try:
|
|
user=self.require_auth(db)
|
|
if not user:return
|
|
org=user["organization_id"]
|
|
if path=="/api/v1/auth/me": return self.send_json(200,{"id":user["id"],"email":user["email"],"role":user["role"],"organization_id":org})
|
|
if path=="/api/v1/admin/users":
|
|
if user["role"] not in {"owner","admin"}: return self.send_json(403,{"error":"forbidden"})
|
|
return self.send_json(200,{"items":[dict(r) for r in db.execute("SELECT id,email,role,organization_id,created_at FROM users WHERE organization_id=? ORDER BY id",(org,))]})
|
|
if path=="/api/v1/dashboard/summary":
|
|
row=db.execute("SELECT COUNT(*) businesses,COALESCE(AVG(score),0) average_score FROM businesses WHERE organization_id=?",(org,)).fetchone(); return self.send_json(200,{"organization_id":org,"businesses":row["businesses"],"average_score":round(row["average_score"],2),"suppressed":db.execute("SELECT COUNT(*) FROM suppressions WHERE organization_id=?",(org,)).fetchone()[0]})
|
|
if path=="/api/v1/businesses": return self.list_businesses(db,org,parse_qs(parsed.query))
|
|
if path=="/api/v1/merge-history": return self.list_merge_history(db,org)
|
|
if path=="/api/v1/sources": return self.list_sources(db,org)
|
|
if path=="/api/v1/discovery-queries": return self.list_queries(db,org)
|
|
if path=="/api/v1/source-records": return self.list_source_records(db,org,parse_qs(parsed.query))
|
|
if path=="/api/v1/jobs": return self.list_jobs(db,org,parse_qs(parsed.query))
|
|
if path=="/api/v1/domain-checks": return self.list_domain_checks(db,org,parse_qs(parsed.query))
|
|
if path=="/api/v1/website-scans": return self.list_website_scans(db,org,parse_qs(parsed.query))
|
|
if path.startswith("/api/v1/jobs/"): return self.get_job_route(db,org,path,parse_qs(parsed.query))
|
|
if path.startswith("/api/v1/businesses/"):
|
|
bits=path.split("/"); ident=bits[4] if len(bits)>4 else ""
|
|
if not ident.isdigit(): return self.send_json(404,{"error":"not_found"})
|
|
row=self.business(db,int(ident),org)
|
|
if not row:return self.send_json(404,{"error":"not_found"})
|
|
if len(bits)==7 and bits[5:]==["websites","scan"]: return self.get_latest_website_scan(int(ident),db,user)
|
|
if len(bits)==7 and bits[5:]==["domains","check"]: return self.get_domain_check(int(ident),db,user,parse_qs(parsed.query))
|
|
if len(bits)==6 and bits[5]=="domain-candidates": return self.list_domain_candidates(int(ident),db,org)
|
|
if len(bits)==6 and bits[5]=="matches": return self.matches(int(ident),db,org)
|
|
payload=row_json(row); payload.update(self.nested(db,int(ident),org)); return self.send_json(200,payload)
|
|
return self.send_json(404,{"error":"not_found"})
|
|
finally: db.close()
|
|
def _domain_result(self, row, cache_hit=False):
|
|
try: result=json.loads(row["result_json"] or "{}")
|
|
except (TypeError,ValueError): result={}
|
|
result.update({"id":row["id"],"business_id":row["business_id"],"domain":row["domain"],"status":row["status"],"cache_hit":cache_hit,"checked_at":row["checked_at"],"cache_expires_at":row["cache_expires_at"]})
|
|
return result
|
|
|
|
def _check_domain(self, bid, domain, db, user):
|
|
normalized=normalize_registrable_domain(domain)
|
|
if normalized == "unknown": return self.send_json(400,{"error":"unsupported_domain","status":"unknown"})
|
|
now=datetime.now(timezone.utc).replace(microsecond=0).isoformat()
|
|
cache_key=normalized+":a_aaaa:v1"
|
|
cached=db.execute("SELECT * FROM domain_checks WHERE organization_id=? AND business_id=? AND cache_key=? AND cache_expires_at>? ORDER BY id DESC LIMIT 1",(user["organization_id"],bid,cache_key,now)).fetchone()
|
|
if cached:return self.send_json(200,self._domain_result(cached,True))
|
|
result=resolve_domain(normalized)
|
|
try: cur=db.execute("INSERT INTO domain_checks(organization_id,business_id,domain,status,result_json,cache_key,checked_at,cache_expires_at) VALUES(?,?,?,?,?,?,?,?)",(user["organization_id"],bid,normalized,result["status"],json.dumps(result,sort_keys=True),cache_key,result["checked_at"],result["cache_expires_at"]))
|
|
except sqlite3.IntegrityError:
|
|
existing=db.execute("SELECT id FROM domain_checks WHERE organization_id=? AND business_id=? AND cache_key=?",(user["organization_id"],bid,cache_key)).fetchone()
|
|
if not existing: return self.send_json(409,{"error":"domain_check_conflict"})
|
|
db.execute("UPDATE domain_checks SET domain=?,status=?,result_json=?,checked_at=?,cache_expires_at=? WHERE id=?",(normalized,result["status"],json.dumps(result,sort_keys=True),result["checked_at"],result["cache_expires_at"],existing["id"]))
|
|
self.audit(db,user,"domain.checked",f"{bid}:{normalized}:{result['status']}"); db.commit()
|
|
return self.send_json(200,self._domain_result(db.execute("SELECT * FROM domain_checks WHERE id=?",(existing["id"],)).fetchone()))
|
|
self.audit(db,user,"domain.checked",f"{bid}:{normalized}:{result['status']}"); db.commit()
|
|
return self.send_json(200,self._domain_result(db.execute("SELECT * FROM domain_checks WHERE id=?",(cur.lastrowid,)).fetchone()))
|
|
|
|
def post_domain_check(self,bid,payload,db,user):
|
|
if not self.business(db,bid,user["organization_id"]): return self.send_json(404,{"error":"not_found"})
|
|
domain=payload.get("domain") or self.business(db,bid,user["organization_id"])["website_domain"]
|
|
if not str(domain).strip(): return self.send_json(400,{"error":"domain_required"})
|
|
return self._check_domain(bid,str(domain),db,user)
|
|
|
|
def get_domain_check(self,bid,db,user,query):
|
|
domain=(query.get("domain") or [""])[0]
|
|
if not domain:return self.send_json(400,{"error":"domain_required"})
|
|
return self._check_domain(bid,domain,db,user)
|
|
|
|
def list_domain_checks(self,db,org,query):
|
|
try: limit=max(1,min(int((query.get("page_size") or [50])[0]),100)); offset=max(0,int((query.get("offset") or [0])[0]))
|
|
except (ValueError,TypeError): return self.send_json(400,{"error":"invalid_pagination"})
|
|
rows=db.execute("SELECT * FROM domain_checks WHERE organization_id=? ORDER BY id DESC LIMIT ? OFFSET ?",(org,limit+1,offset)).fetchall()
|
|
return self.send_json(200,{"organization_id":org,"items":[self._domain_result(r) for r in rows[:limit]],"limit":limit,"offset":offset,"has_more":len(rows)>limit})
|
|
|
|
def _website_scan_result(self, row, cache_hit=False):
|
|
result = json.loads(row["result_json"] or "{}")
|
|
result.update({"id": row["id"], "business_id": row["business_id"], "website_id": row["website_id"], "input_url": row["input_url"], "classification": row["classification"], "scanned_at": row["scanned_at"], "cache_expires_at": row["cache_expires_at"], "cache_hit": cache_hit})
|
|
return result
|
|
|
|
def list_website_scans(self, db, org, query):
|
|
try:
|
|
limit = int((query.get("page_size") or [50])[0]); offset = max(0, int((query.get("offset") or [0])[0]))
|
|
if limit < 1 or limit > WEBSITE_SCAN_PAGE_SIZE: raise ValueError
|
|
except (ValueError, TypeError): return self.send_json(400, {"error": "invalid_pagination"})
|
|
params = [org]; where = ["organization_id=?"]
|
|
if (query.get("business_id") or [""])[0].isdigit(): where.append("business_id=?"); params.append(int(query["business_id"][0]))
|
|
rows = db.execute("SELECT * FROM website_scans WHERE " + " AND ".join(where) + " ORDER BY id DESC LIMIT ? OFFSET ?", params + [limit + 1, offset]).fetchall()
|
|
return self.send_json(200, {"organization_id": org, "items": [self._website_scan_result(r) for r in rows[:limit]], "limit": limit, "offset": offset, "has_more": len(rows) > limit})
|
|
|
|
def get_latest_website_scan(self, bid, db, user):
|
|
row = db.execute("SELECT * FROM website_scans WHERE business_id=? AND organization_id=? ORDER BY id DESC LIMIT 1", (bid, user["organization_id"])).fetchone()
|
|
if not row: return self.send_json(404, {"error": "scan_not_found"})
|
|
return self.send_json(200, self._website_scan_result(row, False))
|
|
|
|
def scan_business_website(self, bid, payload, db, user):
|
|
org = user["organization_id"]; business = self.business(db, bid, org)
|
|
if not business: return self.send_json(404, {"error": "not_found"})
|
|
requested = str(payload.get("url", "")).strip() if isinstance(payload, dict) else ""
|
|
if not requested:
|
|
child = db.execute("SELECT * FROM websites WHERE business_id=? AND organization_id=? ORDER BY id LIMIT 1", (bid, org)).fetchone()
|
|
requested = (child["url"] if child else business["website"]) or ""
|
|
try: safe_url = validate_url(requested)
|
|
except ValueError as exc:
|
|
self.audit(db, user, "website.scan.rejected", f"{bid}:{str(exc)}"); db.commit()
|
|
return self.send_json(400, {"error": "unsafe_url", "reason": str(exc)})
|
|
website = db.execute("SELECT id FROM websites WHERE business_id=? AND organization_id=? AND url=? ORDER BY id LIMIT 1", (bid, org, safe_url)).fetchone()
|
|
cache_key = hashlib.sha256(safe_url.encode()).hexdigest(); now = datetime.now(timezone.utc).replace(microsecond=0); expires = now + timedelta(seconds=WEBSITE_SCAN_CACHE_SECONDS)
|
|
cached = db.execute("SELECT * FROM website_scans WHERE organization_id=? AND business_id=? AND cache_key=? AND cache_expires_at>? ORDER BY id DESC LIMIT 1", (org, bid, cache_key, now.isoformat())).fetchone()
|
|
if cached:
|
|
self.audit(db, user, "website.scan.cache_hit", f"{bid}:{safe_url}"); db.commit()
|
|
return self.send_json(200, self._website_scan_result(cached, True))
|
|
result = scan_website(safe_url); result["business_id"] = bid
|
|
cur = db.execute("INSERT INTO website_scans(organization_id,business_id,website_id,input_url,classification,result_json,cache_key,scanned_at,cache_expires_at) VALUES(?,?,?,?,?,?,?,?,?)", (org, bid, website["id"] if website else None, safe_url, result["classification"], json.dumps(result, sort_keys=True), cache_key, now.isoformat(), expires.isoformat()))
|
|
self.audit(db, user, "website.scanned", f"{bid}:{result['classification']}"); db.commit()
|
|
return self.send_json(201, self._website_scan_result(db.execute("SELECT * FROM website_scans WHERE id=?", (cur.lastrowid,)).fetchone()))
|
|
|
|
def list_domain_candidates(self,bid,db,org):
|
|
business=self.business(db,bid,org)
|
|
if not business:return self.send_json(404,{"error":"not_found"})
|
|
rows=db.execute("SELECT * FROM domain_candidates WHERE organization_id=? AND business_id=? ORDER BY rank,id",(org,bid)).fetchall()
|
|
if not rows:
|
|
generated=generate_candidate_domains(business["name"],business["description"],business["city"])
|
|
for rank,domain in enumerate(generated,1):
|
|
db.execute("INSERT OR IGNORE INTO domain_candidates(organization_id,business_id,domain,rank) VALUES(?,?,?,?)",(org,bid,domain,rank))
|
|
db.commit(); rows=db.execute("SELECT * FROM domain_candidates WHERE organization_id=? AND business_id=? ORDER BY rank,id",(org,bid)).fetchall()
|
|
return self.send_json(200,{"business_id":bid,"items":[row_json(r) for r in rows]})
|
|
|
|
def check_availability(self,bid,payload,db,user):
|
|
if not self.business(db,bid,user["organization_id"]):return self.send_json(404,{"error":"not_found"})
|
|
candidates=db.execute("SELECT domain FROM domain_candidates WHERE organization_id=? AND business_id=? ORDER BY rank,id",(user["organization_id"],bid)).fetchall()
|
|
domains=[r["domain"] for r in candidates]
|
|
if isinstance(payload.get("domains"),list): domains=[normalize_registrable_domain(x) for x in payload["domains"] if normalize_registrable_domain(x)!="unknown"][:20]
|
|
self.audit(db,user,"domain.availability.checked",str(bid)); db.commit()
|
|
return self.send_json(200,{"business_id":bid,"status":"unknown","reason":"not_configured","provider_configured":False,"items":[{"domain":d,"status":"unknown","reason":"not_configured"} for d in domains]})
|
|
|
|
def list_jobs(self, db, org, query):
|
|
try: limit=max(1,min(int(query.get("page_size",[50])[0]),JOB_PAGE_SIZE)); offset=max(0,int(query.get("offset",[0])[0]))
|
|
except (ValueError, TypeError): return self.send_json(400,{"error":"invalid_pagination"})
|
|
rows=db.execute("SELECT * FROM jobs WHERE organization_id=? ORDER BY id DESC LIMIT ? OFFSET ?",(org,limit+1,offset)).fetchall(); more=len(rows)>limit
|
|
return self.send_json(200,{"organization_id":org,"items":[job_json(r) for r in rows[:limit]],"limit":limit,"offset":offset,"has_more":more})
|
|
|
|
def get_job_route(self, db, org, path, query):
|
|
bits=path.split("/")
|
|
if len(bits)<5 or not bits[4].isdigit(): return self.send_json(404,{"error":"not_found"})
|
|
job=db.execute("SELECT * FROM jobs WHERE id=? AND organization_id=?",(int(bits[4]),org)).fetchone()
|
|
if not job:return self.send_json(404,{"error":"not_found"})
|
|
if len(bits)==5:return self.send_json(200,job_json(job))
|
|
if len(bits)==6 and bits[5]=="events":
|
|
try: after=max(0,int(query.get("after",[0])[0]))
|
|
except (ValueError,TypeError): return self.send_json(400,{"error":"invalid_sequence"})
|
|
events=[row_json(r) for r in db.execute("SELECT * FROM job_events WHERE job_id=? AND organization_id=? AND sequence>? ORDER BY sequence",(job["id"],org,after))]
|
|
return self.send_json(200,{"items":events,"after":after})
|
|
if len(bits)==7 and bits[5]=="events" and bits[6]=="stream":
|
|
try: after=max(0,int(query.get("after",[0])[0]))
|
|
except (ValueError,TypeError): return self.send_json(400,{"error":"invalid_sequence"})
|
|
events=[row_json(r) for r in db.execute("SELECT * FROM job_events WHERE job_id=? AND organization_id=? AND sequence>? ORDER BY sequence",(job["id"],org,after))]
|
|
body=b"".join((b"event: "+str(e["event_type"]).encode()+b"\\ndata: "+json.dumps(e,sort_keys=True).encode()+b"\\n\\n") for e in events)
|
|
self.send_response(200);self.send_header("Content-Type","text/event-stream");self.send_header("Cache-Control","no-cache");self.send_header("Content-Length",str(len(body)));self.end_headers();self.wfile.write(body);return
|
|
return self.send_json(404,{"error":"not_found"})
|
|
|
|
def create_job(self, payload, db, user):
|
|
kind=str(payload.get("type","")).strip(); key=str(payload.get("idempotency_key","")).strip(); data=payload.get("payload",{})
|
|
if kind not in JOB_TYPES:return self.send_json(400,{"error":"invalid_job_type"})
|
|
if not key or len(key)>200 or not isinstance(data,dict):return self.send_json(400,{"error":"invalid_job_request"})
|
|
safe=json.dumps(redact(data),sort_keys=True,separators=(",",":")); max_attempts=max(1,min(int(payload.get("max_attempts",3)),5)) if str(payload.get("max_attempts",3)).isdigit() else 3
|
|
try:
|
|
cur=db.execute("INSERT INTO jobs(organization_id,idempotency_key,type,payload,max_attempts) VALUES(?,?,?,?,?)",(user["organization_id"],key,kind,safe,max_attempts)); jid=cur.lastrowid
|
|
self.add_job_event(db,jid,user["organization_id"],"queued","Job queued",0); self.audit(db,user,"job.created",str(jid)); db.commit(); getattr(self.server,"job_wakeup",threading.Event()).set()
|
|
return self.send_json(202 if payload.get("_accepted") else 201,job_json(db.execute("SELECT * FROM jobs WHERE id=?",(jid,)).fetchone()))
|
|
except sqlite3.IntegrityError:
|
|
row=db.execute("SELECT * FROM jobs WHERE organization_id=? AND idempotency_key=?",(user["organization_id"],key)).fetchone(); return self.send_json(200,job_json(row))
|
|
|
|
def add_job_event(self, db, jid, org, event_type, message, progress=0, error_code=None):
|
|
seq=db.execute("SELECT COALESCE(MAX(sequence),0)+1 FROM job_events WHERE job_id=?",(jid,)).fetchone()[0]
|
|
db.execute("INSERT INTO job_events(job_id,organization_id,sequence,event_type,message,progress,error_code) VALUES(?,?,?,?,?,?,?)",(jid,org,seq,event_type,str(message)[:500],progress,error_code)); return seq
|
|
|
|
def job_action(self, db, user, path):
|
|
bits=path.split("/"); jid=int(bits[4]) if len(bits)>4 and bits[4].isdigit() else -1; action=bits[5] if len(bits)>5 else ""
|
|
job=db.execute("SELECT * FROM jobs WHERE id=? AND organization_id=?",(jid,user["organization_id"])).fetchone()
|
|
if not job:return self.send_json(404,{"error":"not_found"})
|
|
if action=="cancel":
|
|
if job["status"] in ("queued","running"): db.execute("UPDATE jobs SET status='cancelled',completed_at=CURRENT_TIMESTAMP,updated_at=CURRENT_TIMESTAMP WHERE id=?",(jid,));self.add_job_event(db,jid,user["organization_id"],"cancelled","Job cancelled",job["progress"])
|
|
self.audit(db,user,"job.cancelled",str(jid));db.commit();return self.send_json(200,job_json(db.execute("SELECT * FROM jobs WHERE id=?",(jid,)).fetchone()))
|
|
if action=="retry":
|
|
if job["status"]!="failed":return self.send_json(409,{"error":"job_not_failed"})
|
|
db.execute("UPDATE jobs SET status='queued',error_code=NULL,completed_at=NULL,updated_at=CURRENT_TIMESTAMP WHERE id=?",(jid,));self.add_job_event(db,jid,user["organization_id"],"retry","Job retry queued",job["progress"]);self.audit(db,user,"job.retried",str(jid));db.commit();getattr(self.server,"job_wakeup",threading.Event()).set();return self.send_json(200,job_json(db.execute("SELECT * FROM jobs WHERE id=?",(jid,)).fetchone()))
|
|
return self.send_json(404,{"error":"not_found"})
|
|
|
|
def list_businesses(self,db,org,query):
|
|
def number(name, default=None):
|
|
raw=query.get(name,[None])[0]
|
|
if raw is None:return default
|
|
try:return int(raw)
|
|
except ValueError: raise ValueError
|
|
try:
|
|
page=number("page",1); size=number("page_size",50); score=number("score_min",None); cursor=number("cursor",0)
|
|
except ValueError:return self.send_json(400,{"error":"invalid_pagination"})
|
|
if page<1 or size<1 or size>100 or cursor<0:return self.send_json(400,{"error":"invalid_pagination"})
|
|
params=[org]; where=["b.organization_id=?"]; q=query.get("q",[""])[0].strip(); website_class=query.get("website_class",[""])[0].strip(); stage=query.get("pipeline_stage",[""])[0].strip()
|
|
if score is not None: where.append("b.score>=?"); params.append(score)
|
|
if website_class: where.append("b.website_class=?"); params.append(website_class)
|
|
if q: where.append("(b.name LIKE ? OR b.website_domain LIKE ? OR b.email LIKE ?)"); params += [f"%{q}%"]*3
|
|
if stage: where.append("EXISTS (SELECT 1 FROM pipeline_entries p WHERE p.business_id=b.id AND p.organization_id=b.organization_id AND p.stage=?)"); params.append(stage)
|
|
offset=(number("cursor",0) or 0)+(page-1)*size
|
|
rows=db.execute("SELECT b.* FROM businesses b WHERE "+" AND ".join(where)+" ORDER BY b.score DESC,b.id LIMIT ? OFFSET ?",params+[size+1,offset]).fetchall(); more=len(rows)>size; rows=rows[:size]
|
|
return self.send_json(200,{"organization_id":org,"items":[row_json(r) for r in rows],"page":page,"page_size":size,"next_cursor":str(offset+size) if more else None})
|
|
def do_POST(self):
|
|
path=urlparse(self.path).path.rstrip("/")
|
|
if path=="/api/v1/auth/login":return self.login(self.read_json())
|
|
db=self.db()
|
|
try:
|
|
user=self.require_auth(db)
|
|
if not user:return
|
|
if path=="/api/v1/auth/logout":
|
|
c=SimpleCookie();c.load(self.headers.get("Cookie",""));t=c.get("session");
|
|
if t:db.execute("DELETE FROM sessions WHERE token_hash=?",(hashlib.sha256(t.value.encode()).hexdigest(),))
|
|
self.audit(db,user,"logout");db.commit();return self.send_json(200,{"ok":True},{"Set-Cookie":self.auth_cookie("",0)})
|
|
if path=="/api/v1/jobs":
|
|
if user["role"] not in MUTATING_ROLES:return self.send_json(403,{"error":"forbidden"})
|
|
return self.create_job(self.read_json(),db,user)
|
|
if user["role"] not in MUTATING_ROLES:return self.send_json(403,{"error":"forbidden"})
|
|
payload=self.read_json(); org=user["organization_id"]
|
|
if path.startswith("/api/v1/jobs/"):
|
|
return self.job_action(db,user,path)
|
|
if path=="/api/v1/businesses":return self.create_business(payload,db,user)
|
|
if path=="/api/v1/sources":return self.create_source(payload,db,user)
|
|
if path=="/api/v1/discovery-queries":return self.create_query(payload,db,user)
|
|
if path=="/api/v1/suppressions":return self.create_suppression(payload,db,user)
|
|
if path=="/api/v1/imports/preview":return self.preview_import(payload,db,org)
|
|
if len(path.split("/"))==7 and path.split("/")[3:6]==["businesses",path.split("/")[4],"websites"] and path.split("/")[6]=="scan": return self.scan_business_website(int(path.split("/")[4]) if path.split("/")[4].isdigit() else -1,payload,db,user)
|
|
if len(path.split("/"))==7 and path.split("/")[3:6]==["businesses",path.split("/")[4],"domains"] and path.split("/")[6]=="check": return self.post_domain_check(int(path.split("/")[4]) if path.split("/")[4].isdigit() else -1,payload,db,user)
|
|
if len(path.split("/"))==7 and path.split("/")[3:6]==["businesses",path.split("/")[4],"domain-candidates"] and path.split("/")[6]=="check-availability": return self.check_availability(int(path.split("/")[4]) if path.split("/")[4].isdigit() else -1,payload,db,user)
|
|
if path.startswith("/api/v1/merge-history/") and path.endswith("/reverse"):
|
|
ident=path.split("/")[4]
|
|
return self.reverse_merge(int(ident) if ident.isdigit() else -1,db,user)
|
|
bits=path.split("/")
|
|
if len(bits)==6 and bits[3] == "sources" and bits[4].isdigit() and bits[5] in {"test","ingest"}:
|
|
return self.test_source(int(bits[4]),db,user) if bits[5]=="test" else self.ingest_source(int(bits[4]),payload,db,user)
|
|
if len(bits)==6 and bits[3] == "discovery-queries" and bits[4].isdigit() and bits[5]=="run": return self.run_query(int(bits[4]),db,user)
|
|
if len(bits)==7 and bits[:4]==["","api","v1","businesses"] and bits[5] in CHILD_TABLES and bits[6]=="": pass
|
|
if len(bits)==6 and bits[:4]==["","api","v1","businesses"] and bits[5] in CHILD_TABLES:return self.create_child(int(bits[4]) if bits[4].isdigit() else -1,bits[5],payload,db,user)
|
|
if len(bits)==6 and bits[:4]==["","api","v1","businesses"] and bits[5]=="verify":return self.verify_business(int(bits[4]) if bits[4].isdigit() else -1,payload,db,user)
|
|
if len(bits)==6 and bits[:4]==["","api","v1","businesses"] and bits[5]=="merge":return self.merge_business(int(bits[4]) if bits[4].isdigit() else -1,payload,db,user)
|
|
return self.send_json(404,{"error":"not_found"})
|
|
finally:db.close()
|
|
def do_PATCH(self):
|
|
path=urlparse(self.path).path.rstrip("/"); db=self.db()
|
|
try:
|
|
user=self.require_auth(db)
|
|
if not user:return
|
|
if user["role"] not in MUTATING_ROLES:return self.send_json(403,{"error":"forbidden"})
|
|
bits=path.split("/")
|
|
if len(bits)==5 and bits[:4]==["","api","v1","sources"] and bits[4].isdigit(): return self.update_source(int(bits[4]),self.read_json(),db,user)
|
|
if len(bits)==6 and bits[:4]==["","api","v1","businesses"] and bits[5]=="pipeline":return self.update_pipeline(int(bits[4]) if bits[4].isdigit() else -1,self.read_json(),db,user)
|
|
return self.send_json(404,{"error":"not_found"})
|
|
finally:db.close()
|
|
def login(self,payload):
|
|
db=self.db(); email=str(payload.get("email"," ")).strip().lower(); password=str(payload.get("password","")); user=db.execute("SELECT * FROM users WHERE email=?",(email,)).fetchone()
|
|
try:
|
|
if not user or not verify_password(password,user["password_hash"],user["password_salt"]):return self.send_json(401,{"error":"invalid_credentials"})
|
|
token=secrets.token_urlsafe(32); expires=datetime.now(timezone.utc)+timedelta(days=SESSION_DAYS);db.execute("INSERT INTO sessions(user_id,token_hash,expires_at) VALUES(?,?,?)",(user["id"],hashlib.sha256(token.encode()).hexdigest(),expires.replace(microsecond=0).isoformat()));self.audit(db,user,"login");db.commit();return self.send_json(200,{"id":user["id"],"email":user["email"],"role":user["role"],"organization_id":user["organization_id"]},{"Set-Cookie":self.auth_cookie(token,int(timedelta(days=SESSION_DAYS).total_seconds()))})
|
|
finally:db.close()
|
|
def create_business(self,payload,db,user):
|
|
org=user["organization_id"]
|
|
if not str(payload.get("name","")).strip():return self.send_json(400,{"error":"name_required"})
|
|
b=normalize_business(payload); suppressions=[dict(r) for r in db.execute("SELECT kind,value FROM suppressions WHERE organization_id=?",(org,))]
|
|
if is_suppressed(b,suppressions):return self.send_json(409,{"error":"suppressed"})
|
|
fields=[(c,b[c]) for c in ("website_domain","email","phone") if b[c]]
|
|
if fields and db.execute("SELECT id FROM businesses WHERE organization_id=? AND ("+" OR ".join(f"{c}=?" for c,_ in fields)+")",[org]+[v for _,v in fields]).fetchone():return self.send_json(409,{"error":"duplicate"})
|
|
scored=score_business(b);cur=db.execute("INSERT INTO businesses(organization_id,name,website,website_domain,email,phone,description,province,city,suburb,score,score_version,score_factors,website_class) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)",(org,b["name"],b["website"],b["website_domain"],b["email"],b["phone"],str(b.get("description","")),b["province"],b["city"],b["suburb"],scored["score"],scored["score_version"],json.dumps(scored["factors"]),scored["website_class"])); self.audit(db,user,"business.created",str(cur.lastrowid));db.commit();return self.send_json(201,row_json(db.execute("SELECT * FROM businesses WHERE id=?",(cur.lastrowid,)).fetchone()))
|
|
def create_suppression(self,payload,db,user):
|
|
kind,value=payload.get("kind"),str(payload.get("value","")).strip().lower()
|
|
if kind not in {"email","domain","phone"} or not value:return self.send_json(400,{"error":"invalid_suppression"})
|
|
try:db.execute("INSERT INTO suppressions(organization_id,kind,value) VALUES(?,?,?)",(user["organization_id"],kind,value))
|
|
except sqlite3.IntegrityError:pass
|
|
self.audit(db,user,"suppression.created",kind);db.commit();return self.send_json(201,dict(db.execute("SELECT * FROM suppressions WHERE organization_id=? AND kind=? AND value=?",(user["organization_id"],kind,value)).fetchone()))
|
|
def child_business(self,db,bid,user):return self.business(db,bid,user["organization_id"])
|
|
def create_child(self,bid,table,payload,db,user):
|
|
if not self.child_business(db,bid,user):return self.send_json(404,{"error":"not_found"})
|
|
if table=="contacts":
|
|
email=str(payload.get("email","")).strip().lower(); phone=normalize_phone(payload.get("phone"));
|
|
if email and not re.match(r"^[^@\s]+@[^@\s]+\.[^@\s]+$",email):return self.send_json(400,{"error":"invalid_contact"})
|
|
suppressed=is_suppressed({"email":email,"phone":phone},[dict(r) for r in db.execute("SELECT kind,value FROM suppressions WHERE organization_id=?",(user["organization_id"],))]); values=(str(payload.get("name","")).strip(),email,phone,str(payload.get("title","")).strip(),int(bool(payload.get("do_not_contact"))) or int(suppressed))
|
|
elif table=="domains":
|
|
value=normalize_domain(payload.get("domain"));
|
|
if not value:return self.send_json(400,{"error":"invalid_domain"})
|
|
values=(value,str(payload.get("kind","other")).strip() or "other")
|
|
elif table=="websites":
|
|
value=str(payload.get("url","")).strip();
|
|
if not urlparse(value).scheme or not urlparse(value).netloc:return self.send_json(400,{"error":"invalid_website"})
|
|
values=(value,str(payload.get("website_class","business_site")).strip() or "business_site")
|
|
elif table=="evidence":
|
|
if not str(payload.get("kind","")).strip():return self.send_json(400,{"error":"invalid_evidence"})
|
|
values=(str(payload["kind"]).strip(),str(payload.get("url","")).strip(),str(payload.get("claim","")).strip())
|
|
else:
|
|
if not str(payload.get("body","")).strip():return self.send_json(400,{"error":"body_required"})
|
|
values=(str(payload["body"]).strip(),)
|
|
columns=CHILD_TABLES[table]; db.execute(f"INSERT INTO {table}(business_id,organization_id,{','.join(columns)}) VALUES(?, ?, {','.join('?' for _ in columns)})",(bid,user["organization_id"])+values); rid=db.execute("SELECT last_insert_rowid()").fetchone()[0];self.audit(db,user,f"{table}.created",str(rid));db.commit();return self.send_json(201,row_json(db.execute(f"SELECT * FROM {table} WHERE id=?",(rid,)).fetchone()))
|
|
def update_pipeline(self,bid,payload,db,user):
|
|
if not self.child_business(db,bid,user) or not str(payload.get("stage","")).strip():return self.send_json(404 if not self.child_business(db,bid,user) else 400,{"error":"not_found" if not self.child_business(db,bid,user) else "stage_required"})
|
|
stage=str(payload["stage"]).strip();status=str(payload.get("status","active")).strip() or "active";db.execute("INSERT INTO pipeline_entries(business_id,organization_id,stage,status) VALUES(?,?,?,?)",(bid,user["organization_id"],stage,status));rid=db.execute("SELECT last_insert_rowid()").fetchone()[0];self.audit(db,user,"pipeline.updated",stage);db.commit();return self.send_json(200,row_json(db.execute("SELECT * FROM pipeline_entries WHERE id=?",(rid,)).fetchone()))
|
|
def verify_business(self,bid,payload,db,user):
|
|
if not self.child_business(db,bid,user):return self.send_json(404,{"error":"not_found"})
|
|
verified=bool(payload.get("verified",True)); now=datetime.now(timezone.utc).replace(microsecond=0).isoformat();db.execute("UPDATE businesses SET verified=?,verified_at=?,updated_at=CURRENT_TIMESTAMP WHERE id=? AND organization_id=?",(int(verified),now if verified else None,bid,user["organization_id"]));self.audit(db,user,"business.verified",str(verified));db.commit();row=self.business(db,bid,user["organization_id"]);return self.send_json(200,row_json(row))
|
|
def preview_import(self,payload,db,org):
|
|
rows=payload.get("rows",[])
|
|
if not isinstance(rows,list):return self.send_json(400,{"error":"rows_required"})
|
|
normalized=deduplicate_businesses([r for r in rows if isinstance(r,dict) and str(r.get("name","")).strip()]); suppressions=[dict(r) for r in db.execute("SELECT kind,value FROM suppressions WHERE organization_id=?",(org,))];existing=[row_json(r) for r in db.execute("SELECT * FROM businesses WHERE organization_id=?",(org,))];seen=set();accepted=[];suppressed=0;existing_keys={deduplication_key(x) for x in existing}
|
|
for b in normalized:
|
|
key=deduplication_key(b)
|
|
if is_suppressed(b,suppressions):suppressed+=1
|
|
elif key in existing_keys or key in seen:continue
|
|
else:seen.add(key);accepted.append(b)
|
|
return self.send_json(200,{"accepted":len(accepted),"duplicates":len(rows)-len(normalized)+len(normalized)-len(accepted)-suppressed,"suppressed":suppressed,"rows":accepted})
|
|
def list_sources(self,db,org):
|
|
cols='id,organization_id,name,kind,enabled,health_status,consecutive_failures,circuit_open,last_success_at,last_failure_at,last_error,created_at,updated_at'
|
|
return self.send_json(200,{"organization_id":org,"items":[row_json(r) for r in db.execute(f"SELECT {cols} FROM sources WHERE organization_id=? ORDER BY id",(org,))]})
|
|
def list_queries(self,db,org):
|
|
return self.send_json(200,{"organization_id":org,"items":[row_json(r) for r in db.execute("SELECT * FROM discovery_queries WHERE organization_id=? ORDER BY id",(org,))]})
|
|
def list_source_records(self,db,org,q):
|
|
try:
|
|
limit=int(q.get('page_size',[50])[0]); offset=max(0,int(q.get('offset',[0])[0]))
|
|
if limit<1 or limit>100: raise ValueError
|
|
except (ValueError,TypeError): return self.send_json(400,{"error":"invalid_pagination"})
|
|
rows=db.execute("SELECT * FROM source_records WHERE organization_id=? ORDER BY id DESC LIMIT ? OFFSET ?",(org,limit+1,offset)).fetchall(); out=[]
|
|
for r in rows[:limit]:
|
|
x=row_json(r)
|
|
for k in ('raw_json','normalized_json','query_context_json','cursor_json','rate_policy_json'):
|
|
try:x[k]=json.loads(x[k])
|
|
except (ValueError,TypeError):pass
|
|
out.append(x)
|
|
return self.send_json(200,{"organization_id":org,"items":out,"limit":limit,"offset":offset,"has_more":len(rows)>limit})
|
|
def create_source(self,payload,db,user):
|
|
name=str(payload.get('name','')).strip(); kind=str(payload.get('kind','')).strip().lower(); config=payload.get('config',{})
|
|
if not name or kind not in ('csv','manual') or not isinstance(config,dict):return self.send_json(400,{"error":"invalid_source"})
|
|
if contains_secret(config):return self.send_json(400,{"error":"secret_not_permitted"})
|
|
try:
|
|
validation=adapter_for(kind).validate(config)
|
|
if config and not validation.valid:return self.send_json(400,{"error":"invalid_source_config","details":validation.errors})
|
|
cur=db.execute("INSERT INTO sources(organization_id,name,kind,enabled,config_json) VALUES(?,?,?,?,?)",(user['organization_id'],name,kind,int(bool(payload.get('enabled',False))),json.dumps(config,sort_keys=True)))
|
|
except sqlite3.IntegrityError:return self.send_json(409,{"error":"duplicate_source"})
|
|
self.audit(db,user,'source.created',str(cur.lastrowid));db.commit();return self.send_json(201,row_json(db.execute("SELECT id,organization_id,name,kind,enabled,health_status,consecutive_failures,circuit_open,last_success_at,last_failure_at,last_error,created_at,updated_at FROM sources WHERE id=?",(cur.lastrowid,)).fetchone()))
|
|
def update_source(self,sid,payload,db,user):
|
|
if not db.execute("SELECT id FROM sources WHERE id=? AND organization_id=?",(sid,user['organization_id'])).fetchone():return self.send_json(404,{"error":"not_found"})
|
|
if 'enabled' not in payload:return self.send_json(400,{"error":"enabled_required"})
|
|
value=int(bool(payload['enabled']));db.execute("UPDATE sources SET enabled=?,updated_at=CURRENT_TIMESTAMP WHERE id=?",(value,sid));self.audit(db,user,'source.enabled' if value else 'source.disabled',str(sid));db.commit();return self.send_json(200,row_json(db.execute("SELECT * FROM sources WHERE id=?",(sid,)).fetchone()))
|
|
def create_query(self,payload,db,user):
|
|
sid=payload.get('source_id');name=str(payload.get('name','')).strip();query=payload.get('query',{})
|
|
if not isinstance(sid,int) or not name or not isinstance(query,dict) or contains_secret(query):return self.send_json(400,{"error":"invalid_query"})
|
|
if not db.execute("SELECT id FROM sources WHERE id=? AND organization_id=?",(sid,user['organization_id'])).fetchone():return self.send_json(404,{"error":"not_found"})
|
|
try:cur=db.execute("INSERT INTO discovery_queries(organization_id,source_id,name,query_json) VALUES(?,?,?,?)",(user['organization_id'],sid,name,json.dumps(query,sort_keys=True)))
|
|
except sqlite3.IntegrityError:return self.send_json(409,{"error":"duplicate_query"})
|
|
self.audit(db,user,'discovery_query.created',str(cur.lastrowid));db.commit();return self.send_json(201,row_json(db.execute("SELECT * FROM discovery_queries WHERE id=?",(cur.lastrowid,)).fetchone()))
|
|
def run_query(self,qid,db,user):
|
|
if not db.execute("SELECT id FROM discovery_queries WHERE id=? AND organization_id=?",(qid,user['organization_id'])).fetchone():return self.send_json(404,{"error":"not_found"})
|
|
return self.create_job({"type":"source_discovery","_accepted":True,"payload":{"discovery_query_id":qid},"idempotency_key":f"discovery-query-{qid}-{int(time.time())}"},db,user)
|
|
def test_source(self,sid,db,user):
|
|
source=db.execute("SELECT * FROM sources WHERE id=? AND organization_id=?",(sid,user['organization_id'])).fetchone()
|
|
if not source:return self.send_json(404,{"error":"not_found"})
|
|
try: result=adapter_for(source['kind']).validate(json.loads(source['config_json'])); ok=result.valid; error='; '.join(result.errors) if not ok else None
|
|
except Exception as exc:ok=False;error=str(exc)[:300]
|
|
if ok:db.execute("UPDATE sources SET health_status='healthy',consecutive_failures=0,circuit_open=0,last_success_at=CURRENT_TIMESTAMP,last_error=NULL WHERE id=?",(sid,));action='source.test.succeeded'
|
|
else:db.execute("UPDATE sources SET health_status='unhealthy',consecutive_failures=consecutive_failures+1,circuit_open=CASE WHEN consecutive_failures+1>=3 THEN 1 ELSE circuit_open END,last_failure_at=CURRENT_TIMESTAMP,last_error=? WHERE id=?",(error,sid));action='source.test.failed'
|
|
self.audit(db,user,action,str(sid));db.commit();return self.send_json(200,{"ok":ok,"errors":[] if ok else [error]})
|
|
def ingest_source(self,sid,payload,db,user):
|
|
source=db.execute("SELECT * FROM sources WHERE id=? AND organization_id=?",(sid,user['organization_id'])).fetchone()
|
|
if not source:return self.send_json(404,{"error":"not_found"})
|
|
if not source['enabled'] or source['circuit_open']:return self.send_json(409,{"error":"source_disabled"})
|
|
if contains_secret(payload):return self.send_json(400,{"error":"secret_not_permitted"})
|
|
config={k:v for k,v in payload.items() if k not in ('source_url','query_context','cursor','rate_policy')}
|
|
if len(json.dumps(config).encode())>5*1024*1024:return self.send_json(400,{"error":"ingest_limits"})
|
|
if isinstance(config.get('rows'),list) and (len(config['rows'])>1000 or any(not isinstance(r,dict) or len(r)>50 or any(len(str(v))>10000 for v in r.values()) for r in config['rows'])):return self.send_json(400,{"error":"ingest_limits"})
|
|
if isinstance(config.get('csv'),str) and config['csv'].count('\n')>1001:return self.send_json(400,{"error":"ingest_limits"})
|
|
try:page=adapter_for(source['kind']).discover(config)
|
|
except (ValueError,KeyError) as exc:return self.send_json(400,{"error":"invalid_ingest","detail":str(exc)})
|
|
inserted=0
|
|
for record in page.records[:1000]:
|
|
raw=json.dumps(record,sort_keys=True,separators=(',',':'));digest=hashlib.sha256(raw.encode()).hexdigest()
|
|
try:db.execute("INSERT INTO source_records(organization_id,source_id,content_hash,raw_json,normalized_json,source_url,query_context_json,cursor_json,rate_policy_json) VALUES(?,?,?,?,?,?,?,?,?)",(user['organization_id'],sid,digest,raw,raw,str(payload.get('source_url','')),json.dumps(payload.get('query_context',{}),sort_keys=True),json.dumps(payload.get('cursor',{}),sort_keys=True),json.dumps(payload.get('rate_policy',{}),sort_keys=True)));inserted+=1
|
|
except sqlite3.IntegrityError:pass
|
|
db.execute("UPDATE sources SET health_status='healthy',consecutive_failures=0,last_success_at=CURRENT_TIMESTAMP,last_error=NULL WHERE id=?",(sid,));self.audit(db,user,'source.ingested',f'{sid}:{inserted}');db.commit();return self.send_json(201 if inserted else 200,{"inserted":inserted,"records":len(page.records)})
|
|
def matches(self,bid,db,org):
|
|
source=self.business(db,bid,org)
|
|
if not source:return self.send_json(404,{"error":"not_found"})
|
|
try: threshold=float(parse_qs(urlparse(self.path).query).get("threshold",["0.72"])[0])
|
|
except ValueError:return self.send_json(400,{"error":"invalid_threshold"})
|
|
rows=[row_json(r) for r in db.execute("SELECT * FROM businesses WHERE organization_id=? AND id<>? AND merge_status='active' ORDER BY id",(org,bid))]
|
|
return self.send_json(200,{"business_id":bid,"threshold":threshold,"items":match_businesses(row_json(source),rows,threshold)})
|
|
|
|
def list_merge_history(self,db,org):
|
|
rows=[row_json(r) for r in db.execute("SELECT * FROM merge_history WHERE organization_id=? ORDER BY id DESC",(org,))]
|
|
for row in rows:
|
|
for key in ("source_snapshot_json","child_reassignment_json"):
|
|
try: row[key]=json.loads(row[key])
|
|
except (TypeError,ValueError): pass
|
|
return self.send_json(200,{"organization_id":org,"items":rows})
|
|
|
|
def merge_business(self,bid,payload,db,user):
|
|
org=user["organization_id"]; target_id=payload.get("target_business_id",payload.get("target_id"))
|
|
if not isinstance(target_id,int) or target_id==bid:return self.send_json(400,{"error":"target_required"})
|
|
source=self.business(db,bid,org); target=self.business(db,target_id,org)
|
|
if not source or not target:return self.send_json(404,{"error":"not_found"})
|
|
if source["merge_status"] != "active":return self.send_json(409,{"error":"source_already_merged"})
|
|
snapshot={"business":row_json(source),"children":{}}
|
|
child_meta={}
|
|
for table in ("business_identifiers","contacts","domains","websites","evidence","pipeline_entries","interactions","notes"):
|
|
rows=[row_json(r) for r in db.execute(f"SELECT * FROM {table} WHERE business_id=? AND organization_id=? ORDER BY id",(bid,org))]
|
|
snapshot["children"][table]=rows; child_meta[table]={"ids":[r["id"] for r in rows],"count":len(rows)}
|
|
if rows: db.execute(f"UPDATE {table} SET business_id=? WHERE business_id=? AND organization_id=?",(target_id,bid,org))
|
|
cur=db.execute("INSERT INTO merge_history(organization_id,source_business_id,target_business_id,source_snapshot_json,child_reassignment_json,actor_user_id) VALUES(?,?,?,?,?,?)",(org,bid,target_id,json.dumps(snapshot,sort_keys=True),json.dumps(child_meta,sort_keys=True),user["id"]))
|
|
db.execute("UPDATE businesses SET merge_status='merged',merged_into_id=?,updated_at=CURRENT_TIMESTAMP WHERE id=? AND organization_id=?",(target_id,bid,org))
|
|
self.audit(db,user,"business.merged",f"{bid}->{target_id}");db.commit()
|
|
return self.send_json(200,{"merge_history_id":cur.lastrowid,"source_business_id":bid,"target_business_id":target_id,"status":"merged","reassigned":child_meta})
|
|
|
|
def reverse_merge(self,hid,db,user):
|
|
row=db.execute("SELECT * FROM merge_history WHERE id=? AND organization_id=?",(hid,user["organization_id"])).fetchone()
|
|
if not row:return self.send_json(404,{"error":"not_found"})
|
|
if not row["reversible"]:return self.send_json(409,{"error":"merge_not_reversible"})
|
|
source=self.business(db,row["source_business_id"],user["organization_id"]); target=self.business(db,row["target_business_id"],user["organization_id"])
|
|
if not source or not target:return self.send_json(409,{"error":"business_missing"})
|
|
snapshot=json.loads(row["source_snapshot_json"]); ids=json.loads(row["child_reassignment_json"])
|
|
for table, meta in ids.items():
|
|
if not meta.get("ids"):continue
|
|
marks=",".join("?" for _ in meta["ids"])
|
|
db.execute(f"UPDATE {table} SET business_id=? WHERE business_id=? AND organization_id=? AND id IN ({marks})",[source["id"],target["id"],user["organization_id"]]+meta["ids"])
|
|
db.execute("UPDATE businesses SET merge_status='active',merged_into_id=NULL,updated_at=CURRENT_TIMESTAMP WHERE id=? AND organization_id=?",(source["id"],user["organization_id"]))
|
|
db.execute("UPDATE merge_history SET reversible=0,reversed_at=CURRENT_TIMESTAMP WHERE id=? AND organization_id=?",(hid,user["organization_id"]))
|
|
self.audit(db,user,"business.merge_reversed",str(hid));db.commit()
|
|
return self.send_json(200,{"id":hid,"status":"reversed","source_business_id":source["id"],"target_business_id":target["id"]})
|
|
|
|
def log_message(self,*_):pass
|
|
|
|
def _job_worker(server):
|
|
while not server.job_stop.is_set():
|
|
db=connect(server.db_path)
|
|
try:
|
|
job=db.execute("SELECT * FROM jobs WHERE status='queued' ORDER BY id LIMIT 1").fetchone()
|
|
if not job:
|
|
db.close(); server.job_wakeup.wait(.1); server.job_wakeup.clear(); continue
|
|
changed=db.execute("UPDATE jobs SET status='running',attempts=attempts+1,started_at=COALESCE(started_at,CURRENT_TIMESTAMP),updated_at=CURRENT_TIMESTAMP WHERE id=? AND status='queued'",(job["id"],)).rowcount
|
|
if not changed: db.close(); continue
|
|
db.commit(); org=job["organization_id"]; jid=job["id"]; server_handler=object.__new__(ApiHandler)
|
|
server_handler.add_job_event(db,jid,org,"started","Job started",0); db.commit()
|
|
try: payload=json.loads(job["payload"] or "{}")
|
|
except ValueError: payload={}
|
|
try: steps=1 if job["type"]=="noop" else max(1,min(int(payload.get("steps",5)),20))
|
|
except (ValueError,TypeError): steps=5
|
|
cancelled=False
|
|
for i in range(steps):
|
|
time.sleep(.01)
|
|
fresh=db.execute("SELECT status FROM jobs WHERE id=?",(jid,)).fetchone()
|
|
if not fresh or fresh["status"]=="cancelled": cancelled=True; break
|
|
progress=int((i+1)*100/steps); db.execute("UPDATE jobs SET progress=?,updated_at=CURRENT_TIMESTAMP WHERE id=? AND status='running'",(progress,jid)); server_handler.add_job_event(db,jid,org,"progress",f"Job progress {progress}%",progress); db.commit()
|
|
if cancelled: continue
|
|
if payload.get("force_fail") or (payload.get("fail_once") and job["attempts"] == 0):
|
|
db.execute("UPDATE jobs SET status='failed',error_code='DEMO_FAILURE',completed_at=CURRENT_TIMESTAMP,updated_at=CURRENT_TIMESTAMP WHERE id=?",(jid,));server_handler.add_job_event(db,jid,org,"failed","Job failed",job["progress"],"DEMO_FAILURE")
|
|
else:
|
|
db.execute("UPDATE jobs SET status='succeeded',progress=100,completed_at=CURRENT_TIMESTAMP,updated_at=CURRENT_TIMESTAMP WHERE id=?",(jid,));server_handler.add_job_event(db,jid,org,"succeeded","Job completed",100)
|
|
db.commit()
|
|
finally: db.close()
|
|
|
|
def create_server(host="127.0.0.1",port=8000,db_path="prospects.db"):
|
|
server=ThreadingHTTPServer((host,port),ApiHandler);server.db_path=db_path;connect(db_path).close();server.job_stop=threading.Event();server.job_wakeup=threading.Event();server.job_thread=threading.Thread(target=_job_worker,args=(server,),daemon=True);server.job_thread.start()
|
|
original_close=server.server_close
|
|
def close():
|
|
server.job_stop.set();server.job_wakeup.set();server.job_thread.join(timeout=2);original_close()
|
|
server.server_close=close
|
|
return server
|
|
if __name__=="__main__":
|
|
parser=argparse.ArgumentParser();parser.add_argument("--host",default="127.0.0.1");parser.add_argument("--port",type=int,default=int(os.environ.get("PROSPECT_API_PORT","8000")));parser.add_argument("--db",default=os.environ.get("PROSPECT_API_DB","prospects.db"));args=parser.parse_args();server=create_server(args.host,args.port,args.db);print(f"Prospect API listening on http://{args.host}:{args.port}",flush=True)
|
|
try:server.serve_forever()
|
|
except KeyboardInterrupt:pass
|
|
finally:server.server_close()
|