add configurable qualification scoring
This commit is contained in:
Marco0300
@@ -12,12 +12,14 @@ if __package__ in (None, ""):
|
||||
from app.domain_intelligence import normalize_registrable_domain, resolve_domain, generate_candidate_domains
|
||||
from app.website_scanner import scan_website, validate_url
|
||||
from app.contact_extractor import extract_contacts, MAX_HTML_BYTES, MAX_RESULTS
|
||||
from app.scoring import DEFAULT_RULES, signals_for_business, evaluate_score, SCORE_VERSION
|
||||
else:
|
||||
from .domain import deduplication_key, deduplicate_businesses, is_suppressed, normalize_business, score_business, normalize_domain, normalize_phone, match_businesses
|
||||
from .sources import adapter_for, contains_secret
|
||||
from .domain_intelligence import normalize_registrable_domain, resolve_domain, generate_candidate_domains
|
||||
from .website_scanner import scan_website, validate_url
|
||||
from .contact_extractor import extract_contacts, MAX_HTML_BYTES, MAX_RESULTS
|
||||
from .scoring import DEFAULT_RULES, signals_for_business, evaluate_score, SCORE_VERSION
|
||||
ORGANIZATION_ID = "demo-tenant"
|
||||
SCHEMA = Path(__file__).resolve().parents[1] / "schema.sql"
|
||||
SESSION_DAYS = 7
|
||||
@@ -57,6 +59,9 @@ def connect(db_path: str) -> sqlite3.Connection:
|
||||
if col not in cols: db.execute(f"ALTER TABLE businesses ADD COLUMN {col} {definition}")
|
||||
db.execute("UPDATE businesses SET updated_at=COALESCE(updated_at,created_at) WHERE updated_at IS NULL")
|
||||
db.execute("INSERT OR IGNORE INTO organizations (id,name) VALUES (?,?)", (ORGANIZATION_ID, "Demo organization"))
|
||||
for organization in db.execute("SELECT id FROM organizations").fetchall():
|
||||
for rule in DEFAULT_RULES:
|
||||
db.execute("INSERT OR IGNORE INTO score_rules(organization_id,code,name,description,condition_json,points,max_applications,enabled,version) VALUES(?,?,?,?,?,?,?,?,?)", (organization["id"], rule["code"], rule["name"], rule["description"], json.dumps(rule["condition_json"], sort_keys=True), rule["points"], rule["max_applications"], rule["enabled"], rule["version"]))
|
||||
email, password = os.environ.get("BOOTSTRAP_ADMIN_EMAIL"), os.environ.get("BOOTSTRAP_ADMIN_PASSWORD")
|
||||
if email and password and not db.execute("SELECT id FROM users WHERE email=?", (email.strip().lower(),)).fetchone():
|
||||
ph, salt = hash_password(password); db.execute("INSERT INTO users (organization_id,email,password_hash,password_salt,role) VALUES (?,?,?,?,?)", (ORGANIZATION_ID,email.strip().lower(),ph,salt,"owner"))
|
||||
@@ -120,6 +125,8 @@ class ApiHandler(BaseHTTPRequestHandler):
|
||||
return self.send_json(200,{"items":[dict(r) for r in db.execute("SELECT id,email,role,organization_id,created_at FROM users WHERE organization_id=? ORDER BY id",(org,))]})
|
||||
if path=="/api/v1/dashboard/summary":
|
||||
row=db.execute("SELECT COUNT(*) businesses,COALESCE(AVG(score),0) average_score FROM businesses WHERE organization_id=?",(org,)).fetchone(); return self.send_json(200,{"organization_id":org,"businesses":row["businesses"],"average_score":round(row["average_score"],2),"suppressed":db.execute("SELECT COUNT(*) FROM suppressions WHERE organization_id=?",(org,)).fetchone()[0]})
|
||||
if path=="/api/v1/score-rules": return self.list_score_rules(db,org)
|
||||
if path=="/api/v1/scoring/summary": return self.scoring_summary(db,org)
|
||||
if path=="/api/v1/businesses": return self.list_businesses(db,org,parse_qs(parsed.query))
|
||||
if path=="/api/v1/merge-history": return self.list_merge_history(db,org)
|
||||
if path=="/api/v1/sources": return self.list_sources(db,org)
|
||||
@@ -356,6 +363,81 @@ class ApiHandler(BaseHTTPRequestHandler):
|
||||
db.execute("UPDATE jobs SET status='queued',error_code=NULL,completed_at=NULL,updated_at=CURRENT_TIMESTAMP WHERE id=?",(jid,));self.add_job_event(db,jid,user["organization_id"],"retry","Job retry queued",job["progress"]);self.audit(db,user,"job.retried",str(jid));db.commit();getattr(self.server,"job_wakeup",threading.Event()).set();return self.send_json(200,job_json(db.execute("SELECT * FROM jobs WHERE id=?",(jid,)).fetchone()))
|
||||
return self.send_json(404,{"error":"not_found"})
|
||||
|
||||
def ensure_score_rules(self, db, org):
|
||||
for rule in DEFAULT_RULES:
|
||||
db.execute("INSERT OR IGNORE INTO score_rules(organization_id,code,name,description,condition_json,points,max_applications,enabled,version) VALUES(?,?,?,?,?,?,?,?,?)", (org, rule["code"], rule["name"], rule["description"], json.dumps(rule["condition_json"], sort_keys=True), rule["points"], rule["max_applications"], rule["enabled"], rule["version"]))
|
||||
|
||||
def list_score_rules(self, db, org):
|
||||
self.ensure_score_rules(db, org); db.commit()
|
||||
rows = db.execute("SELECT * FROM score_rules WHERE organization_id=? ORDER BY code,id", (org,)).fetchall()
|
||||
items = []
|
||||
for row in rows:
|
||||
item = row_json(row)
|
||||
try: item["condition_json"] = json.loads(item["condition_json"])
|
||||
except (TypeError, ValueError): item["condition_json"] = {}
|
||||
item["enabled"] = bool(item["enabled"]); items.append(item)
|
||||
return self.send_json(200, {"organization_id": org, "items": items})
|
||||
|
||||
def create_score_rule(self, payload, db, user):
|
||||
code = str(payload.get("code", "")).strip(); name = str(payload.get("name", "")).strip(); condition = payload.get("condition_json", payload.get("condition", {}))
|
||||
try: points = int(payload.get("points", 0)); maximum = int(payload.get("max_applications", 1)); version = int(payload.get("version", 1))
|
||||
except (TypeError, ValueError): return self.send_json(400, {"error": "invalid_rule"})
|
||||
if not code or not name or not isinstance(condition, dict) or maximum < 1 or version < 1 or points < -100 or points > 100: return self.send_json(400, {"error": "invalid_rule"})
|
||||
try:
|
||||
cur = db.execute("INSERT INTO score_rules(organization_id,code,name,description,condition_json,points,max_applications,enabled,version) VALUES(?,?,?,?,?,?,?,?,?)", (user["organization_id"], code, name, str(payload.get("description", "")), json.dumps(condition, sort_keys=True), points, maximum, int(bool(payload.get("enabled", True))), version))
|
||||
except sqlite3.IntegrityError: return self.send_json(409, {"error": "duplicate_rule"})
|
||||
self.audit(db, user, "score_rule.created", code); db.commit()
|
||||
row = db.execute("SELECT * FROM score_rules WHERE id=?", (cur.lastrowid,)).fetchone(); item = row_json(row); item["condition_json"] = condition; item["enabled"] = bool(item["enabled"])
|
||||
return self.send_json(201, item)
|
||||
|
||||
def update_score_rule(self, rid, payload, db, user):
|
||||
row = db.execute("SELECT * FROM score_rules WHERE id=? AND organization_id=?", (rid, user["organization_id"])).fetchone()
|
||||
if not row: return self.send_json(404, {"error": "not_found"})
|
||||
allowed = {"name", "description", "condition_json", "points", "max_applications", "enabled", "version"}; values = {k: payload[k] for k in allowed if k in payload}
|
||||
if not values: return self.send_json(400, {"error": "no_changes"})
|
||||
if "condition_json" in values and not isinstance(values["condition_json"], dict): return self.send_json(400, {"error": "invalid_rule"})
|
||||
if "points" in values:
|
||||
try: values["points"] = int(values["points"])
|
||||
except (TypeError, ValueError): return self.send_json(400, {"error": "invalid_rule"})
|
||||
columns=[]; params=[]
|
||||
for key, value in values.items(): columns.append(key + "=?"); params.append(json.dumps(value, sort_keys=True) if key == "condition_json" else (int(bool(value)) if key == "enabled" else value))
|
||||
params += [rid, user["organization_id"]]; db.execute("UPDATE score_rules SET " + ",".join(columns) + ",updated_at=CURRENT_TIMESTAMP WHERE id=? AND organization_id=?", params); self.audit(db, user, "score_rule.updated", str(rid)); db.commit()
|
||||
item = row_json(db.execute("SELECT * FROM score_rules WHERE id=?", (rid,)).fetchone())
|
||||
try: item["condition_json"] = json.loads(item["condition_json"])
|
||||
except (TypeError, ValueError): item["condition_json"] = {}
|
||||
item["enabled"] = bool(item["enabled"]); return self.send_json(200, item)
|
||||
|
||||
def recalculate_score(self, bid, payload, db, user):
|
||||
org = user["organization_id"]; business = self.business(db, bid, org)
|
||||
if not business: return self.send_json(404, {"error": "not_found"})
|
||||
scans = db.execute("SELECT result_json,classification,scanned_at FROM website_scans WHERE business_id=? AND organization_id=? ORDER BY id DESC LIMIT 1", (bid, org)).fetchone(); website = {"classification": business["website_class"]}
|
||||
if scans:
|
||||
try: website.update(json.loads(scans["result_json"] or "{}"))
|
||||
except (TypeError, ValueError): pass
|
||||
website["classification"] = scans["classification"]
|
||||
contacts = [dict(r) for r in db.execute("SELECT public_business,suppressed,do_not_contact FROM contact_extractions WHERE business_id=? AND organization_id=?", (bid, org))]
|
||||
drow = db.execute("SELECT status,result_json,checked_at FROM domain_checks WHERE business_id=? AND organization_id=? ORDER BY id DESC LIMIT 1", (bid, org)).fetchone(); domain = dict(drow) if drow else {}
|
||||
if drow:
|
||||
try: domain.update(json.loads(drow["result_json"] or "{}"))
|
||||
except (TypeError, ValueError): pass
|
||||
suppressed = is_suppressed(dict(business), [dict(r) for r in db.execute("SELECT kind,value FROM suppressions WHERE organization_id=?", (org,))])
|
||||
signals = signals_for_business(dict(business), website, contacts, domain, suppressed); self.ensure_score_rules(db, org); rules = [dict(r) for r in db.execute("SELECT * FROM score_rules WHERE organization_id=?", (org,))]; result = evaluate_score(signals, rules)
|
||||
override_score = payload.get("override_score"); override_eligible = payload.get("override_eligible")
|
||||
if override_score is not None or override_eligible is not None:
|
||||
reason = str(payload.get("override_reason", "")).strip()
|
||||
if not reason: return self.send_json(400, {"error": "override_reason_required"})
|
||||
if override_score is not None: result["score"] = max(0, min(100, int(override_score)))
|
||||
if override_eligible is not None and not suppressed: result["eligible"] = bool(override_eligible)
|
||||
result["priority_band"] = "ineligible" if not result["eligible"] else ("high" if result["score"] >= 70 else "medium" if result["score"] >= 40 else "low")
|
||||
db.execute("UPDATE businesses SET score=?,score_version=?,score_factors=?,updated_at=CURRENT_TIMESTAMP WHERE id=? AND organization_id=?", (result["score"], SCORE_VERSION, json.dumps(result["explanations"], sort_keys=True), bid, org))
|
||||
cur=db.execute("INSERT INTO score_history(organization_id,business_id,score,eligible,priority_band,score_version,explanations_json,signals_json,override_score,override_eligible,override_reason,actor_user_id) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)", (org,bid,result["score"],int(result["eligible"]),result["priority_band"],SCORE_VERSION,json.dumps(result["explanations"],sort_keys=True),json.dumps(signals,sort_keys=True),override_score,override_eligible,payload.get("override_reason"),user["id"]))
|
||||
self.audit(db,user,"business.score_recalculated",f"{bid}:{result['score']}"); db.commit(); result.update({"business_id": bid, "history_id": cur.lastrowid}); return self.send_json(200, result)
|
||||
|
||||
def scoring_summary(self, db, org):
|
||||
row=db.execute("SELECT COUNT(*) businesses,COALESCE(AVG(score),0) average_score,SUM(CASE WHEN score>=70 THEN 1 ELSE 0 END) high_priority FROM businesses WHERE organization_id=? AND merge_status='active'",(org,)).fetchone()
|
||||
bands={r["priority_band"]:r["count"] for r in db.execute("SELECT priority_band,COUNT(*) count FROM score_history WHERE organization_id=? GROUP BY priority_band",(org,))}
|
||||
return self.send_json(200,{"organization_id":org,"businesses":row["businesses"],"average_score":round(row["average_score"],2),"high_priority":row["high_priority"] or 0,"bands":bands,"history_count":db.execute("SELECT COUNT(*) FROM score_history WHERE organization_id=?",(org,)).fetchone()[0]})
|
||||
|
||||
def list_businesses(self,db,org,query):
|
||||
def number(name, default=None):
|
||||
raw=query.get(name,[None])[0]
|
||||
@@ -390,6 +472,8 @@ class ApiHandler(BaseHTTPRequestHandler):
|
||||
return self.create_job(self.read_json(),db,user)
|
||||
if user["role"] not in MUTATING_ROLES:return self.send_json(403,{"error":"forbidden"})
|
||||
payload=self.read_json(); org=user["organization_id"]
|
||||
if path=="/api/v1/score-rules": return self.create_score_rule(payload,db,user)
|
||||
if len(path.split("/"))==7 and path.split("/")[3:]==["businesses",path.split("/")[4],"score","recalculate"]: return self.recalculate_score(int(path.split("/")[4]) if path.split("/")[4].isdigit() else -1,payload,db,user)
|
||||
if path.startswith("/api/v1/jobs/"):
|
||||
return self.job_action(db,user,path)
|
||||
if path=="/api/v1/businesses":return self.create_business(payload,db,user)
|
||||
@@ -422,6 +506,7 @@ class ApiHandler(BaseHTTPRequestHandler):
|
||||
if not user:return
|
||||
if user["role"] not in MUTATING_ROLES:return self.send_json(403,{"error":"forbidden"})
|
||||
bits=path.split("/")
|
||||
if len(bits)==5 and bits[:4]==["","api","v1","score-rules"] and bits[4].isdigit(): return self.update_score_rule(int(bits[4]),self.read_json(),db,user)
|
||||
if len(bits)==5 and bits[:4]==["","api","v1","sources"] and bits[4].isdigit(): return self.update_source(int(bits[4]),self.read_json(),db,user)
|
||||
if len(bits)==6 and bits[:4]==["","api","v1","businesses"] and bits[5]=="pipeline":return self.update_pipeline(int(bits[4]) if bits[4].isdigit() else -1,self.read_json(),db,user)
|
||||
return self.send_json(404,{"error":"not_found"})
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Deterministic, explainable qualification scoring."""
|
||||
from __future__ import annotations
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
|
||||
SCORE_VERSION = "phase10-1"
|
||||
DEFAULT_RULES = [
|
||||
{"code": "business_name", "name": "Named business", "description": "Business has a usable name", "condition_json": {"signal": "business.name", "operator": "present"}, "points": 15, "max_applications": 1, "enabled": 1, "version": 1},
|
||||
{"code": "business_site", "name": "Business website", "description": "Business has a non-social website", "condition_json": {"signal": "business.website_class", "operator": "equals", "value": "business_site"}, "points": 20, "max_applications": 1, "enabled": 1, "version": 1},
|
||||
{"code": "website_healthy", "name": "Healthy website", "description": "Latest website scan is healthy", "condition_json": {"signal": "website.classification", "operator": "equals", "value": "healthy"}, "points": 15, "max_applications": 1, "enabled": 1, "version": 1},
|
||||
{"code": "contact_email", "name": "Email contact", "description": "A direct business email is available", "condition_json": {"signal": "business.email", "operator": "present"}, "points": 15, "max_applications": 1, "enabled": 1, "version": 1},
|
||||
{"code": "contact_phone", "name": "Phone contact", "description": "A business phone is available", "condition_json": {"signal": "business.phone", "operator": "present"}, "points": 10, "max_applications": 1, "enabled": 1, "version": 1},
|
||||
{"code": "extracted_contact", "name": "Extracted contact", "description": "A public, non-suppressed contact was extracted", "condition_json": {"signal": "contacts.public_count", "operator": "gte", "value": 1}, "points": 10, "max_applications": 1, "enabled": 1, "version": 1},
|
||||
{"code": "domain_verified", "name": "Domain check", "description": "Domain check resolved successfully", "condition_json": {"signal": "domain.status", "operator": "in", "value": ["resolved", "ok", "healthy"]}, "points": 5, "max_applications": 1, "enabled": 1, "version": 1},
|
||||
{"code": "verified_business", "name": "Verified business", "description": "Business has been verified", "condition_json": {"signal": "state.verified", "operator": "truthy"}, "points": 10, "max_applications": 1, "enabled": 1, "version": 1},
|
||||
]
|
||||
|
||||
def _get(data, path):
|
||||
value = data
|
||||
for part in str(path).split("."):
|
||||
if not isinstance(value, dict): return None
|
||||
value = value.get(part)
|
||||
return value
|
||||
|
||||
def _match(condition, signals):
|
||||
if not isinstance(condition, dict): return False
|
||||
if "all" in condition: return all(_match(c, signals) for c in condition["all"])
|
||||
if "any" in condition: return any(_match(c, signals) for c in condition["any"])
|
||||
if "not" in condition: return not _match(condition["not"], signals)
|
||||
value = _get(signals, condition.get("signal", "")); op = condition.get("operator", "truthy"); expected = condition.get("value")
|
||||
section = signals.get(str(condition.get("signal", "")).split(".")[0], {})
|
||||
if isinstance(section, dict) and (section.get("stale") or section.get("uncertain")): return False
|
||||
if op in ("truthy", "present"): return bool(value) if op == "truthy" else value not in (None, "", [], {})
|
||||
if op == "equals": return value == expected
|
||||
if op == "in": return value in (expected if isinstance(expected, list) else [expected])
|
||||
if op in ("gte", "lte", "gt", "lt"):
|
||||
try: return {"gte": value >= expected, "lte": value <= expected, "gt": value > expected, "lt": value < expected}[op]
|
||||
except (TypeError, ValueError): return False
|
||||
return False
|
||||
|
||||
def evaluate_score(signals, rules):
|
||||
total = 0; explanations = []
|
||||
ordered = sorted((dict(r) for r in rules), key=lambda r: (str(r.get("code", "")), int(r.get("id", 0) or 0)))
|
||||
for rule in ordered:
|
||||
enabled = bool(rule.get("enabled", 1)); applied = enabled and _match(_condition(rule), signals)
|
||||
points = int(rule.get("points", 0) or 0) if applied else 0
|
||||
total += points
|
||||
explanations.append({"code": rule.get("code", ""), "name": rule.get("name", rule.get("code", "")), "version": int(rule.get("version", 1) or 1), "enabled": enabled, "applied": applied, "points": points, "reason": (rule.get("description") or rule.get("name") or rule.get("code") or "Rule") + (" (matched)" if applied else " (not matched)")})
|
||||
total = max(0, min(100, total)); state = signals.get("state", {}) if isinstance(signals, dict) else {}
|
||||
eligible = not bool(state.get("suppressed")) and str(state.get("merge_status", "active")) == "active"
|
||||
if not eligible: band = "ineligible"
|
||||
elif total >= 70: band = "high"
|
||||
elif total >= 40: band = "medium"
|
||||
else: band = "low"
|
||||
return {"score": total, "score_version": SCORE_VERSION, "eligible": eligible, "priority_band": band, "explanations": explanations}
|
||||
|
||||
def _condition(rule):
|
||||
raw = rule.get("condition_json", {})
|
||||
if isinstance(raw, str):
|
||||
try: return json.loads(raw)
|
||||
except (TypeError, ValueError): return {}
|
||||
return raw
|
||||
|
||||
def signals_for_business(business, website=None, contacts=None, domain=None, suppressed=False):
|
||||
b = dict(business); website = website or {}; contacts = contacts or []; domain = domain or {}
|
||||
public = [c for c in contacts if c.get("public_business") and not c.get("suppressed") and not c.get("do_not_contact")]
|
||||
return {"business": {"name": b.get("name", ""), "email": b.get("email", ""), "phone": b.get("phone", ""), "description": b.get("description", ""), "website_domain": b.get("website_domain", ""), "website_class": b.get("website_class", "")}, "website": website, "contacts": {"count": len(contacts), "public_count": len(public)}, "domain": domain, "state": {"verified": bool(b.get("verified")), "suppressed": bool(suppressed), "merge_status": b.get("merge_status", "active"), "merged": b.get("merge_status") == "merged"}}
|
||||
Reference in New Issue
Block a user