separate reusable source config from discovery criteria

apps/api/app/main.py

@@ -9,7 +9,7 @@ from urllib.request import Request, urlopen
 if __package__ in (None, ""):
     sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
     from app.domain import deduplication_key, deduplicate_businesses, is_suppressed, normalize_business, normalize_domain, normalize_phone, match_businesses
-    from app.sources import adapter_for, contains_secret, available_adapters, normalize_record, circuit_is_open
+    from app.sources import adapter_for, contains_secret, available_adapters, normalize_record, circuit_is_open, DISCOVERY_CRITERIA_FIELDS
     from app.domain_intelligence import normalize_registrable_domain, resolve_domain, generate_candidate_domains
     from app.website_scanner import scan_website, validate_url
     from app.contact_extractor import extract_contacts, MAX_HTML_BYTES, MAX_RESULTS
@@ -22,7 +22,7 @@ if __package__ in (None, ""):
     from app.provider_config import validate_payload as validate_remote_provider, encrypt as encrypt_provider_secret, decrypt as decrypt_provider_secret, safe_status as remote_provider_status, test_connectivity as test_remote_connectivity
 else:
     from .domain import deduplication_key, deduplicate_businesses, is_suppressed, normalize_business, normalize_domain, normalize_phone, match_businesses
-    from .sources import adapter_for, contains_secret, available_adapters, normalize_record, circuit_is_open
+    from .sources import adapter_for, contains_secret, available_adapters, normalize_record, circuit_is_open, DISCOVERY_CRITERIA_FIELDS
     from .domain_intelligence import normalize_registrable_domain, resolve_domain, generate_candidate_domains
     from .website_scanner import scan_website, validate_url
     from .contact_extractor import extract_contacts, MAX_HTML_BYTES, MAX_RESULTS
@@ -1446,6 +1446,7 @@ class ApiHandler(BaseHTTPRequestHandler):
         source_code=str(config.get('provider','')).strip().lower() if requested == 'approved_directory' and str(config.get('provider','')).strip().lower() in {'openstreetmap','wikidata','common_crawl'} else requested
         if not name or requested not in ('csv','manual','google_places','bing_local','approved_directory','openstreetmap','wikidata','common_crawl','public_website','permitted_social','ct_logs','dns','rdap') or not isinstance(config,dict):return self.send_json(400,{"error":"invalid_source"})
         if contains_secret(config):return self.send_json(400,{"error":"secret_not_permitted"})
+        if any(field in config for field in DISCOVERY_CRITERIA_FIELDS):return self.send_json(400,{"error":"source_configuration_contains_criteria"})
         try:
             validation=adapter_for(source_code).validate_config(config)
             # Registration may precede the actual local payload or optional provider
@@ -1660,15 +1661,19 @@ def _run_source_discovery(db, job, handler):
         handler.add_job_event(db,job["id"],org,"source.started",f"Starting {source['display_name'] or source['kind']}",5)
         try:
             config=json.loads(source["config_json"] or "{}")
-            if query and query["source_id"]==source["id"]:
-                config.update(json.loads(query["query_json"] or "{}"))
             quota=json.loads(source["quota_json"] or "{}")
             daily_limit=int(quota.get("daily_limit", payload.get("daily_limit", 100000)))
             per_run_limit=int(quota.get("per_run_limit", max_records))
             used_today=db.execute("SELECT COUNT(*) FROM source_records WHERE organization_id=? AND source_id=? AND date(created_at)=date('now')",(org,source["id"])).fetchone()[0]
             if used_today >= daily_limit or per_run_limit <= 0:
                 handler.add_job_event(db,job["id"],org,"source.quota_exceeded",f"Quota reached for {source['kind']}",10,"SOURCE_QUOTA_EXCEEDED"); continue
-            page=adapter_for(source["source_code"] or source["kind"]).discover(source_config_with_credentials(db, source, config))
+            criteria = json.loads(query["query_json"] or "{}") if query else payload.get("criteria", {})
+            limits = {
+                "max_records": max_records,
+                "daily_limit": daily_limit,
+                "per_run_limit": per_run_limit,
+            }
+            page=adapter_for(source["source_code"] or source["kind"]).discover(source_config_with_credentials(db, source, config), criteria=criteria, limits=limits)
         except Exception as exc:
             blocked_count+=1; failures=int(source["consecutive_failures"])+1
             db.execute("UPDATE sources SET health_status='unhealthy',consecutive_failures=?,circuit_open=?,last_failure_at=CURRENT_TIMESTAMP,last_error=?,updated_at=CURRENT_TIMESTAMP WHERE id=?",(failures,int(circuit_is_open(failures)),str(exc)[:300],source["id"]))