root/MarketingTool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

separate reusable source config from discovery criteria
CI / compose (push) Failing after 5m3s
Details

Browse Source
This commit is contained in:
Marco0300
2026-09-04 18:53:30 +02:00
parent dff9ccb5ea
commit 3a9b553440
6 changed files with 120 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/app/sources.py
+18 -18
View File
@@ -18,6 +18,7 @@ except ImportError:
SECRET_KEYS = {"password", "passwd", "secret", "token", "api_key", "apikey", "authorization", "private_key", "credential"}
NETWORK_KINDS = {"google_places", "bing_local", "approved_directory", "public_website", "permitted_social", "ct_logs", "dns", "rdap"}
DISCOVERY_CRITERIA_FIELDS = {"query", "category", "city", "location", "keywords", "province", "country", "language", "search", "phrase", "industry", "keyword"}
def contains_secret(value: Any, path: str = "") -> str | None:
@@ -123,7 +124,7 @@ class _Base:
def health_check(self, config):
result = self.validate_config(config)
return SourceHealth("healthy" if result.valid else "unhealthy", last_error=None if result.valid else "; ".join(result.errors))
def discover(self, config, cursor=None):
def discover(self, config, cursor=None, criteria=None, limits=None):
result = self.validate_config(config)
if not result.valid: raise ValueError(result.errors[0])
raise RuntimeError("source_not_configured")
@@ -135,7 +136,7 @@ class ManualSource(_Base):
if not result.valid: return result
if not isinstance(config.get("rows"), list): return ValidationResult(False, ["rows must be a list"])
return ValidationResult(True)
def discover(self, config, cursor=None):
def discover(self, config, cursor=None, criteria=None, limits=None):
result = self.validate_config(config)
if not result.valid: raise ValueError(result.errors[0])
return DiscoveryPage([normalize_record(row) for row in config["rows"] if isinstance(row, Mapping)], metadata={"adapter":self.source_code})
@@ -151,7 +152,7 @@ class CsvSource(_Base):
if not reader.fieldnames: return ValidationResult(False, ["CSV header is required"])
except csv.Error as exc: return ValidationResult(False, [f"invalid CSV: {exc}"])
return ValidationResult(True)
def discover(self, config, cursor=None):
def discover(self, config, cursor=None, criteria=None, limits=None):
result = self.validate_config(config)
if not result.valid: raise ValueError(result.errors[0])
reader = csv.DictReader(io.StringIO(config["csv"].replace("\r\n", "\n")))
@@ -194,7 +195,7 @@ class PublicWebsiteSource(_Base):
except ValueError: return ValidationResult(False, ["unsafe public website URL"])
return ValidationResult(True)
def discover(self, config, cursor=None):
def discover(self, config, cursor=None, criteria=None, limits=None):
result = self.validate_config(config)
if not result.valid: raise ValueError(result.errors[0])
urls = config.get("urls", config.get("url"))
@@ -225,10 +226,10 @@ class CtLogsSource(_HttpJsonSource):
return ValidationResult(False, ["domain or query is required"])
return ValidationResult(True)
def discover(self, config, cursor=None):
def discover(self, config, cursor=None, criteria=None, limits=None):
result = self.validate_config(config)
if not result.valid: raise ValueError(result.errors[0])
query = str(config.get("domain", config.get("query"))).strip()
query = str(config.get("domain", config.get("query", ""))).strip()
endpoint = "https://crt.sh/?" + urlencode({"q": "%25." + query.lstrip("%.") if not query.startswith("%") else query, "output": "json"})
payload, source_url = self._get_json(endpoint)
if not isinstance(payload, list): raise ValueError("invalid_ct_response")
@@ -254,7 +255,7 @@ class DnsSource(_Base):
if not isinstance(domains, list) or not domains or len(domains) > 100: return ValidationResult(False, ["domains must contain 1 to 100 names"])
return ValidationResult(True)
def discover(self, config, cursor=None):
def discover(self, config, cursor=None, criteria=None, limits=None):
result = self.validate_config(config)
if not result.valid: raise ValueError(result.errors[0])
import socket
@@ -280,7 +281,7 @@ class RdapSource(_HttpJsonSource):
if not domain or "." not in domain: return ValidationResult(False, ["domain is required"])
return ValidationResult(True)
def discover(self, config, cursor=None):
def discover(self, config, cursor=None, criteria=None, limits=None):
result = self.validate_config(config)
if not result.valid: raise ValueError(result.errors[0])
domain = str(config["domain"]).strip().lower().rstrip(".")
@@ -303,7 +304,7 @@ class GatedSource(_Base):
if not isinstance(config.get("rate_limit", 1), (int, float)) or config.get("rate_limit", 1) <= 0:
return ValidationResult(False, ["positive rate_limit is required"])
return ValidationResult(True)
def discover(self, config, cursor=None):
def discover(self, config, cursor=None, criteria=None, limits=None):
result = self.validate_config(config)
if not result.valid: raise ValueError(result.errors[0])
# Network execution is delegated to an explicitly approved provider; never guess.
@@ -322,10 +323,9 @@ class ApprovedDirectorySource(GatedSource):
if not result.valid:return result
provider=str(config.get("provider", "")).strip().lower()
if provider not in {"openstreetmap", "wikidata", "common_crawl"}: return ValidationResult(False,["provider must be openstreetmap, wikidata, or common_crawl"])
if not str(config.get("query", "")).strip(): return ValidationResult(False,["query is required"])
return ValidationResult(True)
def discover(self, config, cursor=None):
def discover(self, config, cursor=None, criteria=None, limits=None):
result=self.validate_config(config)
if not result.valid: raise ValueError(result.errors[0])
provider=str(config["provider"]).lower(); query=str(config["query"]).strip(); limit=max(1,min(100,int(config.get("max_records",50))))
@@ -363,7 +363,7 @@ class GooglePlacesSource(GatedSource):
if not str(config.get("query", "")).strip(): return ValidationResult(False, ["query is required"])
return ValidationResult(True)
def discover(self, config, cursor=None):
def discover(self, config, cursor=None, criteria=None, limits=None):
result = self.validate_config(config)
if not result.valid: raise ValueError(result.errors[0])
api_key = str(config.get("_api_key", "")).strip()
@@ -383,24 +383,24 @@ class OpenStreetMapSource(ApprovedDirectorySource):
kind = source_code = "openstreetmap"
display_name = "OpenStreetMap / Overpass"
optional = False
def discover(self, config, cursor=None):
page = super().discover({**dict(config), "provider": "openstreetmap"}, cursor)
def discover(self, config, cursor=None, criteria=None, limits=None):
page = super().discover({**dict(config), "provider": "openstreetmap"}, cursor, criteria=criteria, limits=limits)
return DiscoveryPage(page.records, page.next_cursor, {**page.metadata, "adapter": self.source_code})
class WikidataSource(ApprovedDirectorySource):
kind = source_code = "wikidata"
display_name = "Wikidata"
optional = False
def discover(self, config, cursor=None):
page = super().discover({**dict(config), "provider": "wikidata"}, cursor)
def discover(self, config, cursor=None, criteria=None, limits=None):
page = super().discover({**dict(config), "provider": "wikidata"}, cursor, criteria=criteria, limits=limits)
return DiscoveryPage(page.records, page.next_cursor, {**page.metadata, "adapter": self.source_code})
class CommonCrawlSource(ApprovedDirectorySource):
kind = source_code = "common_crawl"
display_name = "Common Crawl index"
optional = False
def discover(self, config, cursor=None):
page = super().discover({**dict(config), "provider": "common_crawl"}, cursor)
def discover(self, config, cursor=None, criteria=None, limits=None):
page = super().discover({**dict(config), "provider": "common_crawl"}, cursor, criteria=criteria, limits=limits)
return DiscoveryPage(page.records, page.next_cursor, {**page.metadata, "adapter": self.source_code})
def _gated(code, name):