root/MarketingTool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add governed source adapter discovery framework

Browse Source
This commit is contained in:
Marco0300
2026-09-03 22:49:05 +02:00
parent a99e6b26dc
commit a9213c0282
3 changed files with 228 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/app/sources.py
+124 -34
View File
@@ -1,10 +1,17 @@
"""Deterministic, network-free discovery source contracts and adapters."""
"""Safe, tenant-neutral source adapter contracts.
Network adapters are intentionally capability gated: configuration must explicitly
approve public access, terms, rate limits and credentials (where required).
Adapters never emit or persist credential values.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any, Mapping, Protocol, Sequence
import csv, io, re
import csv, io, random, time
SECRET_KEYS = {"password", "passwd", "secret", "token", "api_key", "apikey", "authorization", "private_key", "credential"}
NETWORK_KINDS = {"google_places", "bing_local", "approved_directory", "public_website", "permitted_social", "ct_logs", "dns", "rdap"}
def contains_secret(value: Any, path: str = "") -> str | None:
if isinstance(value, Mapping):
@@ -38,57 +45,140 @@ class SourceHealth:
circuit_open: bool = False
last_error: str | None = None
@dataclass(frozen=True)
class NormalizedRecord:
name: str = ""
website: str = ""
email: str = ""
phone: str = ""
description: str = ""
location: str = ""
source_url: str = ""
provenance: str = ""
raw: Mapping[str, Any] = field(default_factory=dict)
class DiscoverySource(Protocol):
kind: str
def validate(self, config: Mapping[str, Any]) -> ValidationResult: ...
source_code: str
display_name: str
def validate_config(self, config: Mapping[str, Any]) -> ValidationResult: ...
def discover(self, config: Mapping[str, Any], cursor: str | None = None) -> DiscoveryPage: ...
def health_check(self, config: Mapping[str, Any]) -> SourceHealth: ...
_FIELDS = ("name", "website", "email", "phone", "description")
_FIELDS = ("name", "website", "email", "phone", "description", "location")
def normalize_record(row: Mapping[str, Any]) -> dict[str, str]:
result = {field: str(row.get(field, "")).strip() for field in _FIELDS}
# Accept common CSV spellings without retaining arbitrary sensitive fields.
aliases = {"company": "name", "url": "website", "domain": "website"}
result = {field: str(row.get(field, "") or "").strip() for field in _FIELDS}
aliases = {"company":"name", "business":"name", "url":"website", "domain":"website", "address":"location", "address_line":"location"}
for key, target in aliases.items():
if not result[target] and row.get(key) is not None: result[target] = str(row[key]).strip()
return result
class ManualSource:
kind = "manual"
def validate(self, config: Mapping[str, Any]) -> ValidationResult:
if not isinstance(config, Mapping): return ValidationResult(False, ["config must be an object"])
found = contains_secret(config)
if found: return ValidationResult(False, [f"secret field is not permitted: {found}"])
rows = config.get("rows")
if not isinstance(rows, list): return ValidationResult(False, ["rows must be a list"])
return ValidationResult(True)
def discover(self, config: Mapping[str, Any], cursor: str | None = None) -> DiscoveryPage:
validation = self.validate(config)
if not validation.valid: raise ValueError(validation.errors[0])
rows = [normalize_record(row) for row in config["rows"] if isinstance(row, Mapping)]
return DiscoveryPage(rows, None, {"adapter": self.kind})
def normalized_record(row: Mapping[str, Any], *, source_url="", provenance="") -> NormalizedRecord:
x = normalize_record(row)
return NormalizedRecord(**x, source_url=source_url, provenance=provenance, raw=dict(row))
class CsvSource:
kind = "csv"
def validate(self, config: Mapping[str, Any]) -> ValidationResult:
def exponential_backoff(attempt: int, base: float = 0.5, maximum: float = 30.0, jitter: float = 0.25) -> float:
"""Bounded exponential delay with symmetric jitter; no sleeping occurs here."""
delay = min(maximum, base * (2 ** max(0, int(attempt))))
return max(0.0, delay + random.uniform(-jitter * delay, jitter * delay))
def backoff_delay(attempt: int, base: float = 0.5, maximum: float = 30.0, jitter: float = 0.25) -> float:
return exponential_backoff(attempt, base, maximum, jitter)
def circuit_is_open(consecutive_failures: int, threshold: int = 3) -> bool:
return int(consecutive_failures) >= threshold
def quota_allowed(used: int, limit: int | None) -> bool:
return limit is None or (limit >= 0 and used < limit)
def rate_limit_delay(last_request: float | None, min_interval: float) -> float:
if last_request is None: return 0.0
return max(0.0, float(min_interval) - (time.monotonic() - last_request))
class _Base:
kind = ""
source_code = ""
display_name = ""
def validate_config(self, config):
if not isinstance(config, Mapping): return ValidationResult(False, ["config must be an object"])
found = contains_secret(config)
if found: return ValidationResult(False, [f"secret field is not permitted: {found}"])
return ValidationResult(True)
validate = validate_config
def health_check(self, config):
result = self.validate_config(config)
return SourceHealth("healthy" if result.valid else "unhealthy", last_error=None if result.valid else "; ".join(result.errors))
def discover(self, config, cursor=None):
result = self.validate_config(config)
if not result.valid: raise ValueError(result.errors[0])
raise RuntimeError("source_not_configured")
class ManualSource(_Base):
kind = source_code = "manual"; display_name = "Manual records"
def validate_config(self, config):
result = super().validate_config(config)
if not result.valid: return result
if not isinstance(config.get("rows"), list): return ValidationResult(False, ["rows must be a list"])
return ValidationResult(True)
def discover(self, config, cursor=None):
result = self.validate_config(config)
if not result.valid: raise ValueError(result.errors[0])
return DiscoveryPage([normalize_record(row) for row in config["rows"] if isinstance(row, Mapping)], metadata={"adapter":self.source_code})
class CsvSource(_Base):
kind = source_code = "csv"; display_name = "CSV import"
def validate_config(self, config):
result = super().validate_config(config)
if not result.valid: return result
if not isinstance(config.get("csv"), str): return ValidationResult(False, ["csv must be text"])
try:
reader = csv.DictReader(io.StringIO(config["csv"]));
reader = csv.DictReader(io.StringIO(config["csv"]))
if not reader.fieldnames: return ValidationResult(False, ["CSV header is required"])
except csv.Error as exc: return ValidationResult(False, [f"invalid CSV: {exc}"])
return ValidationResult(True)
def discover(self, config: Mapping[str, Any], cursor: str | None = None) -> DiscoveryPage:
validation = self.validate(config)
if not validation.valid: raise ValueError(validation.errors[0])
def discover(self, config, cursor=None):
result = self.validate_config(config)
if not result.valid: raise ValueError(result.errors[0])
reader = csv.DictReader(io.StringIO(config["csv"].replace("\r\n", "\n")))
records = [normalize_record({str(k).strip().lower(): v for k, v in row.items()}) for row in reader]
return DiscoveryPage(records, None, {"adapter": self.kind, "columns": reader.fieldnames or []})
return DiscoveryPage([normalize_record({str(k).strip().lower():v for k,v in row.items()}) for row in reader], metadata={"adapter":self.source_code,"columns":reader.fieldnames or []})
ADAPTERS = {"manual": ManualSource, "csv": CsvSource}
class GatedSource(_Base):
required = "approved"
def validate_config(self, config):
result = super().validate_config(config)
if not result.valid: return result
if config.get("approved") is not True: return ValidationResult(False, ["source approval is required"])
if config.get("public_access") is not True: return ValidationResult(False, ["public_access approval is required"])
if config.get("terms_accepted") is not True: return ValidationResult(False, ["terms_accepted is required"])
if self.source_code in {"google_places", "bing_local"} and not config.get("credential_ref"):
return ValidationResult(False, ["approved credential_ref is required"])
if not isinstance(config.get("rate_limit", 1), (int, float)) or config.get("rate_limit", 1) <= 0:
return ValidationResult(False, ["positive rate_limit is required"])
return ValidationResult(True)
def discover(self, config, cursor=None):
result = self.validate_config(config)
if not result.valid: raise ValueError(result.errors[0])
# Network execution is delegated to an explicitly approved provider; never guess.
raise RuntimeError("network_adapter_not_configured")
def _gated(code, name):
return type(name.replace(" ", ""), (GatedSource,), {"kind":code, "source_code":code, "display_name":name})
GooglePlacesSource = _gated("google_places", "Google Places")
BingLocalSource = _gated("bing_local", "Bing / approved local API")
ApprovedDirectorySource = _gated("approved_directory", "Approved directory")
PublicWebsiteSource = _gated("public_website", "Public website")
PermittedSocialSource = _gated("permitted_social", "Permitted social")
CtLogsSource = _gated("ct_logs", "Certificate transparency logs")
DnsSource = _gated("dns", "DNS")
RdapSource = _gated("rdap", "RDAP")
ADAPTERS = {x.source_code: x for x in (ManualSource, CsvSource, GooglePlacesSource, BingLocalSource, ApprovedDirectorySource, PublicWebsiteSource, PermittedSocialSource, CtLogsSource, DnsSource, RdapSource)}
# common aliases used by clients
ADAPTER_REGISTRY = ADAPTERS
def adapter_for(kind: str) -> DiscoverySource:
try: return ADAPTERS[kind]()
try: return ADAPTERS[str(kind).strip().lower()]()
except KeyError: raise ValueError("unsupported source kind")
def available_adapters() -> list[dict[str, str]]:
return [{"source_code": cls.source_code, "display_name": cls.display_name} for cls in ADAPTERS.values()]