"""Safe, tenant-neutral source adapter contracts. Network adapters are intentionally capability gated: configuration must explicitly approve public access, terms, rate limits and credentials (where required). Adapters never emit or persist credential values. """ from __future__ import annotations from dataclasses import dataclass, field from typing import Any, Mapping, Protocol, Sequence import csv, io, random, time SECRET_KEYS = {"password", "passwd", "secret", "token", "api_key", "apikey", "authorization", "private_key", "credential"} NETWORK_KINDS = {"google_places", "bing_local", "approved_directory", "public_website", "permitted_social", "ct_logs", "dns", "rdap"} def contains_secret(value: Any, path: str = "") -> str | None: if isinstance(value, Mapping): for key, child in value.items(): key_text = str(key).lower() if key_text in SECRET_KEYS or any(x in key_text for x in ("password", "token", "secret", "api_key", "private_key")): return path + str(key) found = contains_secret(child, path + str(key) + ".") if found: return found elif isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): for index, child in enumerate(value): found = contains_secret(child, path + str(index) + ".") if found: return found return None @dataclass(frozen=True) class ValidationResult: valid: bool errors: list[str] = field(default_factory=list) @dataclass(frozen=True) class DiscoveryPage: records: list[dict[str, str]] next_cursor: str | None = None metadata: dict[str, Any] = field(default_factory=dict) @dataclass(frozen=True) class SourceHealth: status: str = "unknown" consecutive_failures: int = 0 circuit_open: bool = False last_error: str | None = None @dataclass(frozen=True) class NormalizedRecord: name: str = "" website: str = "" email: str = "" phone: str = "" description: str = "" location: str = "" source_url: str = "" provenance: str = "" raw: Mapping[str, Any] = field(default_factory=dict) class DiscoverySource(Protocol): source_code: str display_name: str def validate_config(self, config: Mapping[str, Any]) -> ValidationResult: ... def discover(self, config: Mapping[str, Any], cursor: str | None = None) -> DiscoveryPage: ... def health_check(self, config: Mapping[str, Any]) -> SourceHealth: ... _FIELDS = ("name", "website", "email", "phone", "description", "location") def normalize_record(row: Mapping[str, Any]) -> dict[str, str]: result = {field: str(row.get(field, "") or "").strip() for field in _FIELDS} aliases = {"company":"name", "business":"name", "url":"website", "domain":"website", "address":"location", "address_line":"location"} for key, target in aliases.items(): if not result[target] and row.get(key) is not None: result[target] = str(row[key]).strip() return result def normalized_record(row: Mapping[str, Any], *, source_url="", provenance="") -> NormalizedRecord: x = normalize_record(row) return NormalizedRecord(**x, source_url=source_url, provenance=provenance, raw=dict(row)) def exponential_backoff(attempt: int, base: float = 0.5, maximum: float = 30.0, jitter: float = 0.25) -> float: """Bounded exponential delay with symmetric jitter; no sleeping occurs here.""" delay = min(maximum, base * (2 ** max(0, int(attempt)))) return max(0.0, delay + random.uniform(-jitter * delay, jitter * delay)) def backoff_delay(attempt: int, base: float = 0.5, maximum: float = 30.0, jitter: float = 0.25) -> float: return exponential_backoff(attempt, base, maximum, jitter) def circuit_is_open(consecutive_failures: int, threshold: int = 3) -> bool: return int(consecutive_failures) >= max(1, int(threshold)) def quota_remaining(used: int, limit: int | None) -> int | None: """Return remaining quota, clamped so malformed values fail closed.""" if limit is None: return None return max(0, int(limit) - max(0, int(used))) def quota_allowed(used: int, limit: int | None) -> bool: remaining = quota_remaining(used, limit) return remaining is None or remaining > 0 def rate_limit_delay(last_request: float | None, min_interval: float) -> float: if last_request is None: return 0.0 return max(0.0, float(min_interval) - (time.monotonic() - last_request)) class _Base: kind = "" source_code = "" display_name = "" def validate_config(self, config): if not isinstance(config, Mapping): return ValidationResult(False, ["config must be an object"]) found = contains_secret(config) if found: return ValidationResult(False, [f"secret field is not permitted: {found}"]) return ValidationResult(True) validate = validate_config def health_check(self, config): result = self.validate_config(config) return SourceHealth("healthy" if result.valid else "unhealthy", last_error=None if result.valid else "; ".join(result.errors)) def discover(self, config, cursor=None): result = self.validate_config(config) if not result.valid: raise ValueError(result.errors[0]) raise RuntimeError("source_not_configured") class ManualSource(_Base): kind = source_code = "manual"; display_name = "Manual records" def validate_config(self, config): result = super().validate_config(config) if not result.valid: return result if not isinstance(config.get("rows"), list): return ValidationResult(False, ["rows must be a list"]) return ValidationResult(True) def discover(self, config, cursor=None): result = self.validate_config(config) if not result.valid: raise ValueError(result.errors[0]) return DiscoveryPage([normalize_record(row) for row in config["rows"] if isinstance(row, Mapping)], metadata={"adapter":self.source_code}) class CsvSource(_Base): kind = source_code = "csv"; display_name = "CSV import" def validate_config(self, config): result = super().validate_config(config) if not result.valid: return result if not isinstance(config.get("csv"), str): return ValidationResult(False, ["csv must be text"]) try: reader = csv.DictReader(io.StringIO(config["csv"])) if not reader.fieldnames: return ValidationResult(False, ["CSV header is required"]) except csv.Error as exc: return ValidationResult(False, [f"invalid CSV: {exc}"]) return ValidationResult(True) def discover(self, config, cursor=None): result = self.validate_config(config) if not result.valid: raise ValueError(result.errors[0]) reader = csv.DictReader(io.StringIO(config["csv"].replace("\r\n", "\n"))) return DiscoveryPage([normalize_record({str(k).strip().lower():v for k,v in row.items()}) for row in reader], metadata={"adapter":self.source_code,"columns":reader.fieldnames or []}) class GatedSource(_Base): required = "approved" def validate_config(self, config): result = super().validate_config(config) if not result.valid: return result if config.get("approved") is not True: return ValidationResult(False, ["source approval is required"]) if config.get("public_access") is not True: return ValidationResult(False, ["public_access approval is required"]) if config.get("terms_accepted") is not True: return ValidationResult(False, ["terms_accepted is required"]) if self.source_code in {"google_places", "bing_local"} and not config.get("credential_ref"): return ValidationResult(False, ["approved credential_ref is required"]) if not isinstance(config.get("rate_limit", 1), (int, float)) or config.get("rate_limit", 1) <= 0: return ValidationResult(False, ["positive rate_limit is required"]) return ValidationResult(True) def discover(self, config, cursor=None): result = self.validate_config(config) if not result.valid: raise ValueError(result.errors[0]) # Network execution is delegated to an explicitly approved provider; never guess. raise RuntimeError("network_adapter_not_configured") def _gated(code, name): return type(name.replace(" ", ""), (GatedSource,), {"kind":code, "source_code":code, "display_name":name}) GooglePlacesSource = _gated("google_places", "Google Places") BingLocalSource = _gated("bing_local", "Bing / approved local API") ApprovedDirectorySource = _gated("approved_directory", "Approved directory") PublicWebsiteSource = _gated("public_website", "Public website") PermittedSocialSource = _gated("permitted_social", "Permitted social") CtLogsSource = _gated("ct_logs", "Certificate transparency logs") DnsSource = _gated("dns", "DNS") RdapSource = _gated("rdap", "RDAP") ADAPTERS = {x.source_code: x for x in (ManualSource, CsvSource, GooglePlacesSource, BingLocalSource, ApprovedDirectorySource, PublicWebsiteSource, PermittedSocialSource, CtLogsSource, DnsSource, RdapSource)} # common aliases used by clients ADAPTER_REGISTRY = ADAPTERS def adapter_for(kind: str) -> DiscoverySource: try: return ADAPTERS[str(kind).strip().lower()]() except KeyError: raise ValueError("unsupported source kind") def available_adapters() -> list[dict[str, str]]: return [{"source_code": cls.source_code, "display_name": cls.display_name} for cls in ADAPTERS.values()]