191 lines
9.2 KiB
Python
191 lines
9.2 KiB
Python
"""Safe, tenant-neutral source adapter contracts.
|
|
|
|
Network adapters are intentionally capability gated: configuration must explicitly
|
|
approve public access, terms, rate limits and credentials (where required).
|
|
Adapters never emit or persist credential values.
|
|
"""
|
|
from __future__ import annotations
|
|
from dataclasses import dataclass, field
|
|
from typing import Any, Mapping, Protocol, Sequence
|
|
import csv, io, random, time
|
|
|
|
SECRET_KEYS = {"password", "passwd", "secret", "token", "api_key", "apikey", "authorization", "private_key", "credential"}
|
|
NETWORK_KINDS = {"google_places", "bing_local", "approved_directory", "public_website", "permitted_social", "ct_logs", "dns", "rdap"}
|
|
|
|
|
|
def contains_secret(value: Any, path: str = "") -> str | None:
|
|
if isinstance(value, Mapping):
|
|
for key, child in value.items():
|
|
key_text = str(key).lower()
|
|
if key_text in SECRET_KEYS or any(x in key_text for x in ("password", "token", "secret", "api_key", "private_key")):
|
|
return path + str(key)
|
|
found = contains_secret(child, path + str(key) + ".")
|
|
if found: return found
|
|
elif isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
|
|
for index, child in enumerate(value):
|
|
found = contains_secret(child, path + str(index) + ".")
|
|
if found: return found
|
|
return None
|
|
|
|
@dataclass(frozen=True)
|
|
class ValidationResult:
|
|
valid: bool
|
|
errors: list[str] = field(default_factory=list)
|
|
|
|
@dataclass(frozen=True)
|
|
class DiscoveryPage:
|
|
records: list[dict[str, str]]
|
|
next_cursor: str | None = None
|
|
metadata: dict[str, Any] = field(default_factory=dict)
|
|
|
|
@dataclass(frozen=True)
|
|
class SourceHealth:
|
|
status: str = "unknown"
|
|
consecutive_failures: int = 0
|
|
circuit_open: bool = False
|
|
last_error: str | None = None
|
|
|
|
@dataclass(frozen=True)
|
|
class NormalizedRecord:
|
|
name: str = ""
|
|
website: str = ""
|
|
email: str = ""
|
|
phone: str = ""
|
|
description: str = ""
|
|
location: str = ""
|
|
source_url: str = ""
|
|
provenance: str = ""
|
|
raw: Mapping[str, Any] = field(default_factory=dict)
|
|
|
|
class DiscoverySource(Protocol):
|
|
source_code: str
|
|
display_name: str
|
|
def validate_config(self, config: Mapping[str, Any]) -> ValidationResult: ...
|
|
def discover(self, config: Mapping[str, Any], cursor: str | None = None) -> DiscoveryPage: ...
|
|
def health_check(self, config: Mapping[str, Any]) -> SourceHealth: ...
|
|
|
|
_FIELDS = ("name", "website", "email", "phone", "description", "location")
|
|
def normalize_record(row: Mapping[str, Any]) -> dict[str, str]:
|
|
result = {field: str(row.get(field, "") or "").strip() for field in _FIELDS}
|
|
aliases = {"company":"name", "business":"name", "url":"website", "domain":"website", "address":"location", "address_line":"location"}
|
|
for key, target in aliases.items():
|
|
if not result[target] and row.get(key) is not None: result[target] = str(row[key]).strip()
|
|
return result
|
|
|
|
def normalized_record(row: Mapping[str, Any], *, source_url="", provenance="") -> NormalizedRecord:
|
|
x = normalize_record(row)
|
|
return NormalizedRecord(**x, source_url=source_url, provenance=provenance, raw=dict(row))
|
|
|
|
def exponential_backoff(attempt: int, base: float = 0.5, maximum: float = 30.0, jitter: float = 0.25) -> float:
|
|
"""Bounded exponential delay with symmetric jitter; no sleeping occurs here."""
|
|
delay = min(maximum, base * (2 ** max(0, int(attempt))))
|
|
return max(0.0, delay + random.uniform(-jitter * delay, jitter * delay))
|
|
|
|
def backoff_delay(attempt: int, base: float = 0.5, maximum: float = 30.0, jitter: float = 0.25) -> float:
|
|
return exponential_backoff(attempt, base, maximum, jitter)
|
|
|
|
def circuit_is_open(consecutive_failures: int, threshold: int = 3) -> bool:
|
|
return int(consecutive_failures) >= max(1, int(threshold))
|
|
|
|
def quota_remaining(used: int, limit: int | None) -> int | None:
|
|
"""Return remaining quota, clamped so malformed values fail closed."""
|
|
if limit is None: return None
|
|
return max(0, int(limit) - max(0, int(used)))
|
|
|
|
def quota_allowed(used: int, limit: int | None) -> bool:
|
|
remaining = quota_remaining(used, limit)
|
|
return remaining is None or remaining > 0
|
|
|
|
def rate_limit_delay(last_request: float | None, min_interval: float) -> float:
|
|
if last_request is None: return 0.0
|
|
return max(0.0, float(min_interval) - (time.monotonic() - last_request))
|
|
|
|
class _Base:
|
|
kind = ""
|
|
source_code = ""
|
|
display_name = ""
|
|
def validate_config(self, config):
|
|
if not isinstance(config, Mapping): return ValidationResult(False, ["config must be an object"])
|
|
found = contains_secret(config)
|
|
if found: return ValidationResult(False, [f"secret field is not permitted: {found}"])
|
|
return ValidationResult(True)
|
|
validate = validate_config
|
|
def health_check(self, config):
|
|
result = self.validate_config(config)
|
|
return SourceHealth("healthy" if result.valid else "unhealthy", last_error=None if result.valid else "; ".join(result.errors))
|
|
def discover(self, config, cursor=None):
|
|
result = self.validate_config(config)
|
|
if not result.valid: raise ValueError(result.errors[0])
|
|
raise RuntimeError("source_not_configured")
|
|
|
|
class ManualSource(_Base):
|
|
kind = source_code = "manual"; display_name = "Manual records"
|
|
def validate_config(self, config):
|
|
result = super().validate_config(config)
|
|
if not result.valid: return result
|
|
if not isinstance(config.get("rows"), list): return ValidationResult(False, ["rows must be a list"])
|
|
return ValidationResult(True)
|
|
def discover(self, config, cursor=None):
|
|
result = self.validate_config(config)
|
|
if not result.valid: raise ValueError(result.errors[0])
|
|
return DiscoveryPage([normalize_record(row) for row in config["rows"] if isinstance(row, Mapping)], metadata={"adapter":self.source_code})
|
|
|
|
class CsvSource(_Base):
|
|
kind = source_code = "csv"; display_name = "CSV import"
|
|
def validate_config(self, config):
|
|
result = super().validate_config(config)
|
|
if not result.valid: return result
|
|
if not isinstance(config.get("csv"), str): return ValidationResult(False, ["csv must be text"])
|
|
try:
|
|
reader = csv.DictReader(io.StringIO(config["csv"]))
|
|
if not reader.fieldnames: return ValidationResult(False, ["CSV header is required"])
|
|
except csv.Error as exc: return ValidationResult(False, [f"invalid CSV: {exc}"])
|
|
return ValidationResult(True)
|
|
def discover(self, config, cursor=None):
|
|
result = self.validate_config(config)
|
|
if not result.valid: raise ValueError(result.errors[0])
|
|
reader = csv.DictReader(io.StringIO(config["csv"].replace("\r\n", "\n")))
|
|
return DiscoveryPage([normalize_record({str(k).strip().lower():v for k,v in row.items()}) for row in reader], metadata={"adapter":self.source_code,"columns":reader.fieldnames or []})
|
|
|
|
class GatedSource(_Base):
|
|
required = "approved"
|
|
def validate_config(self, config):
|
|
result = super().validate_config(config)
|
|
if not result.valid: return result
|
|
if config.get("approved") is not True: return ValidationResult(False, ["source approval is required"])
|
|
if config.get("public_access") is not True: return ValidationResult(False, ["public_access approval is required"])
|
|
if config.get("terms_accepted") is not True: return ValidationResult(False, ["terms_accepted is required"])
|
|
if self.source_code in {"google_places", "bing_local"} and not config.get("credential_ref"):
|
|
return ValidationResult(False, ["approved credential_ref is required"])
|
|
if not isinstance(config.get("rate_limit", 1), (int, float)) or config.get("rate_limit", 1) <= 0:
|
|
return ValidationResult(False, ["positive rate_limit is required"])
|
|
return ValidationResult(True)
|
|
def discover(self, config, cursor=None):
|
|
result = self.validate_config(config)
|
|
if not result.valid: raise ValueError(result.errors[0])
|
|
# Network execution is delegated to an explicitly approved provider; never guess.
|
|
raise RuntimeError("network_adapter_not_configured")
|
|
|
|
|
|
def _gated(code, name):
|
|
return type(name.replace(" ", ""), (GatedSource,), {"kind":code, "source_code":code, "display_name":name})
|
|
GooglePlacesSource = _gated("google_places", "Google Places")
|
|
BingLocalSource = _gated("bing_local", "Bing / approved local API")
|
|
ApprovedDirectorySource = _gated("approved_directory", "Approved directory")
|
|
PublicWebsiteSource = _gated("public_website", "Public website")
|
|
PermittedSocialSource = _gated("permitted_social", "Permitted social")
|
|
CtLogsSource = _gated("ct_logs", "Certificate transparency logs")
|
|
DnsSource = _gated("dns", "DNS")
|
|
RdapSource = _gated("rdap", "RDAP")
|
|
|
|
ADAPTERS = {x.source_code: x for x in (ManualSource, CsvSource, GooglePlacesSource, BingLocalSource, ApprovedDirectorySource, PublicWebsiteSource, PermittedSocialSource, CtLogsSource, DnsSource, RdapSource)}
|
|
# common aliases used by clients
|
|
ADAPTER_REGISTRY = ADAPTERS
|
|
|
|
def adapter_for(kind: str) -> DiscoverySource:
|
|
try: return ADAPTERS[str(kind).strip().lower()]()
|
|
except KeyError: raise ValueError("unsupported source kind")
|
|
|
|
def available_adapters() -> list[dict[str, str]]:
|
|
return [{"source_code": cls.source_code, "display_name": cls.display_name} for cls in ADAPTERS.values()]
|