MarketingTool/apps/api/app/sources.py

"""Safe, tenant-neutral source adapter contracts.

Network adapters are intentionally capability gated: configuration must explicitly
approve public access, terms, rate limits and credentials (where required).
Adapters never emit or persist credential values.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any, Mapping, Protocol, Sequence
from html.parser import HTMLParser
import csv, io, os, random, threading, time, json, re
from urllib.parse import urlencode, urlparse
from urllib.request import Request, urlopen

try:
    from .website_scanner import validate_url
except ImportError:
    from website_scanner import validate_url

SECRET_KEYS = {"password", "passwd", "secret", "token", "api_key", "apikey", "authorization", "private_key", "credential"}
NETWORK_KINDS = {"google_places", "bing_local", "approved_directory", "public_website", "permitted_social", "ct_logs", "dns", "rdap"}
DISCOVERY_CRITERIA_FIELDS = {"query", "category", "city", "location", "keywords", "province", "country", "language", "search", "phrase", "industry", "keyword"}


def contains_secret(value: Any, path: str = "") -> str | None:
    if isinstance(value, Mapping):
        for key, child in value.items():
            key_text = str(key).lower()
            if key_text in SECRET_KEYS or any(x in key_text for x in ("password", "token", "secret", "api_key", "private_key")):
                return path + str(key)
            found = contains_secret(child, path + str(key) + ".")
            if found: return found
    elif isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
        for index, child in enumerate(value):
            found = contains_secret(child, path + str(index) + ".")
            if found: return found
    return None

@dataclass(frozen=True)
class ValidationResult:
    valid: bool
    errors: list[str] = field(default_factory=list)

@dataclass(frozen=True)
class DiscoveryPage:
    records: list[dict[str, str]]
    next_cursor: str | None = None
    metadata: dict[str, Any] = field(default_factory=dict)

@dataclass(frozen=True)
class SourceHealth:
    status: str = "unknown"
    consecutive_failures: int = 0
    circuit_open: bool = False
    last_error: str | None = None

@dataclass(frozen=True)
class NormalizedRecord:
    name: str = ""
    website: str = ""
    email: str = ""
    phone: str = ""
    description: str = ""
    location: str = ""
    source_url: str = ""
    provenance: str = ""
    raw: Mapping[str, Any] = field(default_factory=dict)

class DiscoverySource(Protocol):
    source_code: str
    display_name: str
    def validate_config(self, config: Mapping[str, Any]) -> ValidationResult: ...
    def discover(self, config: Mapping[str, Any], cursor: str | None = None) -> DiscoveryPage: ...
    def health_check(self, config: Mapping[str, Any]) -> SourceHealth: ...

_FIELDS = ("name", "website", "email", "phone", "description", "location")
def normalize_record(row: Mapping[str, Any]) -> dict[str, str]:
    result = {field: str(row.get(field, "") or "").strip() for field in _FIELDS}
    aliases = {"company":"name", "business":"name", "url":"website", "domain":"website", "address":"location", "address_line":"location"}
    for key, target in aliases.items():
        if not result[target] and row.get(key) is not None: result[target] = str(row[key]).strip()
    return result

def normalized_record(row: Mapping[str, Any], *, source_url="", provenance="") -> NormalizedRecord:
    x = normalize_record(row)
    return NormalizedRecord(**x, source_url=source_url, provenance=provenance, raw=dict(row))

def exponential_backoff(attempt: int, base: float = 0.5, maximum: float = 30.0, jitter: float = 0.25) -> float:
    """Bounded exponential delay with symmetric jitter; no sleeping occurs here."""
    delay = min(maximum, base * (2 ** max(0, int(attempt))))
    return max(0.0, delay + random.uniform(-jitter * delay, jitter * delay))

def backoff_delay(attempt: int, base: float = 0.5, maximum: float = 30.0, jitter: float = 0.25) -> float:
    return exponential_backoff(attempt, base, maximum, jitter)

def circuit_is_open(consecutive_failures: int, threshold: int = 3) -> bool:
    return int(consecutive_failures) >= max(1, int(threshold))

def quota_remaining(used: int, limit: int | None) -> int | None:
    """Return remaining quota, clamped so malformed values fail closed."""
    if limit is None: return None
    return max(0, int(limit) - max(0, int(used)))

def quota_allowed(used: int, limit: int | None) -> bool:
    remaining = quota_remaining(used, limit)
    return remaining is None or remaining > 0

def rate_limit_delay(last_request: float | None, min_interval: float) -> float:
    if last_request is None: return 0.0
    return max(0.0, float(min_interval) - (time.monotonic() - last_request))

class _Base:
    kind = ""
    source_code = ""
    display_name = ""
    available = True
    optional = False
    requires_credentials = False
    def validate_config(self, config):
        if not isinstance(config, Mapping): return ValidationResult(False, ["config must be an object"])
        found = contains_secret(config)
        if found: return ValidationResult(False, [f"secret field is not permitted: {found}"])
        return ValidationResult(True)
    validate = validate_config
    def health_check(self, config):
        result = self.validate_config(config)
        return SourceHealth("healthy" if result.valid else "unhealthy", last_error=None if result.valid else "; ".join(result.errors))
    def discover(self, config, cursor=None, criteria=None, limits=None):
        result = self.validate_config(config)
        if not result.valid: raise ValueError(result.errors[0])
        raise RuntimeError("source_not_configured")

class ManualSource(_Base):
    kind = source_code = "manual"; display_name = "Manual records"
    def validate_config(self, config):
        result = super().validate_config(config)
        if not result.valid: return result
        if not isinstance(config.get("rows"), list): return ValidationResult(False, ["rows must be a list"])
        return ValidationResult(True)
    def discover(self, config, cursor=None, criteria=None, limits=None):
        result = self.validate_config(config)
        if not result.valid: raise ValueError(result.errors[0])
        return DiscoveryPage([normalize_record(row) for row in config["rows"] if isinstance(row, Mapping)], metadata={"adapter":self.source_code})

class CsvSource(_Base):
    kind = source_code = "csv"; display_name = "CSV import"
    def validate_config(self, config):
        result = super().validate_config(config)
        if not result.valid: return result
        if not isinstance(config.get("csv"), str): return ValidationResult(False, ["csv must be text"])
        try:
            reader = csv.DictReader(io.StringIO(config["csv"]))
            if not reader.fieldnames: return ValidationResult(False, ["CSV header is required"])
        except csv.Error as exc: return ValidationResult(False, [f"invalid CSV: {exc}"])
        return ValidationResult(True)
    def discover(self, config, cursor=None, criteria=None, limits=None):
        result = self.validate_config(config)
        if not result.valid: raise ValueError(result.errors[0])
        reader = csv.DictReader(io.StringIO(config["csv"].replace("\r\n", "\n")))
        records = []
        for row in reader:
            normalized = {str(k).strip().lower(): v for k, v in row.items()}
            if any(str(v or '').strip() for v in normalized.values()):
                records.append(normalize_record(normalized))
        return DiscoveryPage(records, metadata={"adapter":self.source_code,"columns":reader.fieldnames or [],"record_count":len(records)})


class _HttpJsonSource(_Base):
    """Small, bounded JSON client used only for public standards-based sources."""
    max_bytes = 256 * 1024
    timeout = 8

    def _get_json(self, url):
        safe = validate_url(url)
        request = Request(safe, headers={"User-Agent": "ProspectOS/0.1 (+public-source-research)"})
        with urlopen(request, timeout=self.timeout) as response:
            body = response.read(self.max_bytes + 1)
            if len(body) > self.max_bytes:
                raise ValueError("source_response_too_large")
            return json.loads(body.decode("utf-8", "replace")), safe


class PublicWebsiteSource(_Base):
    kind = source_code = "public_website"
    display_name = "Public website"

    def validate_config(self, config):
        result = super().validate_config(config)
        if not result.valid: return result
        urls = config.get("urls", config.get("url", []))
        if isinstance(urls, str): urls = [urls] if urls.strip() else []
        if not isinstance(urls, list) or not urls or len(urls) > 50:
            return ValidationResult(False, ["urls must contain 1 to 50 public HTTP(S) URLs"])
        for value in urls:
            try: validate_url(str(value))
            except ValueError: return ValidationResult(False, ["unsafe public website URL"])
        return ValidationResult(True)

    def discover(self, config, cursor=None, criteria=None, limits=None):
        result = self.validate_config(config)
        if not result.valid: raise ValueError(result.errors[0])
        urls = config.get("urls", config.get("url"))
        if isinstance(urls, str): urls = [urls]
        records = []
        for raw_url in urls[:50]:
            safe = validate_url(str(raw_url))
            request = Request(safe, headers={"User-Agent": "ProspectOS/0.1 (+public-source-research)"})
            with urlopen(request, timeout=8) as response:
                body = response.read(128 * 1024).decode("utf-8", "replace")
                final_url = response.geturl()
            from html.parser import HTMLParser
            parser = HTMLParser()
            title = urlparse(final_url).hostname or safe
            records.append(normalize_record({"name": title, "website": final_url, "description": body[:1000]}))
        return DiscoveryPage(records, metadata={"adapter": self.source_code, "record_count": len(records), "bounded": True})


class CtLogsSource(_HttpJsonSource):
    kind = source_code = "ct_logs"
    display_name = "Certificate transparency logs"

    def validate_config(self, config):
        result = super().validate_config(config)
        if not result.valid: return result
        query = str(config.get("domain", config.get("query", ""))).strip()
        if not query or len(query) > 253 or any(ch in query for ch in "\r\n"):
            return ValidationResult(False, ["domain or query is required"])
        return ValidationResult(True)

    def discover(self, config, cursor=None, criteria=None, limits=None):
        result = self.validate_config(config)
        if not result.valid: raise ValueError(result.errors[0])
        query = str(config.get("domain", config.get("query", ""))).strip()
        endpoint = "https://crt.sh/?" + urlencode({"q": "%25." + query.lstrip("%.") if not query.startswith("%") else query, "output": "json"})
        payload, source_url = self._get_json(endpoint)
        if not isinstance(payload, list): raise ValueError("invalid_ct_response")
        records, seen = [], set()
        for item in payload[:500]:
            names = str(item.get("name_value", "")) if isinstance(item, dict) else ""
            for name in names.splitlines():
                name = name.strip().lower().lstrip("*.")
                if not name or name in seen or "." not in name: continue
                seen.add(name); records.append(normalize_record({"name": name, "website": "https://" + name}))
        return DiscoveryPage(records[:100], metadata={"adapter": self.source_code, "source_url": source_url, "record_count": len(records), "signal_only": True})


class DnsSource(_Base):
    kind = source_code = "dns"
    display_name = "DNS"

    def validate_config(self, config):
        result = super().validate_config(config)
        if not result.valid: return result
        domains = config.get("domains", config.get("domain", []))
        if isinstance(domains, str): domains = [domains] if domains.strip() else []
        if not isinstance(domains, list) or not domains or len(domains) > 100: return ValidationResult(False, ["domains must contain 1 to 100 names"])
        return ValidationResult(True)

    def discover(self, config, cursor=None, criteria=None, limits=None):
        result = self.validate_config(config)
        if not result.valid: raise ValueError(result.errors[0])
        import socket
        domains = config.get("domains", config.get("domain")); domains = [domains] if isinstance(domains, str) else domains
        records = []
        for domain in domains[:100]:
            domain = str(domain).strip().lower().rstrip(".")
            if not domain or "." not in domain: continue
            try: addresses = sorted({item[4][0] for item in socket.getaddrinfo(domain, 443, type=socket.SOCK_STREAM)})
            except socket.gaierror: addresses = []
            records.append(normalize_record({"name": domain, "website": "https://" + domain, "description": json.dumps({"a_aaaa": addresses})}))
        return DiscoveryPage(records, metadata={"adapter": self.source_code, "record_count": len(records), "dns_status_only": True})


class RdapSource(_HttpJsonSource):
    kind = source_code = "rdap"
    display_name = "RDAP"

    def validate_config(self, config):
        result = super().validate_config(config)
        if not result.valid: return result
        domain = str(config.get("domain", "")).strip()
        if not domain or "." not in domain: return ValidationResult(False, ["domain is required"])
        return ValidationResult(True)

    def discover(self, config, cursor=None, criteria=None, limits=None):
        result = self.validate_config(config)
        if not result.valid: raise ValueError(result.errors[0])
        domain = str(config["domain"]).strip().lower().rstrip(".")
        payload, source_url = self._get_json("https://rdap.org/domain/" + domain)
        return DiscoveryPage([normalize_record({"name": domain, "website": "https://" + domain, "description": json.dumps({"rdap": payload}, default=str)[:1000]})], metadata={"adapter": self.source_code, "source_url": source_url, "record_count": 1, "registration_signal_only": True})

class GatedSource(_Base):
    available = False
    optional = True
    requires_credentials = True
    required = "approved"
    def validate_config(self, config):
        result = super().validate_config(config)
        if not result.valid: return result
        if config.get("approved") is not True: return ValidationResult(False, ["source approval is required"])
        if config.get("public_access") is not True: return ValidationResult(False, ["public_access approval is required"])
        if config.get("terms_accepted") is not True: return ValidationResult(False, ["terms_accepted is required"])
        if self.source_code in {"google_places", "bing_local"} and not config.get("credential_ref"):
            return ValidationResult(False, ["approved credential_ref is required"])
        if not isinstance(config.get("rate_limit", 1), (int, float)) or config.get("rate_limit", 1) <= 0:
            return ValidationResult(False, ["positive rate_limit is required"])
        return ValidationResult(True)
    def discover(self, config, cursor=None, criteria=None, limits=None):
        result = self.validate_config(config)
        if not result.valid: raise ValueError(result.errors[0])
        # Network execution is delegated to an explicitly approved provider; never guess.
        raise RuntimeError("network_adapter_not_configured")


class ApprovedDirectorySource(GatedSource):
    kind = source_code = "approved_directory"
    display_name = "Free public directories"
    available = True
    requires_credentials = False
    optional = False

    def validate_config(self, config):
        result=super().validate_config(config)
        if not result.valid:return result
        provider=str(config.get("provider", "")).strip().lower()
        if provider not in {"openstreetmap", "wikidata", "common_crawl"}: return ValidationResult(False,["provider must be openstreetmap, wikidata, or common_crawl"])
        return ValidationResult(True)

    def discover(self, config, cursor=None, criteria=None, limits=None):
        result=self.validate_config(config)
        if not result.valid: raise ValueError(result.errors[0])
        criteria = criteria if isinstance(criteria, Mapping) else {}
        keyword_values = criteria.get("keywords", criteria.get("keyword", criteria.get("query", "")))
        if isinstance(keyword_values, str): keyword_values = [keyword_values]
        terms = [str(value).strip() for value in keyword_values[:10] if str(value).strip()] if isinstance(keyword_values, Sequence) and not isinstance(keyword_values, (bytes, bytearray, str)) else []
        for key in ("category", "industry"):
            value = str(criteria.get(key, "")).strip()
            if value: terms.append(value)
        query = " ".join(dict.fromkeys(terms))[:500]
        if not query: raise ValueError("discovery_criteria_required")
        location_parts = [str(criteria.get(key, "")).strip() for key in ("city", "location", "province", "country")]
        area = next((value for value in location_parts if value), "South Africa")
        limit_source = limits if isinstance(limits, Mapping) else {}
        try: limit=max(1,min(100,int(limit_source.get("per_run_limit", limit_source.get("max_records",50)))))
        except (TypeError, ValueError): raise ValueError("invalid_limits")
        provider=str(config["provider"]).lower()
        if provider == "openstreetmap":
            terms=[token.lower() for token in re.findall(r"[A-Za-z0-9]{2,32}",query)[:5]]
            variants=sorted({variant for term in terms for variant in (term,term[:-1] if term.endswith('s') and len(term)>3 else term)})
            pattern="|".join(re.escape(term) for term in variants)
            overpass='[out:json][timeout:25];area["name"="%s"]->.a;(nwr["name"~"%s",i](area.a);nwr["craft"~"%s",i](area.a);nwr["amenity"~"%s",i](area.a);nwr["shop"~"%s",i](area.a););out center tags;' % ((area.replace('"',''),)+ (pattern,)*4)
            req=Request("https://overpass-api.de/api/interpreter",data=overpass.encode(),method="POST",headers={"Content-Type":"application/x-www-form-urlencoded","User-Agent":"ProspectOS/0.1"})
            with urlopen(req,timeout=30) as response: payload=json.loads(response.read(2*1024*1024).decode("utf-8","replace"))
            records=[]
            for element in payload.get("elements",[]):
                tags=element.get("tags",{}); name=tags.get("name","")
                if not name or not any(term in name.lower() or term in str(tags).lower() for term in variants): continue
                records.append(normalize_record({"name":name,"website":tags.get("website") or tags.get("contact:website", ""),"phone":tags.get("phone") or tags.get("contact:phone", ""),"email":tags.get("email") or tags.get("contact:email", ""),"location":", ".join(x for x in (tags.get("addr:street"),tags.get("addr:city"),tags.get("addr:postcode")) if x),"description":"OpenStreetMap public listing"}))
            return DiscoveryPage(records[:limit],metadata={"adapter":self.source_code,"provider":provider,"record_count":len(records)})
        if provider == "wikidata":
            sparql=query if query.lower().startswith("select") else 'SELECT ?item ?itemLabel ?website WHERE {?item rdfs:label ?itemLabel. FILTER(CONTAINS(LCASE(?itemLabel), LCASE("%s"))). OPTIONAL {?item wdt:P856 ?website} FILTER(LANG(?itemLabel)="en")} LIMIT %d'%(query.replace('"',''),limit)
            url="https://query.wikidata.org/sparql?format=json&"+urlencode({"query":sparql})
            payload, _ = _HttpJsonSource()._get_json(url); records=[normalize_record({"name":x.get("itemLabel",{}).get("value",""),"website":x.get("website",{}).get("value","")}) for x in payload.get("results",{}).get("bindings",[])]
            return DiscoveryPage(records[:limit],metadata={"adapter":self.source_code,"provider":provider,"record_count":len(records)})
        index="https://index.commoncrawl.org/CC-MAIN-2026-30-index?url="+query+"&output=json&filter=status:200&collapse=urlkey"
        payload, _ = _HttpJsonSource()._get_json(index); records=[normalize_record({"name":str(x.get("url","")).split('/')[2] if '://' in str(x.get("url","")) else x.get("url", ""),"website":x.get("url","")}) for x in (payload if isinstance(payload,list) else [])]
        return DiscoveryPage(records[:limit],metadata={"adapter":self.source_code,"provider":provider,"record_count":len(records)})

class GoogleBrowserSearchBlocked(RuntimeError):
    """Fail-closed result for a disabled, rate-limited, or Google-blocked fetch."""

    code = "GOOGLE_BROWSER_BLOCKED"

    def __init__(self, reason: str):
        self.reason = reason
        super().__init__(f"{self.code}:{reason}")

    def as_dict(self) -> dict[str, str]:
        return {"code": self.code, "reason": self.reason}


class _GoogleVisibleResults(HTMLParser):
    """Extract only human-visible heading links from public result HTML."""

    def __init__(self):
        super().__init__(convert_charrefs=True)
        self._href = ""
        self._depth = 0
        self._parts: list[str] = []
        self.results: list[tuple[str, str]] = []

    def handle_starttag(self, tag, attrs):
        attributes = dict(attrs)
        if tag == "a" and not self._href:
            self._href = str(attributes.get("href") or "")
        if tag == "h3" and self._href:
            self._depth = 1
            self._parts = []
        elif self._depth:
            self._depth += 1

    def handle_data(self, data):
        if self._depth:
            self._parts.append(data)

    def handle_endtag(self, tag):
        if not self._depth:
            if tag == "a":
                self._href = ""
            return
        self._depth -= 1
        if tag != "h3" or self._depth:
            return
        title = " ".join("".join(self._parts).split())[:300]
        href = self._href
        self._href = ""
        self._parts = []
        if title and href:
            self.results.append((title, href))


class GoogleBrowserSearchSource(GatedSource):
    """Experimental, feature-flagged public Google result-page connector.

    This connector only requests the public result HTML.  It does not use a
    browser profile, JavaScript execution, login, proxy, CAPTCHA solver, or
    alternative endpoint when Google blocks access.
    """

    kind = source_code = "google_browser_search"
    display_name = "Google Browser Search (experimental)"
    available = True
    optional = True
    requires_credentials = False
    _last_request_at: float | None = None
    _rate_lock = threading.Lock()
    max_response_bytes = 512 * 1024
    timeout = 10
    max_results = 10

    def validate_config(self, config):
        result = super().validate_config(config)
        if not result.valid:
            return result
        rate = config.get("rate_limit")
        if not isinstance(rate, int) or isinstance(rate, bool) or not 1 <= rate <= 12:
            return ValidationResult(False, ["rate_limit must be an integer from 1 to 12 requests per minute"])
        return ValidationResult(True)

    @staticmethod
    def _query(criteria: Mapping[str, Any]) -> str:
        if not isinstance(criteria, Mapping):
            raise ValueError("criteria must be an object")
        values: list[str] = []
        keywords = criteria.get("keywords", criteria.get("keyword", criteria.get("query", "")))
        if isinstance(keywords, str):
            keywords = [keywords]
        if isinstance(keywords, Sequence) and not isinstance(keywords, (bytes, bytearray, str)):
            values.extend(str(value).strip() for value in keywords[:10] if str(value).strip())
        for key in ("category", "industry", "city", "location", "province", "country"):
            value = str(criteria.get(key, "")).strip()
            if value:
                values.append(value)
        query = " ".join(values)
        if not query or len(query) > 500 or any(char in query for char in "\r\n"):
            raise ValueError("bounded discovery criteria are required")
        return query

    @staticmethod
    def _blocked(html: str) -> bool:
        lowered = html.lower()
        markers = ("our systems have detected unusual traffic", "recaptcha", "captcha", "automated queries", "access denied", "sorry...")
        return any(marker in lowered for marker in markers)

    def discover(self, config, cursor=None, criteria=None, limits=None):
        result = self.validate_config(config)
        if not result.valid:
            raise ValueError(result.errors[0])
        if os.environ.get("GOOGLE_BROWSER_SEARCH_ENABLED", "").strip().lower() != "true":
            raise GoogleBrowserSearchBlocked("feature_disabled")
        query = self._query(criteria or {})
        limits = limits if isinstance(limits, Mapping) else {}
        try:
            requested = int(limits.get("per_run_limit", limits.get("max_records", self.max_results)))
        except (TypeError, ValueError):
            raise ValueError("invalid_limits")
        count = max(1, min(self.max_results, requested))
        interval = 60.0 / int(config["rate_limit"])
        with self._rate_lock:
            now = time.monotonic()
            if self._last_request_at is not None and now - self._last_request_at < interval:
                raise GoogleBrowserSearchBlocked("rate_limited")
            self.__class__._last_request_at = now
        url = "https://www.google.com/search?" + urlencode({"q": query, "num": count, "hl": str(criteria.get("language", "en"))[:12] or "en"})
        request = Request(url, headers={"User-Agent": "ProspectPlatform/0.1 public-search (no-login; experimental)", "Accept": "text/html,application/xhtml+xml"})
        try:
            with urlopen(request, timeout=self.timeout) as response:
                html = response.read(self.max_response_bytes + 1)
        except Exception as exc:
            raise GoogleBrowserSearchBlocked("access_denied") from exc
        if len(html) > self.max_response_bytes:
            raise GoogleBrowserSearchBlocked("response_too_large")
        text = html.decode("utf-8", "replace")
        if self._blocked(text):
            raise GoogleBrowserSearchBlocked("google_challenge_or_denial")
        parser = _GoogleVisibleResults()
        parser.feed(text)
        records, seen = [], set()
        for title, href in parser.results:
            parsed = urlparse(href)
            host = (parsed.hostname or "").lower()
            if parsed.scheme not in {"http", "https"} or not host or host.endswith("google.com") or href in seen:
                continue
            seen.add(href)
            records.append(normalize_record({"name": title, "website": href, "description": "Public Google search result"}))
            if len(records) >= count:
                break
        return DiscoveryPage(records, metadata={"adapter": self.source_code, "experimental": True, "public_html_only": True, "record_count": len(records), "query": query})


class GooglePlacesSource(GatedSource):
    kind = source_code = "google_places"
    display_name = "Google Places"
    available = True

    def validate_config(self, config):
        result = super().validate_config(config)
        if not result.valid: return result
        if not str(config.get("query", "")).strip(): return ValidationResult(False, ["query is required"])
        return ValidationResult(True)

    def discover(self, config, cursor=None, criteria=None, limits=None):
        result = self.validate_config(config)
        if not result.valid: raise ValueError(result.errors[0])
        api_key = str(config.get("_api_key", "")).strip()
        if not api_key: raise ValueError("Google Places API key is not configured")
        body = {"textQuery": str(config["query"]).strip(), "pageSize": max(1, min(20, int(config.get("max_records", 20))))}
        if config.get("region_code"): body["regionCode"] = str(config["region_code"]).upper()[:2]
        request = Request("https://places.googleapis.com/v1/places:searchText", data=json.dumps(body).encode(), method="POST", headers={"Content-Type":"application/json", "X-Goog-Api-Key":api_key, "X-Goog-FieldMask":"places.displayName,places.websiteUri,places.nationalPhoneNumber,places.internationalPhoneNumber,places.formattedAddress,places.googleMapsUri"})
        with urlopen(request, timeout=12) as response:
            payload=json.loads(response.read(2*1024*1024).decode("utf-8", "replace"))
        records=[]
        for place in payload.get("places", [])[:20]:
            name=(place.get("displayName") or {}).get("text", "")
            records.append(normalize_record({"name":name, "website":place.get("websiteUri", ""), "phone":place.get("nationalPhoneNumber") or place.get("internationalPhoneNumber", ""), "location":place.get("formattedAddress", ""), "source_url":place.get("googleMapsUri", "") , "description":"Google Places result"}))
        return DiscoveryPage(records, metadata={"adapter":self.source_code,"record_count":len(records),"provider":"google_places"})

class OpenStreetMapSource(ApprovedDirectorySource):
    kind = source_code = "openstreetmap"
    display_name = "OpenStreetMap / Overpass"
    optional = False
    def discover(self, config, cursor=None, criteria=None, limits=None):
        page = super().discover({**dict(config), "provider": "openstreetmap"}, cursor, criteria=criteria, limits=limits)
        return DiscoveryPage(page.records, page.next_cursor, {**page.metadata, "adapter": self.source_code})

class WikidataSource(ApprovedDirectorySource):
    kind = source_code = "wikidata"
    display_name = "Wikidata"
    optional = False
    def discover(self, config, cursor=None, criteria=None, limits=None):
        page = super().discover({**dict(config), "provider": "wikidata"}, cursor, criteria=criteria, limits=limits)
        return DiscoveryPage(page.records, page.next_cursor, {**page.metadata, "adapter": self.source_code})

class CommonCrawlSource(ApprovedDirectorySource):
    kind = source_code = "common_crawl"
    display_name = "Common Crawl index"
    optional = False
    def discover(self, config, cursor=None, criteria=None, limits=None):
        page = super().discover({**dict(config), "provider": "common_crawl"}, cursor, criteria=criteria, limits=limits)
        return DiscoveryPage(page.records, page.next_cursor, {**page.metadata, "adapter": self.source_code})

def _gated(code, name):
    return type(name.replace(" ", ""), (GatedSource,), {"kind":code, "source_code":code, "display_name":name})
BingLocalSource = _gated("bing_local", "Bing / approved local API")
PermittedSocialSource = _gated("permitted_social", "Permitted social")

ADAPTERS = {x.source_code: x for x in (ManualSource, CsvSource, GooglePlacesSource, GoogleBrowserSearchSource, BingLocalSource, ApprovedDirectorySource, OpenStreetMapSource, WikidataSource, CommonCrawlSource, PublicWebsiteSource, PermittedSocialSource, CtLogsSource, DnsSource, RdapSource)}
# common aliases used by clients
ADAPTER_REGISTRY = ADAPTERS

def adapter_for(kind: str) -> DiscoverySource:
    try: return ADAPTERS[str(kind).strip().lower()]()
    except KeyError: raise ValueError("unsupported source kind")

def available_adapters() -> list[dict[str, object]]:
    return [{"source_code": cls.source_code, "display_name": cls.display_name,
             "available": bool(getattr(cls, "available", False)),
             "optional": bool(getattr(cls, "optional", False)),
             "requires_credentials": bool(getattr(cls, "requires_credentials", False))}
            for cls in ADAPTERS.values() if cls.source_code != "approved_directory"]