rebuild source discovery workflows

2026-09-04 20:52:37 +02:00
parent 3a9b553440
commit 6aefbdc1f3
18 changed files with 853 additions and 106 deletions
@@ -7,7 +7,8 @@ Adapters never emit or persist credential values.
 from __future__ import annotations
 from dataclasses import dataclass, field
 from typing import Any, Mapping, Protocol, Sequence
-import csv, io, random, time, json, re
+from html.parser import HTMLParser
+import csv, io, os, random, threading, time, json, re
 from urllib.parse import urlencode, urlparse
 from urllib.request import Request, urlopen

@@ -328,9 +329,22 @@ class ApprovedDirectorySource(GatedSource):
    def discover(self, config, cursor=None, criteria=None, limits=None):
        result=self.validate_config(config)
        if not result.valid: raise ValueError(result.errors[0])
-        provider=str(config["provider"]).lower(); query=str(config["query"]).strip(); limit=max(1,min(100,int(config.get("max_records",50))))
+        criteria = criteria if isinstance(criteria, Mapping) else {}
+        keyword_values = criteria.get("keywords", criteria.get("keyword", criteria.get("query", "")))
+        if isinstance(keyword_values, str): keyword_values = [keyword_values]
+        terms = [str(value).strip() for value in keyword_values[:10] if str(value).strip()] if isinstance(keyword_values, Sequence) and not isinstance(keyword_values, (bytes, bytearray, str)) else []
+        for key in ("category", "industry"):
+            value = str(criteria.get(key, "")).strip()
+            if value: terms.append(value)
+        query = " ".join(dict.fromkeys(terms))[:500]
+        if not query: raise ValueError("discovery_criteria_required")
+        location_parts = [str(criteria.get(key, "")).strip() for key in ("city", "location", "province", "country")]
+        area = next((value for value in location_parts if value), "South Africa")
+        limit_source = limits if isinstance(limits, Mapping) else {}
+        try: limit=max(1,min(100,int(limit_source.get("per_run_limit", limit_source.get("max_records",50)))))
+        except (TypeError, ValueError): raise ValueError("invalid_limits")
+        provider=str(config["provider"]).lower()
        if provider == "openstreetmap":
-            area=str(config.get("location", "South Africa")).strip()
            terms=[token.lower() for token in re.findall(r"[A-Za-z0-9]{2,32}",query)[:5]]
            variants=sorted({variant for term in terms for variant in (term,term[:-1] if term.endswith('s') and len(term)>3 else term)})
            pattern="|".join(re.escape(term) for term in variants)
@@ -352,6 +366,158 @@ class ApprovedDirectorySource(GatedSource):
        payload, _ = _HttpJsonSource()._get_json(index); records=[normalize_record({"name":str(x.get("url","")).split('/')[2] if '://' in str(x.get("url","")) else x.get("url", ""),"website":x.get("url","")}) for x in (payload if isinstance(payload,list) else [])]
        return DiscoveryPage(records[:limit],metadata={"adapter":self.source_code,"provider":provider,"record_count":len(records)})

+class GoogleBrowserSearchBlocked(RuntimeError):
+    """Fail-closed result for a disabled, rate-limited, or Google-blocked fetch."""
+
+    code = "GOOGLE_BROWSER_BLOCKED"
+
+    def __init__(self, reason: str):
+        self.reason = reason
+        super().__init__(f"{self.code}:{reason}")
+
+    def as_dict(self) -> dict[str, str]:
+        return {"code": self.code, "reason": self.reason}
+
+
+class _GoogleVisibleResults(HTMLParser):
+    """Extract only human-visible heading links from public result HTML."""
+
+    def __init__(self):
+        super().__init__(convert_charrefs=True)
+        self._href = ""
+        self._depth = 0
+        self._parts: list[str] = []
+        self.results: list[tuple[str, str]] = []
+
+    def handle_starttag(self, tag, attrs):
+        attributes = dict(attrs)
+        if tag == "a" and not self._href:
+            self._href = str(attributes.get("href") or "")
+        if tag == "h3" and self._href:
+            self._depth = 1
+            self._parts = []
+        elif self._depth:
+            self._depth += 1
+
+    def handle_data(self, data):
+        if self._depth:
+            self._parts.append(data)
+
+    def handle_endtag(self, tag):
+        if not self._depth:
+            if tag == "a":
+                self._href = ""
+            return
+        self._depth -= 1
+        if tag != "h3" or self._depth:
+            return
+        title = " ".join("".join(self._parts).split())[:300]
+        href = self._href
+        self._href = ""
+        self._parts = []
+        if title and href:
+            self.results.append((title, href))
+
+
+class GoogleBrowserSearchSource(GatedSource):
+    """Experimental, feature-flagged public Google result-page connector.
+
+    This connector only requests the public result HTML.  It does not use a
+    browser profile, JavaScript execution, login, proxy, CAPTCHA solver, or
+    alternative endpoint when Google blocks access.
+    """
+
+    kind = source_code = "google_browser_search"
+    display_name = "Google Browser Search (experimental)"
+    available = True
+    optional = True
+    requires_credentials = False
+    _last_request_at: float | None = None
+    _rate_lock = threading.Lock()
+    max_response_bytes = 512 * 1024
+    timeout = 10
+    max_results = 10
+
+    def validate_config(self, config):
+        result = super().validate_config(config)
+        if not result.valid:
+            return result
+        rate = config.get("rate_limit")
+        if not isinstance(rate, int) or isinstance(rate, bool) or not 1 <= rate <= 12:
+            return ValidationResult(False, ["rate_limit must be an integer from 1 to 12 requests per minute"])
+        return ValidationResult(True)
+
+    @staticmethod
+    def _query(criteria: Mapping[str, Any]) -> str:
+        if not isinstance(criteria, Mapping):
+            raise ValueError("criteria must be an object")
+        values: list[str] = []
+        keywords = criteria.get("keywords", criteria.get("keyword", criteria.get("query", "")))
+        if isinstance(keywords, str):
+            keywords = [keywords]
+        if isinstance(keywords, Sequence) and not isinstance(keywords, (bytes, bytearray, str)):
+            values.extend(str(value).strip() for value in keywords[:10] if str(value).strip())
+        for key in ("category", "industry", "city", "location", "province", "country"):
+            value = str(criteria.get(key, "")).strip()
+            if value:
+                values.append(value)
+        query = " ".join(values)
+        if not query or len(query) > 500 or any(char in query for char in "\r\n"):
+            raise ValueError("bounded discovery criteria are required")
+        return query
+
+    @staticmethod
+    def _blocked(html: str) -> bool:
+        lowered = html.lower()
+        markers = ("our systems have detected unusual traffic", "recaptcha", "captcha", "automated queries", "access denied", "sorry...")
+        return any(marker in lowered for marker in markers)
+
+    def discover(self, config, cursor=None, criteria=None, limits=None):
+        result = self.validate_config(config)
+        if not result.valid:
+            raise ValueError(result.errors[0])
+        if os.environ.get("GOOGLE_BROWSER_SEARCH_ENABLED", "").strip().lower() != "true":
+            raise GoogleBrowserSearchBlocked("feature_disabled")
+        query = self._query(criteria or {})
+        limits = limits if isinstance(limits, Mapping) else {}
+        try:
+            requested = int(limits.get("per_run_limit", limits.get("max_records", self.max_results)))
+        except (TypeError, ValueError):
+            raise ValueError("invalid_limits")
+        count = max(1, min(self.max_results, requested))
+        interval = 60.0 / int(config["rate_limit"])
+        with self._rate_lock:
+            now = time.monotonic()
+            if self._last_request_at is not None and now - self._last_request_at < interval:
+                raise GoogleBrowserSearchBlocked("rate_limited")
+            self.__class__._last_request_at = now
+        url = "https://www.google.com/search?" + urlencode({"q": query, "num": count, "hl": str(criteria.get("language", "en"))[:12] or "en"})
+        request = Request(url, headers={"User-Agent": "ProspectPlatform/0.1 public-search (no-login; experimental)", "Accept": "text/html,application/xhtml+xml"})
+        try:
+            with urlopen(request, timeout=self.timeout) as response:
+                html = response.read(self.max_response_bytes + 1)
+        except Exception as exc:
+            raise GoogleBrowserSearchBlocked("access_denied") from exc
+        if len(html) > self.max_response_bytes:
+            raise GoogleBrowserSearchBlocked("response_too_large")
+        text = html.decode("utf-8", "replace")
+        if self._blocked(text):
+            raise GoogleBrowserSearchBlocked("google_challenge_or_denial")
+        parser = _GoogleVisibleResults()
+        parser.feed(text)
+        records, seen = [], set()
+        for title, href in parser.results:
+            parsed = urlparse(href)
+            host = (parsed.hostname or "").lower()
+            if parsed.scheme not in {"http", "https"} or not host or host.endswith("google.com") or href in seen:
+                continue
+            seen.add(href)
+            records.append(normalize_record({"name": title, "website": href, "description": "Public Google search result"}))
+            if len(records) >= count:
+                break
+        return DiscoveryPage(records, metadata={"adapter": self.source_code, "experimental": True, "public_html_only": True, "record_count": len(records), "query": query})
+
+
 class GooglePlacesSource(GatedSource):
    kind = source_code = "google_places"
    display_name = "Google Places"
@@ -408,7 +574,7 @@ def _gated(code, name):
 BingLocalSource = _gated("bing_local", "Bing / approved local API")
 PermittedSocialSource = _gated("permitted_social", "Permitted social")

-ADAPTERS = {x.source_code: x for x in (ManualSource, CsvSource, GooglePlacesSource, BingLocalSource, ApprovedDirectorySource, OpenStreetMapSource, WikidataSource, CommonCrawlSource, PublicWebsiteSource, PermittedSocialSource, CtLogsSource, DnsSource, RdapSource)}
+ADAPTERS = {x.source_code: x for x in (ManualSource, CsvSource, GooglePlacesSource, GoogleBrowserSearchSource, BingLocalSource, ApprovedDirectorySource, OpenStreetMapSource, WikidataSource, CommonCrawlSource, PublicWebsiteSource, PermittedSocialSource, CtLogsSource, DnsSource, RdapSource)}
 # common aliases used by clients
 ADAPTER_REGISTRY = ADAPTERS