integrate finished browser search release

apps/api/app/sources.py

@@ -420,11 +420,11 @@ class _GoogleVisibleResults(HTMLParser):
 
 
 class GoogleBrowserSearchSource(GatedSource):
-    """Experimental, feature-flagged public Google result-page connector.
+    """Experimental, feature-flagged Playwright public Google connector.
 
-    This connector only requests the public result HTML.  It does not use a
-    browser profile, JavaScript execution, login, proxy, CAPTCHA solver, or
-    alternative endpoint when Google blocks access.
+    A fresh headless browser/context is used for each bounded request. No
+    cookies, login, proxy rotation, CAPTCHA solving, or alternate endpoint is
+    permitted. Browser/challenge failures are returned as a safe blocked state.
     """
 
     kind = source_code = "google_browser_search"
@@ -492,21 +492,70 @@ class GoogleBrowserSearchSource(GatedSource):
                 raise GoogleBrowserSearchBlocked("rate_limited")
             self.__class__._last_request_at = now
         url = "https://www.google.com/search?" + urlencode({"q": query, "num": count, "hl": str(criteria.get("language", "en"))[:12] or "en"})
-        request = Request(url, headers={"User-Agent": "ProspectPlatform/0.1 public-search (no-login; experimental)", "Accept": "text/html,application/xhtml+xml"})
         try:
-            with urlopen(request, timeout=self.timeout) as response:
-                html = response.read(self.max_response_bytes + 1)
-        except Exception as exc:
-            raise GoogleBrowserSearchBlocked("access_denied") from exc
-        if len(html) > self.max_response_bytes:
-            raise GoogleBrowserSearchBlocked("response_too_large")
-        text = html.decode("utf-8", "replace")
-        if self._blocked(text):
-            raise GoogleBrowserSearchBlocked("google_challenge_or_denial")
-        parser = _GoogleVisibleResults()
-        parser.feed(text)
+            from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeoutError
+        except ImportError as exc:
+            # Development/test environments may not have the browser package;
+            # retain a bounded public-HTML fallback while production images use
+            # the Playwright path installed by requirements.txt.
+            request = Request(url, headers={"User-Agent": "ProspectPlatform/0.1 public-search (experimental)"})
+            try:
+                with urlopen(request, timeout=self.timeout) as response:
+                    html = response.read(self.max_response_bytes + 1)
+            except Exception as fallback_exc:
+                raise GoogleBrowserSearchBlocked("playwright_unavailable") from fallback_exc
+            if len(html) > self.max_response_bytes:
+                raise GoogleBrowserSearchBlocked("response_too_large")
+            text = html.decode("utf-8", "replace")
+            if self._blocked(text):
+                raise GoogleBrowserSearchBlocked("google_challenge_or_denial")
+            parser = _GoogleVisibleResults(); parser.feed(text)
+            results = parser.results
+            metadata_playwright = False
+        else:
+            metadata_playwright = True
+        if metadata_playwright:
+          try:
+            with sync_playwright() as playwright:
+                launch_options = {"headless": True}
+                executable = os.environ.get("PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH", "").strip()
+                if executable:
+                    launch_options["executable_path"] = executable
+                browser = playwright.chromium.launch(**launch_options)
+                try:
+                    context = browser.new_context(
+                        locale=str(criteria.get("language", "en"))[:12] or "en",
+                        java_script_enabled=True,
+                        service_workers="block",
+                    )
+                    page = context.new_page()
+                    page.set_default_timeout(self.timeout * 1000)
+                    page.goto(url, wait_until="domcontentloaded", timeout=self.timeout * 1000)
+                    page.wait_for_timeout(250)
+                    text = page.locator("body").inner_text(timeout=self.timeout * 1000)
+                    html = page.content()
+                    if self._blocked(text) or self._blocked(html):
+                        raise GoogleBrowserSearchBlocked("google_challenge_or_denial")
+                    anchors = page.locator("a:has(h3)")
+                    results = []
+                    for index in range(min(anchors.count(), count * 3)):
+                        anchor = anchors.nth(index)
+                        title = " ".join((anchor.locator("h3").inner_text() or "").split())[:300]
+                        href = anchor.get_attribute("href") or ""
+                        if title and href:
+                            results.append((title, href))
+                finally:
+                    try: context.close()
+                    except Exception: pass
+                    browser.close()
+          except GoogleBrowserSearchBlocked:
+              raise
+          except PlaywrightTimeoutError as exc:
+              raise GoogleBrowserSearchBlocked("browser_timeout") from exc
+          except Exception as exc:
+              raise GoogleBrowserSearchBlocked("access_denied") from exc
         records, seen = [], set()
-        for title, href in parser.results:
+        for title, href in results:
             parsed = urlparse(href)
             host = (parsed.hostname or "").lower()
             if parsed.scheme not in {"http", "https"} or not host or host.endswith("google.com") or href in seen:
@@ -515,7 +564,7 @@ class GoogleBrowserSearchSource(GatedSource):
             records.append(normalize_record({"name": title, "website": href, "description": "Public Google search result"}))
             if len(records) >= count:
                 break
-        return DiscoveryPage(records, metadata={"adapter": self.source_code, "experimental": True, "public_html_only": True, "record_count": len(records), "query": query})
+        return DiscoveryPage(records, metadata={"adapter": self.source_code, "experimental": True, "playwright": metadata_playwright, "public_html_only": not metadata_playwright, "record_count": len(records), "query": query})
 
 
 class GooglePlacesSource(GatedSource):