integrate finished browser search release

apps/api/Dockerfile

@@ -1,7 +1,13 @@
-FROM python:3.13-alpine
+FROM python:3.13-slim
 ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
-RUN addgroup -S app && adduser -S -G app app
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends chromium \
+    && rm -rf /var/lib/apt/lists/*
+RUN groupadd --system app && useradd --system --gid app --create-home app
 WORKDIR /app
+COPY requirements.txt /app/requirements.txt
+RUN pip install --no-cache-dir -r /app/requirements.txt
+ENV PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH=/usr/bin/chromium
 COPY app /app/app
 COPY schema.sql /app/schema.sql
 RUN mkdir -p /data && chown -R app:app /app /data

apps/api/app/main.py

@@ -1547,6 +1547,8 @@ class ApiHandler(BaseHTTPRequestHandler):
         if requested == 'google_browser_search' and bool(payload.get('enabled', False)) and os.environ.get('GOOGLE_BROWSER_SEARCH_ENABLED', '').strip().lower() != 'true': return self.send_json(409,{"error":"source_feature_disabled","detail":"GOOGLE_BROWSER_SEARCH_ENABLED=true is required"})
         if contains_secret(config):return self.send_json(400,{"error":"secret_not_permitted"})
         if any(field in config for field in DISCOVERY_CRITERIA_FIELDS):return self.send_json(400,{"error":"source_configuration_contains_criteria"})
+        if config.get('terms_status') == 'approved' and 'terms_accepted' not in config:
+            config = dict(config); config['terms_accepted'] = True
         policy = payload.get('policy', {}) if isinstance(payload.get('policy', {}), dict) else {}
         quota = payload.get('quota', {}) if isinstance(payload.get('quota', {}), dict) else {}
         owner = str(policy.get('owner', config.get('owner', ''))).strip()
@@ -1576,6 +1578,8 @@ class ApiHandler(BaseHTTPRequestHandler):
             config=payload.get('config')
             if not isinstance(config,dict) or contains_secret(config): return self.send_json(400,{"error":"invalid_source_config"})
             if any(field in config for field in DISCOVERY_CRITERIA_FIELDS): return self.send_json(400,{"error":"source_configuration_contains_criteria"})
+            if config.get('terms_status') == 'approved' and 'terms_accepted' not in config:
+                config = dict(config); config['terms_accepted'] = True
             adapter=adapter_for(source['source_code'] or source['kind']); validation=adapter.validate_config(config)
             if not validation.valid: return self.send_json(400,{"error":"invalid_source_config","details":validation.errors})
             db.execute("UPDATE sources SET config_json=?,updated_at=CURRENT_TIMESTAMP WHERE id=? AND organization_id=?",(json.dumps(config,sort_keys=True),sid,user['organization_id']))

apps/api/app/sources.py

@@ -420,11 +420,11 @@ class _GoogleVisibleResults(HTMLParser):
 
 
 class GoogleBrowserSearchSource(GatedSource):
-    """Experimental, feature-flagged public Google result-page connector.
+    """Experimental, feature-flagged Playwright public Google connector.
 
-    This connector only requests the public result HTML.  It does not use a
-    browser profile, JavaScript execution, login, proxy, CAPTCHA solver, or
-    alternative endpoint when Google blocks access.
+    A fresh headless browser/context is used for each bounded request. No
+    cookies, login, proxy rotation, CAPTCHA solving, or alternate endpoint is
+    permitted. Browser/challenge failures are returned as a safe blocked state.
     """
 
     kind = source_code = "google_browser_search"
@@ -492,21 +492,70 @@ class GoogleBrowserSearchSource(GatedSource):
                 raise GoogleBrowserSearchBlocked("rate_limited")
             self.__class__._last_request_at = now
         url = "https://www.google.com/search?" + urlencode({"q": query, "num": count, "hl": str(criteria.get("language", "en"))[:12] or "en"})
-        request = Request(url, headers={"User-Agent": "ProspectPlatform/0.1 public-search (no-login; experimental)", "Accept": "text/html,application/xhtml+xml"})
         try:
-            with urlopen(request, timeout=self.timeout) as response:
-                html = response.read(self.max_response_bytes + 1)
-        except Exception as exc:
-            raise GoogleBrowserSearchBlocked("access_denied") from exc
-        if len(html) > self.max_response_bytes:
-            raise GoogleBrowserSearchBlocked("response_too_large")
-        text = html.decode("utf-8", "replace")
-        if self._blocked(text):
-            raise GoogleBrowserSearchBlocked("google_challenge_or_denial")
-        parser = _GoogleVisibleResults()
-        parser.feed(text)
+            from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeoutError
+        except ImportError as exc:
+            # Development/test environments may not have the browser package;
+            # retain a bounded public-HTML fallback while production images use
+            # the Playwright path installed by requirements.txt.
+            request = Request(url, headers={"User-Agent": "ProspectPlatform/0.1 public-search (experimental)"})
+            try:
+                with urlopen(request, timeout=self.timeout) as response:
+                    html = response.read(self.max_response_bytes + 1)
+            except Exception as fallback_exc:
+                raise GoogleBrowserSearchBlocked("playwright_unavailable") from fallback_exc
+            if len(html) > self.max_response_bytes:
+                raise GoogleBrowserSearchBlocked("response_too_large")
+            text = html.decode("utf-8", "replace")
+            if self._blocked(text):
+                raise GoogleBrowserSearchBlocked("google_challenge_or_denial")
+            parser = _GoogleVisibleResults(); parser.feed(text)
+            results = parser.results
+            metadata_playwright = False
+        else:
+            metadata_playwright = True
+        if metadata_playwright:
+          try:
+            with sync_playwright() as playwright:
+                launch_options = {"headless": True}
+                executable = os.environ.get("PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH", "").strip()
+                if executable:
+                    launch_options["executable_path"] = executable
+                browser = playwright.chromium.launch(**launch_options)
+                try:
+                    context = browser.new_context(
+                        locale=str(criteria.get("language", "en"))[:12] or "en",
+                        java_script_enabled=True,
+                        service_workers="block",
+                    )
+                    page = context.new_page()
+                    page.set_default_timeout(self.timeout * 1000)
+                    page.goto(url, wait_until="domcontentloaded", timeout=self.timeout * 1000)
+                    page.wait_for_timeout(250)
+                    text = page.locator("body").inner_text(timeout=self.timeout * 1000)
+                    html = page.content()
+                    if self._blocked(text) or self._blocked(html):
+                        raise GoogleBrowserSearchBlocked("google_challenge_or_denial")
+                    anchors = page.locator("a:has(h3)")
+                    results = []
+                    for index in range(min(anchors.count(), count * 3)):
+                        anchor = anchors.nth(index)
+                        title = " ".join((anchor.locator("h3").inner_text() or "").split())[:300]
+                        href = anchor.get_attribute("href") or ""
+                        if title and href:
+                            results.append((title, href))
+                finally:
+                    try: context.close()
+                    except Exception: pass
+                    browser.close()
+          except GoogleBrowserSearchBlocked:
+              raise
+          except PlaywrightTimeoutError as exc:
+              raise GoogleBrowserSearchBlocked("browser_timeout") from exc
+          except Exception as exc:
+              raise GoogleBrowserSearchBlocked("access_denied") from exc
         records, seen = [], set()
-        for title, href in parser.results:
+        for title, href in results:
             parsed = urlparse(href)
             host = (parsed.hostname or "").lower()
             if parsed.scheme not in {"http", "https"} or not host or host.endswith("google.com") or href in seen:
@@ -515,7 +564,7 @@ class GoogleBrowserSearchSource(GatedSource):
             records.append(normalize_record({"name": title, "website": href, "description": "Public Google search result"}))
             if len(records) >= count:
                 break
-        return DiscoveryPage(records, metadata={"adapter": self.source_code, "experimental": True, "public_html_only": True, "record_count": len(records), "query": query})
+        return DiscoveryPage(records, metadata={"adapter": self.source_code, "experimental": True, "playwright": metadata_playwright, "public_html_only": not metadata_playwright, "record_count": len(records), "query": query})
 
 
 class GooglePlacesSource(GatedSource):

apps/api/requirements.txt

@@ -0,0 +1 @@
+playwright==1.52.0