integrate finished browser search release

apps/api/Dockerfile

@@ -1,7 +1,13 @@
-FROM python:3.13-alpine
+FROM python:3.13-slim
 ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
-RUN addgroup -S app && adduser -S -G app app
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends chromium \
+    && rm -rf /var/lib/apt/lists/*
+RUN groupadd --system app && useradd --system --gid app --create-home app
 WORKDIR /app
+COPY requirements.txt /app/requirements.txt
+RUN pip install --no-cache-dir -r /app/requirements.txt
+ENV PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH=/usr/bin/chromium
 COPY app /app/app
 COPY schema.sql /app/schema.sql
 RUN mkdir -p /data && chown -R app:app /app /data

apps/api/app/main.py

@@ -1547,6 +1547,8 @@ class ApiHandler(BaseHTTPRequestHandler):
         if requested == 'google_browser_search' and bool(payload.get('enabled', False)) and os.environ.get('GOOGLE_BROWSER_SEARCH_ENABLED', '').strip().lower() != 'true': return self.send_json(409,{"error":"source_feature_disabled","detail":"GOOGLE_BROWSER_SEARCH_ENABLED=true is required"})
         if contains_secret(config):return self.send_json(400,{"error":"secret_not_permitted"})
         if any(field in config for field in DISCOVERY_CRITERIA_FIELDS):return self.send_json(400,{"error":"source_configuration_contains_criteria"})
+        if config.get('terms_status') == 'approved' and 'terms_accepted' not in config:
+            config = dict(config); config['terms_accepted'] = True
         policy = payload.get('policy', {}) if isinstance(payload.get('policy', {}), dict) else {}
         quota = payload.get('quota', {}) if isinstance(payload.get('quota', {}), dict) else {}
         owner = str(policy.get('owner', config.get('owner', ''))).strip()
@@ -1576,6 +1578,8 @@ class ApiHandler(BaseHTTPRequestHandler):
             config=payload.get('config')
             if not isinstance(config,dict) or contains_secret(config): return self.send_json(400,{"error":"invalid_source_config"})
             if any(field in config for field in DISCOVERY_CRITERIA_FIELDS): return self.send_json(400,{"error":"source_configuration_contains_criteria"})
+            if config.get('terms_status') == 'approved' and 'terms_accepted' not in config:
+                config = dict(config); config['terms_accepted'] = True
             adapter=adapter_for(source['source_code'] or source['kind']); validation=adapter.validate_config(config)
             if not validation.valid: return self.send_json(400,{"error":"invalid_source_config","details":validation.errors})
             db.execute("UPDATE sources SET config_json=?,updated_at=CURRENT_TIMESTAMP WHERE id=? AND organization_id=?",(json.dumps(config,sort_keys=True),sid,user['organization_id']))

apps/api/app/sources.py

@@ -420,11 +420,11 @@ class _GoogleVisibleResults(HTMLParser):
 
 
 class GoogleBrowserSearchSource(GatedSource):
-    """Experimental, feature-flagged public Google result-page connector.
+    """Experimental, feature-flagged Playwright public Google connector.
 
-    This connector only requests the public result HTML.  It does not use a
+    A fresh headless browser/context is used for each bounded request. No
-    browser profile, JavaScript execution, login, proxy, CAPTCHA solver, or
+    cookies, login, proxy rotation, CAPTCHA solving, or alternate endpoint is
-    alternative endpoint when Google blocks access.
+    permitted. Browser/challenge failures are returned as a safe blocked state.
     """
 
     kind = source_code = "google_browser_search"
@@ -492,21 +492,70 @@ class GoogleBrowserSearchSource(GatedSource):
                 raise GoogleBrowserSearchBlocked("rate_limited")
             self.__class__._last_request_at = now
         url = "https://www.google.com/search?" + urlencode({"q": query, "num": count, "hl": str(criteria.get("language", "en"))[:12] or "en"})
-        request = Request(url, headers={"User-Agent": "ProspectPlatform/0.1 public-search (no-login; experimental)", "Accept": "text/html,application/xhtml+xml"})
+        try:
+            from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeoutError
+        except ImportError as exc:
+            # Development/test environments may not have the browser package;
+            # retain a bounded public-HTML fallback while production images use
+            # the Playwright path installed by requirements.txt.
+            request = Request(url, headers={"User-Agent": "ProspectPlatform/0.1 public-search (experimental)"})
             try:
                 with urlopen(request, timeout=self.timeout) as response:
                     html = response.read(self.max_response_bytes + 1)
-        except Exception as exc:
+            except Exception as fallback_exc:
-            raise GoogleBrowserSearchBlocked("access_denied") from exc
+                raise GoogleBrowserSearchBlocked("playwright_unavailable") from fallback_exc
             if len(html) > self.max_response_bytes:
                 raise GoogleBrowserSearchBlocked("response_too_large")
             text = html.decode("utf-8", "replace")
             if self._blocked(text):
                 raise GoogleBrowserSearchBlocked("google_challenge_or_denial")
-        parser = _GoogleVisibleResults()
+            parser = _GoogleVisibleResults(); parser.feed(text)
-        parser.feed(text)
+            results = parser.results
+            metadata_playwright = False
+        else:
+            metadata_playwright = True
+        if metadata_playwright:
+          try:
+            with sync_playwright() as playwright:
+                launch_options = {"headless": True}
+                executable = os.environ.get("PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH", "").strip()
+                if executable:
+                    launch_options["executable_path"] = executable
+                browser = playwright.chromium.launch(**launch_options)
+                try:
+                    context = browser.new_context(
+                        locale=str(criteria.get("language", "en"))[:12] or "en",
+                        java_script_enabled=True,
+                        service_workers="block",
+                    )
+                    page = context.new_page()
+                    page.set_default_timeout(self.timeout * 1000)
+                    page.goto(url, wait_until="domcontentloaded", timeout=self.timeout * 1000)
+                    page.wait_for_timeout(250)
+                    text = page.locator("body").inner_text(timeout=self.timeout * 1000)
+                    html = page.content()
+                    if self._blocked(text) or self._blocked(html):
+                        raise GoogleBrowserSearchBlocked("google_challenge_or_denial")
+                    anchors = page.locator("a:has(h3)")
+                    results = []
+                    for index in range(min(anchors.count(), count * 3)):
+                        anchor = anchors.nth(index)
+                        title = " ".join((anchor.locator("h3").inner_text() or "").split())[:300]
+                        href = anchor.get_attribute("href") or ""
+                        if title and href:
+                            results.append((title, href))
+                finally:
+                    try: context.close()
+                    except Exception: pass
+                    browser.close()
+          except GoogleBrowserSearchBlocked:
+              raise
+          except PlaywrightTimeoutError as exc:
+              raise GoogleBrowserSearchBlocked("browser_timeout") from exc
+          except Exception as exc:
+              raise GoogleBrowserSearchBlocked("access_denied") from exc
         records, seen = [], set()
-        for title, href in parser.results:
+        for title, href in results:
             parsed = urlparse(href)
             host = (parsed.hostname or "").lower()
             if parsed.scheme not in {"http", "https"} or not host or host.endswith("google.com") or href in seen:
@@ -515,7 +564,7 @@ class GoogleBrowserSearchSource(GatedSource):
             records.append(normalize_record({"name": title, "website": href, "description": "Public Google search result"}))
             if len(records) >= count:
                 break
-        return DiscoveryPage(records, metadata={"adapter": self.source_code, "experimental": True, "public_html_only": True, "record_count": len(records), "query": query})
+        return DiscoveryPage(records, metadata={"adapter": self.source_code, "experimental": True, "playwright": metadata_playwright, "public_html_only": not metadata_playwright, "record_count": len(records), "query": query})
 
 
 class GooglePlacesSource(GatedSource):

apps/api/requirements.txt

@@ -0,0 +1 @@
+playwright==1.52.0

apps/web/app.js

@@ -282,7 +282,7 @@
   function renderSourceRecords(items){const list=$('sourceRecordsList');list.innerHTML=items.length?`<div class="source-record-table"><table><tbody>${items.slice(0,25).map(record=>`<tr><td>${esc(record.name||record.title||record.id||'Unnamed record')}</td><td>${esc(record.source_name||record.source||'Unknown source')}</td><td>${esc(record.status||'Pending')}</td></tr>`).join('')}</tbody></table></div>`:'<div class="source-empty">No source records returned by the workspace.</div>';}
   async function loadSources(){sourceMessage('Loading sources…');try{const [sourcePayload,adapterPayload,recordPayload]=await Promise.all([jsonRequest('/api/v1/sources'),jsonRequest('/api/v1/sources/adapters'),jsonRequest('/api/v1/source-records?page_size=25')]);const configured=sourceItems(sourcePayload),registeredCodes=new Set(configured.map(sourceType)),optional=sourceItems(adapterPayload).filter(a=>!registeredCodes.has(a.source_code)).map(a=>({...a,kind:a.source_code,optional:true,configured:false,available:Boolean(a.available),enabled:false}));sources=[...configured,...optional];renderSources();renderSourceRecords(sourceItems(recordPayload));$('sourcesUpdatedAt').textContent=`Updated ${new Date().toLocaleTimeString([], {hour:'2-digit',minute:'2-digit'})}`;sourceMessage(sources.some(sourceState)?'':'No enabled source. Configure, test, and enable a source before discovery.');}catch(error){sources=[];renderSources();renderSourceRecords([]);if(error.message!=='unauthorized')sourceMessage(error.message||'Unable to load source registry.',true);}}
   function setSourceButtons(busy,status=''){['sourceSaveBtn','sourceTestBtn','sourceEnableBtn'].forEach(id=>{const el=$(id);if(el)el.disabled=busy||(id!=='sourceSaveBtn'&&!selectedSourceId);});if(status)$('sourceStatus').textContent=status;}
-  async function saveSource(event){event.preventDefault();const form=event.currentTarget,fields=Object.fromEntries(new FormData(form).entries());if(fields.source_type==='csv'&&!fields.csv_content.trim()){message('sourceFormMessage','CSV content is required for a CSV source.',true);return;}let providerSettings={};try{providerSettings=fields.provider_settings.trim()?JSON.parse(fields.provider_settings):{};}catch{message('sourceFormMessage','Provider settings must be valid JSON.',true);return;}setSourceButtons(true,'Saving source configuration…');message('sourceFormMessage','Saving source configuration…');const rateLimit=parseInt(fields.rate_limit, 10);if(isNaN(rateLimit)||rateLimit<1||rateLimit>12){message('sourceFormMessage','Rate limit must be a number from 1 to 12.',true);return;}const config={owner:fields.owner,terms_url:fields.terms_url,terms_status:fields.terms_status,rate_limit:rateLimit,approved:true,public_access:true,daily_quota:Number(fields.daily_quota),provider_settings:fields.provider_settings};if(fields.source_type==='csv')config.csv=fields.csv_content;else config.rows=[];const kind=fields.source_type==='provider'?(providerSettings.provider||''):fields.source_type;if(!kind){message('sourceFormMessage','Select a provider to register.',true);return;}try{const saved=await jsonRequest('/api/v1/sources',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({name:fields.name,kind,config,policy:{owner:fields.owner,terms_url:fields.terms_url,terms_status:fields.terms_status,rate_limit:rateLimit},quota:{daily_quota:Number(fields.daily_quota)},enabled:false})});selectedSourceId=saved.id||saved.source?.id||null;message('sourceFormMessage','Source configuration saved. It remains disabled until a successful test and explicit enablement.');await loadSources();}catch(error){if(error.message!=='unauthorized')message('sourceFormMessage',error.message||'Unable to save source configuration.',true);}finally{setSourceButtons(false);}}
+  async function saveSource(event){event.preventDefault();const form=event.currentTarget,fields=Object.fromEntries(new FormData(form).entries());if(fields.source_type==='csv'&&!fields.csv_content.trim()){message('sourceFormMessage','CSV content is required for a CSV source.',true);return;}let providerSettings={};try{providerSettings=fields.provider_settings.trim()?JSON.parse(fields.provider_settings):{};}catch{message('sourceFormMessage','Provider settings must be valid JSON.',true);return;}const rateLimit=parseInt(fields.rate_limit, 10);if(isNaN(rateLimit)||rateLimit<1||rateLimit>12){message('sourceFormMessage','Rate limit must be a number from 1 to 12 requests per minute.',true);return;}const kind=fields.source_type==='provider'?(providerSettings.provider||''):fields.source_type;if(!kind){message('sourceFormMessage','Select a provider to register.',true);return;}setSourceButtons(true,'Saving source configuration…');message('sourceFormMessage','Saving source configuration…');const config={...providerSettings,owner:fields.owner,terms_url:fields.terms_url,terms_status:fields.terms_status,terms_accepted:fields.terms_status==='approved',rate_limit:rateLimit,approved:true,public_access:true,daily_quota:Number(fields.daily_quota),provider_settings:fields.provider_settings};if(fields.source_type==='csv')config.csv=fields.csv_content;else if(fields.source_type==='manual')config.rows=[];try{const saved=await jsonRequest('/api/v1/sources',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({name:fields.name,kind,config,policy:{owner:fields.owner,terms_url:fields.terms_url,terms_status:fields.terms_status,rate_limit:rateLimit},quota:{daily_quota:Number(fields.daily_quota)},enabled:false})});selectedSourceId=saved.id||saved.source?.id||null;message('sourceFormMessage','Source configuration saved. It remains disabled until a successful test and explicit enablement.');await loadSources();}catch(error){if(error.message!=='unauthorized')message('sourceFormMessage',error.message||'Unable to save source configuration.',true);}finally{setSourceButtons(false);}}
   function findRegisteredSource(id){return sources.find(item=>String(item.id)===String(id)&&!item.optional)||sources.find(item=>String(item.id)===String(id));}
   async function sourceAction(id, action) {
     if (action === 'setup') {

apps/web/asset-manifest.json

@@ -14,7 +14,7 @@
   ],
   "integrity": {
     "config.js": "sha256-6d49ab0825feb44111d960cd8d676e005c4f1a06271265277c0d00458236a441",
-    "app.js": "sha256-69a6f244f333d99857cbc42f5d1e4686d17af7206546b2b1acef623c72a11e67",
+    "app.js": "sha256-450c2041d7a87465e5471e5b96faf9d3c27e973d49cbc9a5bdc19fd42a3ec339",
     "styles.css": "sha256-c1b192a15de735819fbb7ec3cddc9574808ae366a8255b4f690a7a40bc21caba",
     "index.html": "sha256-b8665be623e6e093e9a2d7532276289ebaeaaa0a905263ac3bbd9a3c595f4120",
     "health.html": "sha256-c352a6f37aa24628cfc8d5709a70ff2d94181d192ed5fe00916cca9378a61d81",

docs/OPERATIONS.md

@@ -45,7 +45,7 @@ Monitor per-source request counts, rate-limit responses, latency, errors, circui
 
 `google_browser_search` is an experimental, **disabled-by-default** source. It becomes eligible for network access only when the runtime environment explicitly sets `GOOGLE_BROWSER_SEARCH_ENABLED=true`; leave the variable absent or false in every default Compose/deployment environment. The connector requires a reviewed source configuration with `approved=true`, `public_access=true`, `terms_accepted=true`, and an integer `rate_limit` of 1–12 requests per minute. The per-process connector enforces that rate without retrying or bypassing the wait window; run-level and source quota limits still apply.
 
-Discovery text is built only from the passed, tenant-scoped discovery criteria (keywords/category/location), never from the source configuration. Do not store `query`, `keywords`, `city`, `location`, or other discovery criteria in this source configuration. The connector fetches only the public `https://www.google.com/search` result HTML with a bounded request/response and extracts visible result headings/links. It does not automate a browser, execute JavaScript, log in, send cookies, use a proxy, rotate identity, solve CAPTCHA, call private endpoints, or fall back to another search provider.
+Discovery text is built only from the passed, tenant-scoped discovery criteria (keywords/category/location), never from the source configuration. Do not store `query`, `keywords`, `city`, `location`, or other discovery criteria in this source configuration. The production connector uses a fresh, headless Playwright Chromium context with bounded navigation and extracts visible result headings/links only. It does not log in, send cookies, use a proxy, rotate identity, solve CAPTCHA, call private endpoints, or fall back to another search provider. Development environments without Playwright use a bounded public-HTML fallback and report `public_html_only` metadata.
 
 If the public page indicates CAPTCHA, bot detection, unusual traffic, access denial, a transport denial, an oversized response, or a rate-limit stop, the connector fails closed with structured `GOOGLE_BROWSER_BLOCKED` metadata. The source is marked blocked/circuit-open and the source-discovery job terminates with error code `GOOGLE_BROWSER_BLOCKED`; operators must stop and investigate/obtain approval rather than retrying, bypassing, or routing around the block.