integrate finished browser search release

2026-09-05 10:45:52 +02:00
parent 0a1de45586
commit d4f6da80e1
7 changed files with 83 additions and 23 deletions
@@ -1,7 +1,13 @@
-FROM python:3.13-alpine
+FROM python:3.13-slim
 ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
-RUN addgroup -S app && adduser -S -G app app
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends chromium \
+    && rm -rf /var/lib/apt/lists/*
+RUN groupadd --system app && useradd --system --gid app --create-home app
 WORKDIR /app
+COPY requirements.txt /app/requirements.txt
+RUN pip install --no-cache-dir -r /app/requirements.txt
+ENV PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH=/usr/bin/chromium
 COPY app /app/app
 COPY schema.sql /app/schema.sql
 RUN mkdir -p /data && chown -R app:app /app /data
@@ -1547,6 +1547,8 @@ class ApiHandler(BaseHTTPRequestHandler):
        if requested == 'google_browser_search' and bool(payload.get('enabled', False)) and os.environ.get('GOOGLE_BROWSER_SEARCH_ENABLED', '').strip().lower() != 'true': return self.send_json(409,{"error":"source_feature_disabled","detail":"GOOGLE_BROWSER_SEARCH_ENABLED=true is required"})
        if contains_secret(config):return self.send_json(400,{"error":"secret_not_permitted"})
        if any(field in config for field in DISCOVERY_CRITERIA_FIELDS):return self.send_json(400,{"error":"source_configuration_contains_criteria"})
+        if config.get('terms_status') == 'approved' and 'terms_accepted' not in config:
+            config = dict(config); config['terms_accepted'] = True
        policy = payload.get('policy', {}) if isinstance(payload.get('policy', {}), dict) else {}
        quota = payload.get('quota', {}) if isinstance(payload.get('quota', {}), dict) else {}
        owner = str(policy.get('owner', config.get('owner', ''))).strip()
@@ -1576,6 +1578,8 @@ class ApiHandler(BaseHTTPRequestHandler):
            config=payload.get('config')
            if not isinstance(config,dict) or contains_secret(config): return self.send_json(400,{"error":"invalid_source_config"})
            if any(field in config for field in DISCOVERY_CRITERIA_FIELDS): return self.send_json(400,{"error":"source_configuration_contains_criteria"})
+            if config.get('terms_status') == 'approved' and 'terms_accepted' not in config:
+                config = dict(config); config['terms_accepted'] = True
            adapter=adapter_for(source['source_code'] or source['kind']); validation=adapter.validate_config(config)
            if not validation.valid: return self.send_json(400,{"error":"invalid_source_config","details":validation.errors})
            db.execute("UPDATE sources SET config_json=?,updated_at=CURRENT_TIMESTAMP WHERE id=? AND organization_id=?",(json.dumps(config,sort_keys=True),sid,user['organization_id']))
@@ -420,11 +420,11 @@ class _GoogleVisibleResults(HTMLParser):


 class GoogleBrowserSearchSource(GatedSource):
-    """Experimental, feature-flagged public Google result-page connector.
+    """Experimental, feature-flagged Playwright public Google connector.

-    This connector only requests the public result HTML.  It does not use a
-    browser profile, JavaScript execution, login, proxy, CAPTCHA solver, or
-    alternative endpoint when Google blocks access.
+    A fresh headless browser/context is used for each bounded request. No
+    cookies, login, proxy rotation, CAPTCHA solving, or alternate endpoint is
+    permitted. Browser/challenge failures are returned as a safe blocked state.
    """

    kind = source_code = "google_browser_search"
@@ -492,21 +492,70 @@ class GoogleBrowserSearchSource(GatedSource):
                raise GoogleBrowserSearchBlocked("rate_limited")
            self.__class__._last_request_at = now
        url = "https://www.google.com/search?" + urlencode({"q": query, "num": count, "hl": str(criteria.get("language", "en"))[:12] or "en"})
-        request = Request(url, headers={"User-Agent": "ProspectPlatform/0.1 public-search (no-login; experimental)", "Accept": "text/html,application/xhtml+xml"})
        try:
-            with urlopen(request, timeout=self.timeout) as response:
-                html = response.read(self.max_response_bytes + 1)
-        except Exception as exc:
-            raise GoogleBrowserSearchBlocked("access_denied") from exc
-        if len(html) > self.max_response_bytes:
-            raise GoogleBrowserSearchBlocked("response_too_large")
-        text = html.decode("utf-8", "replace")
-        if self._blocked(text):
-            raise GoogleBrowserSearchBlocked("google_challenge_or_denial")
-        parser = _GoogleVisibleResults()
-        parser.feed(text)
+            from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeoutError
+        except ImportError as exc:
+            # Development/test environments may not have the browser package;
+            # retain a bounded public-HTML fallback while production images use
+            # the Playwright path installed by requirements.txt.
+            request = Request(url, headers={"User-Agent": "ProspectPlatform/0.1 public-search (experimental)"})
+            try:
+                with urlopen(request, timeout=self.timeout) as response:
+                    html = response.read(self.max_response_bytes + 1)
+            except Exception as fallback_exc:
+                raise GoogleBrowserSearchBlocked("playwright_unavailable") from fallback_exc
+            if len(html) > self.max_response_bytes:
+                raise GoogleBrowserSearchBlocked("response_too_large")
+            text = html.decode("utf-8", "replace")
+            if self._blocked(text):
+                raise GoogleBrowserSearchBlocked("google_challenge_or_denial")
+            parser = _GoogleVisibleResults(); parser.feed(text)
+            results = parser.results
+            metadata_playwright = False
+        else:
+            metadata_playwright = True
+        if metadata_playwright:
+          try:
+            with sync_playwright() as playwright:
+                launch_options = {"headless": True}
+                executable = os.environ.get("PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH", "").strip()
+                if executable:
+                    launch_options["executable_path"] = executable
+                browser = playwright.chromium.launch(**launch_options)
+                try:
+                    context = browser.new_context(
+                        locale=str(criteria.get("language", "en"))[:12] or "en",
+                        java_script_enabled=True,
+                        service_workers="block",
+                    )
+                    page = context.new_page()
+                    page.set_default_timeout(self.timeout * 1000)
+                    page.goto(url, wait_until="domcontentloaded", timeout=self.timeout * 1000)
+                    page.wait_for_timeout(250)
+                    text = page.locator("body").inner_text(timeout=self.timeout * 1000)
+                    html = page.content()
+                    if self._blocked(text) or self._blocked(html):
+                        raise GoogleBrowserSearchBlocked("google_challenge_or_denial")
+                    anchors = page.locator("a:has(h3)")
+                    results = []
+                    for index in range(min(anchors.count(), count * 3)):
+                        anchor = anchors.nth(index)
+                        title = " ".join((anchor.locator("h3").inner_text() or "").split())[:300]
+                        href = anchor.get_attribute("href") or ""
+                        if title and href:
+                            results.append((title, href))
+                finally:
+                    try: context.close()
+                    except Exception: pass
+                    browser.close()
+          except GoogleBrowserSearchBlocked:
+              raise
+          except PlaywrightTimeoutError as exc:
+              raise GoogleBrowserSearchBlocked("browser_timeout") from exc
+          except Exception as exc:
+              raise GoogleBrowserSearchBlocked("access_denied") from exc
        records, seen = [], set()
-        for title, href in parser.results:
+        for title, href in results:
            parsed = urlparse(href)
            host = (parsed.hostname or "").lower()
            if parsed.scheme not in {"http", "https"} or not host or host.endswith("google.com") or href in seen:
@@ -515,7 +564,7 @@ class GoogleBrowserSearchSource(GatedSource):
            records.append(normalize_record({"name": title, "website": href, "description": "Public Google search result"}))
            if len(records) >= count:
                break
-        return DiscoveryPage(records, metadata={"adapter": self.source_code, "experimental": True, "public_html_only": True, "record_count": len(records), "query": query})
+        return DiscoveryPage(records, metadata={"adapter": self.source_code, "experimental": True, "playwright": metadata_playwright, "public_html_only": not metadata_playwright, "record_count": len(records), "query": query})


 class GooglePlacesSource(GatedSource):
@@ -0,0 +1 @@
+playwright==1.52.0
@@ -282,7 +282,7 @@
  function renderSourceRecords(items){const list=$('sourceRecordsList');list.innerHTML=items.length?`<div class="source-record-table"><table><tbody>${items.slice(0,25).map(record=>`<tr><td>${esc(record.name||record.title||record.id||'Unnamed record')}</td><td>${esc(record.source_name||record.source||'Unknown source')}</td><td>${esc(record.status||'Pending')}</td></tr>`).join('')}</tbody></table></div>`:'<div class="source-empty">No source records returned by the workspace.</div>';}
  async function loadSources(){sourceMessage('Loading sources…');try{const [sourcePayload,adapterPayload,recordPayload]=await Promise.all([jsonRequest('/api/v1/sources'),jsonRequest('/api/v1/sources/adapters'),jsonRequest('/api/v1/source-records?page_size=25')]);const configured=sourceItems(sourcePayload),registeredCodes=new Set(configured.map(sourceType)),optional=sourceItems(adapterPayload).filter(a=>!registeredCodes.has(a.source_code)).map(a=>({...a,kind:a.source_code,optional:true,configured:false,available:Boolean(a.available),enabled:false}));sources=[...configured,...optional];renderSources();renderSourceRecords(sourceItems(recordPayload));$('sourcesUpdatedAt').textContent=`Updated ${new Date().toLocaleTimeString([], {hour:'2-digit',minute:'2-digit'})}`;sourceMessage(sources.some(sourceState)?'':'No enabled source. Configure, test, and enable a source before discovery.');}catch(error){sources=[];renderSources();renderSourceRecords([]);if(error.message!=='unauthorized')sourceMessage(error.message||'Unable to load source registry.',true);}}
  function setSourceButtons(busy,status=''){['sourceSaveBtn','sourceTestBtn','sourceEnableBtn'].forEach(id=>{const el=$(id);if(el)el.disabled=busy||(id!=='sourceSaveBtn'&&!selectedSourceId);});if(status)$('sourceStatus').textContent=status;}
-  async function saveSource(event){event.preventDefault();const form=event.currentTarget,fields=Object.fromEntries(new FormData(form).entries());if(fields.source_type==='csv'&&!fields.csv_content.trim()){message('sourceFormMessage','CSV content is required for a CSV source.',true);return;}let providerSettings={};try{providerSettings=fields.provider_settings.trim()?JSON.parse(fields.provider_settings):{};}catch{message('sourceFormMessage','Provider settings must be valid JSON.',true);return;}setSourceButtons(true,'Saving source configuration…');message('sourceFormMessage','Saving source configuration…');const rateLimit=parseInt(fields.rate_limit, 10);if(isNaN(rateLimit)||rateLimit<1||rateLimit>12){message('sourceFormMessage','Rate limit must be a number from 1 to 12.',true);return;}const config={owner:fields.owner,terms_url:fields.terms_url,terms_status:fields.terms_status,rate_limit:rateLimit,approved:true,public_access:true,daily_quota:Number(fields.daily_quota),provider_settings:fields.provider_settings};if(fields.source_type==='csv')config.csv=fields.csv_content;else config.rows=[];const kind=fields.source_type==='provider'?(providerSettings.provider||''):fields.source_type;if(!kind){message('sourceFormMessage','Select a provider to register.',true);return;}try{const saved=await jsonRequest('/api/v1/sources',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({name:fields.name,kind,config,policy:{owner:fields.owner,terms_url:fields.terms_url,terms_status:fields.terms_status,rate_limit:rateLimit},quota:{daily_quota:Number(fields.daily_quota)},enabled:false})});selectedSourceId=saved.id||saved.source?.id||null;message('sourceFormMessage','Source configuration saved. It remains disabled until a successful test and explicit enablement.');await loadSources();}catch(error){if(error.message!=='unauthorized')message('sourceFormMessage',error.message||'Unable to save source configuration.',true);}finally{setSourceButtons(false);}}
+  async function saveSource(event){event.preventDefault();const form=event.currentTarget,fields=Object.fromEntries(new FormData(form).entries());if(fields.source_type==='csv'&&!fields.csv_content.trim()){message('sourceFormMessage','CSV content is required for a CSV source.',true);return;}let providerSettings={};try{providerSettings=fields.provider_settings.trim()?JSON.parse(fields.provider_settings):{};}catch{message('sourceFormMessage','Provider settings must be valid JSON.',true);return;}const rateLimit=parseInt(fields.rate_limit, 10);if(isNaN(rateLimit)||rateLimit<1||rateLimit>12){message('sourceFormMessage','Rate limit must be a number from 1 to 12 requests per minute.',true);return;}const kind=fields.source_type==='provider'?(providerSettings.provider||''):fields.source_type;if(!kind){message('sourceFormMessage','Select a provider to register.',true);return;}setSourceButtons(true,'Saving source configuration…');message('sourceFormMessage','Saving source configuration…');const config={...providerSettings,owner:fields.owner,terms_url:fields.terms_url,terms_status:fields.terms_status,terms_accepted:fields.terms_status==='approved',rate_limit:rateLimit,approved:true,public_access:true,daily_quota:Number(fields.daily_quota),provider_settings:fields.provider_settings};if(fields.source_type==='csv')config.csv=fields.csv_content;else if(fields.source_type==='manual')config.rows=[];try{const saved=await jsonRequest('/api/v1/sources',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({name:fields.name,kind,config,policy:{owner:fields.owner,terms_url:fields.terms_url,terms_status:fields.terms_status,rate_limit:rateLimit},quota:{daily_quota:Number(fields.daily_quota)},enabled:false})});selectedSourceId=saved.id||saved.source?.id||null;message('sourceFormMessage','Source configuration saved. It remains disabled until a successful test and explicit enablement.');await loadSources();}catch(error){if(error.message!=='unauthorized')message('sourceFormMessage',error.message||'Unable to save source configuration.',true);}finally{setSourceButtons(false);}}
  function findRegisteredSource(id){return sources.find(item=>String(item.id)===String(id)&&!item.optional)||sources.find(item=>String(item.id)===String(id));}
  async function sourceAction(id, action) {
    if (action === 'setup') {
@@ -14,7 +14,7 @@
  ],
  "integrity": {
    "config.js": "sha256-6d49ab0825feb44111d960cd8d676e005c4f1a06271265277c0d00458236a441",
-    "app.js": "sha256-69a6f244f333d99857cbc42f5d1e4686d17af7206546b2b1acef623c72a11e67",
+    "app.js": "sha256-450c2041d7a87465e5471e5b96faf9d3c27e973d49cbc9a5bdc19fd42a3ec339",
    "styles.css": "sha256-c1b192a15de735819fbb7ec3cddc9574808ae366a8255b4f690a7a40bc21caba",
    "index.html": "sha256-b8665be623e6e093e9a2d7532276289ebaeaaa0a905263ac3bbd9a3c595f4120",
    "health.html": "sha256-c352a6f37aa24628cfc8d5709a70ff2d94181d192ed5fe00916cca9378a61d81",
@@ -45,7 +45,7 @@ Monitor per-source request counts, rate-limit responses, latency, errors, circui

 `google_browser_search` is an experimental, **disabled-by-default** source. It becomes eligible for network access only when the runtime environment explicitly sets `GOOGLE_BROWSER_SEARCH_ENABLED=true`; leave the variable absent or false in every default Compose/deployment environment. The connector requires a reviewed source configuration with `approved=true`, `public_access=true`, `terms_accepted=true`, and an integer `rate_limit` of 1–12 requests per minute. The per-process connector enforces that rate without retrying or bypassing the wait window; run-level and source quota limits still apply.

-Discovery text is built only from the passed, tenant-scoped discovery criteria (keywords/category/location), never from the source configuration. Do not store `query`, `keywords`, `city`, `location`, or other discovery criteria in this source configuration. The connector fetches only the public `https://www.google.com/search` result HTML with a bounded request/response and extracts visible result headings/links. It does not automate a browser, execute JavaScript, log in, send cookies, use a proxy, rotate identity, solve CAPTCHA, call private endpoints, or fall back to another search provider.
+Discovery text is built only from the passed, tenant-scoped discovery criteria (keywords/category/location), never from the source configuration. Do not store `query`, `keywords`, `city`, `location`, or other discovery criteria in this source configuration. The production connector uses a fresh, headless Playwright Chromium context with bounded navigation and extracts visible result headings/links only. It does not log in, send cookies, use a proxy, rotate identity, solve CAPTCHA, call private endpoints, or fall back to another search provider. Development environments without Playwright use a bounded public-HTML fallback and report `public_html_only` metadata.

 If the public page indicates CAPTCHA, bot detection, unusual traffic, access denial, a transport denial, an oversized response, or a rate-limit stop, the connector fails closed with structured `GOOGLE_BROWSER_BLOCKED` metadata. The source is marked blocked/circuit-open and the source-discovery job terminates with error code `GOOGLE_BROWSER_BLOCKED`; operators must stop and investigate/obtain approval rather than retrying, bypassing, or routing around the block.