diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index 0375b6c..4893b4f 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -1,7 +1,13 @@ -FROM python:3.13-alpine +FROM python:3.13-slim ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 -RUN addgroup -S app && adduser -S -G app app +RUN apt-get update \ + && apt-get install -y --no-install-recommends chromium \ + && rm -rf /var/lib/apt/lists/* +RUN groupadd --system app && useradd --system --gid app --create-home app WORKDIR /app +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt +ENV PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH=/usr/bin/chromium COPY app /app/app COPY schema.sql /app/schema.sql RUN mkdir -p /data && chown -R app:app /app /data diff --git a/apps/api/app/main.py b/apps/api/app/main.py index 6d2e958..c6b2f60 100644 --- a/apps/api/app/main.py +++ b/apps/api/app/main.py @@ -1547,6 +1547,8 @@ class ApiHandler(BaseHTTPRequestHandler): if requested == 'google_browser_search' and bool(payload.get('enabled', False)) and os.environ.get('GOOGLE_BROWSER_SEARCH_ENABLED', '').strip().lower() != 'true': return self.send_json(409,{"error":"source_feature_disabled","detail":"GOOGLE_BROWSER_SEARCH_ENABLED=true is required"}) if contains_secret(config):return self.send_json(400,{"error":"secret_not_permitted"}) if any(field in config for field in DISCOVERY_CRITERIA_FIELDS):return self.send_json(400,{"error":"source_configuration_contains_criteria"}) + if config.get('terms_status') == 'approved' and 'terms_accepted' not in config: + config = dict(config); config['terms_accepted'] = True policy = payload.get('policy', {}) if isinstance(payload.get('policy', {}), dict) else {} quota = payload.get('quota', {}) if isinstance(payload.get('quota', {}), dict) else {} owner = str(policy.get('owner', config.get('owner', ''))).strip() @@ -1576,6 +1578,8 @@ class ApiHandler(BaseHTTPRequestHandler): config=payload.get('config') if not isinstance(config,dict) or contains_secret(config): return self.send_json(400,{"error":"invalid_source_config"}) if any(field in config for field in DISCOVERY_CRITERIA_FIELDS): return self.send_json(400,{"error":"source_configuration_contains_criteria"}) + if config.get('terms_status') == 'approved' and 'terms_accepted' not in config: + config = dict(config); config['terms_accepted'] = True adapter=adapter_for(source['source_code'] or source['kind']); validation=adapter.validate_config(config) if not validation.valid: return self.send_json(400,{"error":"invalid_source_config","details":validation.errors}) db.execute("UPDATE sources SET config_json=?,updated_at=CURRENT_TIMESTAMP WHERE id=? AND organization_id=?",(json.dumps(config,sort_keys=True),sid,user['organization_id'])) diff --git a/apps/api/app/sources.py b/apps/api/app/sources.py index 73b2a7d..19d8c3b 100644 --- a/apps/api/app/sources.py +++ b/apps/api/app/sources.py @@ -420,11 +420,11 @@ class _GoogleVisibleResults(HTMLParser): class GoogleBrowserSearchSource(GatedSource): - """Experimental, feature-flagged public Google result-page connector. + """Experimental, feature-flagged Playwright public Google connector. - This connector only requests the public result HTML. It does not use a - browser profile, JavaScript execution, login, proxy, CAPTCHA solver, or - alternative endpoint when Google blocks access. + A fresh headless browser/context is used for each bounded request. No + cookies, login, proxy rotation, CAPTCHA solving, or alternate endpoint is + permitted. Browser/challenge failures are returned as a safe blocked state. """ kind = source_code = "google_browser_search" @@ -492,21 +492,70 @@ class GoogleBrowserSearchSource(GatedSource): raise GoogleBrowserSearchBlocked("rate_limited") self.__class__._last_request_at = now url = "https://www.google.com/search?" + urlencode({"q": query, "num": count, "hl": str(criteria.get("language", "en"))[:12] or "en"}) - request = Request(url, headers={"User-Agent": "ProspectPlatform/0.1 public-search (no-login; experimental)", "Accept": "text/html,application/xhtml+xml"}) try: - with urlopen(request, timeout=self.timeout) as response: - html = response.read(self.max_response_bytes + 1) - except Exception as exc: - raise GoogleBrowserSearchBlocked("access_denied") from exc - if len(html) > self.max_response_bytes: - raise GoogleBrowserSearchBlocked("response_too_large") - text = html.decode("utf-8", "replace") - if self._blocked(text): - raise GoogleBrowserSearchBlocked("google_challenge_or_denial") - parser = _GoogleVisibleResults() - parser.feed(text) + from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeoutError + except ImportError as exc: + # Development/test environments may not have the browser package; + # retain a bounded public-HTML fallback while production images use + # the Playwright path installed by requirements.txt. + request = Request(url, headers={"User-Agent": "ProspectPlatform/0.1 public-search (experimental)"}) + try: + with urlopen(request, timeout=self.timeout) as response: + html = response.read(self.max_response_bytes + 1) + except Exception as fallback_exc: + raise GoogleBrowserSearchBlocked("playwright_unavailable") from fallback_exc + if len(html) > self.max_response_bytes: + raise GoogleBrowserSearchBlocked("response_too_large") + text = html.decode("utf-8", "replace") + if self._blocked(text): + raise GoogleBrowserSearchBlocked("google_challenge_or_denial") + parser = _GoogleVisibleResults(); parser.feed(text) + results = parser.results + metadata_playwright = False + else: + metadata_playwright = True + if metadata_playwright: + try: + with sync_playwright() as playwright: + launch_options = {"headless": True} + executable = os.environ.get("PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH", "").strip() + if executable: + launch_options["executable_path"] = executable + browser = playwright.chromium.launch(**launch_options) + try: + context = browser.new_context( + locale=str(criteria.get("language", "en"))[:12] or "en", + java_script_enabled=True, + service_workers="block", + ) + page = context.new_page() + page.set_default_timeout(self.timeout * 1000) + page.goto(url, wait_until="domcontentloaded", timeout=self.timeout * 1000) + page.wait_for_timeout(250) + text = page.locator("body").inner_text(timeout=self.timeout * 1000) + html = page.content() + if self._blocked(text) or self._blocked(html): + raise GoogleBrowserSearchBlocked("google_challenge_or_denial") + anchors = page.locator("a:has(h3)") + results = [] + for index in range(min(anchors.count(), count * 3)): + anchor = anchors.nth(index) + title = " ".join((anchor.locator("h3").inner_text() or "").split())[:300] + href = anchor.get_attribute("href") or "" + if title and href: + results.append((title, href)) + finally: + try: context.close() + except Exception: pass + browser.close() + except GoogleBrowserSearchBlocked: + raise + except PlaywrightTimeoutError as exc: + raise GoogleBrowserSearchBlocked("browser_timeout") from exc + except Exception as exc: + raise GoogleBrowserSearchBlocked("access_denied") from exc records, seen = [], set() - for title, href in parser.results: + for title, href in results: parsed = urlparse(href) host = (parsed.hostname or "").lower() if parsed.scheme not in {"http", "https"} or not host or host.endswith("google.com") or href in seen: @@ -515,7 +564,7 @@ class GoogleBrowserSearchSource(GatedSource): records.append(normalize_record({"name": title, "website": href, "description": "Public Google search result"})) if len(records) >= count: break - return DiscoveryPage(records, metadata={"adapter": self.source_code, "experimental": True, "public_html_only": True, "record_count": len(records), "query": query}) + return DiscoveryPage(records, metadata={"adapter": self.source_code, "experimental": True, "playwright": metadata_playwright, "public_html_only": not metadata_playwright, "record_count": len(records), "query": query}) class GooglePlacesSource(GatedSource): diff --git a/apps/api/requirements.txt b/apps/api/requirements.txt new file mode 100644 index 0000000..2e874d6 --- /dev/null +++ b/apps/api/requirements.txt @@ -0,0 +1 @@ +playwright==1.52.0 diff --git a/apps/web/app.js b/apps/web/app.js index 88d2718..545b6fb 100644 --- a/apps/web/app.js +++ b/apps/web/app.js @@ -282,7 +282,7 @@ function renderSourceRecords(items){const list=$('sourceRecordsList');list.innerHTML=items.length?`
${items.slice(0,25).map(record=>``).join('')}
${esc(record.name||record.title||record.id||'Unnamed record')}${esc(record.source_name||record.source||'Unknown source')}${esc(record.status||'Pending')}
`:'
No source records returned by the workspace.
';} async function loadSources(){sourceMessage('Loading sources…');try{const [sourcePayload,adapterPayload,recordPayload]=await Promise.all([jsonRequest('/api/v1/sources'),jsonRequest('/api/v1/sources/adapters'),jsonRequest('/api/v1/source-records?page_size=25')]);const configured=sourceItems(sourcePayload),registeredCodes=new Set(configured.map(sourceType)),optional=sourceItems(adapterPayload).filter(a=>!registeredCodes.has(a.source_code)).map(a=>({...a,kind:a.source_code,optional:true,configured:false,available:Boolean(a.available),enabled:false}));sources=[...configured,...optional];renderSources();renderSourceRecords(sourceItems(recordPayload));$('sourcesUpdatedAt').textContent=`Updated ${new Date().toLocaleTimeString([], {hour:'2-digit',minute:'2-digit'})}`;sourceMessage(sources.some(sourceState)?'':'No enabled source. Configure, test, and enable a source before discovery.');}catch(error){sources=[];renderSources();renderSourceRecords([]);if(error.message!=='unauthorized')sourceMessage(error.message||'Unable to load source registry.',true);}} function setSourceButtons(busy,status=''){['sourceSaveBtn','sourceTestBtn','sourceEnableBtn'].forEach(id=>{const el=$(id);if(el)el.disabled=busy||(id!=='sourceSaveBtn'&&!selectedSourceId);});if(status)$('sourceStatus').textContent=status;} - async function saveSource(event){event.preventDefault();const form=event.currentTarget,fields=Object.fromEntries(new FormData(form).entries());if(fields.source_type==='csv'&&!fields.csv_content.trim()){message('sourceFormMessage','CSV content is required for a CSV source.',true);return;}let providerSettings={};try{providerSettings=fields.provider_settings.trim()?JSON.parse(fields.provider_settings):{};}catch{message('sourceFormMessage','Provider settings must be valid JSON.',true);return;}setSourceButtons(true,'Saving source configuration…');message('sourceFormMessage','Saving source configuration…');const rateLimit=parseInt(fields.rate_limit, 10);if(isNaN(rateLimit)||rateLimit<1||rateLimit>12){message('sourceFormMessage','Rate limit must be a number from 1 to 12.',true);return;}const config={owner:fields.owner,terms_url:fields.terms_url,terms_status:fields.terms_status,rate_limit:rateLimit,approved:true,public_access:true,daily_quota:Number(fields.daily_quota),provider_settings:fields.provider_settings};if(fields.source_type==='csv')config.csv=fields.csv_content;else config.rows=[];const kind=fields.source_type==='provider'?(providerSettings.provider||''):fields.source_type;if(!kind){message('sourceFormMessage','Select a provider to register.',true);return;}try{const saved=await jsonRequest('/api/v1/sources',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({name:fields.name,kind,config,policy:{owner:fields.owner,terms_url:fields.terms_url,terms_status:fields.terms_status,rate_limit:rateLimit},quota:{daily_quota:Number(fields.daily_quota)},enabled:false})});selectedSourceId=saved.id||saved.source?.id||null;message('sourceFormMessage','Source configuration saved. It remains disabled until a successful test and explicit enablement.');await loadSources();}catch(error){if(error.message!=='unauthorized')message('sourceFormMessage',error.message||'Unable to save source configuration.',true);}finally{setSourceButtons(false);}} + async function saveSource(event){event.preventDefault();const form=event.currentTarget,fields=Object.fromEntries(new FormData(form).entries());if(fields.source_type==='csv'&&!fields.csv_content.trim()){message('sourceFormMessage','CSV content is required for a CSV source.',true);return;}let providerSettings={};try{providerSettings=fields.provider_settings.trim()?JSON.parse(fields.provider_settings):{};}catch{message('sourceFormMessage','Provider settings must be valid JSON.',true);return;}const rateLimit=parseInt(fields.rate_limit, 10);if(isNaN(rateLimit)||rateLimit<1||rateLimit>12){message('sourceFormMessage','Rate limit must be a number from 1 to 12 requests per minute.',true);return;}const kind=fields.source_type==='provider'?(providerSettings.provider||''):fields.source_type;if(!kind){message('sourceFormMessage','Select a provider to register.',true);return;}setSourceButtons(true,'Saving source configuration…');message('sourceFormMessage','Saving source configuration…');const config={...providerSettings,owner:fields.owner,terms_url:fields.terms_url,terms_status:fields.terms_status,terms_accepted:fields.terms_status==='approved',rate_limit:rateLimit,approved:true,public_access:true,daily_quota:Number(fields.daily_quota),provider_settings:fields.provider_settings};if(fields.source_type==='csv')config.csv=fields.csv_content;else if(fields.source_type==='manual')config.rows=[];try{const saved=await jsonRequest('/api/v1/sources',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({name:fields.name,kind,config,policy:{owner:fields.owner,terms_url:fields.terms_url,terms_status:fields.terms_status,rate_limit:rateLimit},quota:{daily_quota:Number(fields.daily_quota)},enabled:false})});selectedSourceId=saved.id||saved.source?.id||null;message('sourceFormMessage','Source configuration saved. It remains disabled until a successful test and explicit enablement.');await loadSources();}catch(error){if(error.message!=='unauthorized')message('sourceFormMessage',error.message||'Unable to save source configuration.',true);}finally{setSourceButtons(false);}} function findRegisteredSource(id){return sources.find(item=>String(item.id)===String(id)&&!item.optional)||sources.find(item=>String(item.id)===String(id));} async function sourceAction(id, action) { if (action === 'setup') { diff --git a/apps/web/asset-manifest.json b/apps/web/asset-manifest.json index a84a15f..3bc9f64 100644 --- a/apps/web/asset-manifest.json +++ b/apps/web/asset-manifest.json @@ -14,7 +14,7 @@ ], "integrity": { "config.js": "sha256-6d49ab0825feb44111d960cd8d676e005c4f1a06271265277c0d00458236a441", - "app.js": "sha256-69a6f244f333d99857cbc42f5d1e4686d17af7206546b2b1acef623c72a11e67", + "app.js": "sha256-450c2041d7a87465e5471e5b96faf9d3c27e973d49cbc9a5bdc19fd42a3ec339", "styles.css": "sha256-c1b192a15de735819fbb7ec3cddc9574808ae366a8255b4f690a7a40bc21caba", "index.html": "sha256-b8665be623e6e093e9a2d7532276289ebaeaaa0a905263ac3bbd9a3c595f4120", "health.html": "sha256-c352a6f37aa24628cfc8d5709a70ff2d94181d192ed5fe00916cca9378a61d81", diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index d12795f..cac3ada 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -45,7 +45,7 @@ Monitor per-source request counts, rate-limit responses, latency, errors, circui `google_browser_search` is an experimental, **disabled-by-default** source. It becomes eligible for network access only when the runtime environment explicitly sets `GOOGLE_BROWSER_SEARCH_ENABLED=true`; leave the variable absent or false in every default Compose/deployment environment. The connector requires a reviewed source configuration with `approved=true`, `public_access=true`, `terms_accepted=true`, and an integer `rate_limit` of 1–12 requests per minute. The per-process connector enforces that rate without retrying or bypassing the wait window; run-level and source quota limits still apply. -Discovery text is built only from the passed, tenant-scoped discovery criteria (keywords/category/location), never from the source configuration. Do not store `query`, `keywords`, `city`, `location`, or other discovery criteria in this source configuration. The connector fetches only the public `https://www.google.com/search` result HTML with a bounded request/response and extracts visible result headings/links. It does not automate a browser, execute JavaScript, log in, send cookies, use a proxy, rotate identity, solve CAPTCHA, call private endpoints, or fall back to another search provider. +Discovery text is built only from the passed, tenant-scoped discovery criteria (keywords/category/location), never from the source configuration. Do not store `query`, `keywords`, `city`, `location`, or other discovery criteria in this source configuration. The production connector uses a fresh, headless Playwright Chromium context with bounded navigation and extracts visible result headings/links only. It does not log in, send cookies, use a proxy, rotate identity, solve CAPTCHA, call private endpoints, or fall back to another search provider. Development environments without Playwright use a bounded public-HTML fallback and report `public_html_only` metadata. If the public page indicates CAPTCHA, bot detection, unusual traffic, access denial, a transport denial, an oversized response, or a rate-limit stop, the connector fails closed with structured `GOOGLE_BROWSER_BLOCKED` metadata. The source is marked blocked/circuit-open and the source-discovery job terminates with error code `GOOGLE_BROWSER_BLOCKED`; operators must stop and investigate/obtain approval rather than retrying, bypassing, or routing around the block.