From 6debef6015e5f8efc08b4e8ea3ef6294aeef8464 Mon Sep 17 00:00:00 2001 From: Marco0300 Date: Sat, 5 Sep 2026 14:19:56 +0200 Subject: [PATCH] add safe public search-engine fallback --- README.md | 11 +++++----- api/server.py | 56 ++++++++++++++++++++++++++++++++------------------ web/index.html | 2 +- 3 files changed, 43 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 0f82724..d8a033f 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,12 @@ # Browser-only ProspectOS This is the stripped-down replacement for the larger ProspectOS application. -It has one live source: Google Search through a headless Playwright Chromium -browser. Each discovery run builds a query from category, keywords, and -location, filters results already present in the local catalogue by root -domain, and stores only new prospects. The explorer supports search and -human review states (`pending`, `approved`, `rejected`, `contacted`). +It searches public engines (Google first, then Bing and DuckDuckGo if Google +challenges the server) through a headless Playwright Chromium browser. Each +discovery run builds a query from category, keywords, and location, filters +results already present in the local catalogue by root domain, and stores only +new prospects. The explorer supports search and human review states +(`pending`, `approved`, `rejected`, `contacted`). There is deliberately no source registry, email scraper, AI provider, CRM, outreach sender, or automatic contact action. diff --git a/api/server.py b/api/server.py index 6798c49..21f6464 100644 --- a/api/server.py +++ b/api/server.py @@ -82,18 +82,31 @@ def blocked(text: str) -> bool: def google_search(query: str, requested: int) -> list[dict[str, str]]: + """Search public engines through Playwright, with safe engine fallback. + + Google frequently challenges datacenter IPs. We do not bypass that + challenge; we close the context and try Bing, then DuckDuckGo, still using + a real headless browser and the same bounded result extraction. + """ try: from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeout except ImportError as exc: raise RuntimeError("playwright_not_installed") from exc - url = "https://www.google.com/search?q=" + quote_plus(query) + "&num=" + str(requested) executable = os.environ.get("PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH", "").strip() options: dict[str, Any] = {"headless": True} if executable: options["executable_path"] = executable - try: - with sync_playwright() as p: - browser = p.chromium.launch(**options) + engines = [ + ("google", "https://www.google.com/search?q=" + quote_plus(query) + "&num=" + str(requested), "a:has(h3)", "h3", {"google.com", "google.co.za"}), + ("bing", "https://www.bing.com/search?q=" + quote_plus(query) + "&count=" + str(requested), "li.b_algo h2 a", "h2", {"bing.com", "bingj.com"}), + ("duckduckgo", "https://html.duckduckgo.com/html/?q=" + quote_plus(query), "a.result__a", None, {"duckduckgo.com"}), + ] + errors: list[str] = [] + with sync_playwright() as p: + browser = p.chromium.launch(**options) + try: + for engine, url, selector, title_selector, excluded_hosts in engines: + context = None try: context = browser.new_context( locale="en-ZA", @@ -108,15 +121,17 @@ def google_search(query: str, requested: int) -> list[dict[str, str]]: body = page.locator("body").inner_text() html = page.content() if blocked(body) or blocked(html): - raise RuntimeError("google_blocked") + errors.append(engine + "_blocked") + continue output: list[dict[str, str]] = [] - anchors = page.locator("a:has(h3)") + anchors = page.locator(selector) for i in range(min(anchors.count(), requested * 3)): anchor = anchors.nth(i) - title = " ".join((anchor.locator("h3").inner_text() or "").split())[:200] + title_node = anchor.locator(title_selector) if title_selector else anchor + title = " ".join((title_node.inner_text() or "").split())[:200] href = anchor.get_attribute("href") or "" domain = clean_domain(href) - if not title or not href or domain in {"google.com", "google.co.za"} or not domain: + if not title or not href or domain in excluded_hosts or not domain: continue if not href.startswith(("http://", "https://")): continue @@ -124,19 +139,20 @@ def google_search(query: str, requested: int) -> list[dict[str, str]]: output.append({"name": title, "website": href, "domain": domain}) if len(output) >= requested: break - return output + if output: + return output + errors.append(engine + "_empty") + except PlaywrightTimeout: + errors.append(engine + "_timeout") + except Exception as exc: + errors.append(engine + "_error") finally: - try: - context.close() - except Exception: - pass - browser.close() - except RuntimeError: - raise - except PlaywrightTimeout as exc: - raise RuntimeError("browser_timeout") from exc - except Exception as exc: - raise RuntimeError("browser_error") from exc + if context is not None: + try: context.close() + except Exception: pass + finally: + browser.close() + raise RuntimeError("search_engines_unavailable:" + ",".join(errors[-3:])) def run_discovery(payload: dict[str, Any]) -> dict[str, Any]: diff --git a/web/index.html b/web/index.html index f49e9b0..cb45d58 100644 --- a/web/index.html +++ b/web/index.html @@ -1,6 +1,6 @@ ProspectOS · Browser Discovery -
PROSPECTOS

Browser discovery

Find businesses through Google, remove anything already catalogued, then review the remaining prospects.

● Google · Playwright
+
PROSPECTOS

Browser discovery

Find businesses through public search engines, remove anything already catalogued, then review the remaining prospects.

● Browser search · Playwright
Catalogued prospects0
Pending review0
Approved0
Last run—
DISCOVERY

Find new prospects

Search criteria are used only for this run. The system stores the business name, website, query, and discovery time.