root/MarketingTool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

deploy DOM search v1 update

Browse Source
This commit is contained in:
Marco0300
2026-09-05 23:27:20 +02:00
parent 112ab552e9
commit a8182137fc
4 changed files with 507 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/browser_investigator.py
+347 -1
View File
@@ -11,14 +11,27 @@ import inspect
import json
import os
import re
import time
from typing import Any, Callable
from urllib.parse import urlparse
from urllib.parse import parse_qs, quote_plus, unquote, urlparse
_BLOCKED_HOSTS = {
"facebook.com", "instagram.com", "linkedin.com", "youtube.com", "x.com",
"twitter.com", "google.com", "google.co.za", "bing.com", "brave.com",
"duckduckgo.com", "startpage.com", "yelp.com", "yellowpages.com",
"yellosa.co.za", "snupit.co.za", "zaubee.com", "mapquest.com",
}
_SEARCH_BLOCKED_MARKERS = (
"unusual traffic", "recaptcha", "captcha", "automated queries",
"access denied", "detected unusual traffic", "verify you are human",
)
_GENERIC_NAME_WORDS = {
"and", "the", "services", "service", "solutions", "company", "pty",
"ltd", "inc", "llc", "south", "africa", "sa", "electrical",
"electrician", "electricians", "installation", "installations",
}
@@ -41,6 +54,339 @@ def _normalise_url(value: Any) -> str:
return text[:1000]
def _candidate_domain(url: str) -> str:
host = (urlparse(url).hostname or "").lower().rstrip(".")
return host.removeprefix("www.")
def _normalise_search_href(value: Any) -> str:
"""Turn a search-result href into the real public URL.
Search engines sometimes wrap result links in redirect URLs. We unwrap
the common query-string forms before applying the blocked-host check; the
old implementation rejected those wrappers before it could see the real
business domain.
"""
text = str(value or "").strip()
if text.startswith("//"):
text = "https:" + text
if text.startswith("/"):
return ""
try:
parsed = urlparse(text)
query = parse_qs(parsed.query)
if parsed.hostname and parsed.hostname.lower().endswith(("duckduckgo.com", "bing.com", "bingj.com")):
for key in ("uddg", "url", "u", "r"):
target = query.get(key, [""])[0]
target = unquote(target)
if target.startswith(("http://", "https://")):
text = target
break
except Exception:
return ""
return _normalise_url(text)
def _tokens(value: str, *, include_generic: bool = False) -> list[str]:
values = re.findall(r"[a-z0-9]+", str(value or "").lower())
return [value for value in values if len(value) >= 3 and (include_generic or value not in _GENERIC_NAME_WORDS)]
def _location_from_address(address: str) -> str:
text = str(address or "").strip()
try:
parsed = json.loads(text)
if isinstance(parsed, dict):
return str(parsed.get("city") or parsed.get("town") or parsed.get("municipality") or parsed.get("state") or "").strip()
except (TypeError, ValueError, json.JSONDecodeError):
pass
parts = [part.strip() for part in re.split(r"[,|]", text) if part.strip()]
return parts[-1] if parts else ""
def _search_query_variants(prospect: dict[str, Any]) -> list[str]:
name = str(prospect.get("name") or "").strip()
if not name:
return []
location = _location_from_address(str(prospect.get("address") or ""))
category = str(prospect.get("category") or "").strip()
phone = re.sub(r"[^0-9]+", "", str(prospect.get("phone") or ""))
queries = [
f'"{name}" {location}'.strip(),
f'"{name}" {phone}'.strip() if phone else "",
f'"{name}" site:co.za'.strip(),
f'"{name}" {category} {location}'.strip(),
f'"{name}"'.strip(),
]
output: list[str] = []
seen: set[str] = set()
for query in queries:
query = " ".join(query.split())
if query and query not in seen:
seen.add(query)
output.append(query)
return output[:5]
def _candidate_score(prospect: dict[str, Any], url: str, title: str, context: str) -> tuple[int, list[str]]:
"""Score a result using only visible search-result evidence.
This is intentionally deterministic. It prevents an LLM's final
``done`` message from discarding a domain that was visibly present on the
search page.
"""
domain = _candidate_domain(url)
haystack = " ".join((title, context, domain)).lower()
domain_stem = domain
if domain.endswith(".co.za"):
domain_stem = domain[:-6]
elif domain.count(".") >= 1:
domain_stem = domain.rsplit(".", 1)[0]
compact_domain = re.sub(r"[^a-z0-9]", "", domain_stem)
name = str(prospect.get("name") or "")
name_compact = re.sub(r"[^a-z0-9]", "", name.lower())
name_tokens = _tokens(name)
evidence: list[str] = []
score = 18
if name_compact and (name_compact in compact_domain or compact_domain in name_compact):
score += 52
evidence.append("domain closely matches business name")
matching_tokens = [token for token in name_tokens if token in compact_domain]
if matching_tokens:
score += min(30, 12 * len(matching_tokens))
evidence.append("name token match: " + ", ".join(matching_tokens[:4]))
if any(token in haystack for token in name_tokens):
score += 8
evidence.append("business name appears in result text")
phone = re.sub(r"[^0-9]", "", str(prospect.get("phone") or ""))
context_digits = re.sub(r"[^0-9]", "", context)
if phone and len(phone) >= 7 and phone[-7:] in context_digits:
score += 14
evidence.append("phone appears in result text")
location_tokens = _tokens(_location_from_address(str(prospect.get("address") or "")), include_generic=True)
if location_tokens and any(token in haystack for token in location_tokens):
score += 6
evidence.append("location appears in result text")
if domain.endswith(".co.za"):
score += 5
evidence.append("South African .co.za domain")
return max(0, min(100, score)), evidence
def _extract_visible_result_links(page: Any) -> list[dict[str, str]]:
"""Read result anchors/snippets from the loaded page DOM.
No indexed click targets are used. This is the critical reliability path
for Bing pages whose element indexes can change after navigation.
"""
script = """
() => {
const cardSelectors = ["li.b_algo", "article", ".result", ".result-item", ".w-gl__result"];
const cards = cardSelectors.flatMap(selector => Array.from(document.querySelectorAll(selector)));
const records = [];
const seen = new Set();
const add = (anchor, card) => {
if (!anchor || seen.has(anchor)) return;
const href = anchor.href || anchor.getAttribute("href") || "";
const title = (anchor.innerText || anchor.textContent || anchor.getAttribute("aria-label") || "").trim();
const context = ((card && card.innerText) || title || "").trim();
if (!href || !title || title.length < 3) return;
seen.add(anchor);
records.push({href, title, context: context.slice(0, 1600)});
};
for (const card of cards) {
const anchor = card.querySelector("h2 a, h3 a, a.result__a, a.heading-serpresult, a.w-gl__result-title, a[href]");
add(anchor, card);
}
if (!records.length) {
for (const anchor of Array.from(document.querySelectorAll("a[href]"))) {
const card = anchor.closest("li, article, .result, .result-item, .w-gl__result");
add(anchor, card);
if (records.length >= 80) break;
}
}
return records.slice(0, 80);
}
"""
try:
raw = page.evaluate(script)
except Exception:
return []
if not isinstance(raw, list):
return []
output: list[dict[str, str]] = []
for item in raw:
if not isinstance(item, dict):
continue
output.append({
"href": str(item.get("href") or "")[:1200],
"title": " ".join(str(item.get("title") or "").split())[:300],
"context": " ".join(str(item.get("context") or "").split())[:1800],
})
return output
def _safe_log(log_callback: Callable[[str, str, dict[str, Any]], Any] | None,
kind: str, event: str, details: dict[str, Any]) -> None:
if not log_callback:
return
try:
result = log_callback(kind, event, details)
# The route callback is synchronous. Do not attempt to run an
# arbitrary coroutine here; Browser Use's async path emits its own
# events through ``emit`` below.
if inspect.isawaitable(result):
try:
result.close() # type: ignore[attr-defined]
except Exception:
pass
except Exception:
return
def direct_search_candidates(
prospect: dict[str, Any],
*,
user_agent: str = "ProspectOS/2.0 (+personal-use)",
log_callback: Callable[[str, str, dict[str, Any]], Any] | None = None,
requested: int = 5,
timeout_seconds: int = 90,
) -> list[dict[str, Any]]:
"""Search public result pages and extract links directly from their DOM.
Bing is tried first because it is usually available when Google has
challenged the browser. Other public engines are fallbacks. This path
is deterministic and can return candidates even when Browser Use runs out
of steps or produces malformed JSON.
"""
try:
from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeout
except ImportError:
_safe_log(log_callback, "search", "website_search_unavailable", {"error": "playwright_not_installed"})
return []
queries = _search_query_variants(prospect)
if not queries:
return []
try:
hard_timeout = max(15, min(180, int(timeout_seconds)))
except (TypeError, ValueError):
hard_timeout = 90
deadline = time.monotonic() + hard_timeout
executable = os.environ.get("PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH", "").strip()
options: dict[str, Any] = {"headless": True}
if executable:
options["executable_path"] = executable
engines = (
("bing", "https://www.bing.com/search?q={query}&count=10&cc=ZA&setlang=en-ZA"),
("duckduckgo", "https://html.duckduckgo.com/html/?q={query}"),
("brave", "https://search.brave.com/search?q={query}"),
)
found: dict[str, dict[str, Any]] = {}
try:
with sync_playwright() as playwright:
browser = playwright.chromium.launch(**options)
try:
for engine, template in engines:
for query_index, query in enumerate(queries):
if time.monotonic() >= deadline:
_safe_log(log_callback, "search", "website_search_deadline", {"engine": engine, "query": query})
return sorted(found.values(), key=lambda item: item["confidence"], reverse=True)[:requested]
url = template.format(query=quote_plus(query))
_safe_log(log_callback, "search", "website_search_started", {"engine": engine, "query": query, "url": url})
context = None
try:
context = browser.new_context(
locale="en-ZA", java_script_enabled=True, service_workers="block",
viewport={"width": 1365, "height": 900}, user_agent=user_agent,
)
page = context.new_page()
page.set_default_timeout(8_000)
response = None
navigation_timed_out = False
try:
response = page.goto(url, wait_until="domcontentloaded", timeout=10_000)
except PlaywrightTimeout:
navigation_timed_out = True
try:
page.wait_for_timeout(800)
except Exception:
pass
body = ""
try:
body = page.locator("body").inner_text()[:20_000]
except Exception:
pass
page_title = ""
try:
page_title = str(page.title() or "")[:300]
except Exception:
pass
blocked_page = any(marker in (body + " " + page_title).lower() for marker in _SEARCH_BLOCKED_MARKERS)
visible = [] if blocked_page else _extract_visible_result_links(page)
page_candidates = 0
for item in visible:
href = _normalise_search_href(item.get("href"))
domain = _candidate_domain(href) if href else ""
if not href or not domain or domain in found:
if href and domain in found:
score, evidence = _candidate_score(prospect, href, item.get("title", ""), item.get("context", ""))
existing = found[domain]
existing["confidence"] = max(existing["confidence"], score)
existing["evidence"] = list(dict.fromkeys(existing["evidence"] + evidence))[:8]
existing["queries"] = list(dict.fromkeys(existing.get("queries", []) + [query]))[:8]
continue
score, evidence = _candidate_score(prospect, href, item.get("title", ""), item.get("context", ""))
# A result must have some visible relationship
# to the business; this filters unrelated ads,
# maps, and navigation links.
if score < 30:
continue
candidate = {
"url": href,
"domain": domain,
"confidence": score,
"evidence": evidence + [f"visible on {engine} search results"],
"status": "candidate",
"source": "browser_dom",
"search_engine": engine,
"title": item.get("title", "")[:300],
"snippet": item.get("context", "")[:1200],
"queries": [query],
}
found[domain] = candidate
page_candidates += 1
_safe_log(log_callback, "search", "website_search_candidate", {"engine": engine, "query": query, **candidate})
_safe_log(log_callback, "search", "website_search_page", {
"engine": engine, "query": query, "url": url, "title": page_title,
"http_status": int(getattr(response, "status", 0) or 0) if response else None,
"blocked": blocked_page, "navigation_timed_out": navigation_timed_out,
"visible_result_count": len(visible), "candidate_count": page_candidates,
})
if len(found) >= requested:
return sorted(found.values(), key=lambda item: item["confidence"], reverse=True)[:requested]
# One or two targeted Bing queries are normally
# enough. Do not burn the full browser deadline
# just to fill a five-domain quota when the
# official-looking result has already been found.
if found and query_index >= 1:
return sorted(found.values(), key=lambda item: item["confidence"], reverse=True)[:requested]
except Exception as exc:
_safe_log(log_callback, "search", "website_search_engine_error", {"engine": engine, "query": query, "url": url, "error": str(exc)[:500]})
finally:
if context is not None:
try:
context.close()
except Exception:
pass
finally:
browser.close()
except Exception as exc:
_safe_log(log_callback, "search", "website_search_failed", {"error": str(exc)[:500]})
results = sorted(found.values(), key=lambda item: item["confidence"], reverse=True)[:requested]
_safe_log(log_callback, "search", "website_search_completed", {"candidate_count": len(results), "candidates": results})
return results
def _parse_agent_result(raw: Any) -> list[dict[str, Any]]:
"""Parse and normalise the agent's final JSON candidate list."""
text = str(raw or "").strip()