241 lines
10 KiB
Python
241 lines
10 KiB
Python
"""Optional Browser Use investigator for public website candidates.
|
|
|
|
This module is deliberately isolated and imports browser-use only when called.
|
|
It performs bounded, headless public research. It does not log in, solve
|
|
CAPTCHAs, rotate proxies, or continue after an access challenge.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import inspect
|
|
import json
|
|
import os
|
|
import re
|
|
from typing import Any, Callable
|
|
from urllib.parse import urlparse
|
|
|
|
|
|
_BLOCKED_HOSTS = {
|
|
"facebook.com", "instagram.com", "linkedin.com", "youtube.com", "x.com",
|
|
"twitter.com", "google.com", "google.co.za", "bing.com", "brave.com",
|
|
"duckduckgo.com", "startpage.com", "yelp.com", "yellowpages.com",
|
|
}
|
|
|
|
|
|
def _normalise_url(value: Any) -> str:
|
|
text = str(value or "").strip()
|
|
if not text:
|
|
return ""
|
|
if text.startswith("//"):
|
|
text = "https:" + text
|
|
if not re.match(r"^https?://", text, re.I):
|
|
text = "https://" + text
|
|
parsed = urlparse(text)
|
|
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
|
return ""
|
|
host = parsed.hostname.lower().rstrip(".")
|
|
if host in {"localhost", "127.0.0.1", "::1"} or host.endswith(".local"):
|
|
return ""
|
|
if any(host == blocked or host.endswith("." + blocked) for blocked in _BLOCKED_HOSTS):
|
|
return ""
|
|
return text[:1000]
|
|
|
|
|
|
def _parse_agent_result(raw: Any) -> list[dict[str, Any]]:
|
|
"""Parse and normalise the agent's final JSON candidate list."""
|
|
text = str(raw or "").strip()
|
|
if text.startswith("```"):
|
|
text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text, flags=re.I | re.S).strip()
|
|
try:
|
|
payload = json.loads(text)
|
|
except (TypeError, ValueError, json.JSONDecodeError):
|
|
start, end = text.find("{"), text.rfind("}")
|
|
if start < 0 or end <= start:
|
|
return []
|
|
try:
|
|
payload = json.loads(text[start:end + 1])
|
|
except (TypeError, ValueError, json.JSONDecodeError):
|
|
return []
|
|
if not isinstance(payload, dict):
|
|
return []
|
|
raw_candidates = payload.get("candidates")
|
|
if not isinstance(raw_candidates, list):
|
|
return []
|
|
output: list[dict[str, Any]] = []
|
|
seen: set[str] = set()
|
|
for item in raw_candidates[:10]:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
url = _normalise_url(item.get("url") or item.get("website") or item.get("domain"))
|
|
if not url:
|
|
continue
|
|
domain = (urlparse(url).hostname or "").lower().removeprefix("www.")
|
|
if not domain or domain in seen:
|
|
continue
|
|
seen.add(domain)
|
|
try:
|
|
confidence = max(0, min(100, int(round(float(item.get("confidence", 0))))))
|
|
except (TypeError, ValueError):
|
|
confidence = 0
|
|
evidence = item.get("evidence")
|
|
if not isinstance(evidence, list):
|
|
evidence = []
|
|
output.append({
|
|
"url": url,
|
|
"domain": domain,
|
|
"confidence": confidence,
|
|
"evidence": [str(value).strip()[:240] for value in evidence if str(value).strip()][:8],
|
|
"status": str(item.get("status") or "candidate").strip()[:80],
|
|
})
|
|
return sorted(output, key=lambda item: item["confidence"], reverse=True)
|
|
|
|
|
|
def investigate_websites(
|
|
prospect: dict[str, Any],
|
|
*,
|
|
base_url: str,
|
|
api_key: str,
|
|
model: str,
|
|
user_agent: str,
|
|
log_callback: Callable[[str, str, dict[str, Any]], Any] | None = None,
|
|
enabled: bool | None = None,
|
|
max_steps: int | None = None,
|
|
timeout_seconds: int | None = None,
|
|
llm_timeout_seconds: int | None = None,
|
|
) -> list[dict[str, Any]]:
|
|
"""Run one bounded Browser Use investigation and return candidates."""
|
|
if enabled is None:
|
|
enabled = os.environ.get("BROWSER_USE_ENABLED", "1").strip().lower() not in {"0", "false", "no", "off"}
|
|
if not enabled:
|
|
raise RuntimeError("browser_use_disabled")
|
|
if not base_url or not model:
|
|
raise RuntimeError("browser_use_requires_ai_provider_and_model")
|
|
if not api_key:
|
|
raise RuntimeError("browser_use_requires_ai_api_key")
|
|
try:
|
|
from browser_use import Agent, BrowserProfile, ChatOpenAI
|
|
except ImportError as exc:
|
|
raise RuntimeError("browser_use_not_installed_rebuild_container") from exc
|
|
|
|
name = str(prospect.get("name") or "").strip()
|
|
address = str(prospect.get("address") or "").strip()
|
|
category = str(prospect.get("category") or "").strip()
|
|
phone = str(prospect.get("phone") or "").strip()
|
|
emails = str(prospect.get("emails") or "").strip()
|
|
task = f"""
|
|
You are a bounded public-data website investigator. Find possible official websites for this one business:
|
|
Business name: {name}
|
|
Category/services: {category}
|
|
Address/location: {address}
|
|
Phone: {phone}
|
|
Public email(s): {emails}
|
|
|
|
Use only public, unauthenticated pages. Search at most three result pages and inspect at most five candidate domains.
|
|
Try exact business-name and location/phone searches. Prefer the official business website over social networks,
|
|
directories, review sites, and map pages. Do not log in, submit forms, solve CAPTCHAs, bypass access challenges,
|
|
or use a proxy. If a site blocks the browser, record it and move on.
|
|
|
|
Return ONLY JSON in this shape:
|
|
{{"candidates":[{{"url":"https://example.co.za","confidence":0,"evidence":["name match","phone match"],"status":"candidate"}}]}}
|
|
Confidence must be an integer from 0 to 100. Include only plausible official website URLs and never invent a URL.
|
|
""".strip()
|
|
|
|
async def emit(kind: str, event: str, details: dict[str, Any]) -> None:
|
|
if not log_callback:
|
|
return
|
|
result = log_callback(kind, event, details)
|
|
if inspect.isawaitable(result):
|
|
await result
|
|
|
|
def output_actions(model_output: Any) -> list[Any]:
|
|
try:
|
|
actions = getattr(model_output, "action", []) or []
|
|
values: list[Any] = []
|
|
for action in list(actions)[:8]:
|
|
if hasattr(action, "model_dump"):
|
|
values.append(action.model_dump(mode="json"))
|
|
else:
|
|
values.append(str(action)[:2_000])
|
|
return values
|
|
except Exception:
|
|
return [str(model_output)[:2_000]] if model_output is not None else []
|
|
|
|
async def on_step(browser_state: Any, model_output: Any, step_number: int) -> None:
|
|
details = {
|
|
"step": step_number,
|
|
"url": str(getattr(browser_state, "url", "") or "")[:700],
|
|
"title": str(getattr(browser_state, "title", "") or "")[:300],
|
|
"actions": output_actions(model_output),
|
|
"model": model,
|
|
}
|
|
await emit("browser", "browser_use_step", details)
|
|
await emit("ai", "browser_use_model_step", {"step": step_number, "model": model, "actions": details["actions"]})
|
|
|
|
class LoggedChatOpenAI(ChatOpenAI):
|
|
"""Capture Browser Use LLM turns without recording provider secrets."""
|
|
|
|
async def ainvoke(self, messages: list[Any], output_format: Any = None, **kwargs: Any) -> Any:
|
|
started = asyncio.get_running_loop().time()
|
|
serialised = [{"role": getattr(message, "type", "message"), "content": str(getattr(message, "content", ""))[:4_000]} for message in list(messages)[-20:]]
|
|
await emit("ai", "browser_use_llm_request", {"model": self.model, "messages": serialised, "structured_output": bool(output_format)})
|
|
try:
|
|
result = await super().ainvoke(messages, output_format=output_format, **kwargs)
|
|
await emit("ai", "browser_use_llm_response", {"model": self.model, "completion": str(getattr(result, "completion", result))[:4_000], "duration_ms": int((asyncio.get_running_loop().time() - started) * 1000)})
|
|
return result
|
|
except Exception as exc:
|
|
await emit("ai", "browser_use_llm_error", {"model": self.model, "duration_ms": int((asyncio.get_running_loop().time() - started) * 1000)},)
|
|
raise
|
|
|
|
async def run() -> Any:
|
|
browser_profile = BrowserProfile(
|
|
headless=True,
|
|
executable_path=os.environ.get("PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH", "/usr/bin/chromium"),
|
|
minimum_wait_page_load_time=0.5,
|
|
wait_between_actions=0.7,
|
|
enable_default_extensions=False,
|
|
captcha_solver=False,
|
|
)
|
|
try:
|
|
llm_timeout = max(10, min(300, int(llm_timeout_seconds if llm_timeout_seconds is not None else os.environ.get("BROWSER_USE_LLM_TIMEOUT_SECONDS", "90"))))
|
|
except (TypeError, ValueError):
|
|
llm_timeout = 45
|
|
llm = LoggedChatOpenAI(
|
|
model=model,
|
|
api_key=api_key,
|
|
base_url=base_url,
|
|
default_headers={"User-Agent": user_agent},
|
|
temperature=0.1,
|
|
frequency_penalty=None,
|
|
max_completion_tokens=900,
|
|
max_retries=1,
|
|
timeout=llm_timeout,
|
|
)
|
|
agent = Agent(
|
|
task=task,
|
|
llm=llm,
|
|
browser_profile=browser_profile,
|
|
use_vision=False,
|
|
flash_mode=True,
|
|
use_judge=False,
|
|
enable_planning=False,
|
|
enable_signal_handler=False,
|
|
register_new_step_callback=on_step,
|
|
)
|
|
try:
|
|
resolved_steps = int(max_steps if max_steps is not None else os.environ.get("BROWSER_USE_MAX_STEPS", "10"))
|
|
except (TypeError, ValueError):
|
|
resolved_steps = 10
|
|
return await agent.run(max_steps=max(3, min(50, resolved_steps)))
|
|
|
|
try:
|
|
timeout = int(timeout_seconds if timeout_seconds is not None else os.environ.get("BROWSER_USE_TIMEOUT_SECONDS", "600"))
|
|
except (TypeError, ValueError):
|
|
timeout = 600
|
|
timeout = max(30, min(1800, timeout))
|
|
try:
|
|
history = asyncio.run(asyncio.wait_for(run(), timeout=timeout))
|
|
except asyncio.TimeoutError as exc:
|
|
raise RuntimeError("browser_use_timeout") from exc
|
|
final = history.final_result() if history is not None and hasattr(history, "final_result") else ""
|
|
return _parse_agent_result(final)
|