587 lines
26 KiB
Python
587 lines
26 KiB
Python
"""Optional Browser Use investigator for public website candidates.
|
|
|
|
This module is deliberately isolated and imports browser-use only when called.
|
|
It performs bounded, headless public research. It does not log in, solve
|
|
CAPTCHAs, rotate proxies, or continue after an access challenge.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import inspect
|
|
import json
|
|
import os
|
|
import re
|
|
import time
|
|
from typing import Any, Callable
|
|
from urllib.parse import parse_qs, quote_plus, unquote, urlparse
|
|
|
|
|
|
_BLOCKED_HOSTS = {
|
|
"facebook.com", "instagram.com", "linkedin.com", "youtube.com", "x.com",
|
|
"twitter.com", "google.com", "google.co.za", "bing.com", "brave.com",
|
|
"duckduckgo.com", "startpage.com", "yelp.com", "yellowpages.com",
|
|
"yellosa.co.za", "snupit.co.za", "zaubee.com", "mapquest.com",
|
|
}
|
|
|
|
_SEARCH_BLOCKED_MARKERS = (
|
|
"unusual traffic", "recaptcha", "captcha", "automated queries",
|
|
"access denied", "detected unusual traffic", "verify you are human",
|
|
)
|
|
|
|
_GENERIC_NAME_WORDS = {
|
|
"and", "the", "services", "service", "solutions", "company", "pty",
|
|
"ltd", "inc", "llc", "south", "africa", "sa", "electrical",
|
|
"electrician", "electricians", "installation", "installations",
|
|
}
|
|
|
|
|
|
def _normalise_url(value: Any) -> str:
|
|
text = str(value or "").strip()
|
|
if not text:
|
|
return ""
|
|
if text.startswith("//"):
|
|
text = "https:" + text
|
|
if not re.match(r"^https?://", text, re.I):
|
|
text = "https://" + text
|
|
parsed = urlparse(text)
|
|
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
|
return ""
|
|
host = parsed.hostname.lower().rstrip(".")
|
|
if host in {"localhost", "127.0.0.1", "::1"} or host.endswith(".local"):
|
|
return ""
|
|
if any(host == blocked or host.endswith("." + blocked) for blocked in _BLOCKED_HOSTS):
|
|
return ""
|
|
return text[:1000]
|
|
|
|
|
|
def _candidate_domain(url: str) -> str:
|
|
host = (urlparse(url).hostname or "").lower().rstrip(".")
|
|
return host.removeprefix("www.")
|
|
|
|
|
|
def _normalise_search_href(value: Any) -> str:
|
|
"""Turn a search-result href into the real public URL.
|
|
|
|
Search engines sometimes wrap result links in redirect URLs. We unwrap
|
|
the common query-string forms before applying the blocked-host check; the
|
|
old implementation rejected those wrappers before it could see the real
|
|
business domain.
|
|
"""
|
|
text = str(value or "").strip()
|
|
if text.startswith("//"):
|
|
text = "https:" + text
|
|
if text.startswith("/"):
|
|
return ""
|
|
try:
|
|
parsed = urlparse(text)
|
|
query = parse_qs(parsed.query)
|
|
if parsed.hostname and parsed.hostname.lower().endswith(("duckduckgo.com", "bing.com", "bingj.com")):
|
|
for key in ("uddg", "url", "u", "r"):
|
|
target = query.get(key, [""])[0]
|
|
target = unquote(target)
|
|
if target.startswith(("http://", "https://")):
|
|
text = target
|
|
break
|
|
except Exception:
|
|
return ""
|
|
return _normalise_url(text)
|
|
|
|
|
|
def _tokens(value: str, *, include_generic: bool = False) -> list[str]:
|
|
values = re.findall(r"[a-z0-9]+", str(value or "").lower())
|
|
return [value for value in values if len(value) >= 3 and (include_generic or value not in _GENERIC_NAME_WORDS)]
|
|
|
|
|
|
def _location_from_address(address: str) -> str:
|
|
text = str(address or "").strip()
|
|
try:
|
|
parsed = json.loads(text)
|
|
if isinstance(parsed, dict):
|
|
return str(parsed.get("city") or parsed.get("town") or parsed.get("municipality") or parsed.get("state") or "").strip()
|
|
except (TypeError, ValueError, json.JSONDecodeError):
|
|
pass
|
|
parts = [part.strip() for part in re.split(r"[,|]", text) if part.strip()]
|
|
return parts[-1] if parts else ""
|
|
|
|
|
|
def _search_query_variants(prospect: dict[str, Any]) -> list[str]:
|
|
name = str(prospect.get("name") or "").strip()
|
|
if not name:
|
|
return []
|
|
location = _location_from_address(str(prospect.get("address") or ""))
|
|
category = str(prospect.get("category") or "").strip()
|
|
phone = re.sub(r"[^0-9]+", "", str(prospect.get("phone") or ""))
|
|
queries = [
|
|
f'"{name}" {location}'.strip(),
|
|
f'"{name}" {phone}'.strip() if phone else "",
|
|
f'"{name}" site:co.za'.strip(),
|
|
f'"{name}" {category} {location}'.strip(),
|
|
f'"{name}"'.strip(),
|
|
]
|
|
output: list[str] = []
|
|
seen: set[str] = set()
|
|
for query in queries:
|
|
query = " ".join(query.split())
|
|
if query and query not in seen:
|
|
seen.add(query)
|
|
output.append(query)
|
|
return output[:5]
|
|
|
|
|
|
def _candidate_score(prospect: dict[str, Any], url: str, title: str, context: str) -> tuple[int, list[str]]:
|
|
"""Score a result using only visible search-result evidence.
|
|
|
|
This is intentionally deterministic. It prevents an LLM's final
|
|
``done`` message from discarding a domain that was visibly present on the
|
|
search page.
|
|
"""
|
|
domain = _candidate_domain(url)
|
|
haystack = " ".join((title, context, domain)).lower()
|
|
domain_stem = domain
|
|
if domain.endswith(".co.za"):
|
|
domain_stem = domain[:-6]
|
|
elif domain.count(".") >= 1:
|
|
domain_stem = domain.rsplit(".", 1)[0]
|
|
compact_domain = re.sub(r"[^a-z0-9]", "", domain_stem)
|
|
name = str(prospect.get("name") or "")
|
|
name_compact = re.sub(r"[^a-z0-9]", "", name.lower())
|
|
name_tokens = _tokens(name)
|
|
evidence: list[str] = []
|
|
score = 18
|
|
if name_compact and (name_compact in compact_domain or compact_domain in name_compact):
|
|
score += 52
|
|
evidence.append("domain closely matches business name")
|
|
matching_tokens = [token for token in name_tokens if token in compact_domain]
|
|
if matching_tokens:
|
|
score += min(30, 12 * len(matching_tokens))
|
|
evidence.append("name token match: " + ", ".join(matching_tokens[:4]))
|
|
if any(token in haystack for token in name_tokens):
|
|
score += 8
|
|
evidence.append("business name appears in result text")
|
|
phone = re.sub(r"[^0-9]", "", str(prospect.get("phone") or ""))
|
|
context_digits = re.sub(r"[^0-9]", "", context)
|
|
if phone and len(phone) >= 7 and phone[-7:] in context_digits:
|
|
score += 14
|
|
evidence.append("phone appears in result text")
|
|
location_tokens = _tokens(_location_from_address(str(prospect.get("address") or "")), include_generic=True)
|
|
if location_tokens and any(token in haystack for token in location_tokens):
|
|
score += 6
|
|
evidence.append("location appears in result text")
|
|
if domain.endswith(".co.za"):
|
|
score += 5
|
|
evidence.append("South African .co.za domain")
|
|
return max(0, min(100, score)), evidence
|
|
|
|
|
|
def _extract_visible_result_links(page: Any) -> list[dict[str, str]]:
|
|
"""Read result anchors/snippets from the loaded page DOM.
|
|
|
|
No indexed click targets are used. This is the critical reliability path
|
|
for Bing pages whose element indexes can change after navigation.
|
|
"""
|
|
script = """
|
|
() => {
|
|
const cardSelectors = ["li.b_algo", "article", ".result", ".result-item", ".w-gl__result"];
|
|
const cards = cardSelectors.flatMap(selector => Array.from(document.querySelectorAll(selector)));
|
|
const records = [];
|
|
const seen = new Set();
|
|
const add = (anchor, card) => {
|
|
if (!anchor || seen.has(anchor)) return;
|
|
const href = anchor.href || anchor.getAttribute("href") || "";
|
|
const title = (anchor.innerText || anchor.textContent || anchor.getAttribute("aria-label") || "").trim();
|
|
const context = ((card && card.innerText) || title || "").trim();
|
|
if (!href || !title || title.length < 3) return;
|
|
seen.add(anchor);
|
|
records.push({href, title, context: context.slice(0, 1600)});
|
|
};
|
|
for (const card of cards) {
|
|
const anchor = card.querySelector("h2 a, h3 a, a.result__a, a.heading-serpresult, a.w-gl__result-title, a[href]");
|
|
add(anchor, card);
|
|
}
|
|
if (!records.length) {
|
|
for (const anchor of Array.from(document.querySelectorAll("a[href]"))) {
|
|
const card = anchor.closest("li, article, .result, .result-item, .w-gl__result");
|
|
add(anchor, card);
|
|
if (records.length >= 80) break;
|
|
}
|
|
}
|
|
return records.slice(0, 80);
|
|
}
|
|
"""
|
|
try:
|
|
raw = page.evaluate(script)
|
|
except Exception:
|
|
return []
|
|
if not isinstance(raw, list):
|
|
return []
|
|
output: list[dict[str, str]] = []
|
|
for item in raw:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
output.append({
|
|
"href": str(item.get("href") or "")[:1200],
|
|
"title": " ".join(str(item.get("title") or "").split())[:300],
|
|
"context": " ".join(str(item.get("context") or "").split())[:1800],
|
|
})
|
|
return output
|
|
|
|
|
|
def _safe_log(log_callback: Callable[[str, str, dict[str, Any]], Any] | None,
|
|
kind: str, event: str, details: dict[str, Any]) -> None:
|
|
if not log_callback:
|
|
return
|
|
try:
|
|
result = log_callback(kind, event, details)
|
|
# The route callback is synchronous. Do not attempt to run an
|
|
# arbitrary coroutine here; Browser Use's async path emits its own
|
|
# events through ``emit`` below.
|
|
if inspect.isawaitable(result):
|
|
try:
|
|
result.close() # type: ignore[attr-defined]
|
|
except Exception:
|
|
pass
|
|
except Exception:
|
|
return
|
|
|
|
|
|
def direct_search_candidates(
|
|
prospect: dict[str, Any],
|
|
*,
|
|
user_agent: str = "ProspectOS/2.0 (+personal-use)",
|
|
log_callback: Callable[[str, str, dict[str, Any]], Any] | None = None,
|
|
requested: int = 5,
|
|
timeout_seconds: int = 90,
|
|
) -> list[dict[str, Any]]:
|
|
"""Search public result pages and extract links directly from their DOM.
|
|
|
|
Bing is tried first because it is usually available when Google has
|
|
challenged the browser. Other public engines are fallbacks. This path
|
|
is deterministic and can return candidates even when Browser Use runs out
|
|
of steps or produces malformed JSON.
|
|
"""
|
|
try:
|
|
from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeout
|
|
except ImportError:
|
|
_safe_log(log_callback, "search", "website_search_unavailable", {"error": "playwright_not_installed"})
|
|
return []
|
|
queries = _search_query_variants(prospect)
|
|
if not queries:
|
|
return []
|
|
try:
|
|
hard_timeout = max(15, min(180, int(timeout_seconds)))
|
|
except (TypeError, ValueError):
|
|
hard_timeout = 90
|
|
deadline = time.monotonic() + hard_timeout
|
|
executable = os.environ.get("PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH", "").strip()
|
|
options: dict[str, Any] = {"headless": True}
|
|
if executable:
|
|
options["executable_path"] = executable
|
|
engines = (
|
|
("bing", "https://www.bing.com/search?q={query}&count=10&cc=ZA&setlang=en-ZA"),
|
|
("duckduckgo", "https://html.duckduckgo.com/html/?q={query}"),
|
|
("brave", "https://search.brave.com/search?q={query}"),
|
|
)
|
|
found: dict[str, dict[str, Any]] = {}
|
|
try:
|
|
with sync_playwright() as playwright:
|
|
browser = playwright.chromium.launch(**options)
|
|
try:
|
|
for engine, template in engines:
|
|
for query_index, query in enumerate(queries):
|
|
if time.monotonic() >= deadline:
|
|
_safe_log(log_callback, "search", "website_search_deadline", {"engine": engine, "query": query})
|
|
return sorted(found.values(), key=lambda item: item["confidence"], reverse=True)[:requested]
|
|
url = template.format(query=quote_plus(query))
|
|
_safe_log(log_callback, "search", "website_search_started", {"engine": engine, "query": query, "url": url})
|
|
context = None
|
|
try:
|
|
context = browser.new_context(
|
|
locale="en-ZA", java_script_enabled=True, service_workers="block",
|
|
viewport={"width": 1365, "height": 900}, user_agent=user_agent,
|
|
)
|
|
page = context.new_page()
|
|
page.set_default_timeout(8_000)
|
|
response = None
|
|
navigation_timed_out = False
|
|
try:
|
|
response = page.goto(url, wait_until="domcontentloaded", timeout=10_000)
|
|
except PlaywrightTimeout:
|
|
navigation_timed_out = True
|
|
try:
|
|
page.wait_for_timeout(800)
|
|
except Exception:
|
|
pass
|
|
body = ""
|
|
try:
|
|
body = page.locator("body").inner_text()[:20_000]
|
|
except Exception:
|
|
pass
|
|
page_title = ""
|
|
try:
|
|
page_title = str(page.title() or "")[:300]
|
|
except Exception:
|
|
pass
|
|
blocked_page = any(marker in (body + " " + page_title).lower() for marker in _SEARCH_BLOCKED_MARKERS)
|
|
visible = [] if blocked_page else _extract_visible_result_links(page)
|
|
page_candidates = 0
|
|
for item in visible:
|
|
href = _normalise_search_href(item.get("href"))
|
|
domain = _candidate_domain(href) if href else ""
|
|
if not href or not domain or domain in found:
|
|
if href and domain in found:
|
|
score, evidence = _candidate_score(prospect, href, item.get("title", ""), item.get("context", ""))
|
|
existing = found[domain]
|
|
existing["confidence"] = max(existing["confidence"], score)
|
|
existing["evidence"] = list(dict.fromkeys(existing["evidence"] + evidence))[:8]
|
|
existing["queries"] = list(dict.fromkeys(existing.get("queries", []) + [query]))[:8]
|
|
continue
|
|
score, evidence = _candidate_score(prospect, href, item.get("title", ""), item.get("context", ""))
|
|
# A result must have some visible relationship
|
|
# to the business; this filters unrelated ads,
|
|
# maps, and navigation links.
|
|
if score < 30:
|
|
continue
|
|
candidate = {
|
|
"url": href,
|
|
"domain": domain,
|
|
"confidence": score,
|
|
"evidence": evidence + [f"visible on {engine} search results"],
|
|
"status": "candidate",
|
|
"source": "browser_dom",
|
|
"search_engine": engine,
|
|
"title": item.get("title", "")[:300],
|
|
"snippet": item.get("context", "")[:1200],
|
|
"queries": [query],
|
|
}
|
|
found[domain] = candidate
|
|
page_candidates += 1
|
|
_safe_log(log_callback, "search", "website_search_candidate", {"engine": engine, "query": query, **candidate})
|
|
_safe_log(log_callback, "search", "website_search_page", {
|
|
"engine": engine, "query": query, "url": url, "title": page_title,
|
|
"http_status": int(getattr(response, "status", 0) or 0) if response else None,
|
|
"blocked": blocked_page, "navigation_timed_out": navigation_timed_out,
|
|
"visible_result_count": len(visible), "candidate_count": page_candidates,
|
|
})
|
|
if len(found) >= requested:
|
|
return sorted(found.values(), key=lambda item: item["confidence"], reverse=True)[:requested]
|
|
# One or two targeted Bing queries are normally
|
|
# enough. Do not burn the full browser deadline
|
|
# just to fill a five-domain quota when the
|
|
# official-looking result has already been found.
|
|
if found and query_index >= 1:
|
|
return sorted(found.values(), key=lambda item: item["confidence"], reverse=True)[:requested]
|
|
except Exception as exc:
|
|
_safe_log(log_callback, "search", "website_search_engine_error", {"engine": engine, "query": query, "url": url, "error": str(exc)[:500]})
|
|
finally:
|
|
if context is not None:
|
|
try:
|
|
context.close()
|
|
except Exception:
|
|
pass
|
|
finally:
|
|
browser.close()
|
|
except Exception as exc:
|
|
_safe_log(log_callback, "search", "website_search_failed", {"error": str(exc)[:500]})
|
|
results = sorted(found.values(), key=lambda item: item["confidence"], reverse=True)[:requested]
|
|
_safe_log(log_callback, "search", "website_search_completed", {"candidate_count": len(results), "candidates": results})
|
|
return results
|
|
|
|
|
|
def _parse_agent_result(raw: Any) -> list[dict[str, Any]]:
|
|
"""Parse and normalise the agent's final JSON candidate list."""
|
|
text = str(raw or "").strip()
|
|
if text.startswith("```"):
|
|
text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text, flags=re.I | re.S).strip()
|
|
try:
|
|
payload = json.loads(text)
|
|
except (TypeError, ValueError, json.JSONDecodeError):
|
|
start, end = text.find("{"), text.rfind("}")
|
|
if start < 0 or end <= start:
|
|
return []
|
|
try:
|
|
payload = json.loads(text[start:end + 1])
|
|
except (TypeError, ValueError, json.JSONDecodeError):
|
|
return []
|
|
if not isinstance(payload, dict):
|
|
return []
|
|
raw_candidates = payload.get("candidates")
|
|
if not isinstance(raw_candidates, list):
|
|
return []
|
|
output: list[dict[str, Any]] = []
|
|
seen: set[str] = set()
|
|
for item in raw_candidates[:10]:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
url = _normalise_url(item.get("url") or item.get("website") or item.get("domain"))
|
|
if not url:
|
|
continue
|
|
domain = (urlparse(url).hostname or "").lower().removeprefix("www.")
|
|
if not domain or domain in seen:
|
|
continue
|
|
seen.add(domain)
|
|
try:
|
|
confidence = max(0, min(100, int(round(float(item.get("confidence", 0))))))
|
|
except (TypeError, ValueError):
|
|
confidence = 0
|
|
evidence = item.get("evidence")
|
|
if not isinstance(evidence, list):
|
|
evidence = []
|
|
output.append({
|
|
"url": url,
|
|
"domain": domain,
|
|
"confidence": confidence,
|
|
"evidence": [str(value).strip()[:240] for value in evidence if str(value).strip()][:8],
|
|
"status": str(item.get("status") or "candidate").strip()[:80],
|
|
})
|
|
return sorted(output, key=lambda item: item["confidence"], reverse=True)
|
|
|
|
|
|
def investigate_websites(
|
|
prospect: dict[str, Any],
|
|
*,
|
|
base_url: str,
|
|
api_key: str,
|
|
model: str,
|
|
user_agent: str,
|
|
log_callback: Callable[[str, str, dict[str, Any]], Any] | None = None,
|
|
enabled: bool | None = None,
|
|
max_steps: int | None = None,
|
|
timeout_seconds: int | None = None,
|
|
llm_timeout_seconds: int | None = None,
|
|
) -> list[dict[str, Any]]:
|
|
"""Run one bounded Browser Use investigation and return candidates."""
|
|
if enabled is None:
|
|
enabled = os.environ.get("BROWSER_USE_ENABLED", "1").strip().lower() not in {"0", "false", "no", "off"}
|
|
if not enabled:
|
|
raise RuntimeError("browser_use_disabled")
|
|
if not base_url or not model:
|
|
raise RuntimeError("browser_use_requires_ai_provider_and_model")
|
|
if not api_key:
|
|
raise RuntimeError("browser_use_requires_ai_api_key")
|
|
try:
|
|
from browser_use import Agent, BrowserProfile, ChatOpenAI
|
|
except ImportError as exc:
|
|
raise RuntimeError("browser_use_not_installed_rebuild_container") from exc
|
|
|
|
name = str(prospect.get("name") or "").strip()
|
|
address = str(prospect.get("address") or "").strip()
|
|
category = str(prospect.get("category") or "").strip()
|
|
phone = str(prospect.get("phone") or "").strip()
|
|
emails = str(prospect.get("emails") or "").strip()
|
|
task = f"""
|
|
You are a bounded public-data website investigator. Find possible official websites for this one business:
|
|
Business name: {name}
|
|
Category/services: {category}
|
|
Address/location: {address}
|
|
Phone: {phone}
|
|
Public email(s): {emails}
|
|
|
|
Use only public, unauthenticated pages. Search at most three result pages and inspect at most five candidate domains.
|
|
Try exact business-name and location/phone searches. Prefer the official business website over social networks,
|
|
directories, review sites, and map pages. Do not log in, submit forms, solve CAPTCHAs, bypass access challenges,
|
|
or use a proxy. If a site blocks the browser, record it and move on.
|
|
|
|
Return ONLY JSON in this shape:
|
|
{{"candidates":[{{"url":"https://example.co.za","confidence":0,"evidence":["name match","phone match"],"status":"candidate"}}]}}
|
|
Confidence must be an integer from 0 to 100. Include only plausible official website URLs and never invent a URL.
|
|
""".strip()
|
|
|
|
async def emit(kind: str, event: str, details: dict[str, Any]) -> None:
|
|
if not log_callback:
|
|
return
|
|
result = log_callback(kind, event, details)
|
|
if inspect.isawaitable(result):
|
|
await result
|
|
|
|
def output_actions(model_output: Any) -> list[Any]:
|
|
try:
|
|
actions = getattr(model_output, "action", []) or []
|
|
values: list[Any] = []
|
|
for action in list(actions)[:8]:
|
|
if hasattr(action, "model_dump"):
|
|
values.append(action.model_dump(mode="json"))
|
|
else:
|
|
values.append(str(action)[:2_000])
|
|
return values
|
|
except Exception:
|
|
return [str(model_output)[:2_000]] if model_output is not None else []
|
|
|
|
async def on_step(browser_state: Any, model_output: Any, step_number: int) -> None:
|
|
details = {
|
|
"step": step_number,
|
|
"url": str(getattr(browser_state, "url", "") or "")[:700],
|
|
"title": str(getattr(browser_state, "title", "") or "")[:300],
|
|
"actions": output_actions(model_output),
|
|
"model": model,
|
|
}
|
|
await emit("browser", "browser_use_step", details)
|
|
await emit("ai", "browser_use_model_step", {"step": step_number, "model": model, "actions": details["actions"]})
|
|
|
|
class LoggedChatOpenAI(ChatOpenAI):
|
|
"""Capture Browser Use LLM turns without recording provider secrets."""
|
|
|
|
async def ainvoke(self, messages: list[Any], output_format: Any = None, **kwargs: Any) -> Any:
|
|
started = asyncio.get_running_loop().time()
|
|
serialised = [{"role": getattr(message, "type", "message"), "content": str(getattr(message, "content", ""))[:4_000]} for message in list(messages)[-20:]]
|
|
await emit("ai", "browser_use_llm_request", {"model": self.model, "messages": serialised, "structured_output": bool(output_format)})
|
|
try:
|
|
result = await super().ainvoke(messages, output_format=output_format, **kwargs)
|
|
await emit("ai", "browser_use_llm_response", {"model": self.model, "completion": str(getattr(result, "completion", result))[:4_000], "duration_ms": int((asyncio.get_running_loop().time() - started) * 1000)})
|
|
return result
|
|
except Exception as exc:
|
|
await emit("ai", "browser_use_llm_error", {"model": self.model, "duration_ms": int((asyncio.get_running_loop().time() - started) * 1000)},)
|
|
raise
|
|
|
|
async def run() -> Any:
|
|
browser_profile = BrowserProfile(
|
|
headless=True,
|
|
executable_path=os.environ.get("PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH", "/usr/bin/chromium"),
|
|
minimum_wait_page_load_time=0.5,
|
|
wait_between_actions=0.7,
|
|
enable_default_extensions=False,
|
|
captcha_solver=False,
|
|
)
|
|
try:
|
|
llm_timeout = max(10, min(300, int(llm_timeout_seconds if llm_timeout_seconds is not None else os.environ.get("BROWSER_USE_LLM_TIMEOUT_SECONDS", "90"))))
|
|
except (TypeError, ValueError):
|
|
llm_timeout = 45
|
|
llm = LoggedChatOpenAI(
|
|
model=model,
|
|
api_key=api_key,
|
|
base_url=base_url,
|
|
default_headers={"User-Agent": user_agent},
|
|
temperature=0.1,
|
|
frequency_penalty=None,
|
|
max_completion_tokens=900,
|
|
max_retries=1,
|
|
timeout=llm_timeout,
|
|
)
|
|
agent = Agent(
|
|
task=task,
|
|
llm=llm,
|
|
browser_profile=browser_profile,
|
|
use_vision=False,
|
|
flash_mode=True,
|
|
use_judge=False,
|
|
enable_planning=False,
|
|
enable_signal_handler=False,
|
|
register_new_step_callback=on_step,
|
|
)
|
|
try:
|
|
resolved_steps = int(max_steps if max_steps is not None else os.environ.get("BROWSER_USE_MAX_STEPS", "10"))
|
|
except (TypeError, ValueError):
|
|
resolved_steps = 10
|
|
return await agent.run(max_steps=max(3, min(50, resolved_steps)))
|
|
|
|
try:
|
|
timeout = int(timeout_seconds if timeout_seconds is not None else os.environ.get("BROWSER_USE_TIMEOUT_SECONDS", "600"))
|
|
except (TypeError, ValueError):
|
|
timeout = 600
|
|
timeout = max(30, min(1800, timeout))
|
|
try:
|
|
history = asyncio.run(asyncio.wait_for(run(), timeout=timeout))
|
|
except asyncio.TimeoutError as exc:
|
|
raise RuntimeError("browser_use_timeout") from exc
|
|
final = history.final_result() if history is not None and hasattr(history, "final_result") else ""
|
|
return _parse_agent_result(final)
|