deploy Nous v19 update
This commit is contained in:
Marco0300
@@ -0,0 +1,173 @@
|
||||
"""Optional Browser Use investigator for public website candidates.
|
||||
|
||||
This module is deliberately isolated and imports browser-use only when called.
|
||||
It performs bounded, headless public research. It does not log in, solve
|
||||
CAPTCHAs, rotate proxies, or continue after an access challenge.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
_BLOCKED_HOSTS = {
|
||||
"facebook.com", "instagram.com", "linkedin.com", "youtube.com", "x.com",
|
||||
"twitter.com", "google.com", "google.co.za", "bing.com", "brave.com",
|
||||
"duckduckgo.com", "startpage.com", "yelp.com", "yellowpages.com",
|
||||
}
|
||||
|
||||
|
||||
def _normalise_url(value: Any) -> str:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
return ""
|
||||
if text.startswith("//"):
|
||||
text = "https:" + text
|
||||
if not re.match(r"^https?://", text, re.I):
|
||||
text = "https://" + text
|
||||
parsed = urlparse(text)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
||||
return ""
|
||||
host = parsed.hostname.lower().rstrip(".")
|
||||
if host in {"localhost", "127.0.0.1", "::1"} or host.endswith(".local"):
|
||||
return ""
|
||||
if any(host == blocked or host.endswith("." + blocked) for blocked in _BLOCKED_HOSTS):
|
||||
return ""
|
||||
return text[:1000]
|
||||
|
||||
|
||||
def _parse_agent_result(raw: Any) -> list[dict[str, Any]]:
|
||||
"""Parse and normalise the agent's final JSON candidate list."""
|
||||
text = str(raw or "").strip()
|
||||
if text.startswith("```"):
|
||||
text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text, flags=re.I | re.S).strip()
|
||||
try:
|
||||
payload = json.loads(text)
|
||||
except (TypeError, ValueError, json.JSONDecodeError):
|
||||
start, end = text.find("{"), text.rfind("}")
|
||||
if start < 0 or end <= start:
|
||||
return []
|
||||
try:
|
||||
payload = json.loads(text[start:end + 1])
|
||||
except (TypeError, ValueError, json.JSONDecodeError):
|
||||
return []
|
||||
if not isinstance(payload, dict):
|
||||
return []
|
||||
raw_candidates = payload.get("candidates")
|
||||
if not isinstance(raw_candidates, list):
|
||||
return []
|
||||
output: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for item in raw_candidates[:10]:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
url = _normalise_url(item.get("url") or item.get("website") or item.get("domain"))
|
||||
if not url:
|
||||
continue
|
||||
domain = (urlparse(url).hostname or "").lower().removeprefix("www.")
|
||||
if not domain or domain in seen:
|
||||
continue
|
||||
seen.add(domain)
|
||||
try:
|
||||
confidence = max(0, min(100, int(round(float(item.get("confidence", 0))))))
|
||||
except (TypeError, ValueError):
|
||||
confidence = 0
|
||||
evidence = item.get("evidence")
|
||||
if not isinstance(evidence, list):
|
||||
evidence = []
|
||||
output.append({
|
||||
"url": url,
|
||||
"domain": domain,
|
||||
"confidence": confidence,
|
||||
"evidence": [str(value).strip()[:240] for value in evidence if str(value).strip()][:8],
|
||||
"status": str(item.get("status") or "candidate").strip()[:80],
|
||||
})
|
||||
return sorted(output, key=lambda item: item["confidence"], reverse=True)
|
||||
|
||||
|
||||
def investigate_websites(
|
||||
prospect: dict[str, Any],
|
||||
*,
|
||||
base_url: str,
|
||||
api_key: str,
|
||||
model: str,
|
||||
user_agent: str,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Run one bounded Browser Use investigation and return candidates."""
|
||||
if os.environ.get("BROWSER_USE_ENABLED", "1").strip().lower() in {"0", "false", "no", "off"}:
|
||||
raise RuntimeError("browser_use_disabled")
|
||||
if not base_url or not model:
|
||||
raise RuntimeError("browser_use_requires_ai_provider_and_model")
|
||||
if not api_key:
|
||||
raise RuntimeError("browser_use_requires_ai_api_key")
|
||||
try:
|
||||
from browser_use import Agent, BrowserProfile, ChatOpenAI
|
||||
except ImportError as exc:
|
||||
raise RuntimeError("browser_use_not_installed_rebuild_container") from exc
|
||||
|
||||
name = str(prospect.get("name") or "").strip()
|
||||
address = str(prospect.get("address") or "").strip()
|
||||
category = str(prospect.get("category") or "").strip()
|
||||
phone = str(prospect.get("phone") or "").strip()
|
||||
emails = str(prospect.get("emails") or "").strip()
|
||||
task = f"""
|
||||
You are a bounded public-data website investigator. Find possible official websites for this one business:
|
||||
Business name: {name}
|
||||
Category/services: {category}
|
||||
Address/location: {address}
|
||||
Phone: {phone}
|
||||
Public email(s): {emails}
|
||||
|
||||
Use only public, unauthenticated pages. Search at most three result pages and inspect at most five candidate domains.
|
||||
Try exact business-name and location/phone searches. Prefer the official business website over social networks,
|
||||
directories, review sites, and map pages. Do not log in, submit forms, solve CAPTCHAs, bypass access challenges,
|
||||
or use a proxy. If a site blocks the browser, record it and move on.
|
||||
|
||||
Return ONLY JSON in this shape:
|
||||
{{"candidates":[{{"url":"https://example.co.za","confidence":0,"evidence":["name match","phone match"],"status":"candidate"}}]}}
|
||||
Confidence must be an integer from 0 to 100. Include only plausible official website URLs and never invent a URL.
|
||||
""".strip()
|
||||
|
||||
async def run() -> Any:
|
||||
browser_profile = BrowserProfile(
|
||||
headless=True,
|
||||
executable_path=os.environ.get("PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH", "/usr/bin/chromium"),
|
||||
minimum_wait_page_load_time=0.5,
|
||||
wait_between_actions=0.7,
|
||||
enable_default_extensions=False,
|
||||
captcha_solver=False,
|
||||
)
|
||||
llm = ChatOpenAI(
|
||||
model=model,
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
default_headers={"User-Agent": user_agent},
|
||||
temperature=0.1,
|
||||
frequency_penalty=None,
|
||||
max_completion_tokens=900,
|
||||
max_retries=1,
|
||||
timeout=45,
|
||||
)
|
||||
agent = Agent(
|
||||
task=task,
|
||||
llm=llm,
|
||||
browser_profile=browser_profile,
|
||||
use_vision=False,
|
||||
flash_mode=True,
|
||||
use_judge=False,
|
||||
enable_planning=False,
|
||||
enable_signal_handler=False,
|
||||
)
|
||||
return await agent.run(max_steps=max(3, min(15, int(os.environ.get("BROWSER_USE_MAX_STEPS", "10")))))
|
||||
|
||||
timeout = max(30, min(300, int(os.environ.get("BROWSER_USE_TIMEOUT_SECONDS", "120"))))
|
||||
try:
|
||||
history = asyncio.run(asyncio.wait_for(run(), timeout=timeout))
|
||||
except asyncio.TimeoutError as exc:
|
||||
raise RuntimeError("browser_use_timeout") from exc
|
||||
final = history.final_result() if history is not None and hasattr(history, "final_result") else ""
|
||||
return _parse_agent_result(final)
|
||||
Reference in New Issue
Block a user