deploy Nous v22 update

This commit is contained in:
Marco0300
2026-09-05 22:45:12 +02:00
parent aacb71221b
commit 112ab552e9
10 changed files with 265 additions and 52 deletions
+18 -8
View File
@@ -98,9 +98,15 @@ def investigate_websites(
model: str,
user_agent: str,
log_callback: Callable[[str, str, dict[str, Any]], Any] | None = None,
enabled: bool | None = None,
max_steps: int | None = None,
timeout_seconds: int | None = None,
llm_timeout_seconds: int | None = None,
) -> list[dict[str, Any]]:
"""Run one bounded Browser Use investigation and return candidates."""
if os.environ.get("BROWSER_USE_ENABLED", "1").strip().lower() in {"0", "false", "no", "off"}:
if enabled is None:
enabled = os.environ.get("BROWSER_USE_ENABLED", "1").strip().lower() not in {"0", "false", "no", "off"}
if not enabled:
raise RuntimeError("browser_use_disabled")
if not base_url or not model:
raise RuntimeError("browser_use_requires_ai_provider_and_model")
@@ -189,6 +195,10 @@ Confidence must be an integer from 0 to 100. Include only plausible official web
enable_default_extensions=False,
captcha_solver=False,
)
try:
llm_timeout = max(10, min(300, int(llm_timeout_seconds if llm_timeout_seconds is not None else os.environ.get("BROWSER_USE_LLM_TIMEOUT_SECONDS", "90"))))
except (TypeError, ValueError):
llm_timeout = 45
llm = LoggedChatOpenAI(
model=model,
api_key=api_key,
@@ -198,7 +208,7 @@ Confidence must be an integer from 0 to 100. Include only plausible official web
frequency_penalty=None,
max_completion_tokens=900,
max_retries=1,
timeout=45,
timeout=llm_timeout,
)
agent = Agent(
task=task,
@@ -212,16 +222,16 @@ Confidence must be an integer from 0 to 100. Include only plausible official web
register_new_step_callback=on_step,
)
try:
max_steps = int(os.environ.get("BROWSER_USE_MAX_STEPS", "10"))
resolved_steps = int(max_steps if max_steps is not None else os.environ.get("BROWSER_USE_MAX_STEPS", "10"))
except (TypeError, ValueError):
max_steps = 10
return await agent.run(max_steps=max(3, min(15, max_steps)))
resolved_steps = 10
return await agent.run(max_steps=max(3, min(50, resolved_steps)))
try:
timeout = int(os.environ.get("BROWSER_USE_TIMEOUT_SECONDS", "120"))
timeout = int(timeout_seconds if timeout_seconds is not None else os.environ.get("BROWSER_USE_TIMEOUT_SECONDS", "600"))
except (TypeError, ValueError):
timeout = 120
timeout = max(30, min(300, timeout))
timeout = 600
timeout = max(30, min(1800, timeout))
try:
history = asyncio.run(asyncio.wait_for(run(), timeout=timeout))
except asyncio.TimeoutError as exc:
+16 -9
View File
@@ -24,7 +24,10 @@ EMAIL_RE = re.compile(r"[A-Z0-9._%+\-]+@[A-Z0-9.\-]+\.[A-Z]{2,63}", re.I)
MAX_BODY_BYTES = 700_000
MAX_REDIRECTS = 2
MAX_PAGES = 2
TIMEOUT_SECONDS = max(2, min(30, int(os.environ.get("WEBSITE_SCAN_TIMEOUT", "8"))))
try:
TIMEOUT_SECONDS = max(2, min(60, int(os.environ.get("WEBSITE_SCAN_TIMEOUT", "8"))))
except (TypeError, ValueError):
TIMEOUT_SECONDS = 8
# getaddrinfo has no per-call timeout parameter on all supported platforms.
# Set a bounded process default so a broken DNS server cannot hold a discovery
# worker forever during website analysis.
@@ -105,7 +108,7 @@ def _same_root(left: str, right: str) -> bool:
return bool(a and b and (a == b or a.endswith("." + b) or b.endswith("." + a)))
def _fetch(url: str) -> dict[str, Any]:
def _fetch(url: str, timeout_seconds: int = TIMEOUT_SECONDS) -> dict[str, Any]:
current = url
redirects = 0
started = time.monotonic()
@@ -117,7 +120,7 @@ def _fetch(url: str) -> dict[str, Any]:
return {"error": "unsafe_or_unresolvable_host", "url": current, "latency_ms": int((time.monotonic() - started) * 1000)}
request = Request(current, headers={"User-Agent": "ProspectOS/1.0 (+public-site-check)", "Accept": "text/html,application/xhtml+xml"})
try:
with _opener.open(request, timeout=TIMEOUT_SECONDS) as response:
with _opener.open(request, timeout=timeout_seconds) as response:
last_status = int(getattr(response, "status", 200) or 200)
last_headers = response.headers
body = response.read(MAX_BODY_BYTES)
@@ -147,13 +150,13 @@ def _fetch(url: str) -> dict[str, Any]:
return {"url": current, "initial_url": url, "status": last_status, "headers": last_headers, "body": body, "latency_ms": int((time.monotonic() - started) * 1000), "redirects": redirects, "error": "redirect_limit"}
def _tls_expiry(url: str) -> str:
def _tls_expiry(url: str, timeout_seconds: int = TIMEOUT_SECONDS) -> str:
parsed = urlparse(url)
if parsed.scheme != "https" or not parsed.hostname:
return ""
try:
context = ssl.create_default_context()
with socket.create_connection((parsed.hostname, parsed.port or 443), timeout=TIMEOUT_SECONDS) as raw:
with socket.create_connection((parsed.hostname, parsed.port or 443), timeout=timeout_seconds) as raw:
with context.wrap_socket(raw, server_hostname=parsed.hostname) as secure:
cert = secure.getpeercert()
value = cert.get("notAfter") if cert else ""
@@ -174,11 +177,15 @@ def _emails(text: str) -> list[str]:
return found[:30]
def scan_website(url: str) -> dict[str, Any]:
def scan_website(url: str, timeout_seconds: int | None = None) -> dict[str, Any]:
"""Return bounded website health, metadata, and public email evidence."""
if not url:
return {"website_status": "no_detected_website", "website_checked_at": datetime.now(timezone.utc).replace(microsecond=0).isoformat(), "website_emails": [], "email_evidence": []}
first = _fetch(url)
try:
timeout = max(2, min(60, int(timeout_seconds if timeout_seconds is not None else TIMEOUT_SECONDS)))
except (TypeError, ValueError):
timeout = TIMEOUT_SECONDS
first = _fetch(url, timeout)
status = int(first.get("status") or 0)
final_url = str(first.get("url") or url)
body = bytes(first.get("body") or b"")
@@ -196,7 +203,7 @@ def scan_website(url: str) -> dict[str, Any]:
candidate = urljoin(final_url, href)
label = (href or "").lower()
if any(token in label for token in ("contact", "about", "get-in-touch")) and _same_root(final_url, candidate) and _safe_url(candidate):
page = _fetch(candidate)
page = _fetch(candidate, timeout)
pages.append(page)
emails.extend(_emails(bytes(page.get("body") or b"").decode("utf-8", "replace")))
break
@@ -230,7 +237,7 @@ def scan_website(url: str) -> dict[str, Any]:
"website_title": title,
"website_technology": technologies,
"website_has_viewport": 1 if "viewport" in parser.meta else 0,
"website_tls_expires_at": _tls_expiry(final_url) if urlparse(final_url).scheme == "https" else "",
"website_tls_expires_at": _tls_expiry(final_url, timeout) if urlparse(final_url).scheme == "https" else "",
"website_checked_at": checked,
"website_error": str(first.get("error") or "")[:300],
"website_emails": emails,
+146 -27
View File
@@ -73,17 +73,71 @@ _schema_lock = threading.Lock()
_schema_path: str | None = None
def _saved_settings() -> dict[str, str]:
"""Read persisted runtime settings, falling back cleanly during startup."""
try:
conn = db()
try:
return {str(row["key"]): str(row["value"]) for row in conn.execute("SELECT key,value FROM settings")}
finally:
conn.close()
except Exception:
return {}
def _setting_int(saved: dict[str, str], key: str, env_key: str, default: int, minimum: int, maximum: int) -> int:
try:
value = int(saved.get(key, os.environ.get(env_key, str(default))))
except (TypeError, ValueError):
value = default
return max(minimum, min(maximum, value))
def _setting_float(saved: dict[str, str], key: str, env_key: str, default: float, minimum: float, maximum: float) -> float:
try:
value = float(saved.get(key, os.environ.get(env_key, str(default))))
except (TypeError, ValueError):
value = default
return max(minimum, min(maximum, value))
def _setting_bool(saved: dict[str, str], key: str, env_key: str, default: bool) -> bool:
raw = str(saved.get(key, os.environ.get(env_key, "true" if default else "false"))).strip().lower()
return raw not in {"0", "false", "no", "off", "disabled"}
def browser_use_settings() -> dict[str, Any]:
"""Return the live Browser Use controls from Settings or .env defaults."""
saved = _saved_settings()
return {
"enabled": _setting_bool(saved, "BROWSER_USE_ENABLED", "BROWSER_USE_ENABLED", BROWSER_USE_ENABLED),
"max_steps": _setting_int(saved, "BROWSER_USE_MAX_STEPS", "BROWSER_USE_MAX_STEPS", 10, 3, 50),
"timeout_seconds": _setting_int(saved, "BROWSER_USE_TIMEOUT_SECONDS", "BROWSER_USE_TIMEOUT_SECONDS", 600, 30, 1800),
"llm_timeout_seconds": _setting_int(saved, "BROWSER_USE_LLM_TIMEOUT_SECONDS", "BROWSER_USE_LLM_TIMEOUT_SECONDS", 90, 10, 300),
"headless": True,
"public_only": True,
}
def browser_use_limits() -> tuple[int, int]:
"""Return bounded Browser Use steps/timeout even if .env is malformed."""
try:
steps = int(os.environ.get("BROWSER_USE_MAX_STEPS", "10"))
except (TypeError, ValueError):
steps = 10
try:
timeout = int(os.environ.get("BROWSER_USE_TIMEOUT_SECONDS", "120"))
except (TypeError, ValueError):
timeout = 120
return max(3, min(15, steps)), max(30, min(300, timeout))
"""Return bounded Browser Use steps/timeout for compatibility with callers."""
runtime = browser_use_settings()
return int(runtime["max_steps"]), int(runtime["timeout_seconds"])
def gmaps_runtime_settings() -> dict[str, Any]:
"""Return live Maps worker timing controls from Settings or .env defaults."""
saved = _saved_settings()
return {
"request_timeout_seconds": _setting_int(saved, "GMAPS_REQUEST_TIMEOUT", "GMAPS_REQUEST_TIMEOUT", GMAPS_REQUEST_TIMEOUT, 60, 7200),
"poll_seconds": _setting_float(saved, "GMAPS_POLL_SECONDS", "GMAPS_POLL_SECONDS", GMAPS_POLL_SECONDS, 0.5, 60.0),
"max_depth": _setting_int(saved, "GMAPS_MAX_DEPTH", "GMAPS_MAX_DEPTH", 1, 1, 10),
}
def website_scan_timeout_seconds() -> int:
saved = _saved_settings()
return _setting_int(saved, "WEBSITE_SCAN_TIMEOUT", "WEBSITE_SCAN_TIMEOUT", 8, 2, 60)
def now() -> str:
@@ -730,7 +784,9 @@ def submit_gmaps_job(query: str, include_email: bool = False, extra_reviews: boo
raise RuntimeError("gmaps_not_configured")
include_email = True
lang = os.environ.get("GMAPS_LANG", "en")
depth = max(1, min(10, int(os.environ.get("GMAPS_MAX_DEPTH", "1"))))
maps_runtime = gmaps_runtime_settings()
depth = int(maps_runtime["max_depth"])
request_timeout = int(maps_runtime["request_timeout_seconds"])
if GMAPS_API_MODE == "saas":
endpoint = "/api/v1/scrape"
payload = {
@@ -739,7 +795,7 @@ def submit_gmaps_job(query: str, include_email: bool = False, extra_reviews: boo
"max_depth": depth,
"email": bool(include_email),
"extra_reviews": bool(extra_reviews),
"timeout": max(1, min(300, GMAPS_REQUEST_TIMEOUT)),
"timeout": max(1, min(7200, request_timeout)),
}
else:
# Standard gosom web mode uses the local job API. It has no API-key
@@ -753,7 +809,7 @@ def submit_gmaps_job(query: str, include_email: bool = False, extra_reviews: boo
"depth": depth,
"email": bool(include_email),
"extra_reviews": bool(extra_reviews),
"max_time": max(60, min(3600, GMAPS_REQUEST_TIMEOUT)),
"max_time": max(60, min(7200, request_timeout)),
}
response = http_json("POST", GMAPS_URL + endpoint, payload, gmaps_headers(), timeout=30, error_prefix="gmaps")
job_id = field_value(response, "job_id", "id", "JobID", "ID")
@@ -765,7 +821,9 @@ def submit_gmaps_job(query: str, include_email: bool = False, extra_reviews: boo
def poll_gmaps_job(job_id: str, progress: Callable[[str, int, int], None] | None = None) -> list[dict[str, Any]]:
if not GMAPS_URL:
raise RuntimeError("gmaps_not_configured")
deadline = time.monotonic() + max(60, GMAPS_REQUEST_TIMEOUT + 60)
maps_runtime = gmaps_runtime_settings()
deadline = time.monotonic() + max(60, int(maps_runtime["request_timeout_seconds"]) + 60)
poll_seconds = float(maps_runtime["poll_seconds"])
if GMAPS_API_MODE == "saas":
terminal_ok = {"completed", "succeeded", "success"}
else:
@@ -786,7 +844,7 @@ def poll_gmaps_job(job_id: str, progress: Callable[[str, int, int], None] | None
if status in terminal_bad:
error = field_value(response, "error", "Error")
raise RuntimeError("gmaps_job_" + status + (":" + as_text(error) if error else ""))
time.sleep(max(0.5, GMAPS_POLL_SECONDS))
time.sleep(poll_seconds)
# The worker can finish writing its CSV just after its status endpoint
# exceeds our polling window. Recover a completed/partial result when it
# is available rather than discarding the run as a false failure.
@@ -995,9 +1053,9 @@ def enrich_place(item: dict[str, Any]) -> dict[str, Any]:
if item.get("website"):
scan_started = time.monotonic()
record_log("search", "website_scan_started", prospect_id=as_int(item.get("id")) or None,
details={"business": item.get("name"), "url": _safe_log_url(str(item.get("website")))})
details={"business": item.get("name"), "url": _safe_log_url(str(item.get("website"))), "timeout_seconds": website_scan_timeout_seconds()})
try:
item.update(scan_website(str(item["website"])))
item.update(scan_website(str(item["website"]), timeout_seconds=website_scan_timeout_seconds()))
record_log("search", "website_scan_completed", prospect_id=as_int(item.get("id")) or None,
duration_ms=int((time.monotonic() - scan_started) * 1000),
details={"business": item.get("name"), "url": _safe_log_url(str(item.get("website"))), "status": item.get("website_status"), "emails_found": len(item.get("website_emails") or [])})
@@ -1388,6 +1446,7 @@ class Handler(BaseHTTPRequestHandler):
auth_required = GMAPS_API_MODE == "saas"
ready = backend_mode() == "browser" or (bool(GMAPS_URL) and (not auth_required or bool(GMAPS_API_KEY)))
ai_runtime = ai_runtime_settings()
browser_runtime = browser_use_settings()
return self.send_json(200, {
"backend": backend_mode(),
"gmaps_url": bool(GMAPS_URL),
@@ -1397,7 +1456,7 @@ class Handler(BaseHTTPRequestHandler):
"ready": ready,
"email_extraction_default": True,
"email_extraction_always": True,
"browser_use_enabled": BROWSER_USE_ENABLED,
"browser_use_enabled": browser_runtime["enabled"],
"ai_provider": ai_runtime["provider"],
"ai_configured": bool(ai_runtime["base_url"] and ai_runtime["model"] and (ai_runtime["provider"] != "nous" or ai_runtime["api_key"])),
})
@@ -1442,8 +1501,20 @@ class Handler(BaseHTTPRequestHandler):
if parsed.path == "/api/settings":
ai_runtime = ai_runtime_settings()
models = nous_models() if ai_runtime["provider"] == "nous" else []
browser_steps, browser_timeout = browser_use_limits()
return self.send_json(200, {"maps": {"backend": backend_mode(), "worker_url": GMAPS_URL, "ready": backend_mode() == "browser" or bool(GMAPS_URL)}, "ai": {"provider": ai_runtime["provider"], "base_url": ai_runtime["base_url"], "model": ai_runtime["model"], "configured": bool(ai_runtime["base_url"] and ai_runtime["model"] and (ai_runtime["provider"] != "nous" or ai_runtime["api_key"])), "models": models}, "browser_use": {"enabled": BROWSER_USE_ENABLED, "headless": True, "public_only": True, "max_steps": browser_steps, "timeout_seconds": browser_timeout}, "email_collection": {"always_on": True, "public_only": True}, "limits": {"max_results": MAX_RESULTS, "website_scan_timeout_seconds": 8}})
browser_runtime = browser_use_settings()
maps_runtime = gmaps_runtime_settings()
scan_timeout = website_scan_timeout_seconds()
return self.send_json(200, {
"maps": {
"backend": backend_mode(), "worker_url": GMAPS_URL,
"ready": backend_mode() == "browser" or bool(GMAPS_URL),
**maps_runtime,
},
"ai": {"provider": ai_runtime["provider"], "base_url": ai_runtime["base_url"], "model": ai_runtime["model"], "configured": bool(ai_runtime["base_url"] and ai_runtime["model"] and (ai_runtime["provider"] != "nous" or ai_runtime["api_key"])), "models": models},
"browser_use": browser_runtime,
"email_collection": {"always_on": True, "public_only": True},
"limits": {"max_results": MAX_RESULTS, "website_scan_timeout_seconds": scan_timeout},
})
if parsed.path == "/api/prospects":
params = parse_qs(parsed.query)
q = (params.get("q", [""])[0]).strip().lower()
@@ -1546,15 +1617,58 @@ class Handler(BaseHTTPRequestHandler):
return self.send_json(400, {"error": str(exc)})
if self.path == "/api/settings":
try:
payload = self.read_json(); ai = payload.get("ai") if isinstance(payload.get("ai"), dict) else {}
payload = self.read_json()
ai = payload.get("ai") if isinstance(payload.get("ai"), dict) else {}
browser = payload.get("browser_use") if isinstance(payload.get("browser_use"), dict) else {}
maps = payload.get("maps") if isinstance(payload.get("maps"), dict) else {}
limits = payload.get("limits") if isinstance(payload.get("limits"), dict) else {}
conn = db()
saved_keys: list[str] = []
def save_setting(key: str, value: Any) -> None:
text_value = str(value).strip()[:500]
conn.execute("INSERT INTO settings(key,value,updated_at) VALUES(?,?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value,updated_at=excluded.updated_at", (key, text_value, now()))
saved_keys.append(key)
for key in ("AI_PROVIDER", "AI_BASE_URL", "AI_API_KEY", "AI_MODEL"):
if key in ai:
value = str(ai.get(key) or "").strip()[:500]
conn.execute("INSERT INTO settings(key,value,updated_at) VALUES(?,?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value,updated_at=excluded.updated_at", (key, value, now()))
conn.commit(); saved={str(r["key"]):str(r["value"]) for r in conn.execute("SELECT key,value FROM settings")}; conn.close()
# An empty password means “keep the existing credential”
# so a normal Settings save cannot erase a working key.
if key == "AI_API_KEY" and not str(ai.get(key) or "").strip():
continue
save_setting(key, ai.get(key) or "")
if "enabled" in browser:
enabled_value = str(browser.get("enabled") or "").strip().lower() not in {"0", "false", "no", "off", "disabled"}
save_setting("BROWSER_USE_ENABLED", "true" if enabled_value else "false")
if "max_steps" in browser:
save_setting("BROWSER_USE_MAX_STEPS", max(3, min(50, int(browser.get("max_steps")))))
if "timeout_seconds" in browser:
save_setting("BROWSER_USE_TIMEOUT_SECONDS", max(30, min(1800, int(browser.get("timeout_seconds")))))
if "llm_timeout_seconds" in browser:
save_setting("BROWSER_USE_LLM_TIMEOUT_SECONDS", max(10, min(300, int(browser.get("llm_timeout_seconds")))))
if "request_timeout_seconds" in maps:
save_setting("GMAPS_REQUEST_TIMEOUT", max(60, min(7200, int(maps.get("request_timeout_seconds")))))
if "poll_seconds" in maps:
save_setting("GMAPS_POLL_SECONDS", max(0.5, min(60.0, float(maps.get("poll_seconds")))))
if "max_depth" in maps:
save_setting("GMAPS_MAX_DEPTH", max(1, min(10, int(maps.get("max_depth")))))
if "website_scan_timeout_seconds" in limits:
save_setting("WEBSITE_SCAN_TIMEOUT", max(2, min(60, int(limits.get("website_scan_timeout_seconds")))))
conn.commit()
conn.close()
record_log("action", "settings_updated", details={"keys": saved_keys})
ai_runtime = ai_runtime_settings()
return self.send_json(200, {"saved": True, "ai": {"provider": ai_runtime["provider"], "base_url": ai_runtime["base_url"], "model": ai_runtime["model"], "configured": bool(ai_runtime["base_url"] and ai_runtime["model"] and (ai_runtime["provider"] != "nous" or ai_runtime["api_key"]))}})
return self.send_json(200, {
"saved": True,
"saved_keys": saved_keys,
"ai": {"provider": ai_runtime["provider"], "base_url": ai_runtime["base_url"], "model": ai_runtime["model"], "configured": bool(ai_runtime["base_url"] and ai_runtime["model"] and (ai_runtime["provider"] != "nous" or ai_runtime["api_key"]))},
"browser_use": browser_use_settings(),
"maps": gmaps_runtime_settings(),
"limits": {"website_scan_timeout_seconds": website_scan_timeout_seconds()},
})
except Exception as exc:
return self.send_json(400, {"error": str(exc)})
if urlparse(self.path).path == "/api/ai/test":
@@ -1584,12 +1698,15 @@ class Handler(BaseHTTPRequestHandler):
conn.close()
if not row:
return self.send_json(404, {"error": "not_found"})
if not BROWSER_USE_ENABLED:
browser_runtime = browser_use_settings()
if not browser_runtime["enabled"]:
return self.send_json(503, {"error": "browser_use_disabled"})
try:
runtime = ai_runtime_settings()
record_log("browser", "website_investigation_started", prospect_id=prospect_id,
details={"business": row["name"], "model": runtime["model"], "public_only": True})
details={"business": row["name"], "model": runtime["model"], "public_only": True,
"max_steps": browser_runtime["max_steps"], "timeout_seconds": browser_runtime["timeout_seconds"],
"llm_timeout_seconds": browser_runtime["llm_timeout_seconds"]})
def browser_log(kind: str, event: str, details: dict[str, Any]) -> None:
record_log(kind, event, prospect_id=prospect_id, details=details)
@@ -1597,6 +1714,8 @@ class Handler(BaseHTTPRequestHandler):
candidates = investigate_websites(
dict(row), base_url=runtime["base_url"], api_key=runtime["api_key"],
model=runtime["model"], user_agent=OUTBOUND_USER_AGENT, log_callback=browser_log,
enabled=browser_runtime["enabled"], max_steps=browser_runtime["max_steps"],
timeout_seconds=browser_runtime["timeout_seconds"], llm_timeout_seconds=browser_runtime["llm_timeout_seconds"],
)
record_log("browser", "website_investigation_completed", prospect_id=prospect_id,
details={"candidate_count": len(candidates), "model": runtime["model"]})