deploy Nous v21 update
This commit is contained in:
Marco0300
@@ -24,6 +24,7 @@ BROWSER_USE_ENABLED=true
|
||||
BROWSER_USE_MAX_STEPS=10
|
||||
BROWSER_USE_TIMEOUT_SECONDS=120
|
||||
ANONYMIZED_TELEMETRY=false
|
||||
LOG_RETENTION=20000
|
||||
|
||||
# AI provider for analysis and tailored drafts: nous or custom.
|
||||
AI_PROVIDER=custom
|
||||
|
||||
@@ -5,7 +5,7 @@ the bundled [gosom/google-maps-scraper](https://github.com/gosom/google-maps-scr
|
||||
worker, waits for its asynchronous job to finish, filters places already in
|
||||
the local catalogue, and presents the remaining places for human review.
|
||||
|
||||
The interface is split into Dashboard, Discover, Prospects, Runs, and Settings
|
||||
The interface is split into Dashboard, Discover, Prospects, Runs, Logs, and Settings
|
||||
views. It includes website health and email evidence, explainable scoring,
|
||||
domain ideas, suppression, CSV export, scheduled runs, rescans, and human review. It never sends outreach
|
||||
automatically. The Prospects explorer has an **All high potential** tab and
|
||||
@@ -17,6 +17,11 @@ evidence and your offer; you edit and approve the draft yourself. Drafts now
|
||||
also return the reasoning, stored evidence IDs used, facts that still need
|
||||
confirmation, and a follow-up review note so a polished email is not mistaken
|
||||
for verified information.
|
||||
The **Logs** view separates API calls, user actions, AI requests, searches,
|
||||
Browser Use/tool steps, and errors. AI request payloads and responses are
|
||||
truncated and credentials such as API keys, authorization headers, tokens, and
|
||||
passwords are redacted. Logs are retained up to `LOG_RETENTION` entries (20,000
|
||||
by default).
|
||||
From a prospect's detail drawer, **Assess with AI** stores a contact probability,
|
||||
recommendation, and short evidence-grounded rationale for that specific company.
|
||||
The same drawer can optionally run a bounded Browser Use investigation to look
|
||||
@@ -65,6 +70,7 @@ error rather than remaining stuck forever.
|
||||
| `BROWSER_USE_MAX_STEPS` | `10` | Maximum Browser Use actions for one investigation (3–15) |
|
||||
| `BROWSER_USE_TIMEOUT_SECONDS` | `120` | Hard timeout for one investigation (30–300 seconds) |
|
||||
| `ANONYMIZED_TELEMETRY` | `false` | Keeps Browser Use telemetry disabled in this personal deployment |
|
||||
| `LOG_RETENTION` | `20000` | Maximum operational log entries retained in SQLite |
|
||||
| `AI_PROVIDER` | `custom` | Set to `nous` for the Nous Research Portal, or `custom` for another OpenAI-compatible endpoint |
|
||||
| `AI_BASE_URL` | empty | Optional OpenAI-compatible `/chat/completions` endpoint |
|
||||
| `AI_API_KEY` | empty | Optional provider key; keep server-side |
|
||||
|
||||
@@ -7,10 +7,11 @@ CAPTCHAs, rotate proxies, or continue after an access challenge.
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import inspect
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from typing import Any
|
||||
from typing import Any, Callable
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
@@ -96,6 +97,7 @@ def investigate_websites(
|
||||
api_key: str,
|
||||
model: str,
|
||||
user_agent: str,
|
||||
log_callback: Callable[[str, str, dict[str, Any]], Any] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Run one bounded Browser Use investigation and return candidates."""
|
||||
if os.environ.get("BROWSER_USE_ENABLED", "1").strip().lower() in {"0", "false", "no", "off"}:
|
||||
@@ -132,6 +134,52 @@ Return ONLY JSON in this shape:
|
||||
Confidence must be an integer from 0 to 100. Include only plausible official website URLs and never invent a URL.
|
||||
""".strip()
|
||||
|
||||
async def emit(kind: str, event: str, details: dict[str, Any]) -> None:
|
||||
if not log_callback:
|
||||
return
|
||||
result = log_callback(kind, event, details)
|
||||
if inspect.isawaitable(result):
|
||||
await result
|
||||
|
||||
def output_actions(model_output: Any) -> list[Any]:
|
||||
try:
|
||||
actions = getattr(model_output, "action", []) or []
|
||||
values: list[Any] = []
|
||||
for action in list(actions)[:8]:
|
||||
if hasattr(action, "model_dump"):
|
||||
values.append(action.model_dump(mode="json"))
|
||||
else:
|
||||
values.append(str(action)[:2_000])
|
||||
return values
|
||||
except Exception:
|
||||
return [str(model_output)[:2_000]] if model_output is not None else []
|
||||
|
||||
async def on_step(browser_state: Any, model_output: Any, step_number: int) -> None:
|
||||
details = {
|
||||
"step": step_number,
|
||||
"url": str(getattr(browser_state, "url", "") or "")[:700],
|
||||
"title": str(getattr(browser_state, "title", "") or "")[:300],
|
||||
"actions": output_actions(model_output),
|
||||
"model": model,
|
||||
}
|
||||
await emit("browser", "browser_use_step", details)
|
||||
await emit("ai", "browser_use_model_step", {"step": step_number, "model": model, "actions": details["actions"]})
|
||||
|
||||
class LoggedChatOpenAI(ChatOpenAI):
|
||||
"""Capture Browser Use LLM turns without recording provider secrets."""
|
||||
|
||||
async def ainvoke(self, messages: list[Any], output_format: Any = None, **kwargs: Any) -> Any:
|
||||
started = asyncio.get_running_loop().time()
|
||||
serialised = [{"role": getattr(message, "type", "message"), "content": str(getattr(message, "content", ""))[:4_000]} for message in list(messages)[-20:]]
|
||||
await emit("ai", "browser_use_llm_request", {"model": self.model, "messages": serialised, "structured_output": bool(output_format)})
|
||||
try:
|
||||
result = await super().ainvoke(messages, output_format=output_format, **kwargs)
|
||||
await emit("ai", "browser_use_llm_response", {"model": self.model, "completion": str(getattr(result, "completion", result))[:4_000], "duration_ms": int((asyncio.get_running_loop().time() - started) * 1000)})
|
||||
return result
|
||||
except Exception as exc:
|
||||
await emit("ai", "browser_use_llm_error", {"model": self.model, "duration_ms": int((asyncio.get_running_loop().time() - started) * 1000)},)
|
||||
raise
|
||||
|
||||
async def run() -> Any:
|
||||
browser_profile = BrowserProfile(
|
||||
headless=True,
|
||||
@@ -141,7 +189,7 @@ Confidence must be an integer from 0 to 100. Include only plausible official web
|
||||
enable_default_extensions=False,
|
||||
captcha_solver=False,
|
||||
)
|
||||
llm = ChatOpenAI(
|
||||
llm = LoggedChatOpenAI(
|
||||
model=model,
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
@@ -161,10 +209,19 @@ Confidence must be an integer from 0 to 100. Include only plausible official web
|
||||
use_judge=False,
|
||||
enable_planning=False,
|
||||
enable_signal_handler=False,
|
||||
register_new_step_callback=on_step,
|
||||
)
|
||||
return await agent.run(max_steps=max(3, min(15, int(os.environ.get("BROWSER_USE_MAX_STEPS", "10")))))
|
||||
try:
|
||||
max_steps = int(os.environ.get("BROWSER_USE_MAX_STEPS", "10"))
|
||||
except (TypeError, ValueError):
|
||||
max_steps = 10
|
||||
return await agent.run(max_steps=max(3, min(15, max_steps)))
|
||||
|
||||
timeout = max(30, min(300, int(os.environ.get("BROWSER_USE_TIMEOUT_SECONDS", "120"))))
|
||||
try:
|
||||
timeout = int(os.environ.get("BROWSER_USE_TIMEOUT_SECONDS", "120"))
|
||||
except (TypeError, ValueError):
|
||||
timeout = 120
|
||||
timeout = max(30, min(300, timeout))
|
||||
try:
|
||||
history = asyncio.run(asyncio.wait_for(run(), timeout=timeout))
|
||||
except asyncio.TimeoutError as exc:
|
||||
|
||||
+217
-8
@@ -90,6 +90,63 @@ def now() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
try:
|
||||
LOG_RETENTION = max(2_000, min(50_000, int(os.environ.get("LOG_RETENTION", "20000"))))
|
||||
except (TypeError, ValueError):
|
||||
LOG_RETENTION = 20_000
|
||||
_LOG_SECRET_KEYS = {"api_key", "authorization", "password", "secret", "token", "access_token", "refresh_token"}
|
||||
|
||||
|
||||
def _redact_log_value(value: Any, depth: int = 0) -> Any:
|
||||
"""Keep logs useful without persisting provider credentials."""
|
||||
if depth > 5:
|
||||
return "[truncated]"
|
||||
if isinstance(value, dict):
|
||||
output: dict[str, Any] = {}
|
||||
for key, item in list(value.items())[:80]:
|
||||
key_text = str(key)
|
||||
if key_text.lower() in _LOG_SECRET_KEYS or key_text.lower().endswith("_api_key"):
|
||||
output[key_text] = "[redacted]"
|
||||
else:
|
||||
output[key_text] = _redact_log_value(item, depth + 1)
|
||||
return output
|
||||
if isinstance(value, (list, tuple, set)):
|
||||
return [_redact_log_value(item, depth + 1) for item in list(value)[:100]]
|
||||
if isinstance(value, str):
|
||||
return value[:4_000] + ("…" if len(value) > 4_000 else "")
|
||||
if isinstance(value, (int, float, bool)) or value is None:
|
||||
return value
|
||||
return str(value)[:4_000]
|
||||
|
||||
|
||||
def record_log(kind: str, event: str, *, level: str = "info", method: str = "", path: str = "",
|
||||
status: int | None = None, duration_ms: int | None = None, details: Any = None,
|
||||
error: str = "", run_id: int | None = None, prospect_id: int | None = None) -> None:
|
||||
"""Persist one bounded operational event; logging must never break a request."""
|
||||
try:
|
||||
safe_kind = str(kind or "system").strip().lower()[:30] or "system"
|
||||
safe_level = str(level or "info").strip().lower()[:12] or "info"
|
||||
safe_details = _redact_log_value(details if details is not None else {})
|
||||
encoded = json.dumps(safe_details, ensure_ascii=False, default=str)[:20_000]
|
||||
conn = db()
|
||||
try:
|
||||
cur = conn.execute(
|
||||
"""INSERT INTO logs(created_at,kind,level,event,method,path,status,duration_ms,run_id,prospect_id,details,error)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,?,?)""",
|
||||
(now(), safe_kind, safe_level, str(event or "event")[:240], str(method or "")[:12],
|
||||
str(path or "")[:500], status, duration_ms, run_id, prospect_id, encoded, str(error or "")[:2_000]),
|
||||
)
|
||||
# Retain enough history for investigation without allowing a busy
|
||||
# browser session to grow the SQLite volume indefinitely.
|
||||
if int(cur.lastrowid or 0) % 100 == 0:
|
||||
conn.execute("DELETE FROM logs WHERE id NOT IN (SELECT id FROM logs ORDER BY id DESC LIMIT ?)", (LOG_RETENTION,))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception:
|
||||
return
|
||||
|
||||
|
||||
def backend_mode() -> str:
|
||||
configured = os.environ.get("DISCOVERY_BACKEND", "").strip().lower()
|
||||
if configured in {"gmaps", "browser"}:
|
||||
@@ -205,6 +262,21 @@ def db() -> sqlite3.Connection:
|
||||
raw_json TEXT NOT NULL DEFAULT '{}',
|
||||
UNIQUE(prospect_id, domain)
|
||||
)""")
|
||||
conn.execute("""CREATE TABLE IF NOT EXISTS logs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
created_at TEXT NOT NULL,
|
||||
kind TEXT NOT NULL DEFAULT 'system',
|
||||
level TEXT NOT NULL DEFAULT 'info',
|
||||
event TEXT NOT NULL,
|
||||
method TEXT NOT NULL DEFAULT '',
|
||||
path TEXT NOT NULL DEFAULT '',
|
||||
status INTEGER,
|
||||
duration_ms INTEGER,
|
||||
run_id INTEGER,
|
||||
prospect_id INTEGER,
|
||||
details TEXT NOT NULL DEFAULT '{}',
|
||||
error TEXT NOT NULL DEFAULT ''
|
||||
)""")
|
||||
|
||||
# Migrate databases created by v1-v5 without destroying the user's
|
||||
# catalogue. SQLite permits adding nullable/defaulted columns in place.
|
||||
@@ -268,6 +340,9 @@ def db() -> sqlite3.Connection:
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_prospects_score ON prospects(opportunity_score DESC)")
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_runs_created ON runs(id DESC)")
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_website_candidates_prospect ON website_candidates(prospect_id, confidence DESC)")
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_logs_created ON logs(id DESC)")
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_logs_kind ON logs(kind, id DESC)")
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_logs_level ON logs(level, id DESC)")
|
||||
conn.commit()
|
||||
_schema_path = DB_PATH
|
||||
_schema_lock.release()
|
||||
@@ -507,7 +582,19 @@ def _upstream_error(prefix: str, code: int, detail: str) -> str:
|
||||
return f"{prefix}_http_{code}:{detail}"
|
||||
|
||||
|
||||
def _log_http_kind(error_prefix: str) -> str:
|
||||
return "ai" if error_prefix == "ai" else "search" if error_prefix == "gmaps" else "api"
|
||||
|
||||
|
||||
def _safe_log_url(url: str) -> str:
|
||||
parsed = urlparse(url)
|
||||
return f"{parsed.scheme}://{parsed.netloc}{parsed.path}"[:700]
|
||||
|
||||
|
||||
def http_json(method: str, url: str, payload: Any = None, headers: dict[str, str] | None = None, timeout: float = 20, error_prefix: str = "upstream") -> Any:
|
||||
started = time.monotonic()
|
||||
log_kind = _log_http_kind(error_prefix)
|
||||
log_details = {"url": _safe_log_url(url), "timeout_seconds": timeout, "request": payload}
|
||||
data = None if payload is None else json.dumps(payload).encode("utf-8")
|
||||
request_headers = {"Accept": "application/json", "User-Agent": OUTBOUND_USER_AGENT, **(headers or {})}
|
||||
if data is not None:
|
||||
@@ -516,19 +603,32 @@ def http_json(method: str, url: str, payload: Any = None, headers: dict[str, str
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
body = response.read()
|
||||
response_status = int(getattr(response, "status", 200) or 200)
|
||||
except urllib.error.HTTPError as exc:
|
||||
detail = exc.read().decode("utf-8", "replace")[:300]
|
||||
record_log(log_kind, "upstream_http_error", level="error", method=method, path=_safe_log_url(url), status=int(exc.code),
|
||||
duration_ms=int((time.monotonic() - started) * 1000), details={**log_details, "response": detail}, error=_upstream_error(error_prefix, int(exc.code), detail))
|
||||
raise RuntimeError(_upstream_error(error_prefix, int(exc.code), detail)) from exc
|
||||
except urllib.error.URLError as exc:
|
||||
record_log(log_kind, "upstream_unreachable", level="error", method=method, path=_safe_log_url(url),
|
||||
duration_ms=int((time.monotonic() - started) * 1000), details=log_details, error=str(exc.reason)[:500])
|
||||
raise RuntimeError(f"{error_prefix}_unreachable:{exc.reason}") from exc
|
||||
try:
|
||||
return json.loads(body or b"{}")
|
||||
parsed = json.loads(body or b"{}")
|
||||
record_log(log_kind, "upstream_request", method=method, path=_safe_log_url(url), status=response_status,
|
||||
duration_ms=int((time.monotonic() - started) * 1000), details={**log_details, "response": parsed})
|
||||
return parsed
|
||||
except json.JSONDecodeError as exc:
|
||||
raise RuntimeError("gmaps_invalid_json") from exc
|
||||
record_log(log_kind, "upstream_invalid_json", level="error", method=method, path=_safe_log_url(url), status=response_status,
|
||||
duration_ms=int((time.monotonic() - started) * 1000), details={**log_details, "response": body[:2_000].decode("utf-8", "replace")}, error=str(exc))
|
||||
raise RuntimeError(f"{error_prefix}_invalid_json") from exc
|
||||
|
||||
|
||||
def http_bytes(method: str, url: str, payload: Any = None, headers: dict[str, str] | None = None, timeout: float = 20, error_prefix: str = "upstream") -> bytes:
|
||||
"""Make a small HTTP request where the response is not JSON (CSV download)."""
|
||||
started = time.monotonic()
|
||||
log_kind = _log_http_kind(error_prefix)
|
||||
log_details = {"url": _safe_log_url(url), "timeout_seconds": timeout, "request": payload}
|
||||
data = None if payload is None else json.dumps(payload).encode("utf-8")
|
||||
request_headers = {"Accept": "*/*", "User-Agent": OUTBOUND_USER_AGENT, **(headers or {})}
|
||||
if data is not None:
|
||||
@@ -536,11 +636,19 @@ def http_bytes(method: str, url: str, payload: Any = None, headers: dict[str, st
|
||||
request = urllib.request.Request(url, data=data, headers=request_headers, method=method.upper())
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
return response.read()
|
||||
body = response.read()
|
||||
response_status = int(getattr(response, "status", 200) or 200)
|
||||
record_log(log_kind, "upstream_request", method=method, path=_safe_log_url(url), status=response_status,
|
||||
duration_ms=int((time.monotonic() - started) * 1000), details={**log_details, "response_bytes": len(body)})
|
||||
return body
|
||||
except urllib.error.HTTPError as exc:
|
||||
detail = exc.read().decode("utf-8", "replace")[:300]
|
||||
record_log(log_kind, "upstream_http_error", level="error", method=method, path=_safe_log_url(url), status=int(exc.code),
|
||||
duration_ms=int((time.monotonic() - started) * 1000), details={**log_details, "response": detail}, error=_upstream_error(error_prefix, int(exc.code), detail))
|
||||
raise RuntimeError(_upstream_error(error_prefix, int(exc.code), detail)) from exc
|
||||
except urllib.error.URLError as exc:
|
||||
record_log(log_kind, "upstream_unreachable", level="error", method=method, path=_safe_log_url(url),
|
||||
duration_ms=int((time.monotonic() - started) * 1000), details=log_details, error=str(exc.reason)[:500])
|
||||
raise RuntimeError(f"{error_prefix}_unreachable:{exc.reason}") from exc
|
||||
|
||||
|
||||
@@ -765,7 +873,7 @@ def extract_search_results(page: Any, selector: str, title_selector: str | None,
|
||||
return output
|
||||
|
||||
|
||||
def google_search(query: str, requested: int) -> list[dict[str, Any]]:
|
||||
def google_search(query: str, requested: int, run_id: int | None = None) -> list[dict[str, Any]]:
|
||||
try:
|
||||
from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeout
|
||||
except ImportError as exc:
|
||||
@@ -793,6 +901,7 @@ def google_search(query: str, requested: int) -> list[dict[str, Any]]:
|
||||
break
|
||||
context = None
|
||||
try:
|
||||
record_log("search", "search_engine_started", run_id=run_id, details={"engine": engine, "query": query, "requested": requested})
|
||||
context = browser.new_context(locale="en-ZA", java_script_enabled=True, service_workers="block", viewport={"width": 1280, "height": 900})
|
||||
page = context.new_page()
|
||||
page.set_default_timeout(ENGINE_TIMEOUT_MS)
|
||||
@@ -806,15 +915,20 @@ def google_search(query: str, requested: int) -> list[dict[str, Any]]:
|
||||
html = page.content()
|
||||
if blocked(body) or blocked(html):
|
||||
errors.append(engine + "_blocked")
|
||||
record_log("search", "search_engine_blocked", level="warn", run_id=run_id, details={"engine": engine, "query": query})
|
||||
continue
|
||||
output = extract_search_results(page, selector, title_selector, excluded_hosts, requested)
|
||||
if output:
|
||||
record_log("search", "search_engine_results", run_id=run_id, details={"engine": engine, "query": query, "count": len(output), "results": output})
|
||||
return output
|
||||
errors.append(engine + ("_timeout" if navigation_timed_out else "_empty"))
|
||||
record_log("search", "search_engine_empty", level="warn", run_id=run_id, details={"engine": engine, "query": query, "navigation_timed_out": navigation_timed_out})
|
||||
except PlaywrightTimeout:
|
||||
errors.append(engine + "_timeout")
|
||||
except Exception:
|
||||
record_log("search", "search_engine_timeout", level="warn", run_id=run_id, details={"engine": engine, "query": query})
|
||||
except Exception as exc:
|
||||
errors.append(engine + "_error")
|
||||
record_log("search", "search_engine_error", level="error", run_id=run_id, details={"engine": engine, "query": query}, error=str(exc)[:500])
|
||||
finally:
|
||||
if context is not None:
|
||||
try:
|
||||
@@ -823,7 +937,9 @@ def google_search(query: str, requested: int) -> list[dict[str, Any]]:
|
||||
pass
|
||||
finally:
|
||||
browser.close()
|
||||
raise RuntimeError("search_engines_unavailable:" + ",".join(errors))
|
||||
message = "search_engines_unavailable:" + ",".join(errors)
|
||||
record_log("search", "search_failed", level="error", run_id=run_id, details={"query": query, "errors": errors}, error=message)
|
||||
raise RuntimeError(message)
|
||||
|
||||
|
||||
def validate_payload(payload: dict[str, Any]) -> tuple[str, str, str, int, bool, bool]:
|
||||
@@ -859,6 +975,10 @@ def update_run(run_id: int, **fields: Any) -> None:
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
if any(key in safe for key in ("stage", "status", "error")):
|
||||
record_log("system", "run_state_updated", level="error" if safe.get("status") == "failed" else "info",
|
||||
run_id=run_id, details={key: value for key, value in safe.items() if key in {"stage", "status", "progress", "found", "added", "error", "external_job_id"}},
|
||||
error=str(safe.get("error") or ""))
|
||||
|
||||
|
||||
def _json_list(value: Any) -> list[Any]:
|
||||
@@ -873,10 +993,19 @@ def enrich_place(item: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Add website health, public email evidence, and explainable scoring."""
|
||||
prior_evidence = _json_list(item.get("email_evidence"))
|
||||
if item.get("website"):
|
||||
scan_started = time.monotonic()
|
||||
record_log("search", "website_scan_started", prospect_id=as_int(item.get("id")) or None,
|
||||
details={"business": item.get("name"), "url": _safe_log_url(str(item.get("website")))})
|
||||
try:
|
||||
item.update(scan_website(str(item["website"])))
|
||||
record_log("search", "website_scan_completed", prospect_id=as_int(item.get("id")) or None,
|
||||
duration_ms=int((time.monotonic() - scan_started) * 1000),
|
||||
details={"business": item.get("name"), "url": _safe_log_url(str(item.get("website"))), "status": item.get("website_status"), "emails_found": len(item.get("website_emails") or [])})
|
||||
except Exception as exc: # a single hostile site must not abort a run
|
||||
item.update({"website_status": "unknown", "website_checked_at": now(), "website_error": str(exc)[:300], "website_emails": [], "email_evidence": []})
|
||||
record_log("search", "website_scan_failed", level="error", prospect_id=as_int(item.get("id")) or None,
|
||||
duration_ms=int((time.monotonic() - scan_started) * 1000),
|
||||
details={"business": item.get("name"), "url": _safe_log_url(str(item.get("website")))}, error=str(exc)[:500])
|
||||
else:
|
||||
item.setdefault("website_status", "no_detected_website")
|
||||
item.setdefault("website_emails", [])
|
||||
@@ -935,6 +1064,7 @@ def insert_catalogue(run_id: int, query: str, raw_results: list[dict[str, Any]],
|
||||
progress(index, total)
|
||||
emails = set(normalize_public_emails(item.get("emails")))
|
||||
if item["domain"].lower() in suppressed_domains or emails.intersection(suppressed_emails):
|
||||
record_log("system", "prospect_suppressed", run_id=run_id, details={"name": item.get("name"), "domain": item.get("domain"), "email_match": bool(emails.intersection(suppressed_emails))})
|
||||
continue
|
||||
try:
|
||||
cur = conn.execute("""INSERT OR IGNORE INTO prospects(
|
||||
@@ -961,6 +1091,11 @@ def insert_catalogue(run_id: int, query: str, raw_results: list[dict[str, Any]],
|
||||
continue
|
||||
if cur.rowcount:
|
||||
added += 1
|
||||
conn.commit()
|
||||
record_log("system", "prospect_added", run_id=run_id, details={"name": item.get("name"), "domain": item.get("domain"), "score": item.get("opportunity_score")})
|
||||
else:
|
||||
conn.commit()
|
||||
record_log("system", "prospect_duplicate", run_id=run_id, details={"name": item.get("name"), "domain": item.get("domain")})
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
@@ -972,6 +1107,7 @@ def run_discovery(run_id: int, payload: dict[str, Any]) -> None:
|
||||
# Serialize local runs so two personal browser jobs do not fight over the
|
||||
# same machine's Chromium/RAM. The Maps worker itself remains async.
|
||||
with _discover_lock:
|
||||
record_log("search", "discovery_started", run_id=run_id, details={"query": query, "location": location, "requested": requested, "backend": backend_mode(), "email_collection": include_email})
|
||||
update_run(run_id, status="running", stage="connecting", progress=5, source=backend_mode())
|
||||
try:
|
||||
if backend_mode() == "gmaps":
|
||||
@@ -985,7 +1121,7 @@ def run_discovery(run_id: int, payload: dict[str, Any]) -> None:
|
||||
raw_results = poll_gmaps_job(external_job_id, on_progress)
|
||||
else:
|
||||
update_run(run_id, stage="browser_search", progress=20)
|
||||
raw_results = google_search(query, requested)
|
||||
raw_results = google_search(query, requested, run_id=run_id)
|
||||
update_run(run_id, stage="website_analysis", progress=76, found=len(raw_results))
|
||||
|
||||
def enrichment_progress(index: int, total: int) -> None:
|
||||
@@ -994,9 +1130,11 @@ def run_discovery(run_id: int, payload: dict[str, Any]) -> None:
|
||||
|
||||
results, added = insert_catalogue(run_id, query, raw_results, requested, enrichment_progress, search_category=category)
|
||||
update_run(run_id, found=len(results), added=added, status="succeeded", stage="complete", progress=100, completed_at=now(), error="")
|
||||
record_log("search", "discovery_completed", run_id=run_id, details={"found": len(results), "added": added, "backend": backend_mode()})
|
||||
except Exception as exc:
|
||||
message = str(exc).strip()[:500] or "discovery_failed"
|
||||
update_run(run_id, status="failed", stage="failed", progress=100, error=message, completed_at=now())
|
||||
record_log("search", "discovery_failed", level="error", run_id=run_id, details={"backend": backend_mode()}, error=message)
|
||||
|
||||
|
||||
def create_discovery(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
@@ -1205,6 +1343,10 @@ def ai_assess_prospect(prospect: dict[str, Any]) -> dict[str, Any]:
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
server_version = "PersonalProspectOS/2.0"
|
||||
|
||||
def handle_one_request(self) -> None:
|
||||
self._request_started_at = time.monotonic()
|
||||
super().handle_one_request()
|
||||
|
||||
def send_json(self, status: int, value: Any) -> None:
|
||||
body = json_bytes(value)
|
||||
self.send_response(status)
|
||||
@@ -1213,6 +1355,15 @@ class Handler(BaseHTTPRequestHandler):
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
if str(getattr(self, "path", "")).startswith("/api/"):
|
||||
error_value = value.get("error") if isinstance(value, dict) else ""
|
||||
record_log(
|
||||
"api", f"{getattr(self, 'command', 'HTTP')} {urlparse(self.path).path}",
|
||||
level="error" if status >= 400 else "info", method=getattr(self, "command", "HTTP"),
|
||||
path=urlparse(self.path).path, status=status,
|
||||
duration_ms=int((time.monotonic() - getattr(self, "_request_started_at", time.monotonic())) * 1000),
|
||||
details={"response_status": status}, error=str(error_value or "")[:500],
|
||||
)
|
||||
|
||||
def read_json(self) -> dict[str, Any]:
|
||||
length = min(int(self.headers.get("Content-Length", "0") or 0), 100_000)
|
||||
@@ -1255,6 +1406,39 @@ class Handler(BaseHTTPRequestHandler):
|
||||
if provider == "nous":
|
||||
return self.send_json(200, {"provider": "nous", "base_url": NOUS_BASE_URL, "models": nous_models()})
|
||||
return self.send_json(200, {"provider": provider or "custom", "models": []})
|
||||
if parsed.path == "/api/logs":
|
||||
params = parse_qs(parsed.query)
|
||||
kind = (params.get("kind", [""])[0] or "").strip().lower()
|
||||
level = (params.get("level", [""])[0] or "").strip().lower()
|
||||
query = (params.get("q", [""])[0] or "").strip().lower()
|
||||
try:
|
||||
limit = max(1, min(500, int(params.get("limit", ["200"])[0])))
|
||||
except (TypeError, ValueError):
|
||||
limit = 200
|
||||
clauses: list[str] = []
|
||||
values: list[Any] = []
|
||||
if kind and kind != "all":
|
||||
clauses.append("kind=?"); values.append(kind)
|
||||
if level and level != "all":
|
||||
clauses.append("level=?"); values.append(level)
|
||||
if query:
|
||||
clauses.append("(lower(event) LIKE ? OR lower(path) LIKE ? OR lower(error) LIKE ? OR lower(details) LIKE ?)")
|
||||
needle = "%" + query[:120] + "%"; values.extend([needle, needle, needle, needle])
|
||||
where = " WHERE " + " AND ".join(clauses) if clauses else ""
|
||||
rows = conn.execute("SELECT * FROM logs" + where + " ORDER BY id DESC LIMIT ?", (*values, limit)).fetchall()
|
||||
items: list[dict[str, Any]] = []
|
||||
for row in rows:
|
||||
item = dict(row)
|
||||
try:
|
||||
item["details"] = json.loads(item.get("details") or "{}")
|
||||
except (TypeError, ValueError, json.JSONDecodeError):
|
||||
item["details"] = {"raw": str(item.get("details") or "")[:20_000]}
|
||||
items.append(item)
|
||||
total = int(conn.execute("SELECT COUNT(*) FROM logs" + where, tuple(values)).fetchone()[0])
|
||||
counts = {str(row[0]): int(row[1]) for row in conn.execute("SELECT kind,COUNT(*) FROM logs GROUP BY kind")}
|
||||
error_count = int(conn.execute("SELECT COUNT(*) FROM logs WHERE level='error'").fetchone()[0])
|
||||
counts["errors"] = error_count
|
||||
return self.send_json(200, {"items": items, "total": total, "counts": counts})
|
||||
if parsed.path == "/api/settings":
|
||||
ai_runtime = ai_runtime_settings()
|
||||
models = nous_models() if ai_runtime["provider"] == "nous" else []
|
||||
@@ -1345,6 +1529,21 @@ class Handler(BaseHTTPRequestHandler):
|
||||
self.wfile.write(content)
|
||||
|
||||
def do_POST(self) -> None:
|
||||
if urlparse(self.path).path == "/api/logs":
|
||||
try:
|
||||
payload = self.read_json()
|
||||
action = str(payload.get("action") or "").strip()
|
||||
if not action:
|
||||
return self.send_json(400, {"error": "action_required"})
|
||||
level = str(payload.get("level") or "info").strip().lower()
|
||||
if level not in {"info", "warn", "error"}:
|
||||
level = "info"
|
||||
details = payload.get("details") if isinstance(payload.get("details"), (dict, list, str, int, float, bool)) else {}
|
||||
record_log("action", action, level=level, method="CLIENT", path=str(payload.get("path") or "")[:500], details=details,
|
||||
error=str(payload.get("error") or "")[:500])
|
||||
return self.send_json(202, {"logged": True})
|
||||
except (ValueError, TypeError, json.JSONDecodeError) as exc:
|
||||
return self.send_json(400, {"error": str(exc)})
|
||||
if self.path == "/api/settings":
|
||||
try:
|
||||
payload = self.read_json(); ai = payload.get("ai") if isinstance(payload.get("ai"), dict) else {}
|
||||
@@ -1389,10 +1588,18 @@ class Handler(BaseHTTPRequestHandler):
|
||||
return self.send_json(503, {"error": "browser_use_disabled"})
|
||||
try:
|
||||
runtime = ai_runtime_settings()
|
||||
record_log("browser", "website_investigation_started", prospect_id=prospect_id,
|
||||
details={"business": row["name"], "model": runtime["model"], "public_only": True})
|
||||
|
||||
def browser_log(kind: str, event: str, details: dict[str, Any]) -> None:
|
||||
record_log(kind, event, prospect_id=prospect_id, details=details)
|
||||
|
||||
candidates = investigate_websites(
|
||||
dict(row), base_url=runtime["base_url"], api_key=runtime["api_key"],
|
||||
model=runtime["model"], user_agent=OUTBOUND_USER_AGENT,
|
||||
model=runtime["model"], user_agent=OUTBOUND_USER_AGENT, log_callback=browser_log,
|
||||
)
|
||||
record_log("browser", "website_investigation_completed", prospect_id=prospect_id,
|
||||
details={"candidate_count": len(candidates), "model": runtime["model"]})
|
||||
conn = db()
|
||||
for candidate in candidates:
|
||||
encoded_evidence = json.dumps(candidate.get("evidence") or [], ensure_ascii=False)
|
||||
@@ -1411,8 +1618,10 @@ class Handler(BaseHTTPRequestHandler):
|
||||
rows = conn.execute("SELECT * FROM website_candidates WHERE prospect_id=? ORDER BY confidence DESC, id DESC", (prospect_id,)).fetchall()
|
||||
return self.send_json(200, {"items": [dict(item) for item in rows], "count": len(rows)})
|
||||
except RuntimeError as exc:
|
||||
record_log("browser", "website_investigation_failed", level="error", prospect_id=prospect_id, error=str(exc))
|
||||
return self.send_json(503, {"error": str(exc)})
|
||||
except Exception as exc:
|
||||
record_log("browser", "website_investigation_failed", level="error", prospect_id=prospect_id, error=str(exc)[:500])
|
||||
return self.send_json(502, {"error": "browser_use_failed:" + str(exc)[:240]})
|
||||
finally:
|
||||
try:
|
||||
|
||||
@@ -29,6 +29,7 @@ services:
|
||||
BROWSER_USE_MAX_STEPS: ${BROWSER_USE_MAX_STEPS:-10}
|
||||
BROWSER_USE_TIMEOUT_SECONDS: ${BROWSER_USE_TIMEOUT_SECONDS:-120}
|
||||
ANONYMIZED_TELEMETRY: ${ANONYMIZED_TELEMETRY:-false}
|
||||
LOG_RETENTION: ${LOG_RETENTION:-20000}
|
||||
AI_PROVIDER: ${AI_PROVIDER:-custom}
|
||||
AI_BASE_URL: ${AI_BASE_URL:-}
|
||||
AI_API_KEY: ${AI_API_KEY:-}
|
||||
|
||||
@@ -198,6 +198,19 @@ class BrowserOnlyAppTests(unittest.TestCase):
|
||||
self.assertEqual(json.loads(saved[0]), body["unknowns"])
|
||||
self.assertEqual(json.loads(saved[1]), body["evidence_used"])
|
||||
|
||||
def test_logs_capture_api_actions_and_redact_credentials(self):
|
||||
server.record_log("ai", "test_ai_call", details={"api_key": "do-not-store", "model": "test", "prompt": "hello"})
|
||||
status, body = self.request("POST", "/api/logs", {"action": "ui_test", "details": {"button": "test"}})
|
||||
self.assertEqual(status, 202)
|
||||
status, body = self.request("GET", "/api/logs?kind=ai")
|
||||
self.assertEqual(status, 200)
|
||||
item = next(row for row in body["items"] if row["event"] == "test_ai_call")
|
||||
self.assertEqual(item["details"]["api_key"], "[redacted]")
|
||||
self.assertEqual(item["details"]["model"], "test")
|
||||
status, body = self.request("GET", "/api/logs?kind=action")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertTrue(any(row["event"] == "ui_test" for row in body["items"]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
+18
-2
@@ -1,5 +1,10 @@
|
||||
const $=id=>document.getElementById(id); const esc=v=>String(v??"").replace(/[&<>\"']/g,c=>({'&':'&','<':'<','>':'>','\"':'"',"'":'''}[c]));
|
||||
let prospects=[],runs=[],settings={},selected=null,pollTimer=null;
|
||||
let prospects=[],runs=[],settings={},logs=[],selected=null,pollTimer=null;
|
||||
let activeLogTab='all';
|
||||
const LOG_TABS=[['all','All'],['action','Actions'],['api','API calls'],['ai','AI calls'],['search','Searches'],['browser','Browser/tool steps'],['errors','Errors']];
|
||||
function logAction(action,details={},level='info'){try{fetch('/api/logs',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({action,details,level,path:location.hash.slice(1)||'dashboard'})}).catch(()=>{})}catch{}}
|
||||
function installLogsNav(){const nav=$('nav');if(!nav||nav.querySelector('[data-route="logs"]'))return;const link=document.createElement('a');link.href='#logs';link.dataset.route='logs';link.innerHTML='<span>≋</span>Logs';nav.append(link)}
|
||||
installLogsNav();
|
||||
let activeProspectCategory='all';
|
||||
const HIGH_POTENTIAL_SCORE=70;
|
||||
async function api(path,opt={}){const r=await fetch(path,{...opt,headers:{'Content-Type':'application/json',...(opt.headers||{})}});const b=await r.json().catch(()=>({}));if(!r.ok)throw Error(b.error||`Request failed (${r.status})`);return b}
|
||||
@@ -15,9 +20,14 @@ function renderDashboard(){const total=prospects.length,pending=prospects.filter
|
||||
function renderDiscover(){ $('view').innerHTML=layout('Discover','Set the criteria for this run. Source configuration and provider settings live in Settings.',`<div class="two-col"><article class="panel form-panel"><div class="panel-title"><div><span class="eyebrow">NEW RUN</span><h2>Find potential customers</h2><p>Search Google Maps, deduplicate the catalogue, then enrich each result.</p></div><span class="pill">Email collection always on</span></div><form id="discoverForm"><label>Keywords<input id="keywords" required placeholder="plumbing services"></label><label>Category<input id="category" required placeholder="plumbers"></label><label>Location<input id="location" value="South Africa" required></label><label>Maximum results<select id="max-results"><option>10</option><option selected>25</option><option>50</option></select></label><label class="checkline"><input type="checkbox" id="extra-reviews"> Collect extra reviews <small>May increase run time</small></label><div class="form-row"><button class="primary" type="submit" id="discoverBtn">Run discovery →</button><span id="discoverMessage" class="notice"></span></div></form></article><article class="panel"><div class="panel-title"><div><span class="eyebrow">HOW IT WORKS</span><h2>Evidence-first pipeline</h2></div></div><ol class="steps"><li><b>Search</b><span>Maps worker finds public business listings.</span></li><li><b>Enrich</b><span>Website, HTTPS, DNS hints and public email checks run safely.</span></li><li><b>Review</b><span>You decide what is relevant before any contact.</span></li></ol></article></div>`)}
|
||||
function renderProspects(){ $('view').innerHTML=layout('Prospects','Browse the strongest opportunities first, then switch to a search category when you want a focused list.',`<article class="panel"><div class="toolbar"><div class="search"><span>⌕</span><input id="prospectSearch" placeholder="Search name, address, domain or email"></div><select id="prospectStatus"><option value="">All review statuses</option><option value="pending">Needs review</option><option value="approved">Approved</option><option value="rejected">Rejected</option><option value="contacted">Contacted</option></select><button class="secondary" id="exportCsv">Export CSV</button><button class="secondary" id="refreshProspects">Refresh</button></div><div id="prospectTabs" class="prospect-tabs">${prospectTabs()}</div><div id="prospectTable">${prospectTable()}</div></article><aside id="detail" class="detail-panel" hidden></aside>`)}
|
||||
function renderRuns(){ $('view').innerHTML=layout('Runs','Monitor discovery jobs and inspect errors without leaving the run history.',`<article class="panel"><div class="table-wrap"><table><thead><tr><th>Run</th><th>Status</th><th>Results</th><th>Started</th><th>Source</th></tr></thead><tbody>${runs.map(r=>`<tr><td><b>#${r.id}</b><small>${esc(r.query)}</small></td><td><span class="status ${r.status}">${esc(r.status)}</span><small>${esc(r.stage||'')}</small></td><td>${r.added||0} added / ${r.found||0} found</td><td>${date(r.created_at)}</td><td>${esc(r.source||'google_maps')}</td></tr>`).join('')}</tbody></table></div>${runs.length?'':empty('No discovery runs yet')}</article>`)}
|
||||
function logTabCount(tab,counts){return tab==='errors'?Number(counts?.errors||0):Number(counts?.[tab]||0)}
|
||||
function logTabs(counts={}){return `<div class="logs-tab-list">${LOG_TABS.map(([key,label])=>`<button class="logs-tab ${activeLogTab===key?'active':''}" data-log-tab="${key}">${label}<span>${logTabCount(key,counts)}</span></button>`).join('')}</div>`}
|
||||
function renderLogTable(){const rows=Array.isArray(logs)?logs:[];return rows.length?`<div class="table-wrap log-table-wrap"><table><thead><tr><th>Time</th><th>Type</th><th>Event</th><th>Status</th><th>Duration</th><th>Details</th></tr></thead><tbody>${rows.map(row=>{const details=typeof row.details==='string'?row.details:JSON.stringify(row.details||{},null,2);return `<tr><td>${esc(date(row.created_at))}</td><td><span class="log-kind ${esc(row.level||'info')}">${esc(row.kind||'system')}</span></td><td><b>${esc(row.event||'')}</b>${row.path?`<small>${esc(row.method||'')} ${esc(row.path)}</small>`:''}${row.error?`<small class="log-error">${esc(row.error)}</small>`:''}</td><td>${row.status?esc(row.status):'—'}</td><td>${row.duration_ms!=null?esc(row.duration_ms)+' ms':'—'}</td><td><details><summary>View</summary><pre>${esc(details)}</pre></details></td></tr>`}).join('')}</tbody></table></div>`:empty('No log entries for this tab')}
|
||||
async function refreshLogs(){const status=$('logsStatus');if(status)status.textContent='Loading logs…';try{const params=new URLSearchParams({limit:'250'});if(activeLogTab==='errors')params.set('level','error');else if(activeLogTab!=='all')params.set('kind',activeLogTab);const data=await api('/api/logs?'+params.toString());logs=data.items||[];const tabs=$('logsTabs');if(tabs)tabs.innerHTML=logTabs(data.counts||{});const table=$('logsTable');if(table)table.innerHTML=renderLogTable();if(status)status.textContent=`Showing ${logs.length} of ${data.total||logs.length} entries · credentials are redacted`; }catch(error){if(status)status.textContent=error.message;if($('logsTable'))$('logsTable').innerHTML=empty('Logs could not be loaded')}}
|
||||
function renderLogs(){ $('view').innerHTML=layout('Logs','Inspect user actions, API calls, AI requests, searches, Browser Use steps, and errors in one place.',`<article class="panel logs-panel"><div class="panel-title"><div><span class="eyebrow">OPERATIONS JOURNAL</span><h2>Detailed activity</h2><p>Requests and actions are timestamped. API keys, authorization headers, and passwords are never stored.</p></div><div class="head-actions"><button class="secondary" id="refreshLogs">Refresh</button></div></div><div id="logsTabs" class="logs-tabs">${logTabs()}</div><div id="logsStatus" class="notice">Loading logs…</div><div id="logsTable"></div></article>`);refreshLogs()}
|
||||
function renderSettings(){const ai=settings.ai||{};const sch=settings.schedules||[];const sup=settings.suppressions||[];const provider=ai.provider||'custom';const modelList=Array.isArray(ai.models)?ai.models:[];const selectedModel=ai.model||'';const options=modelList.map(m=>`<option value="${esc(m.id)}">${esc(m.label||m.id)}${m.free?' · free':''}</option>`).join('')+(selectedModel&&!modelList.some(m=>m.id===selectedModel)?`<option value="${esc(selectedModel)}">${esc(selectedModel)}</option>`:'');$('view').innerHTML=layout('Settings','Configure providers, automation and safety controls here. These settings are separate from daily discovery.',`<div class="settings-grid"><article class="panel"><div class="panel-title"><div><span class="eyebrow">SOURCE</span><h2>Google Maps worker</h2><p>The bundled standard gosom worker runs locally and does not require an API key.</p></div><span class="status ${settings.maps?.ready?'succeeded':'failed'}">${settings.maps?.ready?'Ready':'Not connected'}</span></div><div class="setting-line"><span>Connection</span><b>${esc(settings.maps?.worker_url||'Not configured')}</b></div><div class="setting-line"><span>Email extraction</span><b class="good-text">Always on · public pages only</b></div><div class="setting-line"><span>Website scan</span><b>8 second bounded timeout</b></div></article><article class="panel"><div class="panel-title"><div><span class="eyebrow">AI PROVIDER</span><h2>Draft assistant</h2><p>Used only when you click “Draft AI email”.</p></div><span class="status ${ai.configured?'succeeded':'failed'}">${ai.configured?'Configured':'Not configured'}</span></div><form id="settingsForm"><label>Provider<select id="aiProvider"><option value="nous" ${provider==='nous'?'selected':''}>Nous Portal · free models</option><option value="custom" ${provider!=='nous'?'selected':''}>Custom OpenAI-compatible</option></select></label><p class="field-help" id="aiProviderNote">${provider==='nous'?'Nous uses the official inference endpoint. OAuth is supported by Hermes CLI; this standalone app uses a server-side API key.':'Use any OpenAI-compatible /v1 endpoint.'}</p><label>Base URL<input id="aiBase" value="${esc(ai.base_url||'')}" ${provider==='nous'?'readonly':''} placeholder="https://provider.example/v1"></label><label>API key<input id="aiKey" type="password" placeholder="Leave blank to keep current key"></label><label>Model<input id="aiModel" list="nousModelList" value="${esc(selectedModel)}" placeholder="model-name"><datalist id="nousModelList">${options}</datalist></label><p class="field-help">Nous model availability can change. Refresh Settings to update the free catalog.</p><div class="form-row"><button class="primary" type="submit">Save AI settings</button><button class="secondary" id="testAi" type="button">Test connection</button><span id="settingsMessage" class="notice"></span></div></form></article><article class="panel wide-panel"><div class="panel-title"><div><span class="eyebrow">SCHEDULES</span><h2>Recurring discovery</h2><p>Run bounded searches automatically; all results still arrive as needs-review.</p></div></div><form id="scheduleForm" class="inline-form"><input id="sCat" placeholder="Category"><input id="sKey" placeholder="Keywords"><input id="sLoc" value="South Africa" placeholder="Location"><select id="sHours"><option value="24">Daily</option><option value="48">Every 2 days</option><option value="168">Weekly</option></select><button class="secondary">Add schedule</button></form><div class="list">${sch.map(s=>`<div><b>${esc(s.keywords)} · ${esc(s.category)}</b><span>${esc(s.location)} · every ${s.interval_hours}h · next ${date(s.next_run_at)}</span></div>`).join('')||'<p class="muted">No schedules configured.</p>'}</div></article><article class="panel wide-panel"><div class="panel-title"><div><span class="eyebrow">SAFETY</span><h2>Suppression centre</h2><p>Suppressed domains and emails are excluded before new prospects are saved.</p></div></div><form id="suppressionForm" class="inline-form"><select id="supKind"><option value="domain">Domain</option><option value="email">Email</option></select><input id="supValue" placeholder="example.co.za or info@example.co.za"><input id="supReason" placeholder="Reason"><button class="secondary">Suppress</button></form><div class="list">${sup.map(s=>`<div><b>${esc(s.value)}</b><span>${esc(s.kind)} · ${esc(s.reason||'manual')} · ${date(s.created_at)}</span></div>`).join('')||'<p class="muted">No suppressions configured.</p>'}</div></article></div>`)}
|
||||
async function load(){[prospects,runs,settings]=await Promise.all([api('/api/prospects').then(x=>x.items||[]),api('/api/runs').then(x=>x.items||[]),Promise.all([api('/api/settings'),api('/api/schedules'),api('/api/suppressions')]).then(([s,sch,sup])=>({...s,schedules:sch.items||[],suppressions:sup.items||[]}))]);}
|
||||
async function route(){const r=location.hash.slice(1)||'dashboard';document.querySelectorAll('[data-route]').forEach(a=>a.classList.toggle('active',a.dataset.route===r));try{await load();({dashboard:renderDashboard,discover:renderDiscover,prospects:renderProspects,runs:renderRuns,settings:renderSettings}[r]||renderDashboard)();$('updated').textContent='Updated '+new Date().toLocaleTimeString([], {hour:'2-digit',minute:'2-digit'});}catch(e){$('view').innerHTML=empty(e.message)}}
|
||||
async function route(){const r=location.hash.slice(1)||'dashboard';installLogsNav();document.querySelectorAll('[data-route]').forEach(a=>a.classList.toggle('active',a.dataset.route===r));try{await load();({dashboard:renderDashboard,discover:renderDiscover,prospects:renderProspects,runs:renderRuns,logs:renderLogs,settings:renderSettings}[r]||renderDashboard)();$('updated').textContent='Updated '+new Date().toLocaleTimeString([], {hour:'2-digit',minute:'2-digit'});}catch(e){$('view').innerHTML=empty(e.message)}}
|
||||
function suggestedOffer(p){if(p.ai_offer)return p.ai_offer;if(['broken','unreachable','error','slow'].includes(String(p.website_status||'')))return'Move their hosting to our Gold Plus package at R119/month and rebuild the website at no extra design charge.';if(!p.website)return'Offer a professional .co.za domain, reliable hosting from R119/month, professional email, and a free one-page website.';return'Offer a friendly website and hosting review; only suggest a change that is supported by the evidence.'}
|
||||
function jsonArray(value){try{const parsed=JSON.parse(value||'[]');return Array.isArray(parsed)?parsed:[]}catch{return[]}}
|
||||
function showDraftNotes(d){let notes=$('draftNotes');if(!notes){notes=document.createElement('div');notes.id='draftNotes';notes.className='draft-notes';const status=$('messageStatus');if(status)status.parentNode.insertBefore(notes,status)}if(!notes)return;const unknowns=Array.isArray(d?.unknowns)?d.unknowns:[];const used=Array.isArray(d?.evidence_used)?d.evidence_used:[];notes.innerHTML=`<b>AI review notes</b><p>${esc(d?.rationale||'The draft uses the prospect evidence and your instructions.')}</p>${unknowns.length?`<p><b>Confirm before sending:</b> ${esc(unknowns.join(' · '))}</p>`:''}${used.length?`<p class="muted">Evidence used: ${esc(used.join(', '))}</p>`:''}${d?.follow_up?`<p class="muted">${esc(d.follow_up)}</p>`:''}`}
|
||||
@@ -39,3 +49,9 @@ async function findWebsiteCandidates(id,button){if(button){button.disabled=true;
|
||||
document.addEventListener('click',async e=>{const find=e.target.closest('[data-find-websites]');if(find){e.preventDefault();e.stopPropagation();await findWebsiteCandidates(find.dataset.findWebsites,find);return}const accept=e.target.closest('[data-accept-website]');if(accept){e.preventDefault();e.stopPropagation();accept.disabled=true;accept.textContent='Checking…';try{const d=await api(`/api/prospects/${accept.dataset.prospectId}/website-candidates/${accept.dataset.acceptWebsite}`,{method:'PATCH',body:JSON.stringify({status:'accepted'})});if(d.prospect){const i=prospects.findIndex(p=>String(p.id)===String(d.prospect.id));if(i>=0)prospects[i]={...prospects[i],...d.prospect};selected=d.prospect}openDrawer(accept.dataset.prospectId);if($('prospectTable'))$('prospectTable').innerHTML=prospectTable()}catch(error){accept.disabled=false;accept.textContent='Accept domain';const panel=$('websiteCandidatesPanel');if(panel){const notice=document.createElement('p');notice.className='notice error';notice.textContent=error.message;panel.append(notice)}}return}const reject=e.target.closest('[data-reject-website]');if(reject){e.preventDefault();e.stopPropagation();reject.disabled=true;try{const d=await api(`/api/prospects/${reject.dataset.prospectId}/website-candidates/${reject.dataset.rejectWebsite}`,{method:'PATCH',body:JSON.stringify({status:'rejected'})});const current=await api(`/api/prospects/${reject.dataset.prospectId}/website-candidates`);renderWebsiteCandidates(current.items||[],reject.dataset.prospectId)}catch(error){reject.disabled=false;const panel=$('websiteCandidatesPanel');if(panel){const notice=document.createElement('p');notice.className='notice error';notice.textContent=error.message;panel.append(notice)}}}});
|
||||
window.addEventListener('hashchange',route);$('refreshAll').addEventListener('click',route);route();
|
||||
document.addEventListener('click',e=>{const tab=e.target.closest('[data-category-tab]');if(!tab)return;e.preventDefault();e.stopPropagation();activeProspectCategory=tab.dataset.categoryTab||'all';if($('prospectTabs'))$('prospectTabs').innerHTML=prospectTabs();if($('prospectTable'))$('prospectTable').innerHTML=prospectTable()});
|
||||
document.addEventListener('click',e=>{const tab=e.target.closest('[data-log-tab]');if(tab){e.preventDefault();activeLogTab=tab.dataset.logTab||'all';refreshLogs();return}if(e.target.id==='refreshLogs')refreshLogs()});
|
||||
document.addEventListener('click',e=>{const target=e.target.closest('button,a,select,input,textarea,tr,[role="button"]');if(!target)return;const details={tag:target.tagName.toLowerCase(),id:target.id||'',label:target.getAttribute('aria-label')||target.getAttribute('title')||'',route:location.hash.slice(1)||'dashboard'};for(const name of ['open','message','assess','findWebsites','acceptWebsite','rejectWebsite','categoryTab','logTab','review'])if(target.dataset[name])details[name]=target.dataset[name];logAction('ui_click',details)},true);
|
||||
document.addEventListener('change',e=>{const target=e.target;if(!target||!target.id&&!target.dataset.review)return;logAction('ui_change',{tag:target.tagName.toLowerCase(),id:target.id||'',review_id:target.dataset.review||'',route:location.hash.slice(1)||'dashboard'})},true);
|
||||
document.addEventListener('submit',e=>{if(e.target?.id)logAction('ui_submit',{form:e.target.id,route:location.hash.slice(1)||'dashboard'})},true);
|
||||
window.addEventListener('error',e=>logAction('client_error',{message:String(e.message||'Unknown browser error').slice(0,500),source:String(e.filename||'').slice(0,500),line:e.lineno||0},'error'));
|
||||
window.addEventListener('unhandledrejection',e=>logAction('client_unhandled_rejection',{message:String(e.reason?.message||e.reason||'Unhandled promise rejection').slice(0,500)},'error'));
|
||||
|
||||
+1
-1
@@ -1,3 +1,3 @@
|
||||
<!doctype html>
|
||||
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><meta name="theme-color" content="#0d1726"><title>ProspectOS</title><link rel="stylesheet" href="/style.css?v=ui-v16"></head>
|
||||
<body><div class="app-shell"><aside class="sidebar"><div class="brand"><span class="brand-mark">P</span><div><b>ProspectOS</b><small>Growth workspace</small></div></div><nav id="nav"><a href="#dashboard" data-route="dashboard"><span>⌂</span>Dashboard</a><a href="#discover" data-route="discover"><span>⌁</span>Discover</a><a href="#prospects" data-route="prospects"><span>▦</span>Prospects</a><a href="#runs" data-route="runs"><span>◷</span>Runs</a><a href="#settings" data-route="settings"><span>⚙</span>Settings</a></nav><div class="sidebar-foot"><span class="status-dot" id="sourceDot"></span><div><b id="sourceLabel">Checking worker</b><small id="sourceSub">Connecting…</small></div></div></aside><main class="main"><header class="topbar"><div><span class="crumb">Workspace</span><span class="slash">/</span><b id="pageName">Dashboard</b></div><div class="top-actions"><span id="updated">—</span><button class="icon-btn" id="refreshAll" title="Refresh">↻</button></div></header><div id="view"></div><footer>ProspectOS · public data only · manual review before contact</footer></main></div><aside id="detailDrawer" class="detail-drawer" aria-hidden="true"><div class="drawer-head"><div><span class="eyebrow">PROSPECT DETAIL</span><h2 id="drawerName">Prospect</h2></div><button class="icon-btn" id="closeDrawer">×</button></div><div id="drawerBody"></div></aside><dialog id="messageDialog"><form method="dialog" id="messageForm"><div class="dialog-head"><div><label class="eyebrow">AI ASSISTED DRAFT</label><h2 id="messageTitle">Tailor a message</h2><p>Use the evidence and your offer to create an editable draft.</p></div><button class="icon-btn" value="cancel">×</button></div><label for="offerInput">Your offer or instruction</label><textarea id="offerInput" rows="4" placeholder="Move hosting to Gold Plus for R119/month and rebuild the website free."></textarea><div id="messageStatus" class="notice"></div><button class="primary" id="generateMessage" type="button">Generate subject and email</button><label for="subjectOutput">Subject</label><input id="subjectOutput"><label for="bodyOutput">Email draft</label><textarea id="bodyOutput" rows="12"></textarea><div class="dialog-actions"><button class="secondary" id="copyMessage" type="button">Copy draft</button><button class="secondary" value="cancel">Close</button></div></form></dialog><script src="/app.js?v=prospects-tabs-v1"></script></body></html>
|
||||
<body><div class="app-shell"><aside class="sidebar"><div class="brand"><span class="brand-mark">P</span><div><b>ProspectOS</b><small>Growth workspace</small></div></div><nav id="nav"><a href="#dashboard" data-route="dashboard"><span>⌂</span>Dashboard</a><a href="#discover" data-route="discover"><span>⌁</span>Discover</a><a href="#prospects" data-route="prospects"><span>▦</span>Prospects</a><a href="#runs" data-route="runs"><span>◷</span>Runs</a><a href="#settings" data-route="settings"><span>⚙</span>Settings</a></nav><div class="sidebar-foot"><span class="status-dot" id="sourceDot"></span><div><b id="sourceLabel">Checking worker</b><small id="sourceSub">Connecting…</small></div></div></aside><main class="main"><header class="topbar"><div><span class="crumb">Workspace</span><span class="slash">/</span><b id="pageName">Dashboard</b></div><div class="top-actions"><span id="updated">—</span><button class="icon-btn" id="refreshAll" title="Refresh">↻</button></div></header><div id="view"></div><footer>ProspectOS · public data only · manual review before contact</footer></main></div><aside id="detailDrawer" class="detail-drawer" aria-hidden="true"><div class="drawer-head"><div><span class="eyebrow">PROSPECT DETAIL</span><h2 id="drawerName">Prospect</h2></div><button class="icon-btn" id="closeDrawer">×</button></div><div id="drawerBody"></div></aside><dialog id="messageDialog"><form method="dialog" id="messageForm"><div class="dialog-head"><div><label class="eyebrow">AI ASSISTED DRAFT</label><h2 id="messageTitle">Tailor a message</h2><p>Use the evidence and your offer to create an editable draft.</p></div><button class="icon-btn" value="cancel">×</button></div><label for="offerInput">Your offer or instruction</label><textarea id="offerInput" rows="4" placeholder="Move hosting to Gold Plus for R119/month and rebuild the website free."></textarea><div id="messageStatus" class="notice"></div><button class="primary" id="generateMessage" type="button">Generate subject and email</button><label for="subjectOutput">Subject</label><input id="subjectOutput"><label for="bodyOutput">Email draft</label><textarea id="bodyOutput" rows="12"></textarea><div class="dialog-actions"><button class="secondary" id="copyMessage" type="button">Copy draft</button><button class="secondary" value="cancel">Close</button></div></form></dialog><script src="/app.js?v=logs-v1"></script></body></html>
|
||||
|
||||
@@ -106,6 +106,7 @@ input::placeholder { color: #aeb6c5; } input:focus, select:focus { border-color:
|
||||
|
||||
.setup-section { max-width: 1160px; display: grid; grid-template-columns: 1.2fr 1fr; gap: 30px; margin-top: 17px; padding: 22px 24px; border: 1px solid #e3e6ee; border-radius: 13px; background: #fbfcfe; }.setup-section h2 { margin: 5px 0; color: #38445a; font-size: 15px; }.setup-section p, .setup-section ol { margin: 6px 0 0; color: #8792a5; font-size: 11px; }.setup-section ol { padding-left: 17px; line-height: 1.9; }.setup-section code { padding: 2px 4px; border-radius: 4px; background: #eff1f6; color: #5d687d; font-size: 10px; }.footer { max-width: 1160px; display: flex; justify-content: space-between; gap: 20px; padding: 23px 2px 0; color: #a3acbb; font-size: 10px; }
|
||||
.muted { color: #9ca5b5; font-size: 11px; }
|
||||
.logs-panel{min-width:0}.logs-tabs{margin:22px 0 12px;padding:2px 0 0;border-bottom:1px solid #edf0f5}.logs-tab-list{display:flex;gap:5px;overflow-x:auto}.logs-tab{border:0;border-bottom:2px solid transparent;background:transparent;color:#718097;padding:11px 12px;margin-bottom:-1px;white-space:nowrap;font:inherit;font-size:11px;font-weight:800;cursor:pointer}.logs-tab:hover{color:#4f47df}.logs-tab.active{border-color:#635bff;color:#4f47df}.logs-tab span{display:inline-grid;place-items:center;min-width:19px;height:18px;margin-left:4px;padding:0 5px;border-radius:99px;background:#eef1f7;color:#68758a;font-size:9px}.logs-tab.active span{background:#eeecff;color:#574fe2}.log-table-wrap table{min-width:980px}.log-kind{display:inline-block;padding:5px 7px;border-radius:99px;background:#edf1f8;color:#5d6c83;font-size:9px;font-weight:800;text-transform:uppercase}.log-kind.error{background:#ffe7e9;color:#bd4253}.log-kind.warn{background:#fff2d7;color:#a16e17}.log-table-wrap td{vertical-align:top}.log-table-wrap td small{max-width:300px;overflow-wrap:anywhere}.log-error{color:#bd4253!important}.log-table-wrap details{max-width:370px}.log-table-wrap summary{color:#5b50d3;cursor:pointer;font-size:11px;font-weight:750}.log-table-wrap pre{max-height:240px;overflow:auto;margin:8px 0 0;padding:9px;border-radius:7px;background:#f6f7fb;color:#536178;font:10px/1.45 ui-monospace,SFMono-Regular,Menlo,monospace;white-space:pre-wrap;overflow-wrap:anywhere}
|
||||
.draft-notes{padding:12px;border:1px solid #e6eaf2;border-radius:9px;background:#fafbff;color:#536178;font-size:11px;line-height:1.5}.draft-notes b{color:#334158}.draft-notes p{margin:4px 0}.draft-notes .muted{font-size:10px}
|
||||
.prospect-tabs{margin:4px 0 22px;padding:14px 0 0;border-top:1px solid #edf0f5}.prospect-tab-list{display:flex;gap:8px;overflow-x:auto;padding:2px 1px 3px}.prospect-tab{border:1px solid #dfe4ed;background:#fff;color:#536178;border-radius:999px;padding:9px 13px;white-space:nowrap;font:inherit;font-size:12px;font-weight:750;cursor:pointer;transition:.18s}.prospect-tab:hover{border-color:#b9b5ff;background:#f7f6ff;color:#4f47df}.prospect-tab.active{background:#635bff;border-color:#635bff;color:#fff;box-shadow:0 6px 14px #635bff2e}.prospect-tab span{display:inline-grid;place-items:center;min-width:20px;height:20px;margin-left:5px;padding:0 5px;border-radius:99px;background:#eef1f7;color:#647188;font-size:10px}.prospect-tab.active span{background:#fff3;color:#fff}.tab-hint{margin:10px 0 0;color:#8a96a8;font-size:11px}.candidate-list{display:grid;gap:10px;margin-top:15px}.candidate-card{padding:13px;border:1px solid #e6eaf2;border-radius:10px;background:#fbfcff}.candidate-card>div:first-child{display:flex;align-items:center;gap:8px}.candidate-card a{color:#4f47df;font-weight:750;text-decoration:none;font-size:13px}.candidate-card p{font-size:12px!important;color:#68758a!important}.candidate-card ul{padding-left:18px;margin:7px 0}.candidate-card li{font-size:11px!important;padding:2px 0!important}.candidate-actions{display:flex;gap:7px;margin-top:10px}.candidate-actions button{padding:8px 10px;font-size:11px}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user