diff --git a/.env.example b/.env.example index bec2f14..cf0bf96 100644 --- a/.env.example +++ b/.env.example @@ -4,7 +4,8 @@ HOST_PORT=8000 # Maps worker admin/UI port. Keep it private if the server is internet-facing. GMAPS_HOST_PORT=8081 -# Create this in the Maps worker admin screen at http://SERVER:8081. +# Leave empty for the standard gosom worker. Only used with the optional SaaS API. +GMAPS_API_MODE=standard GMAPS_API_KEY= # A single personal run may take several minutes. diff --git a/README.md b/README.md index 7e8ce23..6926874 100644 --- a/README.md +++ b/README.md @@ -19,19 +19,22 @@ docker compose up -d --build ``` Open `http://SERVER:8000` for ProspectOS. The Maps worker is available on -`http://SERVER:8081` for its one-time setup. +`http://SERVER:8081` if you want to inspect its native Web UI. -### Connect the Maps worker +### How the worker connects -1. Open port `8081` in a browser on your private network. -2. Use the worker's admin screen to create an API key for this local app. -3. Put that key in `.env` as `GMAPS_API_KEY=...`. -4. Restart the app with `docker compose up -d --build` and reload ProspectOS. +No Google Cloud API key is required. The bundled standard worker exposes a +local REST API on the Docker network, and ProspectOS calls it directly. -The app calls `/api/v1/scrape` to queue a job and `/api/v1/jobs/{id}` to poll -it. Jobs are deliberately asynchronous because a Maps scrape can take several -minutes. A run that loses its worker connection is shown as failed with a -specific error rather than remaining stuck forever. +The app uses `/api/v1/jobs` to queue a search, polls `/api/v1/jobs/{id}`, and +downloads `/api/v1/jobs/{id}/download` when the job finishes. Jobs are +deliberately asynchronous because a Maps scrape can take several minutes. + +Only set `GMAPS_API_MODE=saas` and `GMAPS_API_KEY=...` if you deliberately +replace the bundled worker with gosom's optional SaaS edition. + +A run that loses its worker connection is shown as failed with a specific +error rather than remaining stuck forever. ## Configuration @@ -39,10 +42,11 @@ specific error rather than remaining stuck forever. |---|---:|---| | `HOST_PORT` | `8000` | Host port for ProspectOS | | `GMAPS_HOST_PORT` | `8081` | Host port for the Maps worker admin UI | -| `GMAPS_API_KEY` | empty | API key created in the Maps worker admin UI | +| `GMAPS_API_MODE` | `standard` | `standard` for the bundled worker; `saas` only for the optional SaaS API | +| `GMAPS_API_KEY` | empty | Not required in standard mode; only used in SaaS mode | | `GMAPS_REQUEST_TIMEOUT` | `300` | Per-job scraper timeout in seconds | | `GMAPS_POLL_SECONDS` | `2` | Poll interval while a job is running | -| `DISCOVERY_BACKEND` | `gmaps` when `GMAPS_URL` is set | Set to `browser` only for the old public-search fallback | +| `DISCOVERY_BACKEND` | `gmaps` when `GMAPS_URL` is set | Set to `browser` only for the bounded public-search fallback | The app stores its catalogue in the `prospect_data` Docker volume. The worker stores its own job data in `gmaps_data`; keep both volumes if you want to retain diff --git a/api/server.py b/api/server.py index 3829c6a..d3c27f4 100644 --- a/api/server.py +++ b/api/server.py @@ -8,7 +8,9 @@ an explicit fallback (DISCOVERY_BACKEND=browser) for local testing. """ from __future__ import annotations +import csv import hashlib +import io import json import os import re @@ -28,6 +30,10 @@ MAX_RESULTS = 50 GMAPS_POLL_SECONDS = float(os.environ.get("GMAPS_POLL_SECONDS", "2")) GMAPS_REQUEST_TIMEOUT = int(os.environ.get("GMAPS_REQUEST_TIMEOUT", "300")) GMAPS_URL = os.environ.get("GMAPS_URL", "").strip().rstrip("/") +# The bundled gosom/google-maps-scraper image exposes a local, unauthenticated +# REST API. The optional SaaS edition uses a different API and requires an +# X-API-Key header; keep support for it only when explicitly selected. +GMAPS_API_MODE = os.environ.get("GMAPS_API_MODE", "standard").strip().lower() GMAPS_API_KEY = os.environ.get("GMAPS_API_KEY", "").strip() ALLOWED_REVIEW = {"pending", "approved", "rejected", "contacted"} _discover_lock = threading.Lock() @@ -306,24 +312,76 @@ def http_json(method: str, url: str, payload: Any = None, headers: dict[str, str raise RuntimeError("gmaps_invalid_json") from exc +def http_bytes(method: str, url: str, payload: Any = None, headers: dict[str, str] | None = None, timeout: float = 20) -> bytes: + """Make a small HTTP request where the response is not JSON (CSV download).""" + data = None if payload is None else json.dumps(payload).encode("utf-8") + request_headers = {"Accept": "*/*", **(headers or {})} + if data is not None: + request_headers["Content-Type"] = "application/json" + request = urllib.request.Request(url, data=data, headers=request_headers, method=method.upper()) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + return response.read() + except urllib.error.HTTPError as exc: + detail = exc.read().decode("utf-8", "replace")[:300] + raise RuntimeError(f"gmaps_http_{exc.code}:{detail}") from exc + except urllib.error.URLError as exc: + raise RuntimeError(f"gmaps_unreachable:{exc.reason}") from exc + + def gmaps_headers() -> dict[str, str]: - if not GMAPS_API_KEY: - raise RuntimeError("gmaps_api_key_missing") - return {"X-API-Key": GMAPS_API_KEY} + if GMAPS_API_MODE == "saas": + if not GMAPS_API_KEY: + raise RuntimeError("gmaps_api_key_missing_for_saas_mode") + return {"X-API-Key": GMAPS_API_KEY} + return {} + + +def download_standard_results(job_id: str) -> list[dict[str, Any]]: + """Download and parse the CSV produced by the standard web worker.""" + body = http_bytes( + "GET", + GMAPS_URL + "/api/v1/jobs/" + quote_plus(str(job_id)) + "/download", + headers=gmaps_headers(), + timeout=30, + ) + try: + text = body.decode("utf-8-sig") + return [dict(row) for row in csv.DictReader(io.StringIO(text))] + except (UnicodeDecodeError, csv.Error) as exc: + raise RuntimeError("gmaps_csv_invalid") from exc def submit_gmaps_job(query: str, include_email: bool = False, extra_reviews: bool = False) -> str: if not GMAPS_URL: raise RuntimeError("gmaps_not_configured") - payload = { - "keyword": query, - "lang": os.environ.get("GMAPS_LANG", "en"), - "max_depth": max(1, min(5, int(os.environ.get("GMAPS_MAX_DEPTH", "1")))), - "email": bool(include_email), - "extra_reviews": bool(extra_reviews), - "timeout": max(1, min(300, GMAPS_REQUEST_TIMEOUT)), - } - response = http_json("POST", GMAPS_URL + "/api/v1/scrape", payload, gmaps_headers(), timeout=30) + lang = os.environ.get("GMAPS_LANG", "en") + depth = max(1, min(10, int(os.environ.get("GMAPS_MAX_DEPTH", "1")))) + if GMAPS_API_MODE == "saas": + endpoint = "/api/v1/scrape" + payload = { + "keyword": query, + "lang": lang, + "max_depth": depth, + "email": bool(include_email), + "extra_reviews": bool(extra_reviews), + "timeout": max(1, min(300, GMAPS_REQUEST_TIMEOUT)), + } + else: + # Standard gosom web mode uses the local job API. It has no API-key + # authentication and returns a UUID in {"id": ...} with HTTP 201. + endpoint = "/api/v1/jobs" + payload = { + "name": query[:180], + "keywords": [query], + "lang": lang, + "zoom": 15, + "depth": depth, + "email": bool(include_email), + "extra_reviews": bool(extra_reviews), + "max_time": max(60, min(3600, GMAPS_REQUEST_TIMEOUT)), + } + response = http_json("POST", GMAPS_URL + endpoint, payload, gmaps_headers(), timeout=30) job_id = response.get("job_id") or response.get("id") if not job_id: raise RuntimeError("gmaps_job_id_missing") @@ -334,7 +392,10 @@ def poll_gmaps_job(job_id: str, progress: Callable[[str, int, int], None] | None if not GMAPS_URL: raise RuntimeError("gmaps_not_configured") deadline = time.monotonic() + max(60, GMAPS_REQUEST_TIMEOUT + 60) - terminal_ok = {"completed", "succeeded", "success"} + if GMAPS_API_MODE == "saas": + terminal_ok = {"completed", "succeeded", "success"} + else: + terminal_ok = {"ok", "completed", "succeeded", "success"} terminal_bad = {"failed", "cancelled", "canceled", "discarded"} while time.monotonic() < deadline: response = http_json("GET", GMAPS_URL + "/api/v1/jobs/" + quote_plus(str(job_id)), headers=gmaps_headers(), timeout=30) @@ -343,7 +404,11 @@ def poll_gmaps_job(job_id: str, progress: Callable[[str, int, int], None] | None if progress: progress(status, result_count, 0) if status in terminal_ok: - return extract_gmaps_results(response.get("results") if "results" in response else response) + if "results" in response: + return extract_gmaps_results(response.get("results")) + if GMAPS_API_MODE == "standard": + return download_standard_results(job_id) + return extract_gmaps_results(response) if status in terminal_bad: raise RuntimeError("gmaps_job_" + status + (":" + as_text(response.get("error")) if response.get("error") else "")) time.sleep(max(0.5, GMAPS_POLL_SECONDS)) @@ -616,14 +681,20 @@ class Handler(BaseHTTPRequestHandler): conn = db() try: if parsed.path == "/api/health": - configured = bool(GMAPS_URL and GMAPS_API_KEY) if backend_mode() == "gmaps" else True - return self.send_json(200, {"status": "ok", "source": backend_mode(), "configured": configured, "maps_service": bool(GMAPS_URL), "maps_api_key": bool(GMAPS_API_KEY)}) + configured = bool(GMAPS_URL) if backend_mode() == "gmaps" else True + if GMAPS_API_MODE == "saas": + configured = configured and bool(GMAPS_API_KEY) + return self.send_json(200, {"status": "ok", "source": backend_mode(), "configured": configured, "maps_service": bool(GMAPS_URL), "maps_api_mode": GMAPS_API_MODE, "maps_api_key": bool(GMAPS_API_KEY)}) if parsed.path == "/api/config": + auth_required = GMAPS_API_MODE == "saas" + ready = backend_mode() == "browser" or (bool(GMAPS_URL) and (not auth_required or bool(GMAPS_API_KEY))) return self.send_json(200, { "backend": backend_mode(), "gmaps_url": bool(GMAPS_URL), + "gmaps_api_mode": GMAPS_API_MODE, + "gmaps_auth_required": auth_required, "gmaps_api_key": bool(GMAPS_API_KEY), - "ready": backend_mode() == "browser" or bool(GMAPS_URL and GMAPS_API_KEY), + "ready": ready, "email_extraction_default": False, }) if parsed.path == "/api/prospects": diff --git a/docker-compose.yml b/docker-compose.yml index ad79c16..fffcb6c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,6 +20,7 @@ services: PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH: /usr/bin/chromium DISCOVERY_BACKEND: ${DISCOVERY_BACKEND:-gmaps} GMAPS_URL: ${GMAPS_URL:-http://gmaps:8080} + GMAPS_API_MODE: ${GMAPS_API_MODE:-standard} GMAPS_API_KEY: ${GMAPS_API_KEY:-} GMAPS_REQUEST_TIMEOUT: ${GMAPS_REQUEST_TIMEOUT:-300} GMAPS_POLL_SECONDS: ${GMAPS_POLL_SECONDS:-2} diff --git a/tests/test_app.py b/tests/test_app.py index 78ba7f4..48f9f39 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -73,6 +73,22 @@ class BrowserOnlyAppTests(unittest.TestCase): status, item = self.request("PATCH", "/api/prospects/1/review", {"status": "approved", "notes": "Looks relevant"}) self.assertEqual(status, 200); self.assertEqual(item["review_status"], "approved") + def test_standard_maps_worker_uses_jobs_api_without_key(self): + csv_body = b"title,address,website,phone,latitude,longitude,review_rating,review_count,link\nAcme Plumbing,Cape Town,https://acme.co.za,+2710, -33.9,18.4,4.5,12,https://maps.google.com/?cid=1\n" + with patch.object(server, "GMAPS_URL", "http://gmaps:8080"), \ + patch.object(server, "GMAPS_API_MODE", "standard"), \ + patch.object(server, "GMAPS_API_KEY", ""), \ + patch("api.server.http_json", side_effect=[{"id": "job-1"}, {"id": "job-1", "status": "pending"}, {"id": "job-1", "status": "working"}, {"id": "job-1", "status": "ok"}]) as http_json, \ + patch("api.server.http_bytes", return_value=csv_body) as http_bytes: + job_id = server.submit_gmaps_job("plumbers Cape Town") + results = server.poll_gmaps_job(job_id) + self.assertEqual(job_id, "job-1") + self.assertEqual(results[0]["title"], "Acme Plumbing") + first_call = http_json.call_args_list[0] + self.assertEqual(first_call.args[0:2], ("POST", "http://gmaps:8080/api/v1/jobs")) + self.assertNotIn("X-API-Key", first_call.args[3]) + self.assertEqual(http_bytes.call_args.args[1], "http://gmaps:8080/api/v1/jobs/job-1/download") + if __name__ == "__main__": unittest.main() diff --git a/web/app.js b/web/app.js index 1a79dda..59bb62d 100644 --- a/web/app.js +++ b/web/app.js @@ -100,11 +100,12 @@ async function loadConfig() { const config = await api("/api/config"); $("backendLabel").textContent = config.backend === "gmaps" ? "Maps worker" : "Browser fallback"; $("sourceLabel").textContent = config.ready ? "Maps worker ready" : "Setup required"; - $("sourceSub").textContent = config.ready ? "Google Maps scraper connected" : "Add your API key to continue"; + $("sourceSub").textContent = config.ready ? "Google Maps scraper connected" : (config.gmaps_auth_required ? "SaaS API key required" : "Start the bundled Maps worker"); $("sourceDot").classList.toggle("offline", !config.ready); $("setupNotice").hidden = Boolean(config.ready); if (!config.gmaps_url) $("setupText").textContent = "Start the bundled Maps worker with Docker, then reload this page."; - else if (!config.gmaps_api_key) $("setupText").textContent = "Create an API key in the Maps worker admin screen, add it as GMAPS_API_KEY, then restart the stack."; + else if (config.gmaps_auth_required && !config.gmaps_api_key) $("setupText").textContent = "SaaS mode is selected but no API key is configured. Use standard mode or add GMAPS_API_KEY."; + else $("setupText").textContent = "The standard worker needs no API key. If a run fails, check the gmaps container logs."; } async function refreshAll() { diff --git a/web/index.html b/web/index.html index d47c13e..2946e14 100644 --- a/web/index.html +++ b/web/index.html @@ -34,7 +34,7 @@
@@ -71,7 +71,7 @@
-
SETUP

One-time worker connection

The bundled Maps worker runs locally. Create an API key in its admin screen, place it in GMAPS_API_KEY, and restart Docker.

  1. Open the Maps worker on port 8081.
  2. Sign in to its admin area and create one API key.
  3. Copy the key into your .env file, then run docker compose up -d --build.
+
SETUP

One-time worker connection

The bundled standard Maps worker runs locally and does not need a Google or scraper API key.

  1. Run docker compose up -d --build.
  2. Keep GMAPS_API_MODE=standard in .env.
  3. Start a discovery run. Open port 8081 only if you want to inspect the worker's native UI.