use standard Maps worker jobs API without key

.env.example

@@ -4,7 +4,8 @@ HOST_PORT=8000
 # Maps worker admin/UI port. Keep it private if the server is internet-facing.
 GMAPS_HOST_PORT=8081
 
-# Create this in the Maps worker admin screen at http://SERVER:8081.
+# Leave empty for the standard gosom worker. Only used with the optional SaaS API.
+GMAPS_API_MODE=standard
 GMAPS_API_KEY=
 
 # A single personal run may take several minutes.

README.md

@@ -19,19 +19,22 @@ docker compose up -d --build
 ```
 
 Open `http://SERVER:8000` for ProspectOS. The Maps worker is available on
-`http://SERVER:8081` for its one-time setup.
+`http://SERVER:8081` if you want to inspect its native Web UI.
 
-### Connect the Maps worker
+### How the worker connects
 
-1. Open port `8081` in a browser on your private network.
-2. Use the worker's admin screen to create an API key for this local app.
-3. Put that key in `.env` as `GMAPS_API_KEY=...`.
-4. Restart the app with `docker compose up -d --build` and reload ProspectOS.
+No Google Cloud API key is required. The bundled standard worker exposes a
+local REST API on the Docker network, and ProspectOS calls it directly.
 
-The app calls `/api/v1/scrape` to queue a job and `/api/v1/jobs/{id}` to poll
-it. Jobs are deliberately asynchronous because a Maps scrape can take several
-minutes. A run that loses its worker connection is shown as failed with a
-specific error rather than remaining stuck forever.
+The app uses `/api/v1/jobs` to queue a search, polls `/api/v1/jobs/{id}`, and
+downloads `/api/v1/jobs/{id}/download` when the job finishes. Jobs are
+deliberately asynchronous because a Maps scrape can take several minutes.
+
+Only set `GMAPS_API_MODE=saas` and `GMAPS_API_KEY=...` if you deliberately
+replace the bundled worker with gosom's optional SaaS edition.
+
+A run that loses its worker connection is shown as failed with a specific
+error rather than remaining stuck forever.
 
 ## Configuration
 
@@ -39,10 +42,11 @@ specific error rather than remaining stuck forever.
 |---|---:|---|
 | `HOST_PORT` | `8000` | Host port for ProspectOS |
 | `GMAPS_HOST_PORT` | `8081` | Host port for the Maps worker admin UI |
-| `GMAPS_API_KEY` | empty | API key created in the Maps worker admin UI |
+| `GMAPS_API_MODE` | `standard` | `standard` for the bundled worker; `saas` only for the optional SaaS API |
+| `GMAPS_API_KEY` | empty | Not required in standard mode; only used in SaaS mode |
 | `GMAPS_REQUEST_TIMEOUT` | `300` | Per-job scraper timeout in seconds |
 | `GMAPS_POLL_SECONDS` | `2` | Poll interval while a job is running |
-| `DISCOVERY_BACKEND` | `gmaps` when `GMAPS_URL` is set | Set to `browser` only for the old public-search fallback |
+| `DISCOVERY_BACKEND` | `gmaps` when `GMAPS_URL` is set | Set to `browser` only for the bounded public-search fallback |
 
 The app stores its catalogue in the `prospect_data` Docker volume. The worker
 stores its own job data in `gmaps_data`; keep both volumes if you want to retain

api/server.py

@@ -8,7 +8,9 @@ an explicit fallback (DISCOVERY_BACKEND=browser) for local testing.
 """
 from __future__ import annotations
 
+import csv
 import hashlib
+import io
 import json
 import os
 import re
@@ -28,6 +30,10 @@ MAX_RESULTS = 50
 GMAPS_POLL_SECONDS = float(os.environ.get("GMAPS_POLL_SECONDS", "2"))
 GMAPS_REQUEST_TIMEOUT = int(os.environ.get("GMAPS_REQUEST_TIMEOUT", "300"))
 GMAPS_URL = os.environ.get("GMAPS_URL", "").strip().rstrip("/")
+# The bundled gosom/google-maps-scraper image exposes a local, unauthenticated
+# REST API. The optional SaaS edition uses a different API and requires an
+# X-API-Key header; keep support for it only when explicitly selected.
+GMAPS_API_MODE = os.environ.get("GMAPS_API_MODE", "standard").strip().lower()
 GMAPS_API_KEY = os.environ.get("GMAPS_API_KEY", "").strip()
 ALLOWED_REVIEW = {"pending", "approved", "rejected", "contacted"}
 _discover_lock = threading.Lock()
@@ -306,24 +312,76 @@ def http_json(method: str, url: str, payload: Any = None, headers: dict[str, str
         raise RuntimeError("gmaps_invalid_json") from exc
 
 
+def http_bytes(method: str, url: str, payload: Any = None, headers: dict[str, str] | None = None, timeout: float = 20) -> bytes:
+    """Make a small HTTP request where the response is not JSON (CSV download)."""
+    data = None if payload is None else json.dumps(payload).encode("utf-8")
+    request_headers = {"Accept": "*/*", **(headers or {})}
+    if data is not None:
+        request_headers["Content-Type"] = "application/json"
+    request = urllib.request.Request(url, data=data, headers=request_headers, method=method.upper())
+    try:
+        with urllib.request.urlopen(request, timeout=timeout) as response:
+            return response.read()
+    except urllib.error.HTTPError as exc:
+        detail = exc.read().decode("utf-8", "replace")[:300]
+        raise RuntimeError(f"gmaps_http_{exc.code}:{detail}") from exc
+    except urllib.error.URLError as exc:
+        raise RuntimeError(f"gmaps_unreachable:{exc.reason}") from exc
+
+
 def gmaps_headers() -> dict[str, str]:
-    if not GMAPS_API_KEY:
-        raise RuntimeError("gmaps_api_key_missing")
-    return {"X-API-Key": GMAPS_API_KEY}
+    if GMAPS_API_MODE == "saas":
+        if not GMAPS_API_KEY:
+            raise RuntimeError("gmaps_api_key_missing_for_saas_mode")
+        return {"X-API-Key": GMAPS_API_KEY}
+    return {}
+
+
+def download_standard_results(job_id: str) -> list[dict[str, Any]]:
+    """Download and parse the CSV produced by the standard web worker."""
+    body = http_bytes(
+        "GET",
+        GMAPS_URL + "/api/v1/jobs/" + quote_plus(str(job_id)) + "/download",
+        headers=gmaps_headers(),
+        timeout=30,
+    )
+    try:
+        text = body.decode("utf-8-sig")
+        return [dict(row) for row in csv.DictReader(io.StringIO(text))]
+    except (UnicodeDecodeError, csv.Error) as exc:
+        raise RuntimeError("gmaps_csv_invalid") from exc
 
 
 def submit_gmaps_job(query: str, include_email: bool = False, extra_reviews: bool = False) -> str:
     if not GMAPS_URL:
         raise RuntimeError("gmaps_not_configured")
-    payload = {
-        "keyword": query,
-        "lang": os.environ.get("GMAPS_LANG", "en"),
-        "max_depth": max(1, min(5, int(os.environ.get("GMAPS_MAX_DEPTH", "1")))),
-        "email": bool(include_email),
-        "extra_reviews": bool(extra_reviews),
-        "timeout": max(1, min(300, GMAPS_REQUEST_TIMEOUT)),
-    }
-    response = http_json("POST", GMAPS_URL + "/api/v1/scrape", payload, gmaps_headers(), timeout=30)
+    lang = os.environ.get("GMAPS_LANG", "en")
+    depth = max(1, min(10, int(os.environ.get("GMAPS_MAX_DEPTH", "1"))))
+    if GMAPS_API_MODE == "saas":
+        endpoint = "/api/v1/scrape"
+        payload = {
+            "keyword": query,
+            "lang": lang,
+            "max_depth": depth,
+            "email": bool(include_email),
+            "extra_reviews": bool(extra_reviews),
+            "timeout": max(1, min(300, GMAPS_REQUEST_TIMEOUT)),
+        }
+    else:
+        # Standard gosom web mode uses the local job API. It has no API-key
+        # authentication and returns a UUID in {"id": ...} with HTTP 201.
+        endpoint = "/api/v1/jobs"
+        payload = {
+            "name": query[:180],
+            "keywords": [query],
+            "lang": lang,
+            "zoom": 15,
+            "depth": depth,
+            "email": bool(include_email),
+            "extra_reviews": bool(extra_reviews),
+            "max_time": max(60, min(3600, GMAPS_REQUEST_TIMEOUT)),
+        }
+    response = http_json("POST", GMAPS_URL + endpoint, payload, gmaps_headers(), timeout=30)
     job_id = response.get("job_id") or response.get("id")
     if not job_id:
         raise RuntimeError("gmaps_job_id_missing")
@@ -334,7 +392,10 @@ def poll_gmaps_job(job_id: str, progress: Callable[[str, int, int], None] | None
     if not GMAPS_URL:
         raise RuntimeError("gmaps_not_configured")
     deadline = time.monotonic() + max(60, GMAPS_REQUEST_TIMEOUT + 60)
-    terminal_ok = {"completed", "succeeded", "success"}
+    if GMAPS_API_MODE == "saas":
+        terminal_ok = {"completed", "succeeded", "success"}
+    else:
+        terminal_ok = {"ok", "completed", "succeeded", "success"}
     terminal_bad = {"failed", "cancelled", "canceled", "discarded"}
     while time.monotonic() < deadline:
         response = http_json("GET", GMAPS_URL + "/api/v1/jobs/" + quote_plus(str(job_id)), headers=gmaps_headers(), timeout=30)
@@ -343,7 +404,11 @@ def poll_gmaps_job(job_id: str, progress: Callable[[str, int, int], None] | None
         if progress:
             progress(status, result_count, 0)
         if status in terminal_ok:
-            return extract_gmaps_results(response.get("results") if "results" in response else response)
+            if "results" in response:
+                return extract_gmaps_results(response.get("results"))
+            if GMAPS_API_MODE == "standard":
+                return download_standard_results(job_id)
+            return extract_gmaps_results(response)
         if status in terminal_bad:
             raise RuntimeError("gmaps_job_" + status + (":" + as_text(response.get("error")) if response.get("error") else ""))
         time.sleep(max(0.5, GMAPS_POLL_SECONDS))
@@ -616,14 +681,20 @@ class Handler(BaseHTTPRequestHandler):
         conn = db()
         try:
             if parsed.path == "/api/health":
-                configured = bool(GMAPS_URL and GMAPS_API_KEY) if backend_mode() == "gmaps" else True
-                return self.send_json(200, {"status": "ok", "source": backend_mode(), "configured": configured, "maps_service": bool(GMAPS_URL), "maps_api_key": bool(GMAPS_API_KEY)})
+                configured = bool(GMAPS_URL) if backend_mode() == "gmaps" else True
+                if GMAPS_API_MODE == "saas":
+                    configured = configured and bool(GMAPS_API_KEY)
+                return self.send_json(200, {"status": "ok", "source": backend_mode(), "configured": configured, "maps_service": bool(GMAPS_URL), "maps_api_mode": GMAPS_API_MODE, "maps_api_key": bool(GMAPS_API_KEY)})
             if parsed.path == "/api/config":
+                auth_required = GMAPS_API_MODE == "saas"
+                ready = backend_mode() == "browser" or (bool(GMAPS_URL) and (not auth_required or bool(GMAPS_API_KEY)))
                 return self.send_json(200, {
                     "backend": backend_mode(),
                     "gmaps_url": bool(GMAPS_URL),
+                    "gmaps_api_mode": GMAPS_API_MODE,
+                    "gmaps_auth_required": auth_required,
                     "gmaps_api_key": bool(GMAPS_API_KEY),
-                    "ready": backend_mode() == "browser" or bool(GMAPS_URL and GMAPS_API_KEY),
+                    "ready": ready,
                     "email_extraction_default": False,
                 })
             if parsed.path == "/api/prospects":

docker-compose.yml

@@ -20,6 +20,7 @@ services:
       PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH: /usr/bin/chromium
       DISCOVERY_BACKEND: ${DISCOVERY_BACKEND:-gmaps}
       GMAPS_URL: ${GMAPS_URL:-http://gmaps:8080}
+      GMAPS_API_MODE: ${GMAPS_API_MODE:-standard}
       GMAPS_API_KEY: ${GMAPS_API_KEY:-}
       GMAPS_REQUEST_TIMEOUT: ${GMAPS_REQUEST_TIMEOUT:-300}
       GMAPS_POLL_SECONDS: ${GMAPS_POLL_SECONDS:-2}

tests/test_app.py

@@ -73,6 +73,22 @@ class BrowserOnlyAppTests(unittest.TestCase):
         status, item = self.request("PATCH", "/api/prospects/1/review", {"status": "approved", "notes": "Looks relevant"})
         self.assertEqual(status, 200); self.assertEqual(item["review_status"], "approved")
 
+    def test_standard_maps_worker_uses_jobs_api_without_key(self):
+        csv_body = b"title,address,website,phone,latitude,longitude,review_rating,review_count,link\nAcme Plumbing,Cape Town,https://acme.co.za,+2710, -33.9,18.4,4.5,12,https://maps.google.com/?cid=1\n"
+        with patch.object(server, "GMAPS_URL", "http://gmaps:8080"), \
+             patch.object(server, "GMAPS_API_MODE", "standard"), \
+             patch.object(server, "GMAPS_API_KEY", ""), \
+             patch("api.server.http_json", side_effect=[{"id": "job-1"}, {"id": "job-1", "status": "pending"}, {"id": "job-1", "status": "working"}, {"id": "job-1", "status": "ok"}]) as http_json, \
+             patch("api.server.http_bytes", return_value=csv_body) as http_bytes:
+            job_id = server.submit_gmaps_job("plumbers Cape Town")
+            results = server.poll_gmaps_job(job_id)
+        self.assertEqual(job_id, "job-1")
+        self.assertEqual(results[0]["title"], "Acme Plumbing")
+        first_call = http_json.call_args_list[0]
+        self.assertEqual(first_call.args[0:2], ("POST", "http://gmaps:8080/api/v1/jobs"))
+        self.assertNotIn("X-API-Key", first_call.args[3])
+        self.assertEqual(http_bytes.call_args.args[1], "http://gmaps:8080/api/v1/jobs/job-1/download")
+
 
 if __name__ == "__main__":
     unittest.main()

web/app.js

@@ -100,11 +100,12 @@ async function loadConfig() {
   const config = await api("/api/config");
   $("backendLabel").textContent = config.backend === "gmaps" ? "Maps worker" : "Browser fallback";
   $("sourceLabel").textContent = config.ready ? "Maps worker ready" : "Setup required";
-  $("sourceSub").textContent = config.ready ? "Google Maps scraper connected" : "Add your API key to continue";
+  $("sourceSub").textContent = config.ready ? "Google Maps scraper connected" : (config.gmaps_auth_required ? "SaaS API key required" : "Start the bundled Maps worker");
   $("sourceDot").classList.toggle("offline", !config.ready);
   $("setupNotice").hidden = Boolean(config.ready);
   if (!config.gmaps_url) $("setupText").textContent = "Start the bundled Maps worker with Docker, then reload this page.";
-  else if (!config.gmaps_api_key) $("setupText").textContent = "Create an API key in the Maps worker admin screen, add it as GMAPS_API_KEY, then restart the stack.";
+  else if (config.gmaps_auth_required && !config.gmaps_api_key) $("setupText").textContent = "SaaS mode is selected but no API key is configured. Use standard mode or add GMAPS_API_KEY.";
+  else $("setupText").textContent = "The standard worker needs no API key. If a run fails, check the gmaps container logs.";
 }
 
 async function refreshAll() {

web/index.html

@@ -34,7 +34,7 @@
       </section>
 
       <section class="setup-notice" id="setupNotice" hidden>
-        <div class="notice-icon">!</div><div><strong>Finish the Maps worker setup</strong><p id="setupText">Add the scraper API key to your environment, then restart the stack. The rest of the workspace is ready.</p></div><a href="#setup" class="text-link">Setup guide ↓</a>
+        <div class="notice-icon">!</div><div><strong>Finish the Maps worker setup</strong><p id="setupText">Start the bundled worker with Docker, then reload this page. The standard worker needs no API key.</p></div><a href="#setup" class="text-link">Setup guide ↓</a>
       </section>
 
       <section class="stats-grid" aria-label="Catalogue summary">
@@ -71,7 +71,7 @@
       </section>
 
       <section class="setup-section" id="setup">
-        <div><span class="eyebrow">SETUP</span><h2>One-time worker connection</h2><p>The bundled Maps worker runs locally. Create an API key in its admin screen, place it in <code>GMAPS_API_KEY</code>, and restart Docker.</p></div><ol><li>Open the Maps worker on port <code>8081</code>.</li><li>Sign in to its admin area and create one API key.</li><li>Copy the key into your <code>.env</code> file, then run <code>docker compose up -d --build</code>.</li></ol>
+            <div><span class="eyebrow">SETUP</span><h2>One-time worker connection</h2><p>The bundled standard Maps worker runs locally and does not need a Google or scraper API key.</p></div><ol><li>Run <code>docker compose up -d --build</code>.</li><li>Keep <code>GMAPS_API_MODE=standard</code> in <code>.env</code>.</li><li>Start a discovery run. Open port <code>8081</code> only if you want to inspect the worker's native UI.</li></ol>
       </section>
 
       <footer class="footer"><span>ProspectOS · personal research workspace</span><span>Public data only · no automated outreach</span></footer>