diff --git a/apps/api/app/main.py b/apps/api/app/main.py index 9c53d98..a06dfa9 100644 --- a/apps/api/app/main.py +++ b/apps/api/app/main.py @@ -107,6 +107,15 @@ def row_json(row): if key in result: result[key] = bool(result[key]) return result +def source_config_with_credentials(db, source, config): + """Inject a credential only into the in-process adapter call; never persist/return it.""" + if source["kind"] != "google_places": return config + row = db.execute("SELECT secret_ref FROM source_credentials WHERE source_id=? AND organization_id=? AND provider=? AND key_name=?", (source["id"], source["organization_id"], "google_places", "api_key")).fetchone() + if not row or not row["secret_ref"]: return config + try: config["_api_key"] = decrypt_provider_secret(row["secret_ref"]) + except Exception: pass + return config + class ApiHandler(BaseHTTPRequestHandler): server_version = "ProspectPlatform/0.1" def send_json(self, status, payload, extra_headers=None): @@ -1095,6 +1104,9 @@ class ApiHandler(BaseHTTPRequestHandler): bits_outreach=path.split("/") if len(bits_outreach)==7 and bits_outreach[:4]==["","api","v1","outreach"] and bits_outreach[4]=="drafts" and bits_outreach[5].isdigit() and bits_outreach[6] in {"approve","send"}: return self.approve_outreach_draft(int(bits_outreach[5]),db,user) if bits_outreach[6]=="approve" else self.send_outreach_draft(int(bits_outreach[5]),db,user) if path=="/api/v1/sources":return self.create_source(payload,db,user) + if path.startswith("/api/v1/sources/") and path.endswith("/credentials"): + bits=path.split("/") + if len(bits)==6 and bits[4].isdigit(): return self.save_source_credential(int(bits[4]),payload,db,user) if path=="/api/v1/discovery":return self.create_scoped_discovery(payload,db,user) if path.startswith("/api/v1/discovery-runs/"): bits=path.split("/") @@ -1320,9 +1332,10 @@ class ApiHandler(BaseHTTPRequestHandler): except (TypeError,ValueError): config={} try: policy=json.loads(raw["policy_json"] or "{}") except (TypeError,ValueError): policy={} + credential_exists = bool(db.execute("SELECT 1 FROM source_credentials WHERE source_id=? AND organization_id=? AND provider=? AND key_name=?", (raw["id"], org, "google_places", "api_key")).fetchone()) item.update({"available": bool(meta.get("available", False)), "optional": bool(meta.get("optional", False)), "configured": bool(meta.get("available", False) and (config.get("csv") or config.get("rows") is not None or raw["approved"])), - "credential_status": "Not required" if not meta.get("requires_credentials") else "Required / not configured", + "credential_status": ("Configured" if credential_exists else "Required / not configured") if meta.get("requires_credentials") else "Not required", "terms_status": "Provided" if policy.get("terms_url") or config.get("terms_url") else "Not reviewed", "owner": policy.get("owner") or config.get("owner") or "Not assigned", "rate_limit": policy.get("rate_limit") or config.get("rate_limit") or "Not set"}) @@ -1331,6 +1344,15 @@ class ApiHandler(BaseHTTPRequestHandler): return self.send_json(200,{"organization_id":org,"items":items}) def list_queries(self,db,org): return self.send_json(200,{"organization_id":org,"items":[row_json(r) for r in db.execute("SELECT * FROM discovery_queries WHERE organization_id=? ORDER BY id",(org,))]}) + def save_source_credential(self, sid, payload, db, user): + source=db.execute("SELECT * FROM sources WHERE id=? AND organization_id=?",(sid,user["organization_id"])).fetchone() + if not source: return self.send_json(404,{"error":"not_found"}) + provider=str(payload.get("provider", "")).strip().lower(); key_name=str(payload.get("key_name", "api_key")).strip().lower(); value=payload.get("api_key") + if source["kind"] != "google_places" or provider != "google_places" or key_name != "api_key" or not isinstance(value,str) or not (8 <= len(value.strip()) <= 4096): return self.send_json(400,{"error":"invalid_source_credential"}) + ciphertext=encrypt_provider_secret(value.strip()) + db.execute("INSERT INTO source_credentials(source_id,organization_id,provider,key_name,secret_ref,metadata_json) VALUES(?,?,?,?,?,?) ON CONFLICT(source_id,provider,key_name) DO UPDATE SET secret_ref=excluded.secret_ref,metadata_json=excluded.metadata_json",(sid,user["organization_id"],provider,key_name,ciphertext,json.dumps({"configured_at":datetime.now(timezone.utc).replace(microsecond=0).isoformat()}))) + self.audit(db,user,"source.credential.updated",str(sid)); db.commit() + return self.send_json(200,{"ok":True,"configured":True,"provider":provider,"key_name":key_name}) def list_source_records(self,db,org,q): try: limit=int(q.get('page_size',[50])[0]); offset=max(0,int(q.get('offset',[0])[0])) @@ -1382,6 +1404,8 @@ class ApiHandler(BaseHTTPRequestHandler): # before they are enabled. if not validation.valid and not (source['kind'] == 'manual' and not config): return self.send_json(409,{"error":"source_not_configured","details":validation.errors}) + if source['kind'] == 'google_places' and not db.execute("SELECT 1 FROM source_credentials WHERE source_id=? AND organization_id=? AND provider=? AND key_name=?", (sid,user['organization_id'],'google_places','api_key')).fetchone(): + return self.send_json(409,{"error":"source_not_configured","details":["Google Places API key is required"]}) db.execute("UPDATE sources SET enabled=?,updated_at=CURRENT_TIMESTAMP WHERE id=?",(value,sid));self.audit(db,user,'source.enabled' if value else 'source.disabled',str(sid));db.commit();return self.send_json(200,row_json(db.execute("SELECT * FROM sources WHERE id=?",(sid,)).fetchone())) def create_query(self,payload,db,user): sid=payload.get('source_id');name=str(payload.get('name','')).strip();query=payload.get('query',{}) @@ -1421,7 +1445,7 @@ class ApiHandler(BaseHTTPRequestHandler): if len(json.dumps(config).encode())>5*1024*1024:return self.send_json(400,{"error":"ingest_limits"}) if isinstance(config.get('rows'),list) and (len(config['rows'])>1000 or any(not isinstance(r,dict) or len(r)>50 or any(len(str(v))>10000 for v in r.values()) for r in config['rows'])):return self.send_json(400,{"error":"ingest_limits"}) if isinstance(config.get('csv'),str) and config['csv'].count('\n')>1001:return self.send_json(400,{"error":"ingest_limits"}) - try:page=adapter_for(source['kind']).discover(config) + try:page=adapter_for(source['kind']).discover(source_config_with_credentials(db, source, config)) except (ValueError,KeyError) as exc:return self.send_json(400,{"error":"invalid_ingest","detail":str(exc)}) inserted=0 for record in page.records[:1000]: @@ -1559,7 +1583,7 @@ def _run_source_discovery(db, job, handler): used_today=db.execute("SELECT COUNT(*) FROM source_records WHERE organization_id=? AND source_id=? AND date(created_at)=date('now')",(org,source["id"])).fetchone()[0] if used_today >= daily_limit or per_run_limit <= 0: handler.add_job_event(db,job["id"],org,"source.quota_exceeded",f"Quota reached for {source['kind']}",10,"SOURCE_QUOTA_EXCEEDED"); continue - page=adapter_for(source["kind"]).discover(config) + page=adapter_for(source["kind"]).discover(source_config_with_credentials(db, source, config)) except Exception as exc: failures=int(source["consecutive_failures"])+1 db.execute("UPDATE sources SET health_status='unhealthy',consecutive_failures=?,circuit_open=?,last_failure_at=CURRENT_TIMESTAMP,last_error=?,updated_at=CURRENT_TIMESTAMP WHERE id=?",(failures,int(circuit_is_open(failures)),str(exc)[:300],source["id"])) diff --git a/apps/api/app/sources.py b/apps/api/app/sources.py index 86b4347..751f88b 100644 --- a/apps/api/app/sources.py +++ b/apps/api/app/sources.py @@ -310,9 +310,35 @@ class GatedSource(_Base): raise RuntimeError("network_adapter_not_configured") +class GooglePlacesSource(GatedSource): + kind = source_code = "google_places" + display_name = "Google Places" + available = True + + def validate_config(self, config): + result = super().validate_config(config) + if not result.valid: return result + if not str(config.get("query", "")).strip(): return ValidationResult(False, ["query is required"]) + return ValidationResult(True) + + def discover(self, config, cursor=None): + result = self.validate_config(config) + if not result.valid: raise ValueError(result.errors[0]) + api_key = str(config.get("_api_key", "")).strip() + if not api_key: raise ValueError("Google Places API key is not configured") + body = {"textQuery": str(config["query"]).strip(), "pageSize": max(1, min(20, int(config.get("max_records", 20))))} + if config.get("region_code"): body["regionCode"] = str(config["region_code"]).upper()[:2] + request = Request("https://places.googleapis.com/v1/places:searchText", data=json.dumps(body).encode(), method="POST", headers={"Content-Type":"application/json", "X-Goog-Api-Key":api_key, "X-Goog-FieldMask":"places.displayName,places.websiteUri,places.nationalPhoneNumber,places.internationalPhoneNumber,places.formattedAddress,places.googleMapsUri"}) + with urlopen(request, timeout=12) as response: + payload=json.loads(response.read(2*1024*1024).decode("utf-8", "replace")) + records=[] + for place in payload.get("places", [])[:20]: + name=(place.get("displayName") or {}).get("text", "") + records.append(normalize_record({"name":name, "website":place.get("websiteUri", ""), "phone":place.get("nationalPhoneNumber") or place.get("internationalPhoneNumber", ""), "location":place.get("formattedAddress", ""), "source_url":place.get("googleMapsUri", "") , "description":"Google Places result"})) + return DiscoveryPage(records, metadata={"adapter":self.source_code,"record_count":len(records),"provider":"google_places"}) + def _gated(code, name): return type(name.replace(" ", ""), (GatedSource,), {"kind":code, "source_code":code, "display_name":name}) -GooglePlacesSource = _gated("google_places", "Google Places") BingLocalSource = _gated("bing_local", "Bing / approved local API") ApprovedDirectorySource = _gated("approved_directory", "Approved directory") PermittedSocialSource = _gated("permitted_social", "Permitted social") diff --git a/apps/api/tests/test_sources_phase5.py b/apps/api/tests/test_sources_phase5.py index a30964c..25c095f 100644 --- a/apps/api/tests/test_sources_phase5.py +++ b/apps/api/tests/test_sources_phase5.py @@ -9,9 +9,11 @@ class SourceAdapterTests(unittest.TestCase): catalog = {item['source_code']: item for item in available_adapters()} for code in ('manual', 'csv', 'public_website', 'ct_logs', 'dns', 'rdap'): self.assertTrue(catalog[code]['available'], code) - for code in ('google_places', 'bing_local', 'approved_directory', 'permitted_social'): + for code in ('bing_local', 'approved_directory', 'permitted_social'): self.assertFalse(catalog[code]['available'], code) self.assertTrue(catalog[code]['optional'], code) + self.assertTrue(catalog['google_places']['available']) + self.assertTrue(catalog['google_places']['optional']) def test_csv_adapter_is_deterministic_and_normalizes(self): src = CsvSource() diff --git a/apps/web/app.js b/apps/web/app.js index 316cb5c..40ab782 100644 --- a/apps/web/app.js +++ b/apps/web/app.js @@ -280,7 +280,7 @@ const sourceText = (source, keys, fallback='Not returned') => { for (const key of keys) if (source?.[key] !== undefined && source[key] !== null && source[key] !== '') return source[key]; return fallback; }; function sourceMessage(text, error = false) { const el = $('sourcesMessage'); if (el) { el.textContent = text || ''; el.className = `sources-message${error ? ' error' : ''}`; } } function renderSourceSelect() { const select = $('discoverySource'); if (select) select.innerHTML = `${sources.filter(source=>!source.optional).map(s => ``).join('')}`; const multi=$('directDiscoverySources'); if(multi) multi.innerHTML=sources.filter(sourceState).map(s=>``).join('') || ''; } - function renderSources() { renderSourceSelect(); const list = $('sourcesList'); if (!sources.length) { list.innerHTML = '
No registered or available sources returned by the workspace.
'; return; } list.innerHTML = sources.map(source => { const policy=sourceJson(source.policy_json||source.policy), quota=sourceJson(source.quota_json||source.quota), config=sourceJson(source.config_json||source.config), status=sourceStatus(source), configured=source.configured ?? (!source.optional && (source.approved || sourceType(source)==='manual'||sourceType(source)==='csv')), available=source.available ?? (!source.optional || Boolean(source.configured)), health=sourceText(source,['health_status','health'],'Not tested'), failures=sourceText(source,['consecutive_failures'],'0'), circuit=source.circuit_open===true||source.circuit_open===1?'Open':'Closed', credential=sourceText(source,['api_credential_status','credential_status'],source.optional?'Required / not configured':'Not required'), terms=sourceText(source,['terms_status','terms_reviewed'],policy.terms_accepted===true?'Accepted':policy.terms_url||config.terms_url?'Provided':'Not reviewed'), owner=sourceText(source,['owner','owner_name'],policy.owner||config.owner||'Not assigned'), rate=sourceText(source,['rate_limit','rate_limit_label'],policy.rate_limit||config.rate_limit||'Not set'), daily=sourceText(source,['daily_quota','daily_limit'],quota.daily_limit||'Not set'), lastHealth=sourceText(source,['last_health_at','last_checked_at','updated_at'],'Not checked'), success=sourceText(source,['last_success_at'],'No successful run'), error=sourceText(source,['last_error','error'],'None recorded'); return `
${esc(sourceLabel(source))}${esc(sourceType(source))}
${source.optional?'Optional adapter · configuration-gated':'Registered workspace source'}
${esc(status==='unavailable'?'Unavailable':status)}
Integration
${esc(source.optional?'Optional adapter':'Registered')}
Configured
${configured?'Yes':'No'}
Available
${available?'Yes':'No'}
Enabled
${sourceState(source)?'Yes':'No'}
API credential
${esc(credential)}
Terms
${esc(terms)}
Owner
${esc(owner)}
Rate limit
${esc(rate)}
Daily quota
${esc(daily)}
Last health
${esc(lastHealth)} · ${esc(health)}
Success / error
${esc(success)}
${esc(error)}
Circuit
${esc(circuit)} · ${esc(failures)} failures
`; }).join(''); } + function renderSources() { renderSourceSelect(); const list = $('sourcesList'); if (!sources.length) { list.innerHTML = '
No registered or available sources returned by the workspace.
'; return; } list.innerHTML = sources.map(source => { const policy=sourceJson(source.policy_json||source.policy), quota=sourceJson(source.quota_json||source.quota), config=sourceJson(source.config_json||source.config), status=sourceStatus(source), configured=source.configured ?? (!source.optional && (source.approved || sourceType(source)==='manual'||sourceType(source)==='csv')), available=source.available ?? (!source.optional || Boolean(source.configured)), health=sourceText(source,['health_status','health'],'Not tested'), failures=sourceText(source,['consecutive_failures'],'0'), circuit=source.circuit_open===true||source.circuit_open===1?'Open':'Closed', credential=sourceText(source,['api_credential_status','credential_status'],source.optional?'Required / not configured':'Not required'), terms=sourceText(source,['terms_status','terms_reviewed'],policy.terms_accepted===true?'Accepted':policy.terms_url||config.terms_url?'Provided':'Not reviewed'), owner=sourceText(source,['owner','owner_name'],policy.owner||config.owner||'Not assigned'), rate=sourceText(source,['rate_limit','rate_limit_label'],policy.rate_limit||config.rate_limit||'Not set'), daily=sourceText(source,['daily_quota','daily_limit'],quota.daily_limit||'Not set'), lastHealth=sourceText(source,['last_health_at','last_checked_at','updated_at'],'Not checked'), success=sourceText(source,['last_success_at'],'No successful run'), error=sourceText(source,['last_error','error'],'None recorded'); return `
${esc(sourceLabel(source))}${esc(sourceType(source))}
${source.optional?'Optional adapter · configuration-gated':'Registered workspace source'}
${esc(status==='unavailable'?'Unavailable':status)}
Integration
${esc(source.optional?'Optional adapter':'Registered')}
Configured
${configured?'Yes':'No'}
Available
${available?'Yes':'No'}
Enabled
${sourceState(source)?'Yes':'No'}
API credential
${esc(credential)}
Terms
${esc(terms)}
Owner
${esc(owner)}
Rate limit
${esc(rate)}
Daily quota
${esc(daily)}
Last health
${esc(lastHealth)} · ${esc(health)}
Success / error
${esc(success)}
${esc(error)}
Circuit
${esc(circuit)} · ${esc(failures)} failures
`; }).join(''); } function renderSourceRecords(items) { const list = $('sourceRecordsList'); if (!items.length) { list.innerHTML = '
No source records returned by the workspace.
'; return; } list.innerHTML = `
${items.slice(0,25).map(record => ``).join('')}
RecordSourceStatusObserved
${esc(record.name || record.title || record.external_id || record.id || 'Unnamed record')}${esc(record.source_name || record.source || 'Unknown source')}${esc(record.status || 'Pending')}${esc(record.observed_at || record.created_at || 'Time unavailable')}
`; } async function loadSources() { sourceMessage('Loading sources…'); $('sourcesList').innerHTML = '
Loading source registry…
'; $('sourceRecordsList').innerHTML = '
Loading source records…
'; try { const [sourcePayload, adapterPayload, recordPayload] = await Promise.all([jsonRequest('/api/v1/sources'), jsonRequest('/api/v1/sources/adapters'), jsonRequest('/api/v1/source-records?page_size=25')]); const configured=sourceItems(sourcePayload), registeredCodes=new Set(configured.map(sourceType)); const optional=sourceItems(adapterPayload).filter(adapter=>!registeredCodes.has(adapter.source_code)).map(adapter=>({...adapter,source_code:adapter.source_code,display_name:adapter.display_name,kind:adapter.source_code,optional:Boolean(adapter.optional),configured:false,available:Boolean(adapter.available),enabled:false})); sources=[...configured,...optional]; renderSources(); renderSourceRecords(sourceItems(recordPayload)); $('sourcesUpdatedAt').textContent = `Updated ${new Date().toLocaleTimeString([], {hour:'2-digit', minute:'2-digit'})}`; sourceMessage(sources.some(sourceState) ? '' : 'No live source is enabled. Configure and enable a ready source to begin discovery.'); } catch (error) { sources = []; renderSources(); renderSourceRecords([]); if (error.message !== 'unauthorized') sourceMessage(error.message || 'Unable to load sources.', true); } } async function saveSource(event) { event.preventDefault(); const form = event.currentTarget, fields = Object.fromEntries(new FormData(form).entries()); if (fields.source_type === 'csv' && !fields.csv_content.trim()) { message('sourceFormMessage', 'CSV content is required for a CSV source.', true); return; } const config = {url:fields.url, terms_url:fields.terms_url, owner:fields.owner, rate_limit:fields.rate_limit}; if (fields.source_type === 'csv') config.csv = fields.csv_content; else config.rows = []; try { await jsonRequest('/api/v1/sources', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify({name:fields.name, kind:fields.source_type, config, enabled:false})}); message('sourceFormMessage', 'Source saved. It remains disabled until explicitly enabled.'); form.reset(); $('sourceCsvField').hidden = true; await loadSources(); } catch (error) { if (error.message !== 'unauthorized') message('sourceFormMessage', error.message || 'Unable to save source.', true); } } @@ -338,6 +338,8 @@ $('aiProviderForm').addEventListener('submit',saveAiProviderSettings);$('aiProvi const copy = sourceSetup.querySelector('.source-setup-copy'); if (badge) badge.textContent = 'Public and operator-controlled sources'; if (copy) copy.textContent = 'Register bounded public sources or operator-controlled imports. New sources start disabled and must be tested before use.'; + sourceSetup.insertAdjacentHTML('afterend', `

OPTIONAL PROVIDER

Google Places discovery

Secure key storage

Add a Google Places API key here. The key is sent directly to the server, encrypted, and never displayed again.

`); + $('googlePlacesForm')?.addEventListener('submit', async event => { event.preventDefault(); const form=event.currentTarget, fields=Object.fromEntries(new FormData(form).entries()); const msg=$('googlePlacesMessage'); msg.textContent='Saving securely…'; msg.className='form-message'; try { const created=await jsonRequest('/api/v1/sources',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({name:'Google Places',kind:'google_places',config:{approved:true,public_access:true,terms_accepted:true,credential_ref:'google_places_api_key',rate_limit:1,query:fields.query.trim(),region_code:(fields.region_code||'ZA').toUpperCase()}})}); await jsonRequest(`/api/v1/sources/${encodeURIComponent(created.id)}/credentials`,{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({provider:'google_places',key_name:'api_key',api_key:fields.api_key})}); msg.textContent='Google Places is registered and the key is encrypted. Test it, then enable the source.'; form.reset(); form.querySelector('[name="region_code"]').value='ZA'; await loadSources(); } catch(error) { msg.textContent=error.message||'Unable to save Google Places settings.'; msg.className='form-message error'; } }); } async function configureSourceFromUi(source, button) { const type = sourceType(source); diff --git a/apps/web/asset-manifest.json b/apps/web/asset-manifest.json index d5984aa..770c284 100644 --- a/apps/web/asset-manifest.json +++ b/apps/web/asset-manifest.json @@ -1,6 +1,6 @@ { "schema": 1, - "version": "phase-22", + "version": "phase-23", "entrypoints": [ "config.js", "app.js", @@ -13,10 +13,10 @@ "healthz" ], "integrity": { - "config.js": "sha256-734a7d93ee125a12ba355206e4e789c6a8e942f0955ad3abcaf604119f1b8b63", - "app.js": "sha256-9073461507ae2e96eb1d62c62d0cc1103bbc427c17f7f42ae7c6f2c52528a87b", + "config.js": "sha256-a6bfa48e9656dfcfdd9a8f03492399bff25411753e1db248817443f42668c5d4", + "app.js": "sha256-da048730cea4ef1409d180b6cb18f29fa06454e3c8b9be8e140d42ab6b14f171", "styles.css": "sha256-a9fd194b001c0de98775bcf5fbba5da676ccfdd607a889eae84bd30a54c15e08", - "index.html": "sha256-9f6838c02890eff9b83e32177af01b56c0423758edf9411bc8f196cb61510b5e", + "index.html": "sha256-82f92ce6dc4822346e876b341f1cbb2839e9c454d414af673fdc8047fc2fb009", "health.html": "sha256-c352a6f37aa24628cfc8d5709a70ff2d94181d192ed5fe00916cca9378a61d81", "error.html": "sha256-f3cc28d2dfc9e8af112d0257c6ee47b9e9146e5da8590a754dd589bf43702aaf", "healthz": "sha256-dc51b8c96c2d745df3bd5590d990230a482fd247123599548e0632fdbf97fc22" diff --git a/apps/web/config.js b/apps/web/config.js index 33685bb..43d6de0 100644 --- a/apps/web/config.js +++ b/apps/web/config.js @@ -1,5 +1,5 @@ /* Public, non-secret runtime configuration. Replace this file at deploy time if needed. */ window.__PROSPECT_CONFIG__ = Object.freeze({ apiBase: '', - assetVersion: 'phase-22' + assetVersion: 'phase-23' }); diff --git a/apps/web/index.html b/apps/web/index.html index 96ad828..698bc94 100644 --- a/apps/web/index.html +++ b/apps/web/index.html @@ -5,7 +5,7 @@ ProspectOS · Pipeline intelligence - +
@@ -130,7 +130,7 @@ - - + +