diff --git a/apps/api/app/sources.py b/apps/api/app/sources.py index 751f88b..e89b620 100644 --- a/apps/api/app/sources.py +++ b/apps/api/app/sources.py @@ -310,6 +310,44 @@ class GatedSource(_Base): raise RuntimeError("network_adapter_not_configured") +class ApprovedDirectorySource(GatedSource): + kind = source_code = "approved_directory" + display_name = "Free public directories" + available = True + requires_credentials = False + + def validate_config(self, config): + result=super().validate_config(config) + if not result.valid:return result + provider=str(config.get("provider", "")).strip().lower() + if provider not in {"openstreetmap", "wikidata", "common_crawl"}: return ValidationResult(False,["provider must be openstreetmap, wikidata, or common_crawl"]) + if not str(config.get("query", "")).strip(): return ValidationResult(False,["query is required"]) + return ValidationResult(True) + + def discover(self, config, cursor=None): + result=self.validate_config(config) + if not result.valid: raise ValueError(result.errors[0]) + provider=str(config["provider"]).lower(); query=str(config["query"]).strip(); limit=max(1,min(100,int(config.get("max_records",50)))) + if provider == "openstreetmap": + area=str(config.get("location", "South Africa")).strip() + overpass='[out:json][timeout:25];area["name"="%s"]->.a;(nwr["name"](area.a););out center tags;'%(area.replace('"','')) + req=Request("https://overpass-api.de/api/interpreter",data=overpass.encode(),method="POST",headers={"Content-Type":"application/x-www-form-urlencoded","User-Agent":"ProspectOS/0.1"}) + with urlopen(req,timeout=30) as response: payload=json.loads(response.read(2*1024*1024).decode("utf-8","replace")) + records=[] + for element in payload.get("elements",[]): + tags=element.get("tags",{}); name=tags.get("name","") + if not name or (query.lower() not in name.lower() and query.lower() not in str(tags).lower()): continue + records.append(normalize_record({"name":name,"website":tags.get("website") or tags.get("contact:website", ""),"phone":tags.get("phone") or tags.get("contact:phone", ""),"email":tags.get("email") or tags.get("contact:email", ""),"location":", ".join(x for x in (tags.get("addr:street"),tags.get("addr:city"),tags.get("addr:postcode")) if x),"description":"OpenStreetMap public listing"})) + return DiscoveryPage(records[:limit],metadata={"adapter":self.source_code,"provider":provider,"record_count":len(records)}) + if provider == "wikidata": + sparql=query if query.lower().startswith("select") else 'SELECT ?item ?itemLabel ?website WHERE {?item rdfs:label ?itemLabel. FILTER(CONTAINS(LCASE(?itemLabel), LCASE("%s"))). OPTIONAL {?item wdt:P856 ?website} FILTER(LANG(?itemLabel)="en")} LIMIT %d'%(query.replace('"',''),limit) + url="https://query.wikidata.org/sparql?format=json&"+urlencode({"query":sparql}) + payload, _ = _HttpJsonSource()._get_json(url); records=[normalize_record({"name":x.get("itemLabel",{}).get("value",""),"website":x.get("website",{}).get("value","")}) for x in payload.get("results",{}).get("bindings",[])] + return DiscoveryPage(records[:limit],metadata={"adapter":self.source_code,"provider":provider,"record_count":len(records)}) + index="https://index.commoncrawl.org/CC-MAIN-2026-30-index?url="+query+"&output=json&filter=status:200&collapse=urlkey" + payload, _ = _HttpJsonSource()._get_json(index); records=[normalize_record({"name":str(x.get("url","")).split('/')[2] if '://' in str(x.get("url","")) else x.get("url", ""),"website":x.get("url","")}) for x in (payload if isinstance(payload,list) else [])] + return DiscoveryPage(records[:limit],metadata={"adapter":self.source_code,"provider":provider,"record_count":len(records)}) + class GooglePlacesSource(GatedSource): kind = source_code = "google_places" display_name = "Google Places" @@ -340,7 +378,6 @@ class GooglePlacesSource(GatedSource): def _gated(code, name): return type(name.replace(" ", ""), (GatedSource,), {"kind":code, "source_code":code, "display_name":name}) BingLocalSource = _gated("bing_local", "Bing / approved local API") -ApprovedDirectorySource = _gated("approved_directory", "Approved directory") PermittedSocialSource = _gated("permitted_social", "Permitted social") ADAPTERS = {x.source_code: x for x in (ManualSource, CsvSource, GooglePlacesSource, BingLocalSource, ApprovedDirectorySource, PublicWebsiteSource, PermittedSocialSource, CtLogsSource, DnsSource, RdapSource)} diff --git a/apps/api/tests/test_sources_phase5.py b/apps/api/tests/test_sources_phase5.py index 25c095f..88bb558 100644 --- a/apps/api/tests/test_sources_phase5.py +++ b/apps/api/tests/test_sources_phase5.py @@ -9,9 +9,11 @@ class SourceAdapterTests(unittest.TestCase): catalog = {item['source_code']: item for item in available_adapters()} for code in ('manual', 'csv', 'public_website', 'ct_logs', 'dns', 'rdap'): self.assertTrue(catalog[code]['available'], code) - for code in ('bing_local', 'approved_directory', 'permitted_social'): + for code in ('bing_local', 'permitted_social'): self.assertFalse(catalog[code]['available'], code) self.assertTrue(catalog[code]['optional'], code) + self.assertTrue(catalog['approved_directory']['available']) + self.assertTrue(catalog['approved_directory']['optional']) self.assertTrue(catalog['google_places']['available']) self.assertTrue(catalog['google_places']['optional']) diff --git a/apps/web/app.js b/apps/web/app.js index 40ab782..3e57561 100644 --- a/apps/web/app.js +++ b/apps/web/app.js @@ -338,6 +338,8 @@ $('aiProviderForm').addEventListener('submit',saveAiProviderSettings);$('aiProvi const copy = sourceSetup.querySelector('.source-setup-copy'); if (badge) badge.textContent = 'Public and operator-controlled sources'; if (copy) copy.textContent = 'Register bounded public sources or operator-controlled imports. New sources start disabled and must be tested before use.'; + sourceSetup.insertAdjacentHTML('afterend', `

NO BILLING REQUIRED

Free public source

Rate limited

Search OpenStreetMap, Wikidata, or Common Crawl without a Google API key.

`); + $('freeSourceForm')?.addEventListener('submit', async event => { event.preventDefault(); const fields=Object.fromEntries(new FormData(event.currentTarget).entries()), msg=$('freeSourceMessage'); msg.textContent='Registering source…'; msg.className='form-message'; try { await jsonRequest('/api/v1/sources',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({name:`${fields.provider} · ${fields.query}`,kind:'approved_directory',config:{provider:fields.provider,query:fields.query.trim(),location:(fields.location||'South Africa').trim(),approved:true,public_access:true,terms_accepted:true,rate_limit:1},policy:{owner:'workspace operator',rate_limit:1,terms_url:'https://www.openstreetmap.org/copyright'}})}); msg.textContent='Source added disabled. Test it, then enable it below.'; event.currentTarget.reset(); event.currentTarget.querySelector('[name="location"]').value='South Africa'; await loadSources(); } catch(error) { msg.textContent=error.message||'Unable to add source.'; msg.className='form-message error'; } }); sourceSetup.insertAdjacentHTML('afterend', `

OPTIONAL PROVIDER

Google Places discovery

Secure key storage

Add a Google Places API key here. The key is sent directly to the server, encrypted, and never displayed again.

`); $('googlePlacesForm')?.addEventListener('submit', async event => { event.preventDefault(); const form=event.currentTarget, fields=Object.fromEntries(new FormData(form).entries()); const msg=$('googlePlacesMessage'); msg.textContent='Saving securely…'; msg.className='form-message'; try { const created=await jsonRequest('/api/v1/sources',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({name:'Google Places',kind:'google_places',config:{approved:true,public_access:true,terms_accepted:true,credential_ref:'google_places_api_key',rate_limit:1,query:fields.query.trim(),region_code:(fields.region_code||'ZA').toUpperCase()}})}); await jsonRequest(`/api/v1/sources/${encodeURIComponent(created.id)}/credentials`,{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({provider:'google_places',key_name:'api_key',api_key:fields.api_key})}); msg.textContent='Google Places is registered and the key is encrypted. Test it, then enable the source.'; form.reset(); form.querySelector('[name="region_code"]').value='ZA'; await loadSources(); } catch(error) { msg.textContent=error.message||'Unable to save Google Places settings.'; msg.className='form-message error'; } }); } diff --git a/apps/web/asset-manifest.json b/apps/web/asset-manifest.json index 770c284..5805bfe 100644 --- a/apps/web/asset-manifest.json +++ b/apps/web/asset-manifest.json @@ -1,6 +1,6 @@ { "schema": 1, - "version": "phase-23", + "version": "phase-24", "entrypoints": [ "config.js", "app.js", @@ -13,10 +13,10 @@ "healthz" ], "integrity": { - "config.js": "sha256-a6bfa48e9656dfcfdd9a8f03492399bff25411753e1db248817443f42668c5d4", - "app.js": "sha256-da048730cea4ef1409d180b6cb18f29fa06454e3c8b9be8e140d42ab6b14f171", + "config.js": "sha256-29953b8de308d7831c606d2116b30ae59744fc0e45f998d970fe71fcbd80fc2a", + "app.js": "sha256-44809719afe72cd0ddc30819b3c73f36019319bfbda4e59d62055fd040b5c6c0", "styles.css": "sha256-a9fd194b001c0de98775bcf5fbba5da676ccfdd607a889eae84bd30a54c15e08", - "index.html": "sha256-82f92ce6dc4822346e876b341f1cbb2839e9c454d414af673fdc8047fc2fb009", + "index.html": "sha256-f72befe537a05b7172f94346b367c75f364b0895b9dac233cc6bd78220a222d9", "health.html": "sha256-c352a6f37aa24628cfc8d5709a70ff2d94181d192ed5fe00916cca9378a61d81", "error.html": "sha256-f3cc28d2dfc9e8af112d0257c6ee47b9e9146e5da8590a754dd589bf43702aaf", "healthz": "sha256-dc51b8c96c2d745df3bd5590d990230a482fd247123599548e0632fdbf97fc22" diff --git a/apps/web/config.js b/apps/web/config.js index 43d6de0..5eba613 100644 --- a/apps/web/config.js +++ b/apps/web/config.js @@ -1,5 +1,5 @@ /* Public, non-secret runtime configuration. Replace this file at deploy time if needed. */ window.__PROSPECT_CONFIG__ = Object.freeze({ apiBase: '', - assetVersion: 'phase-23' + assetVersion: 'phase-24' }); diff --git a/apps/web/index.html b/apps/web/index.html index 698bc94..3e1796a 100644 --- a/apps/web/index.html +++ b/apps/web/index.html @@ -5,7 +5,7 @@ ProspectOS · Pipeline intelligence - +
@@ -130,7 +130,7 @@ - - + +