add free public source adapters

apps/api/app/sources.py

@@ -310,6 +310,44 @@ class GatedSource(_Base):
         raise RuntimeError("network_adapter_not_configured")
 
 
+class ApprovedDirectorySource(GatedSource):
+    kind = source_code = "approved_directory"
+    display_name = "Free public directories"
+    available = True
+    requires_credentials = False
+
+    def validate_config(self, config):
+        result=super().validate_config(config)
+        if not result.valid:return result
+        provider=str(config.get("provider", "")).strip().lower()
+        if provider not in {"openstreetmap", "wikidata", "common_crawl"}: return ValidationResult(False,["provider must be openstreetmap, wikidata, or common_crawl"])
+        if not str(config.get("query", "")).strip(): return ValidationResult(False,["query is required"])
+        return ValidationResult(True)
+
+    def discover(self, config, cursor=None):
+        result=self.validate_config(config)
+        if not result.valid: raise ValueError(result.errors[0])
+        provider=str(config["provider"]).lower(); query=str(config["query"]).strip(); limit=max(1,min(100,int(config.get("max_records",50))))
+        if provider == "openstreetmap":
+            area=str(config.get("location", "South Africa")).strip()
+            overpass='[out:json][timeout:25];area["name"="%s"]->.a;(nwr["name"](area.a););out center tags;'%(area.replace('"',''))
+            req=Request("https://overpass-api.de/api/interpreter",data=overpass.encode(),method="POST",headers={"Content-Type":"application/x-www-form-urlencoded","User-Agent":"ProspectOS/0.1"})
+            with urlopen(req,timeout=30) as response: payload=json.loads(response.read(2*1024*1024).decode("utf-8","replace"))
+            records=[]
+            for element in payload.get("elements",[]):
+                tags=element.get("tags",{}); name=tags.get("name","")
+                if not name or (query.lower() not in name.lower() and query.lower() not in str(tags).lower()): continue
+                records.append(normalize_record({"name":name,"website":tags.get("website") or tags.get("contact:website", ""),"phone":tags.get("phone") or tags.get("contact:phone", ""),"email":tags.get("email") or tags.get("contact:email", ""),"location":", ".join(x for x in (tags.get("addr:street"),tags.get("addr:city"),tags.get("addr:postcode")) if x),"description":"OpenStreetMap public listing"}))
+            return DiscoveryPage(records[:limit],metadata={"adapter":self.source_code,"provider":provider,"record_count":len(records)})
+        if provider == "wikidata":
+            sparql=query if query.lower().startswith("select") else 'SELECT ?item ?itemLabel ?website WHERE {?item rdfs:label ?itemLabel. FILTER(CONTAINS(LCASE(?itemLabel), LCASE("%s"))). OPTIONAL {?item wdt:P856 ?website} FILTER(LANG(?itemLabel)="en")} LIMIT %d'%(query.replace('"',''),limit)
+            url="https://query.wikidata.org/sparql?format=json&"+urlencode({"query":sparql})
+            payload, _ = _HttpJsonSource()._get_json(url); records=[normalize_record({"name":x.get("itemLabel",{}).get("value",""),"website":x.get("website",{}).get("value","")}) for x in payload.get("results",{}).get("bindings",[])]
+            return DiscoveryPage(records[:limit],metadata={"adapter":self.source_code,"provider":provider,"record_count":len(records)})
+        index="https://index.commoncrawl.org/CC-MAIN-2026-30-index?url="+query+"&output=json&filter=status:200&collapse=urlkey"
+        payload, _ = _HttpJsonSource()._get_json(index); records=[normalize_record({"name":str(x.get("url","")).split('/')[2] if '://' in str(x.get("url","")) else x.get("url", ""),"website":x.get("url","")}) for x in (payload if isinstance(payload,list) else [])]
+        return DiscoveryPage(records[:limit],metadata={"adapter":self.source_code,"provider":provider,"record_count":len(records)})
+
 class GooglePlacesSource(GatedSource):
     kind = source_code = "google_places"
     display_name = "Google Places"
@@ -340,7 +378,6 @@ class GooglePlacesSource(GatedSource):
 def _gated(code, name):
     return type(name.replace(" ", ""), (GatedSource,), {"kind":code, "source_code":code, "display_name":name})
 BingLocalSource = _gated("bing_local", "Bing / approved local API")
-ApprovedDirectorySource = _gated("approved_directory", "Approved directory")
 PermittedSocialSource = _gated("permitted_social", "Permitted social")
 
 ADAPTERS = {x.source_code: x for x in (ManualSource, CsvSource, GooglePlacesSource, BingLocalSource, ApprovedDirectorySource, PublicWebsiteSource, PermittedSocialSource, CtLogsSource, DnsSource, RdapSource)}

apps/api/tests/test_sources_phase5.py

@@ -9,9 +9,11 @@ class SourceAdapterTests(unittest.TestCase):
         catalog = {item['source_code']: item for item in available_adapters()}
         for code in ('manual', 'csv', 'public_website', 'ct_logs', 'dns', 'rdap'):
             self.assertTrue(catalog[code]['available'], code)
-        for code in ('bing_local', 'approved_directory', 'permitted_social'):
+        for code in ('bing_local', 'permitted_social'):
             self.assertFalse(catalog[code]['available'], code)
             self.assertTrue(catalog[code]['optional'], code)
+        self.assertTrue(catalog['approved_directory']['available'])
+        self.assertTrue(catalog['approved_directory']['optional'])
         self.assertTrue(catalog['google_places']['available'])
         self.assertTrue(catalog['google_places']['optional'])
 

apps/web/app.js

@@ -338,6 +338,8 @@ $('aiProviderForm').addEventListener('submit',saveAiProviderSettings);$('aiProvi
     const copy = sourceSetup.querySelector('.source-setup-copy');
     if (badge) badge.textContent = 'Public and operator-controlled sources';
     if (copy) copy.textContent = 'Register bounded public sources or operator-controlled imports. New sources start disabled and must be tested before use.';
+    sourceSetup.insertAdjacentHTML('afterend', `<article class="panel source-config-panel free-sources-panel"><div class="panel-heading"><div><p class="eyebrow">NO BILLING REQUIRED</p><h3>Free public source</h3></div><span class="small-label">Rate limited</span></div><p class="muted">Search OpenStreetMap, Wikidata, or Common Crawl without a Google API key.</p><form id="freeSourceForm" class="crm-form"><label>Provider<select name="provider"><option value="openstreetmap">OpenStreetMap / Overpass</option><option value="wikidata">Wikidata</option><option value="common_crawl">Common Crawl</option></select></label><label>Search query<input name="query" required placeholder="e.g. plumbers"></label><label>Location <span class="optional">optional</span><input name="location" value="South Africa" placeholder="e.g. Eastern Cape"></label><p id="freeSourceMessage" class="form-message" role="status"></p><button class="button primary" type="submit">Add free source</button></form></article>`);
+    $('freeSourceForm')?.addEventListener('submit', async event => { event.preventDefault(); const fields=Object.fromEntries(new FormData(event.currentTarget).entries()), msg=$('freeSourceMessage'); msg.textContent='Registering source…'; msg.className='form-message'; try { await jsonRequest('/api/v1/sources',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({name:`${fields.provider} · ${fields.query}`,kind:'approved_directory',config:{provider:fields.provider,query:fields.query.trim(),location:(fields.location||'South Africa').trim(),approved:true,public_access:true,terms_accepted:true,rate_limit:1},policy:{owner:'workspace operator',rate_limit:1,terms_url:'https://www.openstreetmap.org/copyright'}})}); msg.textContent='Source added disabled. Test it, then enable it below.'; event.currentTarget.reset(); event.currentTarget.querySelector('[name="location"]').value='South Africa'; await loadSources(); } catch(error) { msg.textContent=error.message||'Unable to add source.'; msg.className='form-message error'; } });
     sourceSetup.insertAdjacentHTML('afterend', `<article class="panel source-config-panel google-places-panel"><div class="panel-heading"><div><p class="eyebrow">OPTIONAL PROVIDER</p><h3>Google Places discovery</h3></div><span class="small-label">Secure key storage</span></div><p class="muted">Add a Google Places API key here. The key is sent directly to the server, encrypted, and never displayed again.</p><form id="googlePlacesForm" class="crm-form"><label>Search query<input name="query" required placeholder="e.g. plumbers in Cape Town"></label><label>Region code<input name="region_code" value="ZA" maxlength="2"></label><label>Google API key<input name="api_key" type="password" autocomplete="new-password" required placeholder="Paste your key"></label><label class="checkbox-line"><input name="approved" type="checkbox" required> I have enabled Places API, billing, and accepted Google's terms.</label><p id="googlePlacesMessage" class="form-message" role="status"></p><button class="button primary" type="submit">Save key and register source</button></form></article>`);
     $('googlePlacesForm')?.addEventListener('submit', async event => { event.preventDefault(); const form=event.currentTarget, fields=Object.fromEntries(new FormData(form).entries()); const msg=$('googlePlacesMessage'); msg.textContent='Saving securely…'; msg.className='form-message'; try { const created=await jsonRequest('/api/v1/sources',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({name:'Google Places',kind:'google_places',config:{approved:true,public_access:true,terms_accepted:true,credential_ref:'google_places_api_key',rate_limit:1,query:fields.query.trim(),region_code:(fields.region_code||'ZA').toUpperCase()}})}); await jsonRequest(`/api/v1/sources/${encodeURIComponent(created.id)}/credentials`,{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({provider:'google_places',key_name:'api_key',api_key:fields.api_key})}); msg.textContent='Google Places is registered and the key is encrypted. Test it, then enable the source.'; form.reset(); form.querySelector('[name="region_code"]').value='ZA'; await loadSources(); } catch(error) { msg.textContent=error.message||'Unable to save Google Places settings.'; msg.className='form-message error'; } });
   }

apps/web/asset-manifest.json

@@ -1,6 +1,6 @@
 {
   "schema": 1,
-  "version": "phase-23",
+  "version": "phase-24",
   "entrypoints": [
     "config.js",
     "app.js",
@@ -13,10 +13,10 @@
     "healthz"
   ],
   "integrity": {
-    "config.js": "sha256-a6bfa48e9656dfcfdd9a8f03492399bff25411753e1db248817443f42668c5d4",
-    "app.js": "sha256-da048730cea4ef1409d180b6cb18f29fa06454e3c8b9be8e140d42ab6b14f171",
+    "config.js": "sha256-29953b8de308d7831c606d2116b30ae59744fc0e45f998d970fe71fcbd80fc2a",
+    "app.js": "sha256-44809719afe72cd0ddc30819b3c73f36019319bfbda4e59d62055fd040b5c6c0",
     "styles.css": "sha256-a9fd194b001c0de98775bcf5fbba5da676ccfdd607a889eae84bd30a54c15e08",
-    "index.html": "sha256-82f92ce6dc4822346e876b341f1cbb2839e9c454d414af673fdc8047fc2fb009",
+    "index.html": "sha256-f72befe537a05b7172f94346b367c75f364b0895b9dac233cc6bd78220a222d9",
     "health.html": "sha256-c352a6f37aa24628cfc8d5709a70ff2d94181d192ed5fe00916cca9378a61d81",
     "error.html": "sha256-f3cc28d2dfc9e8af112d0257c6ee47b9e9146e5da8590a754dd589bf43702aaf",
     "healthz": "sha256-dc51b8c96c2d745df3bd5590d990230a482fd247123599548e0632fdbf97fc22"

apps/web/config.js

@@ -1,5 +1,5 @@
 /* Public, non-secret runtime configuration. Replace this file at deploy time if needed. */
 window.__PROSPECT_CONFIG__ = Object.freeze({
   apiBase: '',
-  assetVersion: 'phase-23'
+  assetVersion: 'phase-24'
 });

apps/web/index.html

@@ -5,7 +5,7 @@
   <meta name="viewport" content="width=device-width, initial-scale=1">
   <title>ProspectOS · Pipeline intelligence</title>
   <meta name="description" content="Prospect discovery and review dashboard">
-  <link rel="stylesheet" href="styles.css?v=phase-23">
+  <link rel="stylesheet" href="styles.css?v=phase-24">
 </head>
 <body>
   <section class="login-screen" id="loginScreen" aria-labelledby="loginTitle">
@@ -130,7 +130,7 @@
   <div class="merge-dialog" id="mergeDialog" hidden role="dialog" aria-modal="true" aria-labelledby="mergeDialogTitle">
     <div class="merge-dialog-card"><div class="panel-heading"><div><p class="eyebrow">REVIEW REQUIRED</p><h2 id="mergeDialogTitle">Confirm merge</h2></div><button class="icon-button" id="cancelMergeBtn" type="button" aria-label="Close merge confirmation">×</button></div><p id="mergeDialogCopy"></p><div class="merge-warning"><strong>This action is reversible.</strong> The merge will be recorded in history and can be reversed later.</div><p id="mergeDialogMessage" class="form-message" role="alert" aria-live="polite"></p><div class="merge-dialog-actions"><button class="button ghost" id="cancelMergeBtnSecondary" type="button">Cancel</button><button class="button primary" id="confirmMergeBtn" type="button">Confirm merge</button></div></div>
   </div>
-  <script src="config.js?v=phase-23"></script>
-  <script src="app.js?v=phase-23"></script>
+  <script src="config.js?v=phase-24"></script>
+  <script src="app.js?v=phase-24"></script>
 </body>
 </html>