add free public source adapters

apps/api/app/sources.py

@@ -310,6 +310,44 @@ class GatedSource(_Base):
         raise RuntimeError("network_adapter_not_configured")
 
 
+class ApprovedDirectorySource(GatedSource):
+    kind = source_code = "approved_directory"
+    display_name = "Free public directories"
+    available = True
+    requires_credentials = False
+
+    def validate_config(self, config):
+        result=super().validate_config(config)
+        if not result.valid:return result
+        provider=str(config.get("provider", "")).strip().lower()
+        if provider not in {"openstreetmap", "wikidata", "common_crawl"}: return ValidationResult(False,["provider must be openstreetmap, wikidata, or common_crawl"])
+        if not str(config.get("query", "")).strip(): return ValidationResult(False,["query is required"])
+        return ValidationResult(True)
+
+    def discover(self, config, cursor=None):
+        result=self.validate_config(config)
+        if not result.valid: raise ValueError(result.errors[0])
+        provider=str(config["provider"]).lower(); query=str(config["query"]).strip(); limit=max(1,min(100,int(config.get("max_records",50))))
+        if provider == "openstreetmap":
+            area=str(config.get("location", "South Africa")).strip()
+            overpass='[out:json][timeout:25];area["name"="%s"]->.a;(nwr["name"](area.a););out center tags;'%(area.replace('"',''))
+            req=Request("https://overpass-api.de/api/interpreter",data=overpass.encode(),method="POST",headers={"Content-Type":"application/x-www-form-urlencoded","User-Agent":"ProspectOS/0.1"})
+            with urlopen(req,timeout=30) as response: payload=json.loads(response.read(2*1024*1024).decode("utf-8","replace"))
+            records=[]
+            for element in payload.get("elements",[]):
+                tags=element.get("tags",{}); name=tags.get("name","")
+                if not name or (query.lower() not in name.lower() and query.lower() not in str(tags).lower()): continue
+                records.append(normalize_record({"name":name,"website":tags.get("website") or tags.get("contact:website", ""),"phone":tags.get("phone") or tags.get("contact:phone", ""),"email":tags.get("email") or tags.get("contact:email", ""),"location":", ".join(x for x in (tags.get("addr:street"),tags.get("addr:city"),tags.get("addr:postcode")) if x),"description":"OpenStreetMap public listing"}))
+            return DiscoveryPage(records[:limit],metadata={"adapter":self.source_code,"provider":provider,"record_count":len(records)})
+        if provider == "wikidata":
+            sparql=query if query.lower().startswith("select") else 'SELECT ?item ?itemLabel ?website WHERE {?item rdfs:label ?itemLabel. FILTER(CONTAINS(LCASE(?itemLabel), LCASE("%s"))). OPTIONAL {?item wdt:P856 ?website} FILTER(LANG(?itemLabel)="en")} LIMIT %d'%(query.replace('"',''),limit)
+            url="https://query.wikidata.org/sparql?format=json&"+urlencode({"query":sparql})
+            payload, _ = _HttpJsonSource()._get_json(url); records=[normalize_record({"name":x.get("itemLabel",{}).get("value",""),"website":x.get("website",{}).get("value","")}) for x in payload.get("results",{}).get("bindings",[])]
+            return DiscoveryPage(records[:limit],metadata={"adapter":self.source_code,"provider":provider,"record_count":len(records)})
+        index="https://index.commoncrawl.org/CC-MAIN-2026-30-index?url="+query+"&output=json&filter=status:200&collapse=urlkey"
+        payload, _ = _HttpJsonSource()._get_json(index); records=[normalize_record({"name":str(x.get("url","")).split('/')[2] if '://' in str(x.get("url","")) else x.get("url", ""),"website":x.get("url","")}) for x in (payload if isinstance(payload,list) else [])]
+        return DiscoveryPage(records[:limit],metadata={"adapter":self.source_code,"provider":provider,"record_count":len(records)})
+
 class GooglePlacesSource(GatedSource):
     kind = source_code = "google_places"
     display_name = "Google Places"
@@ -340,7 +378,6 @@ class GooglePlacesSource(GatedSource):
 def _gated(code, name):
     return type(name.replace(" ", ""), (GatedSource,), {"kind":code, "source_code":code, "display_name":name})
 BingLocalSource = _gated("bing_local", "Bing / approved local API")
-ApprovedDirectorySource = _gated("approved_directory", "Approved directory")
 PermittedSocialSource = _gated("permitted_social", "Permitted social")
 
 ADAPTERS = {x.source_code: x for x in (ManualSource, CsvSource, GooglePlacesSource, BingLocalSource, ApprovedDirectorySource, PublicWebsiteSource, PermittedSocialSource, CtLogsSource, DnsSource, RdapSource)}