root/MarketingTool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

complete source discovery scope
CI / compose (push) Successful in 14m21s
Details

Browse Source
This commit is contained in:
Marco0300
2026-09-04 13:36:40 +02:00
parent f860d04762
commit b532e39f7c
6 changed files with 103 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/app/sources.py
+26 -1
View File
@@ -315,6 +315,7 @@ class ApprovedDirectorySource(GatedSource):
display_name = "Free public directories"
available = True
requires_credentials = False
optional = False
def validate_config(self, config):
result=super().validate_config(config)
@@ -375,12 +376,36 @@ class GooglePlacesSource(GatedSource):
records.append(normalize_record({"name":name, "website":place.get("websiteUri", ""), "phone":place.get("nationalPhoneNumber") or place.get("internationalPhoneNumber", ""), "location":place.get("formattedAddress", ""), "source_url":place.get("googleMapsUri", "") , "description":"Google Places result"}))
return DiscoveryPage(records, metadata={"adapter":self.source_code,"record_count":len(records),"provider":"google_places"})
class OpenStreetMapSource(ApprovedDirectorySource):
kind = source_code = "openstreetmap"
display_name = "OpenStreetMap / Overpass"
optional = False
def discover(self, config, cursor=None):
page = super().discover({**dict(config), "provider": "openstreetmap"}, cursor)
return DiscoveryPage(page.records, page.next_cursor, {**page.metadata, "adapter": self.source_code})
class WikidataSource(ApprovedDirectorySource):
kind = source_code = "wikidata"
display_name = "Wikidata"
optional = False
def discover(self, config, cursor=None):
page = super().discover({**dict(config), "provider": "wikidata"}, cursor)
return DiscoveryPage(page.records, page.next_cursor, {**page.metadata, "adapter": self.source_code})
class CommonCrawlSource(ApprovedDirectorySource):
kind = source_code = "common_crawl"
display_name = "Common Crawl index"
optional = False
def discover(self, config, cursor=None):
page = super().discover({**dict(config), "provider": "common_crawl"}, cursor)
return DiscoveryPage(page.records, page.next_cursor, {**page.metadata, "adapter": self.source_code})
def _gated(code, name):
return type(name.replace(" ", ""), (GatedSource,), {"kind":code, "source_code":code, "display_name":name})
BingLocalSource = _gated("bing_local", "Bing / approved local API")
PermittedSocialSource = _gated("permitted_social", "Permitted social")
ADAPTERS = {x.source_code: x for x in (ManualSource, CsvSource, GooglePlacesSource, BingLocalSource, ApprovedDirectorySource, PublicWebsiteSource, PermittedSocialSource, CtLogsSource, DnsSource, RdapSource)}
ADAPTERS = {x.source_code: x for x in (ManualSource, CsvSource, GooglePlacesSource, BingLocalSource, ApprovedDirectorySource, OpenStreetMapSource, WikidataSource, CommonCrawlSource, PublicWebsiteSource, PermittedSocialSource, CtLogsSource, DnsSource, RdapSource)}
# common aliases used by clients
ADAPTER_REGISTRY = ADAPTERS