From 0f8c1988b21848ae4b221b6873ac6b936727b9da Mon Sep 17 00:00:00 2001 From: Marco0300 Date: Sat, 5 Sep 2026 11:45:50 +0200 Subject: [PATCH] encode Common Crawl discovery query parameters --- apps/api/app/sources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/app/sources.py b/apps/api/app/sources.py index 19d8c3b..bad9de8 100644 --- a/apps/api/app/sources.py +++ b/apps/api/app/sources.py @@ -362,7 +362,7 @@ class ApprovedDirectorySource(GatedSource): url="https://query.wikidata.org/sparql?format=json&"+urlencode({"query":sparql}) payload, _ = _HttpJsonSource()._get_json(url); records=[normalize_record({"name":x.get("itemLabel",{}).get("value",""),"website":x.get("website",{}).get("value","")}) for x in payload.get("results",{}).get("bindings",[])] return DiscoveryPage(records[:limit],metadata={"adapter":self.source_code,"provider":provider,"record_count":len(records)}) - index="https://index.commoncrawl.org/CC-MAIN-2026-30-index?url="+query+"&output=json&filter=status:200&collapse=urlkey" + index="https://index.commoncrawl.org/CC-MAIN-2026-30-index?"+urlencode({"url":query,"output":"json","filter":"status:200","collapse":"urlkey"}) payload, _ = _HttpJsonSource()._get_json(index); records=[normalize_record({"name":str(x.get("url","")).split('/')[2] if '://' in str(x.get("url","")) else x.get("url", ""),"website":x.get("url","")}) for x in (payload if isinstance(payload,list) else [])] return DiscoveryPage(records[:limit],metadata={"adapter":self.source_code,"provider":provider,"record_count":len(records)})