fix source discovery provider execution
CI / compose (push) Failing after 5m28s

This commit is contained in:
Marco0300
2026-09-04 18:21:05 +02:00
parent 1b060db211
commit dff9ccb5ea
2 changed files with 21 additions and 4 deletions
+6 -3
View File
@@ -7,7 +7,7 @@ Adapters never emit or persist credential values.
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any, Mapping, Protocol, Sequence
import csv, io, random, time, json
import csv, io, random, time, json, re
from urllib.parse import urlencode, urlparse
from urllib.request import Request, urlopen
@@ -331,13 +331,16 @@ class ApprovedDirectorySource(GatedSource):
provider=str(config["provider"]).lower(); query=str(config["query"]).strip(); limit=max(1,min(100,int(config.get("max_records",50))))
if provider == "openstreetmap":
area=str(config.get("location", "South Africa")).strip()
overpass='[out:json][timeout:25];area["name"="%s"]->.a;(nwr["name"](area.a););out center tags;'%(area.replace('"',''))
terms=[token.lower() for token in re.findall(r"[A-Za-z0-9]{2,32}",query)[:5]]
variants=sorted({variant for term in terms for variant in (term,term[:-1] if term.endswith('s') and len(term)>3 else term)})
pattern="|".join(re.escape(term) for term in variants)
overpass='[out:json][timeout:25];area["name"="%s"]->.a;(nwr["name"~"%s",i](area.a);nwr["craft"~"%s",i](area.a);nwr["amenity"~"%s",i](area.a);nwr["shop"~"%s",i](area.a););out center tags;' % ((area.replace('"',''),)+ (pattern,)*4)
req=Request("https://overpass-api.de/api/interpreter",data=overpass.encode(),method="POST",headers={"Content-Type":"application/x-www-form-urlencoded","User-Agent":"ProspectOS/0.1"})
with urlopen(req,timeout=30) as response: payload=json.loads(response.read(2*1024*1024).decode("utf-8","replace"))
records=[]
for element in payload.get("elements",[]):
tags=element.get("tags",{}); name=tags.get("name","")
if not name or (query.lower() not in name.lower() and query.lower() not in str(tags).lower()): continue
if not name or not any(term in name.lower() or term in str(tags).lower() for term in variants): continue
records.append(normalize_record({"name":name,"website":tags.get("website") or tags.get("contact:website", ""),"phone":tags.get("phone") or tags.get("contact:phone", ""),"email":tags.get("email") or tags.get("contact:email", ""),"location":", ".join(x for x in (tags.get("addr:street"),tags.get("addr:city"),tags.get("addr:postcode")) if x),"description":"OpenStreetMap public listing"}))
return DiscoveryPage(records[:limit],metadata={"adapter":self.source_code,"provider":provider,"record_count":len(records)})
if provider == "wikidata":