fix source discovery provider execution
CI / compose (push) Failing after 5m28s

This commit is contained in:
Marco0300
2026-09-04 18:21:05 +02:00
parent 1b060db211
commit dff9ccb5ea
2 changed files with 21 additions and 4 deletions
+6 -3
View File
@@ -7,7 +7,7 @@ Adapters never emit or persist credential values.
from __future__ import annotations from __future__ import annotations
from dataclasses import dataclass, field from dataclasses import dataclass, field
from typing import Any, Mapping, Protocol, Sequence from typing import Any, Mapping, Protocol, Sequence
import csv, io, random, time, json import csv, io, random, time, json, re
from urllib.parse import urlencode, urlparse from urllib.parse import urlencode, urlparse
from urllib.request import Request, urlopen from urllib.request import Request, urlopen
@@ -331,13 +331,16 @@ class ApprovedDirectorySource(GatedSource):
provider=str(config["provider"]).lower(); query=str(config["query"]).strip(); limit=max(1,min(100,int(config.get("max_records",50)))) provider=str(config["provider"]).lower(); query=str(config["query"]).strip(); limit=max(1,min(100,int(config.get("max_records",50))))
if provider == "openstreetmap": if provider == "openstreetmap":
area=str(config.get("location", "South Africa")).strip() area=str(config.get("location", "South Africa")).strip()
overpass='[out:json][timeout:25];area["name"="%s"]->.a;(nwr["name"](area.a););out center tags;'%(area.replace('"','')) terms=[token.lower() for token in re.findall(r"[A-Za-z0-9]{2,32}",query)[:5]]
variants=sorted({variant for term in terms for variant in (term,term[:-1] if term.endswith('s') and len(term)>3 else term)})
pattern="|".join(re.escape(term) for term in variants)
overpass='[out:json][timeout:25];area["name"="%s"]->.a;(nwr["name"~"%s",i](area.a);nwr["craft"~"%s",i](area.a);nwr["amenity"~"%s",i](area.a);nwr["shop"~"%s",i](area.a););out center tags;' % ((area.replace('"',''),)+ (pattern,)*4)
req=Request("https://overpass-api.de/api/interpreter",data=overpass.encode(),method="POST",headers={"Content-Type":"application/x-www-form-urlencoded","User-Agent":"ProspectOS/0.1"}) req=Request("https://overpass-api.de/api/interpreter",data=overpass.encode(),method="POST",headers={"Content-Type":"application/x-www-form-urlencoded","User-Agent":"ProspectOS/0.1"})
with urlopen(req,timeout=30) as response: payload=json.loads(response.read(2*1024*1024).decode("utf-8","replace")) with urlopen(req,timeout=30) as response: payload=json.loads(response.read(2*1024*1024).decode("utf-8","replace"))
records=[] records=[]
for element in payload.get("elements",[]): for element in payload.get("elements",[]):
tags=element.get("tags",{}); name=tags.get("name","") tags=element.get("tags",{}); name=tags.get("name","")
if not name or (query.lower() not in name.lower() and query.lower() not in str(tags).lower()): continue if not name or not any(term in name.lower() or term in str(tags).lower() for term in variants): continue
records.append(normalize_record({"name":name,"website":tags.get("website") or tags.get("contact:website", ""),"phone":tags.get("phone") or tags.get("contact:phone", ""),"email":tags.get("email") or tags.get("contact:email", ""),"location":", ".join(x for x in (tags.get("addr:street"),tags.get("addr:city"),tags.get("addr:postcode")) if x),"description":"OpenStreetMap public listing"})) records.append(normalize_record({"name":name,"website":tags.get("website") or tags.get("contact:website", ""),"phone":tags.get("phone") or tags.get("contact:phone", ""),"email":tags.get("email") or tags.get("contact:email", ""),"location":", ".join(x for x in (tags.get("addr:street"),tags.get("addr:city"),tags.get("addr:postcode")) if x),"description":"OpenStreetMap public listing"}))
return DiscoveryPage(records[:limit],metadata={"adapter":self.source_code,"provider":provider,"record_count":len(records)}) return DiscoveryPage(records[:limit],metadata={"adapter":self.source_code,"provider":provider,"record_count":len(records)})
if provider == "wikidata": if provider == "wikidata":
+15 -1
View File
@@ -1,8 +1,9 @@
import json, os, sqlite3, threading, unittest import json, os, sqlite3, threading, unittest
from unittest.mock import patch
from http.client import HTTPConnection from http.client import HTTPConnection
from tempfile import TemporaryDirectory from tempfile import TemporaryDirectory
from app.main import ORGANIZATION_ID, connect, create_server from app.main import ORGANIZATION_ID, connect, create_server
from app.sources import CsvSource, ManualSource, available_adapters from app.sources import ApprovedDirectorySource, CsvSource, ManualSource, available_adapters
class SourceAdapterTests(unittest.TestCase): class SourceAdapterTests(unittest.TestCase):
def test_adapter_catalog_exposes_ready_and_gated_sources(self): def test_adapter_catalog_exposes_ready_and_gated_sources(self):
@@ -31,6 +32,19 @@ class SourceAdapterTests(unittest.TestCase):
self.assertFalse(result.valid) self.assertFalse(result.valid)
self.assertIn('secret', result.errors[0].lower()) self.assertIn('secret', result.errors[0].lower())
def test_openstreetmap_uses_provider_side_service_tag_query(self):
class Response:
def read(self, _): return b'{"elements":[{"tags":{"name":"Cape Plumber","craft":"plumber"}}]}'
def __enter__(self): return self
def __exit__(self, *_): return False
config={'provider':'openstreetmap','query':'plumbers','location':'Cape Town','approved':True,'public_access':True,'terms_accepted':True,'rate_limit':1}
with patch('app.sources.urlopen',return_value=Response()) as request:
page=ApprovedDirectorySource().discover(config)
query=request.call_args.args[0].data.decode()
self.assertIn('"craft"~"plumber|plumbers",i]',query)
self.assertIn('"shop"~"plumber|plumbers",i]',query)
self.assertEqual(page.records[0]['name'],'Cape Plumber')
def test_legacy_source_kind_constraint_is_migrated(self): def test_legacy_source_kind_constraint_is_migrated(self):
with TemporaryDirectory() as tmp: with TemporaryDirectory() as tmp:
path=os.path.join(tmp,'legacy.db') path=os.path.join(tmp,'legacy.db')