fix source discovery provider execution
CI / compose (push) Failing after 5m28s

This commit is contained in:
Marco0300
2026-09-04 18:21:05 +02:00
parent 1b060db211
commit dff9ccb5ea
2 changed files with 21 additions and 4 deletions
+6 -3
View File
@@ -7,7 +7,7 @@ Adapters never emit or persist credential values.
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any, Mapping, Protocol, Sequence
import csv, io, random, time, json
import csv, io, random, time, json, re
from urllib.parse import urlencode, urlparse
from urllib.request import Request, urlopen
@@ -331,13 +331,16 @@ class ApprovedDirectorySource(GatedSource):
provider=str(config["provider"]).lower(); query=str(config["query"]).strip(); limit=max(1,min(100,int(config.get("max_records",50))))
if provider == "openstreetmap":
area=str(config.get("location", "South Africa")).strip()
overpass='[out:json][timeout:25];area["name"="%s"]->.a;(nwr["name"](area.a););out center tags;'%(area.replace('"',''))
terms=[token.lower() for token in re.findall(r"[A-Za-z0-9]{2,32}",query)[:5]]
variants=sorted({variant for term in terms for variant in (term,term[:-1] if term.endswith('s') and len(term)>3 else term)})
pattern="|".join(re.escape(term) for term in variants)
overpass='[out:json][timeout:25];area["name"="%s"]->.a;(nwr["name"~"%s",i](area.a);nwr["craft"~"%s",i](area.a);nwr["amenity"~"%s",i](area.a);nwr["shop"~"%s",i](area.a););out center tags;' % ((area.replace('"',''),)+ (pattern,)*4)
req=Request("https://overpass-api.de/api/interpreter",data=overpass.encode(),method="POST",headers={"Content-Type":"application/x-www-form-urlencoded","User-Agent":"ProspectOS/0.1"})
with urlopen(req,timeout=30) as response: payload=json.loads(response.read(2*1024*1024).decode("utf-8","replace"))
records=[]
for element in payload.get("elements",[]):
tags=element.get("tags",{}); name=tags.get("name","")
if not name or (query.lower() not in name.lower() and query.lower() not in str(tags).lower()): continue
if not name or not any(term in name.lower() or term in str(tags).lower() for term in variants): continue
records.append(normalize_record({"name":name,"website":tags.get("website") or tags.get("contact:website", ""),"phone":tags.get("phone") or tags.get("contact:phone", ""),"email":tags.get("email") or tags.get("contact:email", ""),"location":", ".join(x for x in (tags.get("addr:street"),tags.get("addr:city"),tags.get("addr:postcode")) if x),"description":"OpenStreetMap public listing"}))
return DiscoveryPage(records[:limit],metadata={"adapter":self.source_code,"provider":provider,"record_count":len(records)})
if provider == "wikidata":
+15 -1
View File
@@ -1,8 +1,9 @@
import json, os, sqlite3, threading, unittest
from unittest.mock import patch
from http.client import HTTPConnection
from tempfile import TemporaryDirectory
from app.main import ORGANIZATION_ID, connect, create_server
from app.sources import CsvSource, ManualSource, available_adapters
from app.sources import ApprovedDirectorySource, CsvSource, ManualSource, available_adapters
class SourceAdapterTests(unittest.TestCase):
def test_adapter_catalog_exposes_ready_and_gated_sources(self):
@@ -31,6 +32,19 @@ class SourceAdapterTests(unittest.TestCase):
self.assertFalse(result.valid)
self.assertIn('secret', result.errors[0].lower())
def test_openstreetmap_uses_provider_side_service_tag_query(self):
class Response:
def read(self, _): return b'{"elements":[{"tags":{"name":"Cape Plumber","craft":"plumber"}}]}'
def __enter__(self): return self
def __exit__(self, *_): return False
config={'provider':'openstreetmap','query':'plumbers','location':'Cape Town','approved':True,'public_access':True,'terms_accepted':True,'rate_limit':1}
with patch('app.sources.urlopen',return_value=Response()) as request:
page=ApprovedDirectorySource().discover(config)
query=request.call_args.args[0].data.decode()
self.assertIn('"craft"~"plumber|plumbers",i]',query)
self.assertIn('"shop"~"plumber|plumbers",i]',query)
self.assertEqual(page.records[0]['name'],'Cape Plumber')
def test_legacy_source_kind_constraint_is_migrated(self):
with TemporaryDirectory() as tmp:
path=os.path.join(tmp,'legacy.db')