diff --git a/apps/api/app/sources.py b/apps/api/app/sources.py index eeb6ab7..05d0238 100644 --- a/apps/api/app/sources.py +++ b/apps/api/app/sources.py @@ -7,7 +7,7 @@ Adapters never emit or persist credential values. from __future__ import annotations from dataclasses import dataclass, field from typing import Any, Mapping, Protocol, Sequence -import csv, io, random, time, json +import csv, io, random, time, json, re from urllib.parse import urlencode, urlparse from urllib.request import Request, urlopen @@ -331,13 +331,16 @@ class ApprovedDirectorySource(GatedSource): provider=str(config["provider"]).lower(); query=str(config["query"]).strip(); limit=max(1,min(100,int(config.get("max_records",50)))) if provider == "openstreetmap": area=str(config.get("location", "South Africa")).strip() - overpass='[out:json][timeout:25];area["name"="%s"]->.a;(nwr["name"](area.a););out center tags;'%(area.replace('"','')) + terms=[token.lower() for token in re.findall(r"[A-Za-z0-9]{2,32}",query)[:5]] + variants=sorted({variant for term in terms for variant in (term,term[:-1] if term.endswith('s') and len(term)>3 else term)}) + pattern="|".join(re.escape(term) for term in variants) + overpass='[out:json][timeout:25];area["name"="%s"]->.a;(nwr["name"~"%s",i](area.a);nwr["craft"~"%s",i](area.a);nwr["amenity"~"%s",i](area.a);nwr["shop"~"%s",i](area.a););out center tags;' % ((area.replace('"',''),)+ (pattern,)*4) req=Request("https://overpass-api.de/api/interpreter",data=overpass.encode(),method="POST",headers={"Content-Type":"application/x-www-form-urlencoded","User-Agent":"ProspectOS/0.1"}) with urlopen(req,timeout=30) as response: payload=json.loads(response.read(2*1024*1024).decode("utf-8","replace")) records=[] for element in payload.get("elements",[]): tags=element.get("tags",{}); name=tags.get("name","") - if not name or (query.lower() not in name.lower() and query.lower() not in str(tags).lower()): continue + if not name or not any(term in name.lower() or term in str(tags).lower() for term in variants): continue records.append(normalize_record({"name":name,"website":tags.get("website") or tags.get("contact:website", ""),"phone":tags.get("phone") or tags.get("contact:phone", ""),"email":tags.get("email") or tags.get("contact:email", ""),"location":", ".join(x for x in (tags.get("addr:street"),tags.get("addr:city"),tags.get("addr:postcode")) if x),"description":"OpenStreetMap public listing"})) return DiscoveryPage(records[:limit],metadata={"adapter":self.source_code,"provider":provider,"record_count":len(records)}) if provider == "wikidata": diff --git a/apps/api/tests/test_sources_phase5.py b/apps/api/tests/test_sources_phase5.py index b349662..208d337 100644 --- a/apps/api/tests/test_sources_phase5.py +++ b/apps/api/tests/test_sources_phase5.py @@ -1,8 +1,9 @@ import json, os, sqlite3, threading, unittest +from unittest.mock import patch from http.client import HTTPConnection from tempfile import TemporaryDirectory from app.main import ORGANIZATION_ID, connect, create_server -from app.sources import CsvSource, ManualSource, available_adapters +from app.sources import ApprovedDirectorySource, CsvSource, ManualSource, available_adapters class SourceAdapterTests(unittest.TestCase): def test_adapter_catalog_exposes_ready_and_gated_sources(self): @@ -31,6 +32,19 @@ class SourceAdapterTests(unittest.TestCase): self.assertFalse(result.valid) self.assertIn('secret', result.errors[0].lower()) + def test_openstreetmap_uses_provider_side_service_tag_query(self): + class Response: + def read(self, _): return b'{"elements":[{"tags":{"name":"Cape Plumber","craft":"plumber"}}]}' + def __enter__(self): return self + def __exit__(self, *_): return False + config={'provider':'openstreetmap','query':'plumbers','location':'Cape Town','approved':True,'public_access':True,'terms_accepted':True,'rate_limit':1} + with patch('app.sources.urlopen',return_value=Response()) as request: + page=ApprovedDirectorySource().discover(config) + query=request.call_args.args[0].data.decode() + self.assertIn('"craft"~"plumber|plumbers",i]',query) + self.assertIn('"shop"~"plumber|plumbers",i]',query) + self.assertEqual(page.records[0]['name'],'Cape Plumber') + def test_legacy_source_kind_constraint_is_migrated(self): with TemporaryDirectory() as tmp: path=os.path.join(tmp,'legacy.db')