Merge pull request #363 from xr843/feat/address-search-amap-v2

xr843 · web-flow · commit 4d400efae09b · 2026-04-06T14:41:51.000+08:00
feat(kg-map): address search + Amap v2 full coverage (+16K temples)
diff --git a/backend/scripts/backfill_address_regeo.py b/backend/scripts/backfill_address_regeo.py
@@ -0,0 +1,165 @@
+"""Backfill province/city/district for CN monasteries via Amap reverse geocoding.
+
+Targets: entity_type='monastery', country='CN', province IS NULL, has lat/lng.
+Uses Amap regeo API with WGS-84→GCJ-02 conversion.
+
+Rate limit: 0.25s between requests. ~1600 entities ≈ 7 min.
+"""
+import argparse
+import asyncio
+import json
+import math
+import os
+import sys
+import time
+import urllib.parse
+import urllib.request
+
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from app.config import settings
+
+AMAP_KEY = "7971e9b134c4684c3b43b6e442475d0e"
+AMAP_REGEO_URL = "https://restapi.amap.com/v3/geocode/regeo"
+USER_AGENT = "FoJinBot/1.0"
+
+
+def wgs84_to_gcj02(lng, lat):
+    """Convert WGS-84 to GCJ-02."""
+    a = 6378245.0
+    ee = 0.00669342162296594323
+
+    dlat = _transform_lat(lng - 105.0, lat - 35.0)
+    dlng = _transform_lon(lng - 105.0, lat - 35.0)
+    radlat = lat / 180.0 * math.pi
+    magic = math.sin(radlat)
+    magic = 1 - ee * magic * magic
+    sqrtmagic = math.sqrt(magic)
+    dlat = (dlat * 180.0) / ((a * (1 - ee)) / (magic * sqrtmagic) * math.pi)
+    dlng = (dlng * 180.0) / (a / sqrtmagic * math.cos(radlat) * math.pi)
+    return lng + dlng, lat + dlat
+
+
+def _transform_lat(x, y):
+    ret = -100.0 + 2.0 * x + 3.0 * y + 0.2 * y * y + 0.1 * x * y + 0.2 * math.sqrt(abs(x))
+    ret += (20.0 * math.sin(6.0 * x * math.pi) + 20.0 * math.sin(2.0 * x * math.pi)) * 2.0 / 3.0
+    ret += (20.0 * math.sin(y * math.pi) + 40.0 * math.sin(y / 3.0 * math.pi)) * 2.0 / 3.0
+    ret += (160.0 * math.sin(y / 12.0 * math.pi) + 320.0 * math.sin(y * math.pi / 30.0)) * 2.0 / 3.0
+    return ret
+
+
+def _transform_lon(x, y):
+    ret = 300.0 + x + 2.0 * y + 0.1 * x * x + 0.1 * x * y + 0.1 * math.sqrt(abs(x))
+    ret += (20.0 * math.sin(6.0 * x * math.pi) + 20.0 * math.sin(2.0 * x * math.pi)) * 2.0 / 3.0
+    ret += (20.0 * math.sin(x * math.pi) + 40.0 * math.sin(x / 3.0 * math.pi)) * 2.0 / 3.0
+    ret += (150.0 * math.sin(x / 12.0 * math.pi) + 300.0 * math.sin(x / 30.0 * math.pi)) * 2.0 / 3.0
+    return ret
+
+
+def regeo(lng_wgs, lat_wgs) -> dict | None:
+    lng_gcj, lat_gcj = wgs84_to_gcj02(lng_wgs, lat_wgs)
+    params = urllib.parse.urlencode({
+        "key": AMAP_KEY,
+        "location": f"{lng_gcj:.6f},{lat_gcj:.6f}",
+        "extensions": "base",
+        "output": "json",
+    })
+    url = f"{AMAP_REGEO_URL}?{params}"
+    req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
+    with urllib.request.urlopen(req, timeout=10) as resp:
+        data = json.loads(resp.read())
+    if data.get("status") != "1":
+        return None
+    comp = data.get("regeocode", {}).get("addressComponent", {})
+    province = comp.get("province", "")
+    city = comp.get("city", "")
+    district = comp.get("district", "")
+    # Amap returns [] for empty fields in municipalities
+    if isinstance(province, list):
+        province = ""
+    if isinstance(city, list):
+        city = ""
+    if isinstance(district, list):
+        district = ""
+    if not province:
+        return None
+    return {"province": province, "city": city or province, "district": district}
+
+
+async def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--dry-run", action="store_true")
+    parser.add_argument("--limit", type=int, default=2000)
+    args = parser.parse_args()
+
+    print("=" * 60)
+    print("FoJin — Backfill Address via Amap Reverse Geocoding")
+    print("=" * 60)
+
+    engine = create_async_engine(settings.database_url)
+    sf = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
+
+    async with sf() as session:
+        result = await session.execute(text("""
+            SELECT id, name_zh,
+                   (properties->>'latitude')::float AS lat,
+                   (properties->>'longitude')::float AS lng
+            FROM kg_entities
+            WHERE entity_type = 'monastery'
+              AND (properties->>'country' = 'CN' OR properties->>'country' = '中国'
+                   OR properties->>'geo_source' LIKE 'osm:CN%' OR properties->>'geo_source' LIKE 'osm:中国%'
+                   OR properties->>'geo_source' LIKE 'osm_ext%')
+              AND (properties->>'province' IS NULL OR properties->>'province' = '')
+              AND properties->>'latitude' IS NOT NULL
+            LIMIT :limit
+        """), {"limit": args.limit})
+        rows = result.fetchall()
+        print(f"Found {len(rows)} monasteries to backfill")
+
+        stats = {"updated": 0, "failed": 0, "skipped": 0}
+
+        for i, (eid, name, lat, lng) in enumerate(rows):
+            try:
+                addr = regeo(lng, lat)
+                time.sleep(0.25)
+            except Exception as e:
+                print(f"  ERR {eid} {name}: {e}")
+                stats["failed"] += 1
+                time.sleep(1)
+                continue
+
+            if not addr:
+                stats["skipped"] += 1
+                continue
+
+            if not args.dry_run:
+                props_patch = json.dumps(addr)
+                await session.execute(text("""
+                    UPDATE kg_entities
+                    SET properties = (properties::jsonb || cast(:patch as jsonb))::json
+                    WHERE id = :id
+                """), {"id": eid, "patch": props_patch})
+
+            stats["updated"] += 1
+
+            if (i + 1) % 200 == 0:
+                if not args.dry_run:
+                    await session.commit()
+                print(f"  [{i+1}/{len(rows)}] updated: {stats['updated']}, failed: {stats['failed']}")
+
+        if not args.dry_run:
+            await session.commit()
+
+        print(f"\n{'='*60}")
+        print(f"Results: {stats}")
+        print(f"{'Dry run' if args.dry_run else 'Committed'}")
+        print("=" * 60)
+
+    await engine.dispose()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/backend/scripts/fetch_amap_temples_v2.py b/backend/scripts/fetch_amap_temples_v2.py
@@ -0,0 +1,186 @@
+"""Fetch Chinese Buddhist temples from Amap POI API — v2: search by city for full coverage.
+
+V1 searched by province and hit 500-result caps in populous provinces.
+V2 searches by prefecture-level city (~340 cities) to avoid truncation.
+
+Output: data/amap_temples_v2.json
+"""
+import json
+import math
+import time
+import urllib.parse
+import urllib.request
+
+AMAP_KEY = "7971e9b134c4684c3b43b6e442475d0e"
+AMAP_URL = "https://restapi.amap.com/v3/place/text"
+AMAP_DISTRICT_URL = "https://restapi.amap.com/v3/config/district"
+USER_AGENT = "FoJinBot/1.0"
+OUTPUT = "data/amap_temples_v2.json"
+
+KEYWORDS = ["寺", "庵", "禅寺", "佛寺", "佛教", "精舍", "佛堂"]
+
+SKIP_WORDS = ["清真", "教堂", "基督", "天主", "道观", "道教", "伊斯兰",
+              "关帝", "妈祖", "城隍", "土地庙", "孔庙", "文庙",
+              "殡仪", "墓", "陵园", "酒店", "宾馆", "饭店", "餐厅",
+              "停车", "公厕", "超市", "药店", "医院", "学校",
+              "公园", "广场", "商场", "写字楼", "小区", "花园"]
+
+
+def _transform_lat(x, y):
+    ret = -100.0 + 2.0 * x + 3.0 * y + 0.2 * y * y + 0.1 * x * y + 0.2 * math.sqrt(abs(x))
+    ret += (20.0 * math.sin(6.0 * x * math.pi) + 20.0 * math.sin(2.0 * x * math.pi)) * 2.0 / 3.0
+    ret += (20.0 * math.sin(y * math.pi) + 40.0 * math.sin(y / 3.0 * math.pi)) * 2.0 / 3.0
+    ret += (160.0 * math.sin(y / 12.0 * math.pi) + 320.0 * math.sin(y * math.pi / 30.0)) * 2.0 / 3.0
+    return ret
+
+
+def _transform_lon(x, y):
+    ret = 300.0 + x + 2.0 * y + 0.1 * x * x + 0.1 * x * y + 0.1 * math.sqrt(abs(x))
+    ret += (20.0 * math.sin(6.0 * x * math.pi) + 20.0 * math.sin(2.0 * x * math.pi)) * 2.0 / 3.0
+    ret += (20.0 * math.sin(x * math.pi) + 40.0 * math.sin(x / 3.0 * math.pi)) * 2.0 / 3.0
+    ret += (150.0 * math.sin(x / 12.0 * math.pi) + 300.0 * math.sin(x / 30.0 * math.pi)) * 2.0 / 3.0
+    return ret
+
+
+def gcj02_to_wgs84(lng, lat):
+    a = 6378245.0
+    ee = 0.00669342162296594323
+    dlat = _transform_lat(lng - 105.0, lat - 35.0)
+    dlng = _transform_lon(lng - 105.0, lat - 35.0)
+    radlat = lat / 180.0 * math.pi
+    magic = math.sin(radlat)
+    magic = 1 - ee * magic * magic
+    sqrtmagic = math.sqrt(magic)
+    dlat = (dlat * 180.0) / ((a * (1 - ee)) / (magic * sqrtmagic) * math.pi)
+    dlng = (dlng * 180.0) / (a / sqrtmagic * math.cos(radlat) * math.pi)
+    return lng - dlng, lat - dlat
+
+
+def get_cities() -> list[dict]:
+    """Fetch all prefecture-level cities from Amap district API."""
+    params = urllib.parse.urlencode({
+        "key": AMAP_KEY,
+        "keywords": "中国",
+        "subdistrict": 2,
+        "extensions": "base",
+    })
+    url = f"{AMAP_DISTRICT_URL}?{params}"
+    req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
+    with urllib.request.urlopen(req, timeout=30) as resp:
+        data = json.loads(resp.read())
+
+    cities = []
+    for province in data.get("districts", [{}])[0].get("districts", []):
+        pname = province.get("name", "")
+        for city in province.get("districts", []):
+            cities.append({
+                "adcode": city.get("adcode", ""),
+                "name": city.get("name", ""),
+                "province": pname,
+            })
+        # Direct-administered municipalities: province itself is the city
+        if not province.get("districts"):
+            cities.append({
+                "adcode": province.get("adcode", ""),
+                "name": pname,
+                "province": pname,
+            })
+    return cities
+
+
+def amap_search(keyword: str, city_code: str, page: int = 1) -> dict:
+    params = urllib.parse.urlencode({
+        "key": AMAP_KEY,
+        "keywords": keyword,
+        "city": city_code,
+        "citylimit": "true",
+        "offset": 25,
+        "page": page,
+        "output": "json",
+        "extensions": "base",
+    })
+    url = f"{AMAP_URL}?{params}"
+    req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
+    with urllib.request.urlopen(req, timeout=15) as resp:
+        return json.loads(resp.read())
+
+
+def main():
+    print("Fetching city list...")
+    cities = get_cities()
+    print(f"Got {len(cities)} cities")
+
+    all_pois: dict[str, dict] = {}
+    total_requests = 0
+
+    for ci, city in enumerate(cities):
+        city_before = len(all_pois)
+        for keyword in KEYWORDS:
+            page = 1
+            while page <= 20:
+                try:
+                    data = amap_search(keyword, city["adcode"], page)
+                    total_requests += 1
+                    time.sleep(0.25)
+                except Exception as e:
+                    print(f"  ERR {city['name']}/{keyword}: {e}")
+                    time.sleep(2)
+                    break
+
+                if data.get("status") != "1":
+                    break
+
+                pois = data.get("pois", [])
+                if not pois:
+                    break
+
+                for poi in pois:
+                    pid = poi.get("id", "")
+                    name = poi.get("name", "")
+                    location = poi.get("location", "")
+                    if not pid or not name or not location:
+                        continue
+                    if any(w in name for w in SKIP_WORDS):
+                        continue
+                    if pid in all_pois:
+                        continue
+                    try:
+                        lng_gcj, lat_gcj = [float(x) for x in location.split(",")]
+                        lng_wgs, lat_wgs = gcj02_to_wgs84(lng_gcj, lat_gcj)
+                    except (ValueError, IndexError):
+                        continue
+
+                    all_pois[pid] = {
+                        "amap_id": pid,
+                        "name": name,
+                        "latitude": round(lat_wgs, 7),
+                        "longitude": round(lng_wgs, 7),
+                        "address": poi.get("address", ""),
+                        "province": poi.get("pname", city["province"]),
+                        "city": poi.get("cityname", city["name"]),
+                        "district": poi.get("adname", ""),
+                        "type": poi.get("type", ""),
+                        "typecode": poi.get("typecode", ""),
+                    }
+
+                count = int(data.get("count", 0))
+                if page * 25 >= count or page * 25 >= 500:
+                    break
+                page += 1
+
+        city_new = len(all_pois) - city_before
+        if (ci + 1) % 20 == 0 or ci == len(cities) - 1:
+            print(f"[{ci+1}/{len(cities)}] {city['province']}/{city['name']}: +{city_new} (total: {len(all_pois)}, reqs: {total_requests})")
+
+    # Save
+    result = list(all_pois.values())
+    with open(OUTPUT, "w", encoding="utf-8") as f:
+        json.dump(result, f, ensure_ascii=False, indent=2)
+    print(f"\n{'='*60}")
+    print(f"Total: {len(result)} POIs, {total_requests} API requests")
+    print(f"Saved to {OUTPUT}")
+    print(f"{'='*60}")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/frontend/src/pages/KGMapPage.tsx b/frontend/src/pages/KGMapPage.tsx
