Update Map Data #1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Update Map Data | |
| on: | |
| schedule: | |
| # Run every Sunday at midnight UTC | |
| - cron: '0 0 * * 0' | |
| workflow_dispatch: | |
| jobs: | |
| update-map-data: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.x' | |
| - name: Install dependencies | |
| run: pip install requests | |
| - name: Fetch CKAN data and generate static files | |
| env: | |
| CKAN_SITE_URL: ${{ vars.CKAN_SITE_URL }} | |
| run: | | |
| python - <<'SCRIPT' | |
| import json | |
| import os | |
| import sys | |
| import requests | |
| CKAN_SITE_URL = os.environ.get("CKAN_SITE_URL", "").rstrip("/") | |
| if not CKAN_SITE_URL: | |
| print("Error: CKAN_SITE_URL repository variable is not set.") | |
| sys.exit(1) | |
| API_URL = f"{CKAN_SITE_URL}/api/3/action/package_search" | |
| OUTPUT_DIR = "map-data" | |
| os.makedirs(OUTPUT_DIR, exist_ok=True) | |
| # --- Build sites.geojson --- | |
| geojson = {"type": "FeatureCollection", "features": []} | |
| rows = 500 | |
| start = 0 | |
| while True: | |
| resp = requests.get(API_URL, params={ | |
| "fq": "type:site", | |
| "rows": rows, | |
| "start": start, | |
| }, timeout=60) | |
| resp.raise_for_status() | |
| result = resp.json().get("result", {}) | |
| records = result.get("results", []) | |
| if not records: | |
| break | |
| start += rows | |
| for row in records: | |
| lat = row.get("latitude") | |
| lon = row.get("longitude") | |
| if lat is None or lon is None: | |
| continue | |
| try: | |
| geojson["features"].append({ | |
| "type": "Feature", | |
| "geometry": { | |
| "type": "Point", | |
| "coordinates": [float(lon), float(lat)], | |
| }, | |
| "properties": { | |
| "id": row.get("id", ""), | |
| "name": row.get("name", ""), | |
| "title": row.get("title", ""), | |
| "url": row.get("url", ""), | |
| "catalog_url": f"{CKAN_SITE_URL}/site/{row.get('name', '')}", | |
| "state": row.get("state", ""), | |
| "description": row.get("notes", ""), | |
| "num_datasets": row.get("num_datasets", 0), | |
| "num_organizations": row.get("num_organizations", 0), | |
| "num_resources": row.get("num_resources", 0), | |
| }, | |
| }) | |
| except (ValueError, TypeError) as e: | |
| print(f"Skipping site with invalid data: {e}") | |
| with open(os.path.join(OUTPUT_DIR, "sites.geojson"), "w") as f: | |
| json.dump(geojson, f) | |
| print(f"Generated sites.geojson with {len(geojson['features'])} features") | |
| # --- Build stats.json --- | |
| ext_resp = requests.get(API_URL, params={ | |
| "fq": "type:extension", | |
| "rows": 0, | |
| }, timeout=30) | |
| ext_resp.raise_for_status() | |
| extensions = ext_resp.json().get("result", {}).get("count", 0) | |
| site_resp = requests.get(API_URL, params={ | |
| "fq": "type:site", | |
| "rows": 0, | |
| }, timeout=30) | |
| site_resp.raise_for_status() | |
| sites = site_resp.json().get("result", {}).get("count", 0) | |
| stats = {"extensions": extensions, "sites": sites} | |
| with open(os.path.join(OUTPUT_DIR, "stats.json"), "w") as f: | |
| json.dump(stats, f) | |
| print(f"Generated stats.json: {stats}") | |
| SCRIPT | |
| - name: Upload map data as artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: map-data | |
| path: | | |
| map-data/sites.geojson | |
| map-data/stats.json | |
| retention-days: 90 |