feat: using lat/lon from all available files for gbfs loc extraction (#1226)

Author: cka-y

functions-python/gbfs_validator/src/gbfs_data_processor.py

@@ -436,27 +436,23 @@ def trigger_location_extraction(self):
         version_id = f"{self.stable_id}_{autodiscovery_version.version}_{autodiscovery_version.extracted_from}"
         endpoints = self.gbfs_endpoints.get(version_id, [])
 
-        # Find the station_information_url endpoint
-        station_information_url = next(
-            (
-                endpoint.url
-                for endpoint in endpoints
-                if endpoint.name == "station_information"
-            ),
-            None,
-        )
-        # If station_information_url is not found, use vehicle_status_url
-        vehicle_status_url = next(
-            (
-                endpoint.url
-                for endpoint in endpoints
-                if endpoint.name == "vehicle_status"
-            ),
-            None,
-        )
-        if not station_information_url and not vehicle_status_url:
+        def get_endpoint_url(name: str) -> Optional[str]:
+            return next(
+                (endpoint.url for endpoint in endpoints if endpoint.name == name), None
+            )
+
+        # Get the URLs for the required endpoints
+        station_information_url = get_endpoint_url("station_information")
+        vehicle_status_url = get_endpoint_url("vehicle_status")
+        free_bike_status_url = get_endpoint_url("free_bike_status")
+
+        if (
+            not station_information_url
+            and not vehicle_status_url
+            and not free_bike_status_url
+        ):
             self.logger.warning(
-                "No station_information_url or vehicle_status_url found."
+                "No station_information_url or vehicle_status_url or free_bike_status_url found."
             )
             return
         client = tasks_v2.CloudTasksClient()
@@ -466,6 +462,7 @@ def trigger_location_extraction(self):
                 "data_type": "gbfs",
                 "station_information_url": station_information_url,
                 "vehicle_status_url": vehicle_status_url,
+                "free_bike_status_url": free_bike_status_url,
             }
         ).encode()
         project_id = os.getenv("PROJECT_ID")

functions-python/reverse_geolocation/.env.rename_me

@@ -5,4 +5,6 @@ PROJECT_ID=${{PROJECT_ID}}
 GCP_REGION=${{GCP_REGION}}
 SERVICE_ACCOUNT_EMAIL=${{SERVICE_ACCOUNT_EMAIL}}
 DATASETS_BUCKET_NAME=${{DATASETS_BUCKET_NAME}}
-PUBSUB_TOPIC_NAME=${{PUBSUB_TOPIC_NAME}}
+PUBSUB_TOPIC_NAME=${{PUBSUB_TOPIC_NAME}}
+DATASET_BUCKET_NAME_GBFS=${{DATASET_BUCKET_NAME_GBFS}}
+DATASET_BUCKET_NAME_GTFS=${{DATASET_BUCKET_NAME_GTFS}}

functions-python/reverse_geolocation/README.md

@@ -58,15 +58,16 @@ This function performs the core reverse geolocation logic. It processes location
 - `stable_id`: Identifies the feed (GTFS or GBFS).
 - `dataset_id`: Required if `data_type` is not provided or is `gtfs`. Identifies the dataset being processed.
 - `stops_url`: Required if `data_type` is not provided or is `gtfs`. URL of the GTFS `stops.txt` file.
-- `station_information_url`: Required if `data_type` is `gbfs` and `vehicle_status_url` is omitted. URL of the GBFS `station_information.json` file.
-- `vehicle_status_url`: Required if `data_type` is `gbfs` and `station_information_url` is omitted. URL of the GBFS `vehicle_status.json` file.
+- `station_information_url`: Required if `data_type` is `gbfs` and `vehicle_status_url` and `free_bike_status_url` are omitted. URL of the GBFS `station_information.json` file.
+- `vehicle_status_url`: Required if `data_type` is `gbfs` and `station_information_url` and `free_bike_status_url` are omitted. URL of the GBFS `vehicle_status.json` file.
+- `free_bike_status_url`: Required if `data_type` is `gbfs` and `station_information_url` and `vehicle_status_url` are omitted. URL of the GBFS `free_bike_status.json` file.
 - `data_type`: Optional. Specifies the type of data being processed. Can be `gtfs` or `gbfs`. If not provided, the function will attempt to determine the type based on the URLs provided.
 
 ### Processing Steps:
 
 1. **Load Location Data**  
    - For GTFS: the function reads `stops.txt` into a Pandas DataFrame, ensuring unique longitude-latitude pairs.  
-   - For GBFS: location data is extracted from `station_information.json` (preferred) or `vehicle_status.json` (fallback), also ensuring uniqueness.
+   - For GBFS: location data is extracted from `station_information.json` and `vehicle_status.json` and `free_bike_status.json`, also ensuring uniqueness.
 
 2. **Updates Bounding Box**  
    - For GTFS: the bounding box is derived from stop coordinates. The dataset's bounding box is updated in the database.

functions-python/reverse_geolocation/src/parse_request.py

@@ -1,6 +1,6 @@
 import io
 import logging
-from typing import Tuple, Optional
+from typing import Tuple, Optional, List
 
 import flask
 import pandas as pd
@@ -10,10 +10,11 @@
 
 def parse_request_parameters(
     request: flask.Request,
-) -> Tuple[pd.DataFrame, str, Optional[str], str, str]:
+) -> Tuple[pd.DataFrame, str, Optional[str], str, List[str]]:
     """
     Parse the request parameters and return a DataFrame with the stops data.
-    @:returns Tuple: A tuple containing the stops DataFrame, stable ID, and dataset ID.
+    @:returns Tuple: A tuple containing the stops DataFrame, stable ID, dataset ID, data type, and a list of URLs that
+    were used to fetch the data.
     """
     logging.info("Parsing request parameters.")
     request_json = request.get_json(silent=True)
@@ -37,13 +38,14 @@ def parse_request_parameters(
     logging.info("Data type: %s", data_type)
     if data_type == "gtfs":
         df, stable_id, dataset_id, url = parse_request_parameters_gtfs(request_json)
+        urls = [url]
     elif data_type == "gbfs":
-        df, stable_id, dataset_id, url = parse_request_parameters_gbfs(request_json)
+        df, stable_id, dataset_id, urls = parse_request_parameters_gbfs(request_json)
     else:
         raise ValueError(
             f"Invalid data_type '{data_type}'. Supported types are 'gtfs' and 'gbfs'."
         )
-    return df, stable_id, dataset_id, data_type, url
+    return df, stable_id, dataset_id, data_type, urls
 
 
 def parse_request_parameters_gtfs(
@@ -82,16 +84,11 @@ def parse_station_information_url(station_information_url) -> pd.DataFrame:
 
     lat_expr = parse("data.stations[*].lat")
     lon_expr = parse("data.stations[*].lon")
-    station_id_expr = parse("data.stations[*].station_id")
 
     lats = [match.value for match in lat_expr.find(data)]
     lons = [match.value for match in lon_expr.find(data)]
-    station_ids = [match.value for match in station_id_expr.find(data)]
 
-    stations_info = [
-        {"station_id": sid, "stop_lat": lat, "stop_lon": lon}
-        for sid, lat, lon in zip(station_ids, lats, lons)
-    ]
+    stations_info = [{"stop_lat": lat, "stop_lon": lon} for lat, lon in zip(lats, lons)]
     return pd.DataFrame(stations_info)
 
 
@@ -103,29 +100,42 @@ def parse_vehicle_status_url(vehicle_status_url) -> pd.DataFrame:
 
     lat_expr = parse("data.vehicles[*].lat")
     lon_expr = parse("data.vehicles[*].lon")
-    vehicle_id_expr = parse("data.vehicles[*].vehicle_id")
 
     lats = [match.value for match in lat_expr.find(data)]
     lons = [match.value for match in lon_expr.find(data)]
-    vehicle_ids = [match.value for match in vehicle_id_expr.find(data)]
 
-    vehicles_info = [
-        {"vehicle_id": vid, "stop_lat": lat, "stop_lon": lon}
-        for vid, lat, lon in zip(vehicle_ids, lats, lons)
-    ]
+    vehicles_info = [{"stop_lat": lat, "stop_lon": lon} for lat, lon in zip(lats, lons)]
 
     return pd.DataFrame(vehicles_info)
 
 
+def parse_free_bike_status_url(free_bike_status_url):
+    """Parse the free bike status URL and return a DataFrame with bike_id, lat, and lon."""
+    response = requests.get(free_bike_status_url)
+    response.raise_for_status()
+    data = response.json()
+
+    lat_expr = parse("data.bikes[*].lat")
+    lon_expr = parse("data.bikes[*].lon")
+
+    lats = [match.value for match in lat_expr.find(data)]
+    lons = [match.value for match in lon_expr.find(data)]
+
+    bikes_info = [{"stop_lat": lat, "stop_lon": lon} for lat, lon in zip(lats, lons)]
+
+    return pd.DataFrame(bikes_info)
+
+
 def parse_request_parameters_gbfs(
     request_json: dict,
-) -> Tuple[pd.DataFrame, str, Optional[str], str]:
+) -> Tuple[pd.DataFrame, str, Optional[str], List[str]]:
     """Parse the request parameters for GBFS data."""
     if (
         not request_json
         or (
             "station_information_url" not in request_json
             and "vehicle_status_url" not in request_json
+            and "free_bike_status_url" not in request_json
         )
         or "stable_id" not in request_json
     ):
@@ -137,10 +147,26 @@ def parse_request_parameters_gbfs(
     stable_id = request_json["stable_id"]
     station_information_url = request_json.get("station_information_url")
     vehicle_status_url = request_json.get("vehicle_status_url")
+    free_bike_status_url = request_json.get("free_bike_status_url")
+    stops_df = pd.DataFrame()
+    urls = []
     if station_information_url:
         logging.info("Parsing station information URL")
-        stops_df = parse_station_information_url(station_information_url)
-    else:
+        stops_df_station_information = parse_station_information_url(
+            station_information_url
+        )
+        stops_df = pd.concat(
+            [stops_df, stops_df_station_information], ignore_index=True
+        )
+        urls.append(station_information_url)
+    if vehicle_status_url:
         logging.info("Parsing vehicle status URL")
-        stops_df = parse_vehicle_status_url(vehicle_status_url)
-    return stops_df, stable_id, None, station_information_url or vehicle_status_url
+        stops_df_vehicle_status = parse_vehicle_status_url(vehicle_status_url)
+        stops_df = pd.concat([stops_df, stops_df_vehicle_status], ignore_index=True)
+        urls.append(vehicle_status_url)
+    if free_bike_status_url:
+        logging.info("Parsing free bike status URL")
+        stops_df_free_bike_status = parse_free_bike_status_url(free_bike_status_url)
+        stops_df = pd.concat([stops_df, stops_df_free_bike_status], ignore_index=True)
+        urls.append(free_bike_status_url)
+    return stops_df, stable_id, None, urls

functions-python/reverse_geolocation/src/reverse_geolocation_processor.py

@@ -227,7 +227,7 @@ def create_geojson_aggregate(
     bounding_box: shapely.Polygon,
     data_type: str,
     logger,
-    extraction_url: str = None,
+    extraction_urls: List[str] = None,
 ) -> None:
     """Create a GeoJSON file with the aggregated locations. This file will be uploaded to GCS and used for
     visualization."""
@@ -250,7 +250,7 @@ def create_geojson_aggregate(
     json_data = {
         "type": "FeatureCollection",
         "extracted_at": datetime.now().isoformat(),
-        "extraction_url": extraction_url,
+        "extraction_url": extraction_urls,
         "features": [
             {
                 "type": "Feature",
@@ -424,7 +424,7 @@ def reverse_geolocation_process(
             stable_id,
             dataset_id,
             data_type,
-            extraction_url,
+            extraction_urls,
         ) = parse_request_parameters(request)
 
         # Remove duplicate lat/lon points
@@ -464,7 +464,7 @@ def reverse_geolocation_process(
             stable_id=stable_id,
             bounding_box=bounding_box,
             data_type=data_type,
-            extraction_url=extraction_url,
+            extraction_urls=extraction_urls,
             logger=logger,
         )
 

functions-python/reverse_geolocation/tests/test_reverse_geolocation_processor.py

@@ -84,14 +84,13 @@ def test_parse_request_parameters_gbfs_station_information(self, requests_mock):
             "data_type": "gbfs",
         }
 
-        df, stable_id, dataset_id, data_type, url = parse_request_parameters(request)
+        df, stable_id, dataset_id, data_type, urls = parse_request_parameters(request)
 
         self.assertEqual(stable_id, "stable123")
         self.assertEqual(dataset_id, None)
         self.assertEqual(data_type, "gbfs")
-        self.assertEqual(url, "http://dummy.url")
-        self.assertEqual(df.shape, (2, 3))
-        self.assertIn("station_id", df.columns)
+        self.assertEqual(urls[0], "http://dummy.url")
+        self.assertEqual(df.shape, (2, 2))
 
     @patch("parse_request.requests")
     def test_parse_request_parameters_gbfs_vehicle_status(self, requests_mock):
@@ -114,14 +113,13 @@ def test_parse_request_parameters_gbfs_vehicle_status(self, requests_mock):
             "data_type": "gbfs",
         }
 
-        df, stable_id, dataset_id, data_type, url = parse_request_parameters(request)
+        df, stable_id, dataset_id, data_type, urls = parse_request_parameters(request)
 
         self.assertEqual(stable_id, "stable456")
         self.assertEqual(dataset_id, None)
         self.assertEqual(data_type, "gbfs")
-        self.assertEqual(url, "http://dummy.vehicle")
-        self.assertEqual(df.shape, (2, 3))
-        self.assertIn("vehicle_id", df.columns)
+        self.assertEqual(urls[0], "http://dummy.vehicle")
+        self.assertEqual(df.shape, (2, 2))
 
     @patch("parse_request.requests")
     def test_parse_request_parameters_invalid_request(self, requests_mock):
@@ -426,7 +424,7 @@ def test_create_geojson_aggregate(self, _, mock_storage_client):
             stable_id="test_stable_id",
             bounding_box=bounding_box,
             data_type="gtfs",
-            extraction_url="test_extraction_url",
+            extraction_urls=["test_extraction_url"],
             logger=logger,
         )