loading float values if exists

Author: davidgamez

functions-python/helpers/transform.py

@@ -77,3 +77,40 @@ def to_enum(value, enum_class=None, default_value=None):
     except (ValueError, TypeError) as e:
         logging.warning("Failed to convert value to enum member: %s", e)
         return default_value
+
+
+def to_float(value, default_value: Optional[float] = None) -> Optional[float]:
+    """
+    Convert a value to a float. If conversion fails, return the default value.
+    """
+    try:
+        return float(value)
+    except (ValueError, TypeError):
+        return default_value
+
+
+def get_safe_value(row, column_name, default_value):
+    """
+    Get a safe value from the row. If the value is missing or empty, return the default value.
+    """
+    import pandas
+
+    value = row.get(column_name, None)
+    if not value or pandas.isna(value) or f"{value}".strip() == "":
+        return default_value
+    return f"{value}".strip()
+
+
+def get_safe_float(row, column_name, default_value=None):
+    """
+    Get a safe float value from the row. If the value is missing or cannot be converted to float,
+    """
+    import pandas
+
+    value = row.get(column_name, None)
+    if not value or pandas.isna(value) or f"{value}".strip() == "":
+        return default_value
+    try:
+        return float(value)
+    except (ValueError, TypeError):
+        return default_value

functions-python/helpers/verifier_common.py

@@ -50,7 +50,9 @@ def download_to_local(
             data = BytesIO(response.content)
             blob.upload_from_file(data, rewind=True)
     else:
-        logging.info(f"Blob already exists: gs://{EMULATOR_STORAGE_BUCKET_NAME}/{blob_path}")
+        logging.info(
+            f"Blob already exists: gs://{EMULATOR_STORAGE_BUCKET_NAME}/{blob_path}"
+        )
 
 
 @with_db_session
@@ -87,24 +89,29 @@ def setup_local_storage_emulator():
     """
     from gcp_storage_emulator.server import create_server
 
-    os.environ["STORAGE_EMULATOR_HOST"] = f"http://{EMULATOR_HOST}:{EMULATOR_STORAGE_PORT}"
+    os.environ[
+        "STORAGE_EMULATOR_HOST"
+    ] = f"http://{EMULATOR_HOST}:{EMULATOR_STORAGE_PORT}"
     os.environ["DATASETS_BUCKET_NAME_GBFS"] = EMULATOR_STORAGE_BUCKET_NAME
     os.environ["DATASETS_BUCKET_NAME_GTFS"] = EMULATOR_STORAGE_BUCKET_NAME
     os.environ["DATASTORE_EMULATOR_HOST"] = "localhost:8081"
     server = create_server(
-        host=EMULATOR_HOST, port=EMULATOR_STORAGE_PORT, in_memory=False, default_bucket=EMULATOR_STORAGE_BUCKET_NAME
+        host=EMULATOR_HOST,
+        port=EMULATOR_STORAGE_PORT,
+        in_memory=False,
+        default_bucket=EMULATOR_STORAGE_BUCKET_NAME,
     )
     server.start()
     return server
 
 
 def shutdown_local_storage_emulator(server):
-    """ Shutdown the Google Cloud Storage emulator."""
+    """Shutdown the Google Cloud Storage emulator."""
     server.stop()
 
 
 def is_datastore_emulator_running(host=EMULATOR_HOST, port=8081):
-    """ Check if the Google Cloud Datastore emulator is running."""
+    """Check if the Google Cloud Datastore emulator is running."""
     try:
         with socket.create_connection((host, port), timeout=2):
             return True
@@ -113,19 +120,25 @@ def is_datastore_emulator_running(host=EMULATOR_HOST, port=8081):
 
 
 def start_datastore_emulator(project_id="test-project"):
-    """ Start the Google Cloud Datastore emulator if it's not already running."""
+    """Start the Google Cloud Datastore emulator if it's not already running."""
     if not is_datastore_emulator_running():
-        process = subprocess.Popen([
-            "gcloud", "beta", "emulators", "datastore", "start",
-            "--project={}".format(project_id),
-            "--host-port=localhost:8081"
-        ])
+        process = subprocess.Popen(
+            [
+                "gcloud",
+                "beta",
+                "emulators",
+                "datastore",
+                "start",
+                "--project={}".format(project_id),
+                "--host-port=localhost:8081",
+            ]
+        )
         return process
     return None  # Already running
 
 
 def shutdown_datastore_emulator(process):
-    """ Shutdown the Google Cloud Datastore emulator."""
+    """Shutdown the Google Cloud Datastore emulator."""
     if process:
         process.terminate()
-        process.wait()
+        process.wait()

functions-python/pmtiles_builder/requirements.txt

@@ -25,4 +25,5 @@ google-cloud-storage
 python-dotenv==1.0.0
 tippecanoe
 psutil
+pandas
 

functions-python/pmtiles_builder/src/csv_cache.py

@@ -15,8 +15,9 @@
 #
 import csv
 import os
-from shared.helpers.logger import get_logger
 
+from shared.helpers.logger import get_logger
+from shared.helpers.transform import get_safe_value, get_safe_float
 
 STOP_TIMES_FILE = "stop_times.txt"
 SHAPES_FILE = "shapes.txt"
@@ -127,10 +128,14 @@ def get_stops_from_trip(self, trip_id):
 
     def get_coordinates_for_stop(self, stop_id) -> tuple[float, float] | None:
         if self.stop_to_coordinates is None:
-            self.stop_to_coordinates = {
-                s["stop_id"]: (float(s["stop_lon"]), float(s["stop_lat"]))
-                for s in self.get_file(STOPS_FILE)
-            }
+            for s in self.get_file(STOPS_FILE):
+                row_stop_id = get_safe_value(s, "stop_id")
+                row_stop_lon = get_safe_float(s, "stop_lon")
+                row_stop_lat = get_safe_float(s, "stop_lat")
+                if row_stop_id is None or row_stop_lon is None or row_stop_lat is None:
+                    self.logger.warning("Invalid stop data: %s", s)
+                    continue
+                self.stop_to_coordinates = {row_stop_id: (row_stop_lon, row_stop_lat)}
         return self.stop_to_coordinates.get(stop_id, None)
 
     def set_workdir(self, workdir):