Enhance logging

Author: davidgamez

functions-python/helpers/utils.py

@@ -254,21 +254,20 @@ def check_maximum_executions(
 
     trace_service = DatasetTraceService()
     trace = trace_service.get_by_execution_and_stable_ids(execution_id, stable_id)
-    logger.info(f"Dataset trace: {trace}")
     executions = len(trace) if trace else 0
     logger.info(
-        f"Dataset executed times={executions}/{maximum_executions} "
+        f"Function executed times={executions}/{maximum_executions} "
         f"in execution=[{execution_id}] "
     )
 
     if executions > 0:
         if executions >= maximum_executions:
-            error_message = (
+            message = (
                 f"Function already executed maximum times "
                 f"in execution: [{execution_id}]"
             )
-            logger.error(error_message)
-            return error_message
+            logger.warning(message)
+            return message
     return None
 
 

functions-python/reverse_geolocation/src/scripts/reverse_geolocation_process_verifier.py

@@ -29,40 +29,44 @@
 
 feeds = [
     {
-        # 1539 stops, NZ, 1 location
+        # 0. 1539 stops, NZ, 1 location
         "stable_id": "local-test-gbfs-flamingo_auckland",
         "station_information_url": "https://data.rideflamingo.com/gbfs/3/auckland/station_information.json",
         "vehicle_status_url": "https://data.rideflamingo.com/gbfs/3/auckland/vehicle_status.json",
         "data_type": "gbfs",
     },
     {
-        # 11777 stops, JP, 241 locations
+        # 1. 11777 stops, JP, 241 locations
         "stable_id": "local-test-gbfs-hellocycling",
         "station_information_url": "https://api-public.odpt.org/api/v4/gbfs/hellocycling/station_information.json",
         "data_type": "gbfs",
     },
     {
-        # 308611, UK aggregated, 225 locations
+        # 2. 308611, UK aggregated, 225 locations
         "stable_id": "local-test-2014",
         "stops_url": "https://storage.googleapis.com/mobilitydata-datasets-prod/mdb-2014/"
         "mdb-2014-202508120303/extracted/stops.txt",
         "data_type": "gtfs",
     },
     {
-        # 663 stops, Europe, 334 locations
+        # 3. 663 stops, Europe, 334 locations
         "stable_id": "local-test-1139",
         "stops_url": "https://storage.googleapis.com/mobilitydata-datasets-prod/mdb-1139/"
         "mdb-1139-202406071559/stops.txt",
         "data_type": "gtfs",
     },
     {
-        # 10985 stops, Spain, duplicate key error(https://github.com/MobilityData/mobility-feed-api/issues/1289)
+        # 4. 10985 stops, Spain, duplicate key error(https://github.com/MobilityData/mobility-feed-api/issues/1289)
         "stable_id": "local-test-gtfs-mdb-2825",
         "stops_url": "https://storage.googleapis.com/mobilitydata-datasets-prod/mdb-2825/"
         "mdb-2825-202508181628/extracted/stops.txt",
         "data_type": "gtfs",
     },
 ]
+run_with_feed_index = (
+    4  # Set to an integer index to run with a specific feed from the list above
+)
+
 
 # Load environment variables from .env.local
 load_dotenv(dotenv_path=".env.local")
@@ -218,7 +222,7 @@ def create_test_data(feed_stable_id: str, feed_dict: Dict, db_session: Session =
 
     strategy = ReverseGeocodingStrategy.PER_POLYGON
 
-    feed_dict = feeds[3]
+    feed_dict = feeds[run_with_feed_index]
     feed_stable_id = feed_dict["stable_id"]
     # create test data in the database if does not exist
     create_test_data(feed_stable_id=feed_stable_id, feed_dict=feed_dict)
@@ -237,7 +241,7 @@ def create_test_data(feed_stable_id: str, feed_dict: Dict, db_session: Session =
         "stops_url": f"http://{HOST}:{PORT}/{BUCKET_NAME}/{feed_stable_id}/stops.txt",
         "strategy": str(strategy.value),
         "data_type": feed_dict["data_type"],
-        "use_cache": False,
+        # "use_cache": False,
         "public": False,
         "maximum_executions": 1000,
     }

functions-python/reverse_geolocation/src/strategy_extraction_per_polygon.py

@@ -76,6 +76,7 @@ def extract_location_aggregates_per_polygon(
     batch_size = max(
         int(total_stop_count / 20), 0
     )  # Process ~5% of the total stops in each batch
+    stop_clustered_total = 0
     while not remaining_stops_df.empty:
         if (last_seen_count - len(remaining_stops_df)) >= batch_size or len(
             remaining_stops_df
@@ -126,6 +127,7 @@ def extract_location_aggregates_per_polygon(
                 highest.admin_level,
                 count_before - len(remaining_stops_df),
             )
+            stop_clustered_total += count_before - len(remaining_stops_df)
         else:
             # If admin_level < locality_admin_level, we assume the polygon is too large to filter points
             # directly, so we just use the first point as a representative
@@ -169,4 +171,6 @@ def extract_location_aggregates_per_polygon(
             location_aggregates[location_aggregate.group_id] = location_aggregate
     # Make sure to commit the changes after processing all points
     db_session.commit()
-    logger.info("Completed processing all points")
+    logger.info(
+        "Completed processing all points with clustered total %d", stop_clustered_total
+    )