fix: Modified populate script to not change the official DB value if csv field is empty. (#1053)

Author: jcpitre

api/src/scripts/populate_db.py

@@ -80,9 +80,28 @@ def get_safe_value(row, column_name, default_value):
         """
         Get a safe value from the row
         """
-        if not row[column_name] or pandas.isna(row[column_name]) or f"{row[column_name]}".strip() == "":
-            return default_value if default_value is not None else None
-        return f"{row[column_name]}".strip()
+        value = row[column_name]
+        if not value or pandas.isna(value) or f"{value}".strip() == "":
+            return default_value
+        return f"{value}".strip()
+
+    @staticmethod
+    def get_safe_boolean_value(row, column_name, default_value: bool | None) -> bool | None:
+        """
+        Get a safe boolean value from the row
+        Only allowed values are "true" and "false" (case insensitive)
+        Anything else returns the default.
+        """
+        value = row[column_name]
+        if value is None or pandas.isna(value) or f"{value}".strip() == "":
+            return default_value
+        # I am not sure if pandas will convert "TRUE" and "FALSE" to boolean, so go back to using a string
+        value = f"{value}".strip().lower()
+        if value == "true":
+            return True
+        if value == "false":
+            return False
+        return default_value
 
     @staticmethod
     def get_location_id(country_code, subdivision_name, municipality):

api/src/scripts/populate_db_gtfs.py

@@ -192,13 +192,10 @@ def populate_db(self, session: "Session"):
             # Create or update the GTFS feed
             data_type = self.get_data_type(row)
             stable_id = self.get_stable_id(row)
-            is_official_from_csv = self.get_safe_value(row, "is_official", "false").lower() == "true"
+            is_official_from_csv = self.get_safe_boolean_value(row, "is_official", None)
             feed = self.query_feed_by_stable_id(session, stable_id, data_type)
             if feed:
                 self.logger.debug(f"Updating {feed.__class__.__name__}: {stable_id}")
-                if feed.official != is_official_from_csv:
-                    feed.official = is_official_from_csv
-                    feed.official_updated_at = datetime.now(pytz.utc)
             else:
                 feed = self.get_model(data_type)(
                     id=generate_unique_id(),
@@ -219,8 +216,12 @@ def populate_db(self, session: "Session"):
                         source="mdb",
                     )
                 ]
-                feed.official = is_official_from_csv
-                feed.official_updated_at = datetime.now(pytz.utc)
+
+            # If the is_official field from the CSV is empty, the value here will be None and we don't touch the DB
+            if is_official_from_csv is not None:
+                if feed.official != is_official_from_csv:
+                    feed.official = is_official_from_csv
+                    feed.official_updated_at = datetime.now(pytz.utc)
 
             # Populate common fields from Feed
             feed.feed_name = self.get_safe_value(row, "name", "")

functions-python/export_csv/src/main.py

@@ -130,6 +130,7 @@ def export_csv(csv_file_path: str):
 
         count = 0
         for feed in fetch_feeds():
+            # We're counting on the writer to leave an empty field in the csv for None values
             writer.writerow(feed)
             count += 1