feat: Added the official flag value to the DB when populating from csv (#981)

jcpitre · web-flow · commit b95177c569a8 · 2025-03-13T13:41:36.000-04:00
diff --git a/api/src/scripts/populate_db_gtfs.py b/api/src/scripts/populate_db_gtfs.py
@@ -188,9 +188,13 @@ def populate_db(self, session: "Session"):
             # Create or update the GTFS feed
             data_type = self.get_data_type(row)
             stable_id = self.get_stable_id(row)
+            is_official_from_csv = self.get_safe_value(row, "is_official", "false").lower() == "true"
             feed = self.query_feed_by_stable_id(session, stable_id, data_type)
             if feed:
                 self.logger.debug(f"Updating {feed.__class__.__name__}: {stable_id}")
+                if feed.official != is_official_from_csv:
+                    feed.official = is_official_from_csv
+                    feed.official_updated_at = datetime.now(pytz.utc)
             else:
                 feed = self.get_model(data_type)(
                     id=generate_unique_id(),
@@ -211,6 +215,9 @@ def populate_db(self, session: "Session"):
                         source="mdb",
                     )
                 ]
+                feed.official = is_official_from_csv
+                feed.official_updated_at = datetime.now(pytz.utc)
+
             # Populate common fields from Feed
             feed.feed_name = self.get_safe_value(row, "name", "")
             feed.note = self.get_safe_value(row, "note", "")
diff --git a/api/tests/integration/test_data/sources_test.csv b/api/tests/integration/test_data/sources_test.csv
@@ -1,5 +1,5 @@
-mdb_source_id,data_type,entity_type,location.country_code,location.subdivision_name,location.municipality,provider,name,note,feed_contact_email,static_reference,urls.direct_download,urls.authentication_type,urls.authentication_info,urls.api_key_parameter_name,urls.latest,urls.license,location.bounding_box.minimum_latitude,location.bounding_box.maximum_latitude,location.bounding_box.minimum_longitude,location.bounding_box.maximum_longitude,location.bounding_box.extracted_on,status,features,redirect.id,redirect.comment
-40,gtfs,,CA,Ontario,London,London Transit Commission,,,croy@londontransit.ca,,http://www.londontransit.ca/gtfsfeed/google_transit.zip,0,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-london-transit-commission-gtfs-2.zip?alt=media,https://www.londontransit.ca/open-data/ltcs-open-data-terms-of-use/,42.905244,43.051188,-81.36311,-81.137591,2022-02-22T19:51:34+00:00,inactive,,,
-50,gtfs,,CA,Ontario,Barrie,ZBarrie Transit,,,,,http://www.myridebarrie.ca/gtfs/Google_transit.zip,,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-barrie-transit-gtfs-3.zip?alt=media,https://www.barrie.ca/services-payments/transportation-parking/barrie-transit/barrie-gtfs,44.3218044,44.42020676,-79.74063237,-79.61089569,2022-03-01T22:43:25+00:00,deprecated,,,
-702,gtfs,,CA,[British Columbia,Whistler],BC Transit (Whistler Transit System),,,,,http://whistler.mapstrat.com/current/google_transit.zip,,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-british-columbia-bc-transit-whistler-transit-system-gtfs-702.zip?alt=media,https://www.bctransit.com/open-data/terms-of-use,50.077122,50.159071,-123.043635,-122.926836,2022-03-16T22:05:05+00:00,development,,,
-1562,gtfs-rt,sa,CA,BC,Vancouver,Vancouver-Transit(éèàçíóúČ),Realtime(ŘŤÜÎ),,,40,http://foo.org/google_transit.zip,0,,,,,,,,,,active,,10,
+mdb_source_id,data_type,entity_type,location.country_code,location.subdivision_name,location.municipality,provider,is_official,name,note,feed_contact_email,static_reference,urls.direct_download,urls.authentication_type,urls.authentication_info,urls.api_key_parameter_name,urls.latest,urls.license,location.bounding_box.minimum_latitude,location.bounding_box.maximum_latitude,location.bounding_box.minimum_longitude,location.bounding_box.maximum_longitude,location.bounding_box.extracted_on,status,features,redirect.id,redirect.comment
+40,gtfs,,CA,Ontario,London,London Transit Commission,TRUE,,,croy@londontransit.ca,,http://www.londontransit.ca/gtfsfeed/google_transit.zip,0,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-london-transit-commission-gtfs-2.zip?alt=media,https://www.londontransit.ca/open-data/ltcs-open-data-terms-of-use/,42.905244,43.051188,-81.36311,-81.137591,2022-02-22T19:51:34+00:00,inactive,,,
+50,gtfs,,CA,Ontario,Barrie,ZBarrie Transit,FALSE,,,,,http://www.myridebarrie.ca/gtfs/Google_transit.zip,,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-barrie-transit-gtfs-3.zip?alt=media,https://www.barrie.ca/services-payments/transportation-parking/barrie-transit/barrie-gtfs,44.3218044,44.42020676,-79.74063237,-79.61089569,2022-03-01T22:43:25+00:00,deprecated,,,
+702,gtfs,,CA,[British Columbia,Whistler],BC Transit (Whistler Transit System),,,,,,http://whistler.mapstrat.com/current/google_transit.zip,,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-british-columbia-bc-transit-whistler-transit-system-gtfs-702.zip?alt=media,https://www.bctransit.com/open-data/terms-of-use,50.077122,50.159071,-123.043635,-122.926836,2022-03-16T22:05:05+00:00,development,,,
+1562,gtfs-rt,sa,CA,BC,Vancouver,Vancouver-Transit(éèàçíóúČ),,Realtime(ŘŤÜÎ),,,40,http://foo.org/google_transit.zip,0,,,,,,,,,,active,,10,
diff --git a/api/tests/test_data/sources_test.csv b/api/tests/test_data/sources_test.csv
@@ -1,6 +1,6 @@
-mdb_source_id,data_type,entity_type,location.country_code,location.subdivision_name,location.municipality,provider,name,note,feed_contact_email,static_reference,urls.direct_download,urls.authentication_type,urls.authentication_info,urls.api_key_parameter_name,urls.latest,urls.license,location.bounding_box.minimum_latitude,location.bounding_box.maximum_latitude,location.bounding_box.minimum_longitude,location.bounding_box.maximum_longitude,location.bounding_box.extracted_on,status,features,redirect.id,redirect.comment
-1,gtfs,,US,mdb-1-subdivision,mdb-1-municipality,mdb-1-MobilityDataTest provider,mdb-1-MobilityDataTest Feed Name,,,,,0,,,,,,,,,,active,Route Colors|Bike Allowed|Headsigns|Sandra Reeves,10|20,
-10,gtfs,,US,mdb-10-subdivision,mdb-10-municipality,mdb-10-MobilityDataTest provider,mdb-10-MobilityDataTest Feed Name,,,,,0,,,,,,,,,,active,,20|30,
-20,gtfs,,US,mdb-20-subdivision,mdb-20-municipality,mdb-20-MobilityDataTest provider,mdb-20-MobilityDataTest Feed Name,,,,,0,,,,,,,,,,active,,,
-30,gtfs,,US,mdb-30-subdivision,mdb-30-municipality,mdb-30-MobilityDataTest provider,mdb-30-MobilityDataTest Feed Name,,,,,0,,,,,,,,,,active,,,
-1561,gtfs-rt,vp,US,,,,,,,1,,0,,,,,,,,,,active,,10,
+mdb_source_id,data_type,entity_type,location.country_code,location.subdivision_name,location.municipality,provider,is_official,name,note,feed_contact_email,static_reference,urls.direct_download,urls.authentication_type,urls.authentication_info,urls.api_key_parameter_name,urls.latest,urls.license,location.bounding_box.minimum_latitude,location.bounding_box.maximum_latitude,location.bounding_box.minimum_longitude,location.bounding_box.maximum_longitude,location.bounding_box.extracted_on,status,features,redirect.id,redirect.comment
+1,gtfs,,US,mdb-1-subdivision,mdb-1-municipality,mdb-1-MobilityDataTest provider,,mdb-1-MobilityDataTest Feed Name,,,,,0,,,,,,,,,,active,Route Colors|Bike Allowed|Headsigns|Sandra Reeves,10|20,
+10,gtfs,,US,mdb-10-subdivision,mdb-10-municipality,mdb-10-MobilityDataTest provider,,mdb-10-MobilityDataTest Feed Name,,,,,0,,,,,,,,,,active,,20|30,
+20,gtfs,,US,mdb-20-subdivision,mdb-20-municipality,mdb-20-MobilityDataTest provider,,mdb-20-MobilityDataTest Feed Name,,,,,0,,,,,,,,,,active,,,
+30,gtfs,,US,mdb-30-subdivision,mdb-30-municipality,mdb-30-MobilityDataTest provider,,mdb-30-MobilityDataTest Feed Name,,,,,0,,,,,,,,,,active,,,
+1561,gtfs-rt,vp,US,,,,,,,,1,,0,,,,,,,,,,active,,10,
diff --git a/functions-python/validation_to_ndjson/src/locations.py b/functions-python/validation_to_ndjson/src/locations.py
diff --git a/functions-python/validation_to_ndjson/src/validation_report_converter.py b/functions-python/validation_to_ndjson/src/validation_report_converter.py
@@ -11,7 +11,7 @@
     load_json_schema,
     filter_json_by_schema,
 )
-from utils.locations import get_feed_location
+from locations import get_feed_location
 
 # Environment variables
 project_id = os.getenv("PROJECT_ID")
diff --git a/functions-python/validation_to_ndjson/tests/test_locations.py b/functions-python/validation_to_ndjson/tests/test_locations.py
@@ -1,7 +1,7 @@
 import unittest
 from unittest.mock import MagicMock
 
-from utils.locations import get_feed_location, Location
+from locations import get_feed_location, Location
 
 
 class TestFeedsLocations(unittest.TestCase):

Original file line number	Diff line number	Diff line change
`@@ -11,7 +11,7 @@`
`11`	`11`	`load_json_schema,`
`12`	`12`	`filter_json_by_schema,`
`13`	`13`	`)`
`14`		`-from utils.locations import get_feed_location`
	`14`	`+from locations import get_feed_location`
`15`	`15`
`16`	`16`	`# Environment variables`
`17`	`17`	`project_id = os.getenv("PROJECT_ID")`