Added feed_license_changes to the list of cascade entries. (#1528)

Author: jcpitre

api/src/scripts/populate_db.py

@@ -45,7 +45,7 @@ class DatabasePopulateHelper:
     Helper class to populate the database
     """
 
-    def __init__(self, filepaths):
+    def __init__(self, filepaths, test_mode=False):
         """
         Specify a list of files to load the csv data from.
         Can also be a single string with a file name.
@@ -60,7 +60,10 @@ def __init__(self, filepaths):
             filepaths = [filepaths]
 
         for filepath in filepaths:
-            new_df = pandas.read_csv(filepath, low_memory=False)
+            if test_mode:
+                new_df = pandas.read_csv(filepath, comment="#", low_memory=False)
+            else:
+                new_df = pandas.read_csv(filepath, low_memory=False)
             self.df = pandas.concat([self.df, new_df])
 
         self.filter_data()

api/src/scripts/populate_db_gbfs.py

@@ -13,8 +13,8 @@
 
 
 class GBFSDatabasePopulateHelper(DatabasePopulateHelper):
-    def __init__(self, filepaths):
-        super().__init__(filepaths)
+    def __init__(self, filepaths, test_mode=False):
+        super().__init__(filepaths, test_mode)
 
     def filter_data(self):
         """Filter out rows with Authentication Info and duplicate System IDs"""

api/src/scripts/populate_db_gtfs.py

@@ -26,12 +26,12 @@ class GTFSDatabasePopulateHelper(DatabasePopulateHelper):
     GTFS - Helper class to populate the database
     """
 
-    def __init__(self, filepaths):
+    def __init__(self, filepaths, test_mode=False):
         """
         Specify a list of files to load the csv data from.
         Can also be a single string with a file name.
         """
-        super().__init__(filepaths)
+        super().__init__(filepaths, test_mode)
         # Keep track of the feeds that have been added to the database
         self.added_gtfs_feeds = []
 

api/src/shared/database/database.py

@@ -75,6 +75,7 @@ def configure_polymorphic_mappers():
         Feed.officialstatushistories,  # officialstatushistory_feed_id_fkey
         Feed.redirectingids,  # redirectingid_source_id_fkey
         Feed.redirectingids_,  # redirectingid_target_id_fkey
+        Feed.feed_license_changes,
     ],
     Gbfsfeed: [
         Gbfsfeed.gbfsversions,  # gbfsversion_feed_id_fkey

api/tests/integration/cascade_delete/test_cascade_delete.py

@@ -25,6 +25,7 @@
     Notice,
     Gtfsfeed,
     Geopolygon,
+    FeedLicenseChange,
 )
 
 from sqlalchemy import text
@@ -188,6 +189,24 @@ def test_delete_feed_cascadeto_redirectingid(test_database):
         assert assoc_count == 0
 
 
+def test_delete_feed_cascadeto_feed_license_changes(test_database):
+
+    with test_database.start_db_session() as session:
+        feed = Feed(id="f1")
+        feed_license_change = FeedLicenseChange(feed_id="f1")
+        session.add_all([feed, feed_license_change])
+        session.commit()
+
+        delete_and_assert(
+            session,
+            [
+                "SELECT COUNT(*) FROM feed where id = 'f1'",
+                "SELECT COUNT(*) FROM feed_license_change where feed_id = 'f1'",
+            ],
+            feed,
+        )
+
+
 def test_delete_gbfsfeed_cascadeto_gbfsversion_cascadeto_gbfsendpoint_cascadeto_gbfsendpointhttpaccesslog(
     test_database,
 ):

api/tests/integration/populate_tests/test_data/sources_test.csv

@@ -1,5 +1,14 @@
+# The contents of this file will be added to the database after the sources_test.csv of the parent directories.
+# So we can replicate some feeds here that are a slight change from the ones in the parent source_test.csv
+# and see the effect.
+# We want to make sure the is_official flag is handled properly in that case.
+# Normally changing to True or False should change the original value but changing to an empty cell should not
 mdb_source_id,data_type,entity_type,location.country_code,location.subdivision_name,location.municipality,provider,is_official,name,note,feed_contact_email,static_reference,urls.direct_download,urls.authentication_type,urls.authentication_info,urls.api_key_parameter_name,urls.latest,urls.license,location.bounding_box.minimum_latitude,location.bounding_box.maximum_latitude,location.bounding_box.minimum_longitude,location.bounding_box.maximum_longitude,location.bounding_box.extracted_on,status,features,redirect.id,redirect.comment
+# For mdb-40, change is_official from TRUE to empty. Should retain True# For mdb-40, change is_official from TRUE to empty. Should retain True
 40,gtfs,,CA,Ontario,London,London Transit Commission,,,,[email protected],,http://www.londontransit.ca/gtfsfeed/google_transit.zip,0,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-london-transit-commission-gtfs-2.zip?alt=media,https://www.londontransit.ca/open-data/ltcs-open-data-terms-of-use/,42.905244,43.051188,-81.36311,-81.137591,2022-02-22T19:51:34+00:00,inactive,,,
+# For mdb-50, change is_official from FALSE to empty. Should retain False
 50,gtfs,,CA,Ontario,Barrie,ZBarrie Transit,,,,,,http://www.myridebarrie.ca/gtfs/Google_transit.zip,,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-barrie-transit-gtfs-3.zip?alt=media,https://www.barrie.ca/services-payments/transportation-parking/barrie-transit/barrie-gtfs,44.3218044,44.42020676,-79.74063237,-79.61089569,2022-03-01T22:43:25+00:00,deprecated,,40|mdb-702,Some|Comment
+# For mdb-1562, change is_official from FALSE to TRUE. Should change to True
 1562,gtfs-rt,sa,CA,BC,Vancouver,Vancouver-Transit(éèàçíóúČ),TRUE,Realtime(ŘŤÜÎ),,,50,http://foo.org/google_transit.zip,0,,,,,,,,,,active,,10,
+# For mdb-1563, change is_official from TRUE to FALSE. Should change to False
 1563,gtfs-rt,tu,US,SomeState,SomeCity,SomeCity Bus,FALSE,RT,,,mdb-50,http://bar.com,0,,,,,,,,,,inactive,,10,

api/tests/integration/populate_twice_gbfs/test_data_part_2/systems_test.csv

@@ -1,4 +1,10 @@
+# We want to test the populate when there is already stuff in the DB and we rerun
+# with changes in systems.csv
 Country Code,Name,Location,System ID,URL,Auto-Discovery URL,Supported Versions,Authentication Info URL, Authentication Type,Authentication Parameter Name
+# Feed 1 stays the same
 AE,Feed 1,Dubai,feed_1,https://www.careem.com/en-ae/careem-bike/,https://some_url/gbfs.json,1.1 ; 2.3,,,
+# Feed 2 was removed. We expect it to be made deprecated in the DB
+# Feed 3 was modified (Name will be Feed 3 modified instead of Feed 3)
 AE,Feed 3 modified,Dubai,feed_3,https://www.careem.com/en-ae/careem-bike/,https://some_url/gbfs.json,1.1 ; 2.3,,,
+# Feed 4 was added
 AE,Feed 4,Dubai,feed_4,https://ridedott.com/,https://some_url/gbfs.json,2.3,,,

api/tests/test_utils/database.py

@@ -72,7 +72,7 @@ def _populate_db_phase(db: Database, data_dirs: List[str]):
         if (filepath := os.path.join(directory, "sources_test.csv")) and os.path.isfile(filepath)
     ]
     if csv_filepaths:
-        gtfs_db_helper = GTFSDatabasePopulateHelper(csv_filepaths)
+        gtfs_db_helper = GTFSDatabasePopulateHelper(csv_filepaths, test_mode=True)
         gtfs_db_helper.initialize(trigger_downstream_tasks=False)
 
     # GBFS
@@ -82,7 +82,9 @@ def _populate_db_phase(db: Database, data_dirs: List[str]):
         if (filepath := os.path.join(directory, "systems_test.csv")) and os.path.isfile(filepath)
     ]
     if gbfs_csv_filepaths:
-        GBFSDatabasePopulateHelper(gbfs_csv_filepaths).initialize(trigger_downstream_tasks=False, fetch_url=False)
+        GBFSDatabasePopulateHelper(gbfs_csv_filepaths, test_mode=True).initialize(
+            trigger_downstream_tasks=False, fetch_url=False
+        )
 
     # Extra test data
     json_filepaths = [