feat: Modified populate script to accept non-integers as feed_reference (#990)

Author: jcpitre

api/src/scripts/populate_db_gtfs.py

@@ -126,7 +126,10 @@ def process_feed_references(self, session: "Session"):
             gtfs_rt_feed = self.query_feed_by_stable_id(session, stable_id, "gtfs_rt")
             static_reference = self.get_safe_value(row, "static_reference", "")
             if static_reference:
-                gtfs_stable_id = f"mdb-{int(float(static_reference))}"
+                try:
+                    gtfs_stable_id = f"mdb-{int(float(static_reference))}"
+                except ValueError:
+                    gtfs_stable_id = static_reference
                 gtfs_feed = self.query_feed_by_stable_id(session, gtfs_stable_id, "gtfs")
                 already_referenced_ids = {ref.id for ref in gtfs_feed.gtfs_rt_feeds}
                 if gtfs_feed and gtfs_rt_feed.id not in already_referenced_ids:
@@ -150,8 +153,9 @@ def process_redirects(self, session: "Session"):
             comments = raw_comments.split("|") if raw_comments is not None else []
             if len(redirects_ids) != len(comments) and len(comments) > 0:
                 self.logger.warning(f"Number of redirect ids and redirect comments differ for feed {stable_id}")
-            for mdb_source_id in redirects_ids:
-                if len(mdb_source_id) == 0:
+            for redirect_id in redirects_ids:
+                redirect_id = redirect_id.strip() if redirect_id else ""
+                if len(redirect_id) == 0:
                     # since there is a 1:1 correspondence between redirect ids and comments, skip also the comment
                     comments = comments[1:]
                     continue
@@ -160,9 +164,9 @@ def process_redirects(self, session: "Session"):
                 else:
                     comment = ""
                 try:
-                    target_stable_id = f"mdb-{int(float(mdb_source_id.strip()))}"
+                    target_stable_id = f"mdb-{int(float(redirect_id))}"
                 except ValueError:
-                    target_stable_id = mdb_source_id.strip()
+                    target_stable_id = redirect_id
                 target_feed = self.query_feed_by_stable_id(session, target_stable_id, None)
                 if not target_feed:
                     self.logger.warning(f"Could not find redirect target feed {target_stable_id} for feed {stable_id}")

api/tests/integration/test_data/sources_test.csv

@@ -1,5 +1,6 @@
-mdb_source_id,data_type,entity_type,location.country_code,location.subdivision_name,location.municipality,provider,is_official,name,note,feed_contact_email,static_reference,urls.direct_download,urls.authentication_type,urls.authentication_info,urls.api_key_parameter_name,urls.latest,urls.license,location.bounding_box.minimum_latitude,location.bounding_box.maximum_latitude,location.bounding_box.minimum_longitude,location.bounding_box.maximum_longitude,location.bounding_box.extracted_on,status,features,redirect.id,redirect.comment
+mdb_source_id,data_type,entity_type,location.country_code,location.subdivision_name,location.municipality,provider,is_offical,name,note,feed_contact_email,static_reference,urls.direct_download,urls.authentication_type,urls.authentication_info,urls.api_key_parameter_name,urls.latest,urls.license,location.bounding_box.minimum_latitude,location.bounding_box.maximum_latitude,location.bounding_box.minimum_longitude,location.bounding_box.maximum_longitude,location.bounding_box.extracted_on,status,features,redirect.id,redirect.comment
 40,gtfs,,CA,Ontario,London,London Transit Commission,TRUE,,,[email protected],,http://www.londontransit.ca/gtfsfeed/google_transit.zip,0,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-london-transit-commission-gtfs-2.zip?alt=media,https://www.londontransit.ca/open-data/ltcs-open-data-terms-of-use/,42.905244,43.051188,-81.36311,-81.137591,2022-02-22T19:51:34+00:00,inactive,,,
-50,gtfs,,CA,Ontario,Barrie,ZBarrie Transit,FALSE,,,,,http://www.myridebarrie.ca/gtfs/Google_transit.zip,,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-barrie-transit-gtfs-3.zip?alt=media,https://www.barrie.ca/services-payments/transportation-parking/barrie-transit/barrie-gtfs,44.3218044,44.42020676,-79.74063237,-79.61089569,2022-03-01T22:43:25+00:00,deprecated,,,
+50,gtfs,,CA,Ontario,Barrie,ZBarrie Transit,FALSE,,,,,http://www.myridebarrie.ca/gtfs/Google_transit.zip,,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-barrie-transit-gtfs-3.zip?alt=media,https://www.barrie.ca/services-payments/transportation-parking/barrie-transit/barrie-gtfs,44.3218044,44.42020676,-79.74063237,-79.61089569,2022-03-01T22:43:25+00:00,deprecated,,40|mdb-702,Some|Comment
 702,gtfs,,CA,[British Columbia,Whistler],BC Transit (Whistler Transit System),,,,,,http://whistler.mapstrat.com/current/google_transit.zip,,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-british-columbia-bc-transit-whistler-transit-system-gtfs-702.zip?alt=media,https://www.bctransit.com/open-data/terms-of-use,50.077122,50.159071,-123.043635,-122.926836,2022-03-16T22:05:05+00:00,development,,,
 1562,gtfs-rt,sa,CA,BC,Vancouver,Vancouver-Transit(éèàçíóúČ),,Realtime(ŘŤÜÎ),,,40,http://foo.org/google_transit.zip,0,,,,,,,,,,active,,10,
+1563,gtfs-rt,tu,US,SomeState,SomeCity,SomeCity Bus,,RT,,,mdb-50,http://bar.com,0,,,,,,,,,,inactive,,10,

api/tests/integration/test_feeds_api.py

@@ -657,23 +657,23 @@ def test_filter_by_subdivision_and_municipality(client):
 @pytest.mark.parametrize(
     "values",
     [
-        {"response_code": 200, "expected_feed_ids": ["mdb-1561", "mdb-1562"]},
+        {"response_code": 200, "expected_feed_ids": ["mdb-1561", "mdb-1562", "mdb-1563"]},
         {"entity_types": "vp", "response_code": 200, "expected_feed_ids": ["mdb-1561"]},
         {"entity_types": "sa,vp", "response_code": 200, "expected_feed_ids": ["mdb-1561", "mdb-1562"]},
-        {"entity_types": "", "response_code": 200, "expected_feed_ids": ["mdb-1561", "mdb-1562"]},
+        {"entity_types": "", "response_code": 200, "expected_feed_ids": ["mdb-1561", "mdb-1562", "mdb-1563"]},
         {"entity_types": "not_valid", "response_code": 422},
         {"country_code": "CA", "response_code": 200, "expected_feed_ids": ["mdb-1562"]},
         {"country_code": "CA", "entity_types": "sa,vp", "response_code": 200, "expected_feed_ids": ["mdb-1562"]},
-        {"country_code": "", "response_code": 200, "expected_feed_ids": ["mdb-1561", "mdb-1562"]},
+        {"country_code": "", "response_code": 200, "expected_feed_ids": ["mdb-1561", "mdb-1562", "mdb-1563"]},
         {"provider": "no-found-provider", "response_code": 200, "expected_feed_ids": []},
         {"provider": "transit", "response_code": 200, "expected_feed_ids": ["mdb-1562"]},
-        {"provider": "", "response_code": 200, "expected_feed_ids": ["mdb-1561", "mdb-1562"]},
+        {"provider": "", "response_code": 200, "expected_feed_ids": ["mdb-1561", "mdb-1562", "mdb-1563"]},
         {"producer_url": "foo.org", "response_code": 200, "expected_feed_ids": ["mdb-1562"]},
-        {"producer_url": "", "response_code": 200, "expected_feed_ids": ["mdb-1561", "mdb-1562"]},
+        {"producer_url": "", "response_code": 200, "expected_feed_ids": ["mdb-1561", "mdb-1562", "mdb-1563"]},
         {"subdivision_name": "bc", "response_code": 200, "expected_feed_ids": ["mdb-1562"]},
-        {"subdivision_name": "", "response_code": 200, "expected_feed_ids": ["mdb-1561", "mdb-1562"]},
+        {"subdivision_name": "", "response_code": 200, "expected_feed_ids": ["mdb-1561", "mdb-1562", "mdb-1563"]},
         {"municipality": "vanco", "response_code": 200, "expected_feed_ids": ["mdb-1562"]},
-        {"municipality": "", "response_code": 200, "expected_feed_ids": ["mdb-1561", "mdb-1562"]},
+        {"municipality": "", "response_code": 200, "expected_feed_ids": ["mdb-1561", "mdb-1562", "mdb-1563"]},
     ],
     ids=[
         "all_none",
@@ -723,3 +723,56 @@ def test_gtfs_rt_filters(client, values):
     assert len(feeds) == len(values["expected_feed_ids"])
     if len(values["expected_feed_ids"]) != 0:
         assert any(feed["id"] in values["expected_feed_ids"] for feed in feeds)
+
+
+@pytest.mark.parametrize(
+    "values",
+    [
+        {"stable_id": "mdb-1562", "target_id": "mdb-40"},
+        {"stable_id": "mdb-1563", "target_id": "mdb-50"},
+    ],
+    ids=[
+        "integer_static_reference",
+        "string_static_reference",
+    ],
+)
+def test_gtfs_rt_reference(client, values):
+    """Test /v1/gtfs_rt_feeds to make sure it returns the proper feed_reference for integer and string references."""
+
+    stable_id = values["stable_id"] if "stable_id" in values else None
+    response = client.request(
+        "GET",
+        f"/v1/gtfs_rt_feeds/{stable_id}",
+        headers=authHeaders,
+    )
+    assert response.status_code == 200
+    feed = response.json()
+    assert feed["feed_references"][0] == values["target_id"]
+
+
+def test_gtfs_redirect(client):
+    """Test that a returned feed contains the proper redirects for integer or string redirects."""
+
+    response = client.request(
+        "GET",
+        "/v1/feeds/mdb-50",
+        headers=authHeaders,
+    )
+    assert response.status_code == 200
+    feed = response.json()
+    assert feed["redirects"][0]["comment"] == "Some"
+    assert feed["redirects"][1]["comment"] == "Comment"
+    # spreadsheet contains '40|mdb-702'
+    assert feed["redirects"][0]["target_id"] == "mdb-40"
+    assert feed["redirects"][1]["target_id"] == "mdb-702"
+
+    response = client.request(
+        "GET",
+        "/v1/feeds/mdb-1562",
+        headers=authHeaders,
+    )
+    assert response.status_code == 200
+    feed = response.json()
+    assert feed["redirects"][0]["comment"] == ""
+    # spreadsheet contains '10'
+    assert feed["redirects"][0]["target_id"] == "mdb-10"

api/tests/integration/test_search_api.py

@@ -133,7 +133,7 @@ def test_search_feeds_provider_one_feed(client: TestClient, search_query: str):
     [
         ("gtfs", 10),
         ("not_valid_gtfs", 0),
-        ("gtfs_rt", 2),
+        ("gtfs_rt", 3),
     ],
 )
 def test_search_feeds_filter_data_type(client: TestClient, data_type: str, expected_results_total: int):
@@ -176,8 +176,8 @@ def test_search_feeds_filter_data_type(client: TestClient, data_type: str, expec
     [
         ("active", 9),
         ("not_valid_status", 0),
-        ("inactive", 1),
-        ("active,inactive", 10),
+        ("inactive", 2),
+        ("active,inactive", 11),
     ],
 )
 def test_search_feeds_filter_status(client: TestClient, status: str, expected_results_total: int):

functions-python/export_csv/src/main.py

@@ -114,7 +114,7 @@ def export_and_upload_csv(request=None):
 
     csv_file_path = csv_default_file_path
     export_csv(csv_file_path)
-    upload_file_to_storage(csv_file_path, "sources_v2.csv")
+    upload_file_to_storage(csv_file_path, "feeds_v2.csv")
 
     logging.info("Export successful")
     return "Export successful"