fix: Added some tests for the populate script (#1065)

Author: jcpitre

api/src/scripts/populate_db.py

@@ -54,7 +54,7 @@ def __init__(self, filepaths):
             filepaths = [filepaths]
 
         for filepath in filepaths:
-            new_df = pandas.read_csv(filepath, low_memory=False)
+            new_df = pandas.read_csv(filepath, comment="#", low_memory=False)
             self.df = pandas.concat([self.df, new_df])
 
         self.filter_data()

api/tests/integration/populate_tests/conftest.py

@@ -0,0 +1,32 @@
+import os
+
+import pytest
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+
+from shared.database.database import Database
+from main import app as application
+from tests.test_utils.database import populate_database
+
+
+@pytest.fixture(scope="package")
+def app() -> FastAPI:
+    application.dependency_overrides = {}
+    return application
+
+
+@pytest.fixture(scope="package")
+def test_database():
+    # Restrict the tests to the test database
+    os.environ["FEEDS_DATABASE_URL"] = "postgresql://postgres:postgres@localhost:54320/MobilityDatabaseTest"
+
+    current_path = os.path.dirname(os.path.abspath(__file__))
+
+    data_dirs = [current_path + "/../../test_data", current_path + "/../test_data", current_path + "/test_data"]
+    with populate_database(Database(), data_dirs) as db:
+        yield db
+
+
+@pytest.fixture(scope="package")
+def client(app, test_database) -> TestClient:
+    return TestClient(app)

api/tests/integration/populate_tests/test_data/sources_test.csv

@@ -0,0 +1,14 @@
+# The contents of this file will be added to the database after the sources_test.csv of the parent directories.
+# So we can replicate some feeds here that are a slight change from the ones in the parent source_test.csv
+# and see the effect.
+# We want to make sure the is_official flag is handled properly in that case.
+# Normally changing to True or False should change the original value but changing to an empty cell should not
+mdb_source_id,data_type,entity_type,location.country_code,location.subdivision_name,location.municipality,provider,is_official,name,note,feed_contact_email,static_reference,urls.direct_download,urls.authentication_type,urls.authentication_info,urls.api_key_parameter_name,urls.latest,urls.license,location.bounding_box.minimum_latitude,location.bounding_box.maximum_latitude,location.bounding_box.minimum_longitude,location.bounding_box.maximum_longitude,location.bounding_box.extracted_on,status,features,redirect.id,redirect.comment
+# For mdb-40, change is_official from TRUE to empty. Should retain True
+40,gtfs,,CA,Ontario,London,London Transit Commission,,,,[email protected],,http://www.londontransit.ca/gtfsfeed/google_transit.zip,0,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-london-transit-commission-gtfs-2.zip?alt=media,https://www.londontransit.ca/open-data/ltcs-open-data-terms-of-use/,42.905244,43.051188,-81.36311,-81.137591,2022-02-22T19:51:34+00:00,inactive,,,
+# For mdb-50, change is_official from FALSE to empty. Should retain False
+50,gtfs,,CA,Ontario,Barrie,ZBarrie Transit,,,,,,http://www.myridebarrie.ca/gtfs/Google_transit.zip,,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-barrie-transit-gtfs-3.zip?alt=media,https://www.barrie.ca/services-payments/transportation-parking/barrie-transit/barrie-gtfs,44.3218044,44.42020676,-79.74063237,-79.61089569,2022-03-01T22:43:25+00:00,deprecated,,40|mdb-702,Some|Comment
+# For mdb-1562, change is_official from FALSE to TRUE. Should change to True
+1562,gtfs-rt,sa,CA,BC,Vancouver,Vancouver-Transit(éèàçíóúČ),TRUE,Realtime(ŘŤÜÎ),,,40,http://foo.org/google_transit.zip,0,,,,,,,,,,active,,10,
+# For mdb-1563, change is_official from TRUE to FALSE. Should change to False
+1563,gtfs-rt,tu,US,SomeState,SomeCity,SomeCity Bus,FALSE,RT,,,mdb-50,http://bar.com,0,,,,,,,,,,inactive,,10,

api/tests/integration/populate_tests/test_populate.py

@@ -0,0 +1,55 @@
+# coding: utf-8
+import pytest
+from fastapi.testclient import TestClient
+
+from tests.test_utils.token import authHeaders
+
+# Test the populate script. The assumption is that the populate script is called to populate the database for these
+# tests, so we can check if certain behaviours are correct.
+
+
+@pytest.mark.parametrize(
+    "values",
+    [
+        {
+            "feed_id": "mdb-40",
+            "expected_official": True,
+            "assert_fail_message": "mdb-40, changed is_official from TRUE to empty. Should retain True in the DB",
+        },
+        {
+            "feed_id": "mdb-50",
+            "expected_official": False,
+            "assert_fail_message": "mdb-50, changed is_official from FALSE to empty. Should retain False in the DB",
+        },
+        {
+            "feed_id": "mdb-1562",
+            "expected_official": True,
+            "assert_fail_message": "mdb-1562, changed is_official from FALSE to TRUE. Should change to True in the DB",
+        },
+        {
+            "feed_id": "mdb-1563",
+            "expected_official": False,
+            "assert_fail_message": "mdb-1563, changed is_official from TRUE to FALSE. Should change to False in the DB",
+        },
+    ],
+    ids=[
+        "official_change_true_to_empty",
+        "official_change_false_to_empty",
+        "official_change_false_to_true",
+        "official_change_true_to_false",
+    ],
+)
+def test_is_official_overwrite(client: TestClient, values):
+    """Test case for feeds_gtfs_id_get with a non-existent feed"""
+    feed_id = values["feed_id"]
+    expected_official = values["expected_official"]
+
+    response = client.request(
+        "GET",
+        "/v1/feeds/{id}".format(id=feed_id),
+        headers=authHeaders,
+    )
+
+    assert response.status_code == 200
+    json_response = response.json()
+    assert json_response["official"] is expected_official, values["assert_fail_message"]