Skip to content

Commit 32ec99a

Browse files
cka-ycka-y
committed
Merge branch 'main' into feat/999
2 parents 7c32e05 + fabc85c commit 32ec99a

File tree

32 files changed

+1417
-234
lines changed

32 files changed

+1417
-234
lines changed

‎api/src/scripts/populate_db_gtfs.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,10 @@ def populate_db(self, session: "Session", fetch_url: bool = True):
202202
feed = self.query_feed_by_stable_id(session, stable_id, data_type)
203203
if feed:
204204
self.logger.debug(f"Updating {feed.__class__.__name__}: {stable_id}")
205+
# Always set the deprecated status if found in the csv
206+
csv_status = self.get_safe_value(row, "status", "active")
207+
if csv_status.lower() == "deprecated":
208+
feed.status = "deprecated"
205209
else:
206210
feed = self.get_model(data_type)(
207211
id=generate_unique_id(),
@@ -223,7 +227,6 @@ def populate_db(self, session: "Session", fetch_url: bool = True):
223227
source="mdb",
224228
)
225229
]
226-
227230
# If the is_official field from the CSV is empty, the value here will be None and we don't touch the DB
228231
if is_official_from_csv is not None:
229232
if feed.official != is_official_from_csv:

‎api/src/scripts/populate_db_test_data.py‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@
1515
t_feedsearch,
1616
Location,
1717
Officialstatushistory,
18+
Gbfsversion,
19+
Gbfsendpoint,
20+
Gbfsfeed,
1821
)
1922
from scripts.populate_db import set_up_configs, DatabasePopulateHelper
2023
from shared.common.logging_utils import Logger
@@ -47,7 +50,6 @@ def populate_test_datasets(self, filepath, db_session: "Session"):
4750
"""
4851
Populate the database with the test datasets
4952
"""
50-
# TODO: parse GBFS versions
5153
# Load the JSON file
5254
with open(filepath) as f:
5355
data = json.load(f)
@@ -125,6 +127,27 @@ def populate_test_datasets(self, filepath, db_session: "Session"):
125127
db_session.query(Feature).filter(Feature.name == report_features["feature_name"]).first()
126128
)
127129

130+
# GBFS version
131+
if "gbfs_versions" in data:
132+
for version in data["gbfs_versions"]:
133+
gbfs_feed = db_session.query(Gbfsfeed).filter(Gbfsfeed.stable_id == version["feed_id"]).one_or_none()
134+
if not gbfs_feed:
135+
self.logger.error(f"No feed found with stable_id: {version['feed_id']}")
136+
continue
137+
gbfs_version = Gbfsversion(
138+
id=version["id"], version=version["version"], url=version["url"], latest=version["latest"]
139+
)
140+
if version.get("endpoints"):
141+
for endpoint in version["endpoints"]:
142+
gbfs_endpoint = Gbfsendpoint(
143+
id=endpoint["id"],
144+
url=endpoint["url"],
145+
language=endpoint.get("language"),
146+
name=endpoint["name"],
147+
)
148+
gbfs_version.gbfsendpoints.append(gbfs_endpoint)
149+
gbfs_feed.gbfsversions.append(gbfs_version)
150+
128151
db_session.commit()
129152
db_session.execute(text(f"REFRESH MATERIALIZED VIEW CONCURRENTLY {t_feedsearch.name}"))
130153

‎api/tests/integration/test_feeds_api.py‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -842,3 +842,104 @@ def test_hard_limits(client, monkeypatch, values):
842842
params={"limit": hard_limit + 1},
843843
)
844844
assert response.status_code != 200
845+
846+
847+
@pytest.mark.parametrize(
848+
"values",
849+
[
850+
{"response_code": 200, "expected_feed_ids": ["gbfs-system_id_1", "gbfs-system_id_2", "gbfs-system_id_3"]},
851+
{
852+
"provider": "Provider Name 1",
853+
"response_code": 200,
854+
"expected_feed_ids": ["gbfs-system_id_1", "gbfs-system_id_2"],
855+
},
856+
{"provider": "Provider Name 2", "response_code": 200, "expected_feed_ids": ["gbfs-system_id_3"]},
857+
{
858+
"country_code": "CA",
859+
"response_code": 200,
860+
"expected_feed_ids": ["gbfs-system_id_1", "gbfs-system_id_2", "gbfs-system_id_3"],
861+
},
862+
{"country_code": "US", "response_code": 200, "expected_feed_ids": []},
863+
{"municipality": "Laval", "response_code": 200, "expected_feed_ids": ["gbfs-system_id_2"]},
864+
{
865+
"producer_url": "https://www.example.com/gbfs_feed_3/",
866+
"response_code": 200,
867+
"expected_feed_ids": ["gbfs-system_id_3"],
868+
},
869+
{"system_id": "system_id_1", "response_code": 200, "expected_feed_ids": ["gbfs-system_id_1"]},
870+
{"version": "3.0", "response_code": 200, "expected_feed_ids": ["gbfs-system_id_1"]},
871+
{"version": "2.3", "response_code": 200, "expected_feed_ids": ["gbfs-system_id_1", "gbfs-system_id_2"]},
872+
{"version": "1.0", "response_code": 200, "expected_feed_ids": []},
873+
{"system_id": "system_id_1", "response_code": 200, "expected_feed_ids": ["gbfs-system_id_1"]},
874+
{"system_id": "doesnt_exist", "response_code": 200, "expected_feed_ids": []},
875+
],
876+
ids=[
877+
"all_none",
878+
"provider_name_1",
879+
"provider_name_2",
880+
"country_code_ca",
881+
"country_code_us",
882+
"municipality_laval",
883+
"producer_url",
884+
"system_id",
885+
"version_3.0",
886+
"version_2.3",
887+
"version_1.0",
888+
"system_id_1",
889+
"system_id_doesnt_exist",
890+
],
891+
)
892+
def test_gbfs_filters(client, values):
893+
"""Test /v1/gbfs_feeds filters by system_id"""
894+
params = {
895+
"provider": values["provider"] if "provider" in values else None,
896+
"country_code": values["country_code"] if "country_code" in values else None,
897+
"municipality": values["municipality"] if "municipality" in values else None,
898+
"producer_url": values["producer_url"] if "producer_url" in values else None,
899+
"system_id": values["system_id"] if "system_id" in values else None,
900+
"version": values["version"] if "version" in values else None,
901+
}
902+
params = {k: v for k, v in params.items() if v is not None}
903+
904+
response = client.request(
905+
"GET",
906+
"/v1/gbfs_feeds",
907+
headers=authHeaders,
908+
params=params,
909+
)
910+
assert response.status_code == values["response_code"]
911+
if values["response_code"] != 200:
912+
return
913+
914+
feeds = response.json()
915+
assert isinstance(feeds, list), "Response should be a list."
916+
assert len(feeds) == len(values["expected_feed_ids"]), (
917+
f"Expected {len(values['expected_feed_ids'])} feeds, " f"got {len(feeds)}."
918+
)
919+
if len(values["expected_feed_ids"]) != 0:
920+
assert any(feed["id"] in values["expected_feed_ids"] for feed in feeds)
921+
922+
923+
@pytest.mark.parametrize(
924+
"values",
925+
[
926+
{"response_code": 200, "expected_feed_ids": ["gbfs-system_id_1"]},
927+
{"response_code": 200, "expected_feed_ids": ["gbfs-system_id_2"]},
928+
{"response_code": 200, "expected_feed_ids": ["gbfs-system_id_3"]},
929+
{"response_code": 404},
930+
],
931+
ids=["valid_id", "valid_id_2", "valid_id_3", "invalid_id"],
932+
)
933+
def test_gbfs_feed_id_get(client: TestClient, values):
934+
"""Test case for gbfs_feed_id_get"""
935+
test_id = values.get("expected_feed_ids", ["dummy_id"])[0]
936+
response = client.request(
937+
"GET",
938+
"/v1/gbfs_feeds/{id}".format(id=test_id),
939+
headers=authHeaders,
940+
)
941+
942+
assert response.status_code == values["response_code"]
943+
if values["response_code"] != 200:
944+
return
945+
assert response.json()["id"] == test_id

‎api/tests/test_data/extra_test_data.json‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -759,5 +759,74 @@
759759
"severity": "ERROR",
760760
"total_notices": 2
761761
}
762+
],
763+
"gbfs_versions": [
764+
{
765+
"feed_id": "gbfs-system_id_1",
766+
"id": "gbfs-system_id_1-2.3",
767+
"version": "2.3",
768+
"url": "https://www.example.com/gbfs_feed_1/2.3/gbfs.json",
769+
"latest": false,
770+
"endpoints": [
771+
{
772+
"id": "gbfs-system_id_1-2.3-system_information",
773+
"url": "https://www.example.com/gbfs_feed_1/2.3/system_information.json",
774+
"language": "en",
775+
"name": "system_information",
776+
"is_feature": false
777+
},
778+
{
779+
"id": "gbfs-system_id_1-2.3-vehicle_position",
780+
"url": "https://www.example.com/gbfs_feed_1/2.3/vehicule_position.json",
781+
"language": "en",
782+
"name": "vehicle_position",
783+
"is_feature": false
784+
}
785+
]
786+
},
787+
{
788+
"feed_id": "gbfs-system_id_1",
789+
"id": "gbfs-system_id_1-3.0",
790+
"version": "3.0",
791+
"url": "https://www.example.com/gbfs_feed_1/3.0/gbfs.json",
792+
"latest": true,
793+
"endpoints": [
794+
{
795+
"id": "gbfs-system_id_1-3.0-system_information",
796+
"url": "https://www.example.com/gbfs_feed_1/3.0/system_information.json",
797+
"name": "system_information",
798+
"is_feature": false
799+
},
800+
{
801+
"id": "gbfs-system_id_1-3.0-vehicle_position",
802+
"url": "https://www.example.com/gbfs_feed_1/3.0/vehicule_position.json",
803+
"name": "vehicle_position",
804+
"is_feature": false
805+
}
806+
]
807+
},
808+
{
809+
"feed_id": "gbfs-system_id_2",
810+
"id": "gbfs-system_id_2-2.3",
811+
"version": "2.3",
812+
"url": "https://www.example.com/gbfs_feed_1/2.3/gbfs.json",
813+
"latest": false,
814+
"endpoints": [
815+
{
816+
"id": "gbfs-system_id_2-2.3-system_information",
817+
"url": "https://www.example.com/gbfs_feed_2/2.3/system_information.json",
818+
"language": "en",
819+
"name": "system_information",
820+
"is_feature": false
821+
},
822+
{
823+
"id": "gbfs-system_id_2-2.3-vehicle_position",
824+
"url": "https://www.example.com/gbfs_feed_2/2.3/vehicule_position.json",
825+
"language": "en",
826+
"name": "vehicle_position",
827+
"is_feature": false
828+
}
829+
]
830+
}
762831
]
763832
}

‎functions-python/process_validation_report/src/main.py‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,9 @@ def parse_json_report(json_report):
108108
try:
109109
dt = json_report["summary"]["validatedAt"]
110110
validated_at = datetime.fromisoformat(dt.replace("Z", "+00:00"))
111-
version = json_report["summary"]["validatorVersion"]
111+
version = None
112+
if "validatorVersion" in json_report["summary"]:
113+
version = json_report["summary"]["validatorVersion"]
112114
logging.info(
113115
f"Validation report validated at {validated_at} with version {version}."
114116
)
@@ -141,9 +143,11 @@ def generate_report_entities(
141143
json_report_url = (
142144
f"{FILES_ENDPOINT}/{feed_stable_id}/{dataset_stable_id}/report_{version}.json"
143145
)
144-
if get_validation_report(report_id, session): # Check if report already exists
146+
# Check if report already exists
147+
# If exists, the function should graceful finish avoiding retry mechanism to trigger again
148+
if get_validation_report(report_id, session):
145149
logging.warning(f"Validation report {report_id} already exists. Terminating.")
146-
raise Exception(f"Validation report {report_id} already exists.")
150+
return []
147151

148152
validation_report_entity = Validationreport(
149153
id=report_id,
@@ -248,15 +252,16 @@ def create_validation_report_entities(
248252
return json_report, code
249253

250254
try:
251-
validated_at, version = parse_json_report(json_report)
255+
validated_at, version_from_json = parse_json_report(json_report)
252256
except Exception as error:
253257
return str(error), 500
254258

255259
try:
256260
# Generate the database entities required for the report
257261
# If an error is thrown we should let the retry mechanism to do its work
258262
entities = generate_report_entities(
259-
version,
263+
# default to the version parameter
264+
version_from_json if version_from_json else version,
260265
validated_at,
261266
json_report,
262267
dataset_stable_id,

0 commit comments

Comments
 (0)