fix: add descending order to gtfs datasets endpoint (#1062)

Author: davidgamez

api/src/feeds/impl/datasets_api_impl.py

@@ -28,13 +28,17 @@ class DatasetsApiImpl(BaseDatasetsApi):
 
     @staticmethod
     def create_dataset_query() -> Query:
-        return Query(
-            [
-                Gtfsdataset,
-                Gtfsdataset.bounding_box.ST_AsGeoJSON(),
-                Feed.stable_id,
-            ]
-        ).join(Feed, Feed.id == Gtfsdataset.feed_id)
+        return (
+            Query(
+                [
+                    Gtfsdataset,
+                    Gtfsdataset.bounding_box.ST_AsGeoJSON(),
+                    Feed.stable_id,
+                ]
+            )
+            .join(Feed, Feed.id == Gtfsdataset.feed_id)
+            .order_by(Gtfsdataset.downloaded_at.desc())
+        )
 
     @staticmethod
     def get_datasets_gtfs(query: Query, session: Session, limit: int = None, offset: int = None) -> List[GtfsDataset]:

api/tests/integration/test_feeds_api.py

@@ -419,7 +419,8 @@ def test_get_gtfs_feed_datasets_with_limit(client: TestClient):
 
     assert response.status_code == 200
     assert len(response.json()) == 1
-    assert response.json()[0]["id"] == TEST_DATASET_STABLE_IDS[0]
+    # the second dataset should be returned as it is the most recent
+    assert response.json()[0]["id"] == TEST_DATASET_STABLE_IDS[1]
 
 
 def test_get_gtfs_feed_datasets_with_offset(client: TestClient):
@@ -434,7 +435,8 @@ def test_get_gtfs_feed_datasets_with_offset(client: TestClient):
 
     assert response.status_code == 200
     assert len(response.json()) == 1
-    assert response.json()[0]["id"] == TEST_DATASET_STABLE_IDS[1]
+    # The first dataset should be returned as it is the oldest
+    assert response.json()[0]["id"] == TEST_DATASET_STABLE_IDS[0]
 
 
 def test_get_gtfs_feed_datasets_with_downloaded_after_after(client: TestClient):
@@ -478,7 +480,7 @@ def test_get_gtfs_feed_datasets_with_downloaded_after_first(client: TestClient):
     Expected result: the full list of datasets
     """
     datasets = get_all_datasets(client)
-    datasets.pop(0)
+    datasets.pop(len(datasets) - 1)
     date = datasets_download_first_date + timedelta(days=1)
     response = client.request(
         "GET",

docs/DatabaseCatalogAPI.yaml

@@ -178,7 +178,7 @@ paths:
     parameters:
       - $ref: "#/components/parameters/feed_id_of_datasets_path_param"
     get:
-      description: Get a list of datasets related to a GTFS feed. Once a week, we check if the latest dataset has been updated and, if so, we update it in our system accordingly.
+      description: Get a list of datasets associated with a GTFS feed. Once a day, we check whether the latest dataset has changed; if it has, we update it in our system. The list is sorted from newest to oldest.
       tags:
         - "feeds"
       operationId: getGtfsFeedDatasets

liquibase/changelog.xml

@@ -43,4 +43,5 @@
     <include file="changes/feat_966.sql" relativeToChangelogFile="true"/>
     <include file="changes/feat_946.sql" relativeToChangelogFile="true"/>
     <include file="changes/add_idxs.sql" relativeToChangelogFile="true"/>
+    <include file="changes/feat_1046.sql" relativeToChangelogFile="true"/>
 </databaseChangeLog>

liquibase/changes/feat_1046.sql

@@ -0,0 +1,2 @@
+-- Index to improve the filtering by feed_id and downloaded_at
+CREATE INDEX idx_gtfsdataset_feed_id_downloaded_at_desc ON GTFSDataset(feed_id, downloaded_at DESC);

web-app/public/locales/en/feeds.json

@@ -130,7 +130,7 @@
     }
   },
   "datasetHistory": "Dataset History",
-  "datasetHistoryDescription": "The Mobility Database fetches and stores new datasets twice a week, on Mondays and Thursdays at midnight EST.",
+  "datasetHistoryDescription": "The Mobility Database fetches and stores new datasets once a day at midnight UTC.",
   "datasets": "Datasets",
   "validationReportNotAvailable": "Validation report not available",
   "runValidationReportYourself": "Run Validator Yourself",

web-app/src/app/screens/FAQ.tsx

@@ -174,9 +174,9 @@ export default function FAQ(): React.ReactElement {
           How often do you check for feed updates?
         </Typography>
         <Typography className='answer'>
-          The Mobility Database checks for feed updates twice a week using the
-          producer&apos;s URL, on Mondays and Thursdays. We store the new feed
-          version if we detect a change.
+          The Mobility Database checks for feed updates once a day at midnight
+          UTC using the producer&apos;s URL. We store the new feed version if we
+          detect a change.
         </Typography>
       </ColoredContainer>
     </Container>

web-app/src/app/services/feeds/types.ts

@@ -44,7 +44,7 @@ export interface paths {
     };
   };
   '/v1/gtfs_feeds/{id}/datasets': {
-    /** @description Get a list of datasets related to a GTFS feed. Once a week, we check if the latest dataset has been updated and, if so, we update it in our system accordingly. */
+    /** @description Get a list of datasets associated with a GTFS feed. Once a day, we check whether the latest dataset has changed; if it has, we update it in our system. The list is sorted from newest to oldest. */
     get: operations['getGtfsFeedDatasets'];
     parameters: {
       path: {
@@ -714,7 +714,7 @@ export interface operations {
       };
     };
   };
-  /** @description Get a list of datasets related to a GTFS feed. Once a week, we check if the latest dataset has been updated and, if so, we update it in our system accordingly. */
+  /** @description Get a list of datasets associated with a GTFS feed. Once a day, we check whether the latest dataset has changed; if it has, we update it in our system. The list is sorted from newest to oldest. */
   getGtfsFeedDatasets: {
     parameters: {
       query?: {