Feat: dataset pagination (#1075)

Author: Alessandro100

web-app/cypress/e2e/feeds.cy.ts

@@ -10,7 +10,7 @@ describe('Feed page', () => {
     cy.intercept('GET', `${apiBaseUrl}/v1/gtfs_feeds/test-516`, gtfsFeedJson);
     cy.intercept(
       'GET',
-      `${apiBaseUrl}/v1/gtfs_feeds/test-516/datasets`,
+      `${apiBaseUrl}/v1/gtfs_feeds/test-516/datasets?offset=0&limit=10`,
       datasetsFeedJson,
     );
     cy.visit('feeds/test-516');
@@ -25,7 +25,10 @@ describe('Feed page', () => {
   });
 
   it('should render the last updated date', () => {
-    cy.get('[data-testid="last-updated"]').should('contain', 'Quality report updated');
+    cy.get('[data-testid="last-updated"]').should(
+      'contain',
+      'Quality report updated',
+    );
   });
 
   it('should render download button', () => {

web-app/public/locales/en/feeds.json

@@ -131,6 +131,7 @@
   },
   "datasetHistory": "Dataset History",
   "datasetHistoryDescription": "The Mobility Database fetches and stores new datasets once a day at midnight UTC.",
+  "allDatasetsLoaded": "All datasets loaded",
   "datasets": "Datasets",
   "validationReportNotAvailable": "Validation report not available",
   "runValidationReportYourself": "Run Validator Yourself",

web-app/src/app/screens/Feed/PreviousDatasets.tsx

@@ -34,6 +34,7 @@ import {
   selectBoundingBoxFromLatestDataset,
   selectDatasetsData,
   selectDatasetsLoadingStatus,
+  selectHasLoadedAllDatasets,
   selectLatestDatasetsData,
 } from '../../store/dataset-selectors';
 import PreviousDatasets from './PreviousDatasets';
@@ -121,6 +122,7 @@ export default function Feed(): React.ReactElement {
   const relatedFeeds = useSelector(selectRelatedFeedsData);
   const relatedGtfsRtFeeds = useSelector(selectRelatedGtfsRTFeedsData);
   const datasets = useSelector(selectDatasetsData);
+  const hasLoadedAllDatasets = useSelector(selectHasLoadedAllDatasets);
   const latestDataset = useSelector(selectLatestDatasetsData);
   const boundingBox = useSelector(selectBoundingBoxFromLatestDataset);
   const gtfsFeedData = useSelector(selectGTFSFeedData);
@@ -130,13 +132,26 @@ export default function Feed(): React.ReactElement {
   const isAuthenticatedOrAnonymous =
     useSelector(selectIsAuthenticated) || useSelector(selectIsAnonymous);
   const sortedProviders = formatProvidersSorted(feed?.provider ?? '');
+  const DATASET_CALL_LIMIT = 10;
+
+  const loadDatasets = (offset: number): void => {
+    if (feedId != undefined && hasLoadedAllDatasets === false) {
+      dispatch(
+        loadingDataset({
+          feedId,
+          offset,
+          limit: DATASET_CALL_LIMIT,
+        }),
+      );
+    }
+  };
 
   useEffect(() => {
     if (user != undefined && feedId != undefined && needsToLoadFeed) {
       dispatch(clearDataset());
       dispatch(loadingFeed({ feedId, feedDataType }));
       if (feedDataType === 'gtfs') {
-        dispatch(loadingDataset({ feedId }));
+        loadDatasets(0);
       }
     }
   }, [isAuthenticatedOrAnonymous, needsToLoadFeed]);
@@ -167,7 +182,7 @@ export default function Feed(): React.ReactElement {
       feedLoadingStatus === 'loaded' &&
       datasets == undefined
     ) {
-      dispatch(loadingDataset({ feedId }));
+      loadDatasets(0);
     }
     return () => {
       document.title = 'Mobility Database';
@@ -176,7 +191,9 @@ export default function Feed(): React.ReactElement {
 
   // The feedId parameter doesn't match the feedId in the store, so we need to load the feed and only render the loading message.
   const areDatasetsLoading =
-    feed?.data_type === 'gtfs' && datasetLoadingStatus === 'loading';
+    feed?.data_type === 'gtfs' &&
+    datasetLoadingStatus === 'loading' &&
+    datasets == undefined;
   const isCurrenltyLoadingFeed =
     feedLoadingStatus === 'loading' || areDatasetsLoading;
   if (needsToLoadFeed || isCurrenltyLoadingFeed) {
@@ -483,7 +500,16 @@ export default function Feed(): React.ReactElement {
       </Grid>
       {feed?.data_type === 'gtfs' && hasDatasets && (
         <Grid item xs={12}>
-          <PreviousDatasets datasets={datasets} />
+          <PreviousDatasets
+            datasets={datasets}
+            isLoadingDatasets={datasetLoadingStatus === 'loading'}
+            hasloadedAllDatasets={
+              hasLoadedAllDatasets != undefined && hasLoadedAllDatasets
+            }
+            loadMoreDatasets={(offset: number) => {
+              loadDatasets(offset);
+            }}
+          />
         </Grid>
       )}
     </Box>,

web-app/src/app/screens/Feed/index.tsx

@@ -1,20 +1,23 @@
 import { createSlice, type PayloadAction } from '@reduxjs/toolkit';
 import { type FeedErrors, FeedErrorSource, type FeedError } from '../types';
 import { type paths } from '../services/feeds/types';
+import { mergeAndSortDatasets } from '../utils/dataset';
 
 interface DatasetState {
   status: 'loading' | 'loaded';
   datasetId: string | undefined;
   data:
     | paths['/v1/gtfs_feeds/{id}/datasets']['get']['responses'][200]['content']['application/json']
     | undefined;
+  loadedAllData?: boolean;
   errors: FeedErrors;
 }
 
 const initialState: DatasetState = {
   status: 'loading',
   datasetId: undefined,
   data: undefined,
+  loadedAllData: false,
   errors: {
     [FeedErrorSource.DatabaseAPI]: null,
   },
@@ -26,6 +29,7 @@ export const datasetSlice = createSlice({
   reducers: {
     clearDataset: (state) => {
       state.data = initialState.data;
+      state.loadedAllData = initialState.loadedAllData;
       state.errors = initialState.errors;
       state.status = initialState.status;
       state.datasetId = initialState.datasetId;
@@ -42,10 +46,11 @@ export const datasetSlice = createSlice({
       state,
       action: PayloadAction<{
         feedId: string;
+        offset?: number;
+        limit?: number;
       }>,
     ) => {
       state.status = 'loading';
-      state.data = undefined;
       state.errors = {
         ...state.errors,
         DatabaseAPI: initialState.errors.DatabaseAPI,
@@ -55,18 +60,12 @@ export const datasetSlice = createSlice({
       state,
       action: PayloadAction<{
         data: paths['/v1/gtfs_feeds/{id}/datasets']['get']['responses'][200]['content']['application/json'];
+        loadedAllData?: boolean;
       }>,
     ) => {
       state.status = 'loaded';
-      state.data = action.payload?.data.sort((a, b) => {
-        if (a.downloaded_at !== undefined && b.downloaded_at !== undefined) {
-          const dateB = new Date(b.downloaded_at).getTime();
-          const dateA = new Date(a.downloaded_at).getTime();
-          return dateB - dateA;
-        }
-        return 0;
-      });
-      // state.datasetId = action.payload.data?.id;
+      state.loadedAllData = action.payload?.loadedAllData;
+      state.data = mergeAndSortDatasets(action.payload?.data, state.data);
       state.errors = {
         ...state.errors,
         DatabaseAPI: initialState.errors.DatabaseAPI,

web-app/src/app/store/dataset-reducer.ts

@@ -21,6 +21,12 @@ export const selectLatestDatasetsData = (
   return state.dataset.data !== undefined ? state.dataset.data[0] : undefined;
 };
 
+export const selectHasLoadedAllDatasets = (
+  state: RootState,
+): boolean | undefined => {
+  return state.dataset.loadedAllData;
+};
+
 export const selectBoundingBoxFromLatestDataset = createSelector(
   [selectLatestDatasetsData],
   (latestDataset): LatLngExpression[] | undefined => {

web-app/src/app/store/dataset-selectors.ts

@@ -7,20 +7,33 @@ import { getGtfsFeedDatasets } from '../../services/feeds';
 import { type paths } from '../../services/feeds/types';
 import { loadingDatasetSuccess } from '../dataset-reducer';
 import { getUserAccessToken } from '../../services';
+import { areAllDatasetsLoaded } from '../../utils/dataset';
 
 function* getDatasetSaga({
-  payload: { feedId },
-}: PayloadAction<{ feedId: string }>): Generator {
+  payload: { feedId, offset, limit },
+}: PayloadAction<{
+  feedId: string;
+  offset?: number;
+  limit?: number;
+}>): Generator {
   try {
     if (feedId !== undefined) {
       const accessToken = (yield call(getUserAccessToken)) as string;
-      const datasets = (yield call(
-        getGtfsFeedDatasets,
-        feedId,
-        accessToken,
-        {},
-      )) as paths['/v1/gtfs_feeds/{id}/datasets']['get']['responses'][200]['content']['application/json'];
-      yield put(loadingDatasetSuccess({ data: datasets }));
+      const datasets = (yield call(getGtfsFeedDatasets, feedId, accessToken, {
+        offset,
+        limit,
+      })) as paths['/v1/gtfs_feeds/{id}/datasets']['get']['responses'][200]['content']['application/json'];
+      const hasLoadedAllData = areAllDatasetsLoaded(
+        datasets.length,
+        limit,
+        offset,
+      );
+      yield put(
+        loadingDatasetSuccess({
+          data: datasets,
+          loadedAllData: hasLoadedAllData,
+        }),
+      );
     }
   } catch (error) {
     yield put(loadingFeedFail(getAppError(error) as FeedError));

web-app/src/app/store/saga/dataset-saga.ts

@@ -0,0 +1,72 @@
+import { type paths } from '../services/feeds/types';
+import { areAllDatasetsLoaded, mergeAndSortDatasets } from './dataset';
+
+type Datasets =
+  paths['/v1/gtfs_feeds/{id}/datasets']['get']['responses'][200]['content']['application/json'];
+
+const newDatasets = [
+  { id: 1, downloaded_at: '2023-01-02T00:00:00Z' },
+  { id: 2, downloaded_at: '2023-02-01T00:00:00Z' },
+] as unknown as Datasets;
+
+const existingDatasets = [
+  { id: 3, downloaded_at: '2023-01-12T00:00:00Z' },
+  { id: 4, downloaded_at: '2023-03-01T00:00:00Z' },
+] as unknown as Datasets;
+
+const duplicateDatasets = [
+  { id: 2, downloaded_at: '2023-02-01T00:00:00Z' },
+  { id: 5, downloaded_at: '2023-05-12T00:00:00Z' },
+] as unknown as Datasets;
+
+describe('Dataset utils', () => {
+  describe('mergeAndSortDatasets', () => {
+    it('should return the sorted datasets when no existing datasets are provided', () => {
+      const result = mergeAndSortDatasets(newDatasets, undefined);
+      expect(result).toEqual([
+        { id: 2, downloaded_at: '2023-02-01T00:00:00Z' },
+        { id: 1, downloaded_at: '2023-01-02T00:00:00Z' },
+      ]);
+    });
+
+    it('should return the merged and sorted datasets when existing datasets are provided', () => {
+      const result = mergeAndSortDatasets(newDatasets, existingDatasets);
+      expect(result).toEqual([
+        { id: 3, downloaded_at: '2023-01-12T00:00:00Z' },
+        { id: 4, downloaded_at: '2023-03-01T00:00:00Z' },
+        { id: 2, downloaded_at: '2023-02-01T00:00:00Z' },
+        { id: 1, downloaded_at: '2023-01-02T00:00:00Z' },
+      ]);
+    });
+
+    it('should filter out duplicates and return the merged and sorted datasets', () => {
+      const result = mergeAndSortDatasets(newDatasets, duplicateDatasets);
+      expect(result).toEqual([
+        { id: 2, downloaded_at: '2023-02-01T00:00:00Z' },
+        { id: 5, downloaded_at: '2023-05-12T00:00:00Z' },
+        { id: 1, downloaded_at: '2023-01-02T00:00:00Z' },
+      ]);
+    });
+  });
+  describe('areAllDatasetsLoaded', () => {
+    it('should return true if offset and limit are undefined', () => {
+      const result = areAllDatasetsLoaded(3, undefined, undefined);
+      expect(result).toBe(true);
+    });
+
+    it('should return true if the number of datasets returned is less than the limit', () => {
+      const result = areAllDatasetsLoaded(3, 5, undefined);
+      expect(result).toBe(true);
+    });
+
+    it('should return undefined if offset is defined and limit is undefined', () => {
+      const result = areAllDatasetsLoaded(3, undefined, 0);
+      expect(result).toBe(undefined);
+    });
+
+    it('should return false if the number of datasets returned is greater than the limit', () => {
+      const result = areAllDatasetsLoaded(3, 2, 5);
+      expect(result).toBe(false);
+    });
+  });
+});

web-app/src/app/utils/dataset.spec.ts

@@ -0,0 +1,58 @@
+import { type paths } from '../services/feeds/types';
+
+type Datasets =
+  paths['/v1/gtfs_feeds/{id}/datasets']['get']['responses'][200]['content']['application/json'];
+
+export function mergeAndSortDatasets(
+  newDatasets: Datasets,
+  existingDatasets: Datasets | undefined,
+): Datasets {
+  let formattedDatasets: Datasets = [];
+  if (existingDatasets === undefined) {
+    formattedDatasets = newDatasets.sort((a, b) => {
+      if (a.downloaded_at !== undefined && b.downloaded_at !== undefined) {
+        const dateB = new Date(b.downloaded_at).getTime();
+        const dateA = new Date(a.downloaded_at).getTime();
+        return dateB - dateA;
+      }
+      return 0;
+    });
+  } else {
+    const existingIds = new Set(existingDatasets.map((item) => item.id));
+    const newFilteredData = newDatasets.filter(
+      (item) => !existingIds.has(item.id),
+    );
+    const sortedNewFilteredData = newFilteredData.sort((a, b) => {
+      if (a.downloaded_at !== undefined && b.downloaded_at !== undefined) {
+        const dateB = new Date(b.downloaded_at).getTime();
+        const dateA = new Date(a.downloaded_at).getTime();
+        return dateB - dateA;
+      }
+      return 0;
+    });
+    formattedDatasets = [...existingDatasets, ...sortedNewFilteredData];
+  }
+  return formattedDatasets;
+}
+
+/*
+ Function to determine if all datasets are loaded given the offset and limit
+ False is if the datasets are not finished loading due to offset / limit
+ True is if the datasets are all loaded for the given feed
+ Undefined is if there is not enough information to determine if the datasets are all loaded
+*/
+export function areAllDatasetsLoaded(
+  numberOfDatasetsLoaded: number,
+  limit?: number,
+  offset?: number,
+): boolean | undefined {
+  let hasLoadedAllData: boolean | undefined = false;
+  if (offset == undefined && limit == undefined) {
+    hasLoadedAllData = true;
+  } else if (limit != undefined && numberOfDatasetsLoaded < limit) {
+    hasLoadedAllData = true;
+  } else if (offset != undefined && limit == undefined) {
+    hasLoadedAllData = undefined;
+  }
+  return hasLoadedAllData;
+}