feat: add handling for paginated data and allow loading all pages at once (#77)

mariusheine · anbraten · web-flow · commit d43c2211bbae · 2022-08-22T22:36:12.000+02:00
* add handling of paginated requests/responses

* run prettier

* fix branch testing coverage

* add test case for single entity to fix branch testing coverage

* fix eslint issues

* add option for chunking and add some comments to the code

* fix typecheck error

* fix unit tests

* rename test cases a bit

Co-authored-by: Anbraten &lt;anton@ju60.de&gt;

* make unit test title more specific

* restructured page handling

* fix unit test

Co-authored-by: Anbraten &lt;anton@ju60.de&gt;
diff --git a/src/useFind.ts b/src/useFind.ts
@@ -1,9 +1,10 @@
+import { AdapterService } from '@feathersjs/adapter-commons/lib';
 import type { FeathersError } from '@feathersjs/errors';
-import type { Application, FeathersService, Params, ServiceMethods } from '@feathersjs/feathers';
+import type { Application, FeathersService, Paginated, Params, Query, ServiceMethods } from '@feathersjs/feathers';
 import sift from 'sift';
 import { getCurrentInstance, onBeforeUnmount, Ref, ref, watch } from 'vue';
 
-import { getId, ServiceModel, ServiceTypes } from './utils';
+import { getId, isPaginated, ServiceModel, ServiceTypes } from './utils';
 
 function loadServiceEventHandlers<
   CustomApplication extends Application,
@@ -87,21 +88,32 @@ export type UseFindFunc<CustomApplication> = <
   params?: Ref<Params | undefined | null>,
 ) => UseFind<M>;
 
+type Options = {
+  disableUnloadingEventHandlers: boolean;
+  loadAllPages: boolean;
+};
+
+const defaultOptions: Options = { disableUnloadingEventHandlers: false, loadAllPages: false };
+
 export default <CustomApplication extends Application>(feathers: CustomApplication) =>
   <T extends keyof ServiceTypes<CustomApplication>, M = ServiceModel<CustomApplication, T>>(
     serviceName: T,
     params: Ref<Params | undefined | null> = ref({ paginate: false, query: {} }),
-    { disableUnloadingEventHandlers } = { disableUnloadingEventHandlers: false },
+    options: Partial<Options> = {},
   ): UseFind<M> => {
+    const { disableUnloadingEventHandlers, loadAllPages } = { ...defaultOptions, ...options };
     // type cast is fine here (source: https://github.com/vuejs/vue-next/issues/2136#issuecomment-693524663)
     const data = ref<M[]>([]) as Ref<M[]>;
     const isLoading = ref(false);
     const error = ref<FeathersError>();
 
     const service = feathers.service(serviceName as string);
     const unloadEventHandlers = loadServiceEventHandlers(service, params, data);
+    let unloaded = false;
+
+    const currentFindCall = ref(0);
 
-    const find = async () => {
+    const find = async (call: number) => {
       isLoading.value = true;
       error.value = undefined;
 
@@ -112,10 +124,44 @@ export default <CustomApplication extends Application>(feathers: CustomApplicati
       }
 
       try {
+        const originalParams: Params = params.value;
+        const originalQuery: Query & { $limit?: number } = originalParams.query || {};
         // TODO: the typecast below is necessary due to the prerelease state of feathers v5. The problem there is
         // that the AdapterService interface is not yet updated and is not compatible with the ServiceMethods interface.
-        const res = await (service as unknown as ServiceMethods<M>).find(params.value);
-        data.value = Array.isArray(res) ? res : [res];
+        const res = await (service as unknown as ServiceMethods<M> | AdapterService<M>).find(originalParams);
+        if (call !== currentFindCall.value) {
+          // stop handling response since there already is a new find call running within this composition
+          return;
+        }
+        if (isPaginated(res) && !loadAllPages) {
+          data.value = [...res.data];
+        } else if (!isPaginated(res)) {
+          data.value = Array.isArray(res) ? res : [res];
+        } else {
+          // extract data from page response
+          let loadedPage: Paginated<M> = res;
+          let loadedItemsCount = loadedPage.data.length;
+          data.value = [...loadedPage.data];
+          // limit might not be specified in the original query if default pagination from backend is applied, that's why we use this fallback pattern
+          const limit: number = originalQuery.$limit || loadedPage.data.length;
+          // if chunking is enabled we go on requesting all following pages until all data have been received
+          while (!unloaded && loadedPage.total > loadedItemsCount) {
+            // skip can be a string in cases where key based chunking/pagination is done e.g. in DynamoDb via `LastEvaluatedKey`
+            const skip: string | number =
+              typeof loadedPage.skip === 'string' ? loadedPage.skip : loadedPage.skip + limit;
+            // request next page
+            loadedPage = (await (service as unknown as ServiceMethods<M> | AdapterService<M>).find({
+              ...originalParams,
+              query: { ...originalQuery, $skip: skip, $limit: limit },
+            })) as Paginated<M>;
+            if (call !== currentFindCall.value) {
+              // stop handling/requesting further pages since there already is a new find call running within this composition
+              return;
+            }
+            loadedItemsCount += loadedPage.data.length;
+            data.value = [...data.value, ...loadedPage.data];
+          }
+        }
       } catch (_error) {
         error.value = _error as FeathersError;
       }
@@ -124,10 +170,12 @@ export default <CustomApplication extends Application>(feathers: CustomApplicati
     };
 
     const load = () => {
-      void find();
+      currentFindCall.value = currentFindCall.value + 1;
+      void find(currentFindCall.value);
     };
 
     const unload = () => {
+      unloaded = true;
       unloadEventHandlers();
       feathers.off('connect', load);
     };
diff --git a/src/utils.ts b/src/utils.ts
@@ -1,5 +1,5 @@
 import type { AdapterService } from '@feathersjs/adapter-commons';
-import type { Application, Id, ServiceMethods } from '@feathersjs/feathers';
+import type { Application, Id, Paginated, ServiceMethods } from '@feathersjs/feathers';
 
 export type PotentialIds = {
   id?: Id;
@@ -29,3 +29,8 @@ export type ServiceModel<
   : ServiceTypes<CustomApplication>[T] extends ServiceMethods<infer M2>
   ? M2
   : never;
+
+export function isPaginated<T>(response: T | T[] | Paginated<T>): response is Paginated<T> {
+  const { total, limit, skip, data } = response as Paginated<T>;
+  return total !== undefined && limit !== undefined && skip !== undefined && data !== undefined && Array.isArray(data);
+}
diff --git a/test/useFind.test.ts b/test/useFind.test.ts
@@ -1,5 +1,5 @@
 import { FeathersError, GeneralError } from '@feathersjs/errors';
-import type { Application, Params } from '@feathersjs/feathers';
+import type { Application, Paginated, Params } from '@feathersjs/feathers';
 import { flushPromises } from '@vue/test-utils';
 import { beforeEach, describe, expect, it, vi } from 'vitest';
 import { nextTick, ref } from 'vue';
@@ -423,6 +423,36 @@ describe('Find composition', () => {
     expect(findComposition && findComposition.error.value).toBeTruthy();
   });
 
+  it('should load single entity response', async () => {
+    expect.assertions(3);
+
+    // given
+    const serviceFind = vi.fn(() => testModel);
+
+    const feathersMock = {
+      service: () => ({
+        find: serviceFind,
+        on: vi.fn(),
+        off: vi.fn(),
+      }),
+      on: vi.fn(),
+      off: vi.fn(),
+    } as unknown as Application;
+    const useFind = useFindOriginal(feathersMock);
+
+    // when
+    let findComposition = null as UseFind<TestModel> | null;
+    mountComposition(() => {
+      findComposition = useFind('testModels');
+    });
+    await nextTick();
+
+    // then
+    expect(serviceFind).toHaveBeenCalledTimes(1);
+    expect(findComposition).toBeTruthy();
+    expect(findComposition && findComposition.data.value).toStrictEqual([testModel]);
+  });
+
   describe('Event Handlers', () => {
     it('should listen to "create" events', () => {
       expect.assertions(2);
@@ -935,4 +965,173 @@ describe('Find composition', () => {
       expect(serviceOff).toHaveBeenCalledTimes(4); // unload of: created, updated, patched, removed events
     });
   });
+
+  describe('pagination', () => {
+    it('should handle paginated data', async () => {
+      expect.assertions(3);
+
+      // given
+      let startItemIndex = 0;
+      const serviceFind = vi.fn(() => {
+        const page: Paginated<TestModel> = {
+          total: testModels.length,
+          skip: startItemIndex,
+          limit: 1,
+          data: testModels.slice(startItemIndex, startItemIndex + 1),
+        };
+        startItemIndex++;
+        return page;
+      });
+
+      const feathersMock = {
+        service: () => ({
+          find: serviceFind,
+          on: vi.fn(),
+          off: vi.fn(),
+        }),
+        on: vi.fn(),
+        off: vi.fn(),
+      } as unknown as Application;
+      const useFind = useFindOriginal(feathersMock);
+
+      // when
+      let findComposition = null as UseFind<TestModel> | null;
+      mountComposition(() => {
+        findComposition = useFind('testModels');
+      });
+      await nextTick();
+
+      // then
+      expect(serviceFind).toHaveBeenCalledTimes(1);
+      expect(findComposition).toBeTruthy();
+      expect(findComposition && findComposition.data.value).toStrictEqual(testModels.slice(0, 1));
+    });
+
+    it('should load all data with chunking', async () => {
+      expect.assertions(3);
+
+      // given
+      let startItemIndex = 0;
+      const serviceFind = vi.fn(() => {
+        const page: Paginated<TestModel> = {
+          total: testModels.length,
+          skip: startItemIndex,
+          limit: 1,
+          data: testModels.slice(startItemIndex, startItemIndex + 1),
+        };
+        startItemIndex++;
+        return page;
+      });
+
+      const feathersMock = {
+        service: () => ({
+          find: serviceFind,
+          on: vi.fn(),
+          off: vi.fn(),
+        }),
+        on: vi.fn(),
+        off: vi.fn(),
+      } as unknown as Application;
+      const useFind = useFindOriginal(feathersMock);
+
+      // when
+      let findComposition = null as UseFind<TestModel> | null;
+      mountComposition(() => {
+        findComposition = useFind('testModels', undefined, { loadAllPages: true });
+      });
+      await nextTick();
+
+      // then
+      expect(serviceFind).toHaveBeenCalledTimes(2);
+      expect(findComposition).toBeTruthy();
+      expect(findComposition && findComposition.data.value).toStrictEqual(testModels);
+    });
+
+    it('should load data with pagination using lastEvaluatedKey patterns', async () => {
+      expect.assertions(3);
+
+      // given
+      const serviceFind = vi.fn((params?: Params) => {
+        const startItemIndex = testModels.findIndex(({ _id }) => _id === params?.query?.$skip) + 1;
+        const data = testModels.slice(startItemIndex, startItemIndex + 1);
+        const page: Paginated<TestModel> = {
+          total: testModels.length,
+          skip: data[data.length - 1]._id as unknown as number,
+          limit: 1,
+          data,
+        };
+        return page;
+      });
+
+      const feathersMock = {
+        service: () => ({
+          find: serviceFind,
+          on: vi.fn(),
+          off: vi.fn(),
+        }),
+        on: vi.fn(),
+        off: vi.fn(),
+      } as unknown as Application;
+      const useFind = useFindOriginal(feathersMock);
+
+      // when
+      let findComposition = null as UseFind<TestModel> | null;
+      mountComposition(() => {
+        findComposition = useFind('testModels', undefined, { loadAllPages: true });
+      });
+      await nextTick();
+
+      // then
+      expect(serviceFind).toHaveBeenCalledTimes(2);
+      expect(findComposition).toBeTruthy();
+      expect(findComposition && findComposition.data.value).toStrictEqual(testModels);
+    });
+
+    it('should stop further page requests if find was retriggered due to a change to params or connection reset', async () => {
+      expect.assertions(3);
+
+      // given
+      let startItemIndex = 0;
+      let data = [additionalTestModel, ...testModels];
+      const serviceFind = vi.fn(() => {
+        const page: Paginated<TestModel> = {
+          total: data.length,
+          skip: startItemIndex,
+          limit: 1,
+          data: data.slice(startItemIndex, startItemIndex + 1),
+        };
+        startItemIndex++;
+        return page;
+      });
+      const emitter = eventHelper();
+      const feathersMock = {
+        service: () => ({
+          find: serviceFind,
+          on: vi.fn(),
+          off: vi.fn(),
+        }),
+        on: emitter.on,
+        off: vi.fn(),
+      } as unknown as Application;
+      const useFind = useFindOriginal(feathersMock);
+      let findComposition = null as UseFind<TestModel> | null;
+      mountComposition(() => {
+        findComposition = useFind('testModels', undefined, { loadAllPages: true });
+      });
+      await nextTick();
+      serviceFind.mockClear();
+      data = testModels;
+      startItemIndex = 0;
+
+      // when
+      emitter.emit('connect');
+      await nextTick();
+      await nextTick();
+
+      // then
+      expect(serviceFind).toHaveBeenCalledTimes(2);
+      expect(findComposition).toBeTruthy();
+      expect(findComposition && findComposition.data.value).toStrictEqual(testModels);
+    });
+  });
 });
