feat(data-access): fetch all records on demand (#659)

Please ensure your pull request adheres to the following guidelines:
- [ ] make sure to link the related issues in this description
- [ ] when merging / squashing, make sure the fixed issue references are
visible in the commits, for easy compilation of release notes

## Related Issues


Thanks for contributing!

Author: ekremney

packages/spacecat-shared-data-access/src/models/base/base.collection.js

@@ -248,15 +248,26 @@ class BaseCollection {
         );
       }
 
-      const records = await query.go(queryOptions);
+      // execute the initial query
+      let result = await query.go(queryOptions);
+      let allData = result.data;
+
+      // if the caller requests ALL pages and we're not using limit: 1,
+      // continue to fetch until there is no LastEvaluatedKey.
+      if (options.fetchAllPages && options.limit !== 1) {
+        while (result.lastEvaluatedKey) {
+          // update queryOptions with the start key to fetch the next page
+          queryOptions.ExclusiveStartKey = result.lastEvaluatedKey;
+          // eslint-disable-next-line no-await-in-loop
+          result = await query.go(queryOptions);
+          allData = allData.concat(result.data);
+        }
+      }
 
       if (options.limit === 1) {
-        if (records.data?.length === 0) {
-          return null;
-        }
-        return this.#createInstance(records.data[0]);
+        return allData.length ? this.#createInstance(allData[0]) : null;
       } else {
-        return this.#createInstances(records.data);
+        return this.#createInstances(allData);
       }
     } catch (error) {
       return this.#logAndThrowError('Failed to query', error);

packages/spacecat-shared-data-access/test/unit/models/base/base.collection.test.js

@@ -675,6 +675,30 @@ describe('BaseCollection', () => {
         .to.have.been.calledOnceWithExactly({ pk: 'ALL_MOCKENTITYMODELS' });
       expect(mockGo).to.have.been.calledOnceWithExactly({ order: 'desc', attributes: ['test'] });
     });
+
+    it('handles pagination with fetchAllPages option', async () => {
+      const firstResult = { data: [mockRecord], lastEvaluatedKey: 'key1' };
+      const secondRecord = { id: '2', foo: 'bar' };
+      const secondResult = { data: [secondRecord] };
+
+      const goStub = stub();
+      goStub.onFirstCall().resolves(firstResult);
+      goStub.onSecondCall().resolves(secondResult);
+
+      mockElectroService.entities.mockEntityModel.query.all.returns({
+        go: goStub,
+      });
+
+      const result = await baseCollectionInstance.all({}, { fetchAllPages: true });
+      expect(result).to.be.an('array').that.has.length(2);
+      expect(result[0].record).to.deep.include(mockRecord);
+      expect(result[1].record).to.deep.include(secondRecord);
+
+      expect(goStub.callCount).to.equal(2);
+
+      const secondCallArgs = goStub.secondCall.args[0];
+      expect(secondCallArgs).to.deep.include({ order: 'desc', ExclusiveStartKey: 'key1' });
+    });
   });
 
   describe('allByIndexKeys', () => {