improving paging for ops metrics

Author: ikethecoder

src/services/keystone/batch-service.ts

@@ -61,6 +61,65 @@ export class BatchService {
     return result['data'][query].length == 0 ? [] : result['data'][query];
   }
 
+  public async listAllPages(
+    query: any,
+    fields: string[],
+    where: BatchWhereClause = undefined
+  ) {
+    const records = [];
+
+    const pageSize = 50;
+    const first = pageSize;
+    let skip = 0;
+    let more = true;
+
+    logger.debug('[listAllPages] : %s', query);
+    do {
+      logger.debug('[listAllPages] : %d', skip);
+      let queryString;
+      if (where) {
+        queryString = `query(${where.query}) {
+        ${query}(where: ${where.clause}, first: ${first}, skip: ${skip}) {
+          id, ${fields.join(',')}
+        }
+      }`;
+      } else {
+        queryString = `query {
+        ${query}(first: ${first}, skip: ${skip}) {
+          id, ${fields.join(',')}
+        }
+      }`;
+      }
+      logger.debug('[listAllPages] %s', queryString);
+
+      const result = await this.context.executeGraphQL({
+        query: queryString,
+        variables: where ? where.variables : {},
+      });
+
+      if ('errors' in result) {
+        logger.error('[listAll] RESULT %j', result);
+        return null;
+      }
+
+      more = result['data'][query].length > 0;
+
+      skip += pageSize;
+
+      logger.debug(
+        '[listAllPages] RESULT COUNT %d',
+        result['data'][query].length
+      );
+      records.push(
+        result['data'][query].length == 0 ? [] : result['data'][query]
+      );
+    } while (more);
+
+    logger.info('[listAllPages] TOTAL COUNT %d', records.length);
+
+    return records;
+  }
+
   public async list(
     query: any,
     refKey: string,

src/services/report/ops-metrics.ts

@@ -525,21 +525,15 @@ async function getAllConsumers(ctx: any) {
   const batch = new BatchService(ctx);
 
   // Limiting to 1000 is not great!  We should really recurse until we get to the end!
-  const allConsumers = await batch.listAll(
-    'allServiceAccesses',
-    [
-      'namespace',
-      'active',
-      'consumerType',
-      'consumer { username }',
-      'application { name, owner { name }}',
-      'productEnvironment { namespace, name, flow, product { name, namespace, dataset { title } } }',
-      'createdAt',
-    ],
-    undefined,
-    0,
-    1000
-  );
+  const allConsumers = await batch.listAllPages('allServiceAccesses', [
+    'namespace',
+    'active',
+    'consumerType',
+    'consumer { username }',
+    'application { name, owner { name }}',
+    'productEnvironment { namespace, name, flow, product { name, namespace, dataset { title } } }',
+    'createdAt',
+  ]);
 
   return allConsumers;
 }

src/test/integrated/batchworker/paging.ts

@@ -0,0 +1,91 @@
+/*
+Wire up directly with Keycloak and use the Services
+To run:
+npm run ts-build
+npm run ts-watch
+node dist/test/integrated/batchworker/paging.js
+*/
+
+import InitKeystone from '../keystonejs/init';
+import {
+  getRecords,
+  parseJsonString,
+  transformAllRefID,
+  removeEmpty,
+  removeKeys,
+  syncRecords,
+} from '../../../batch/feed-worker';
+import { o } from '../util';
+import { BatchService } from '../../../services/keystone/batch-service';
+import { newEnvironmentID, newProductID } from '../../../services/identifiers';
+
+(async () => {
+  const keystone = await InitKeystone();
+  console.log('K = ' + keystone);
+
+  const ns = 'platform';
+  const skipAccessControl = false;
+
+  const identity = {
+    id: null,
+    username: 'sample_username',
+    namespace: ns,
+    roles: JSON.stringify(['api-owner']),
+    scopes: [],
+    userId: null,
+  } as any;
+
+  const ctx = keystone.createContext({
+    skipAccessControl,
+    authentication: { item: identity },
+  });
+
+  if (false) {
+    const json = {
+      name: 'Refactor Time Test2',
+      namespace: ns,
+      environments: [
+        {
+          name: 'stage',
+          appId: '0A021EB0',
+          //services: [] as any,
+          //services: ['a-service-for-refactortime'],
+          // services: ['a-service-for-refactortime', 'a-service-for-aps-moh-proto'],
+        },
+      ] as any,
+    };
+    const res = await syncRecords(ctx, 'Product', null, json);
+    o(res);
+  }
+  if (false) {
+    for (let i = 0; i < 1000; i++) {
+      const appId = newProductID();
+      console.log(appId);
+      const json = {
+        name: 'Refactor Time Test 4',
+        appId: appId,
+        namespace: ns,
+        environments: [
+          {
+            name: 'stage',
+            appId: newEnvironmentID(),
+            services: [] as any,
+          },
+        ],
+      };
+      const res = await syncRecords(ctx, 'Product', appId, json);
+      o(res);
+    }
+  }
+
+  const batchService = new BatchService(ctx);
+
+  const res = await batchService.listAllPages('allProducts', [
+    'name',
+    'namespace',
+    'dataset { title }',
+    'createdAt',
+  ]);
+  //console.log(res);
+  await keystone.disconnect();
+})();