dataset selection and ops metrics fix

.github/workflows/aps-cypress-e2e.yaml

@@ -28,7 +28,9 @@ jobs:
           docker build -t gwa-api:e2e .
 
       - name: Checkout Portal
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - name: Build Docker Images
         run: |
@@ -85,6 +87,32 @@ jobs:
           name: code-coverage
           path: ${{ github.workspace }}/e2e/coverage
 
+      - name: Instrument the code for coverage analysis
+        if: always()
+        run: |
+          # Rewrite the paths as the coverage starts with '../app'!
+          sed -e 's/..\/app/./g' ./e2e/coverage/lcov.info > lcov.info
+
+          #cd src
+          #npm install --legacy-peer-deps
+          #npx nyc instrument --compact=false . --in-place
+
+      - name: SonarCloud Scan
+        if: always()
+        uses: sonarsource/sonarqube-scan-action@v3.1.0
+        with:
+          args: >
+            -Dsonar.organization=bcgov-oss
+            -Dsonar.projectKey=aps-portal-e2e
+            -Dsonar.host.url=https://sonarcloud.io
+            -Dsonar.projectBaseDir=src
+            -Dsonar.sources=.
+            -Dsonar.exclusions=nextapp/**,mocks/**,test/**,tools/**,*.json,*.js
+            -Dsonar.javascript.lcov.reportPaths=/github/workspace/lcov.info
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+
       - name: Upload Astra scan results
         uses: actions/upload-artifact@v4
         with:
@@ -114,30 +142,6 @@ jobs:
             exit 1
           fi
 
-      - name: Instrument the code for coverage analysis
-        run: |
-          # Rewrite the paths as the coverage starts with '../app'!
-          sed -e 's/..\/app/./g' ./e2e/coverage/lcov.info > lcov.info
-
-          #cd src
-          #npm install --legacy-peer-deps
-          #npx nyc instrument --compact=false . --in-place
-
-      - name: SonarCloud Scan
-        uses: sonarsource/sonarqube-scan-action@master
-        with:
-          args: >
-            -Dsonar.organization=bcgov-oss
-            -Dsonar.projectKey=aps-portal-e2e
-            -Dsonar.host.url=https://sonarcloud.io
-            -Dsonar.projectBaseDir=src
-            -Dsonar.sources=.
-            -Dsonar.exclusions=nextapp/**,mocks/**,test/**,tools/**,*.json,*.js
-            -Dsonar.javascript.lcov.reportPaths=/github/workspace/lcov.info
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
-
       - name: Set up Python 3.9
         if: failure()
         uses: actions/setup-python@v2
@@ -157,4 +161,4 @@ jobs:
         env:
           JIRA_EMAIL: ${{ secrets.JIRA_EMAIL }}
           JIRA_API_KEY: ${{ secrets.JIRA_API_KEY }}
-          ASTRA_SCAN_RESULTS: ${{ github.workspace }}/e2e/cypress/fixtures/state/scanResult.json
+          ASTRA_SCAN_RESULTS: ${{ github.workspace }}/e2e/cypress/fixtures/state/scanResult.json

.github/workflows/ci-feat-sonar.yaml

@@ -15,7 +15,9 @@ jobs:
     env:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - uses: actions/setup-node@v2
         with:

local/kong/Dockerfile

@@ -1,50 +1,30 @@
-FROM kong:2.8.3
+FROM docker.io/kong:2.8.5
 
 USER root
 
-RUN apk add git ethtool strace
+RUN apk add git ethtool strace unzip
 
-ARG PLUGIN_VERSION=1.1.1-1
-ARG PLUGIN_OIDC_VERSION=1.2.4-2
-ARG PLUGIN_OIDC_CONSUMER_VERSION=0.0.1-0
-ARG PLUGIN_UPSTREAM_BASIC_VERSION=1.0.0-1
-ARG FORCE_BUILD=8
+RUN git clone -b v1.5.0-1 https://github.com/bcgov/kong-oss-plugins.git \
+ && cd kong-oss-plugins/plugins \
+ && (cd jwt-keycloak && luarocks make) \
+ && (cd oidc && luarocks make) \
+ && (cd oidc-consumer && luarocks make)
 
-RUN git clone https://github.com/ikethecoder/kong-plugin-upstream-auth-basic.git
-RUN (cd kong-plugin-upstream-auth-basic && luarocks make && luarocks pack kong-plugin-upstream-auth-basic ${PLUGIN_UPSTREAM_BASIC_VERSION})
-
-RUN luarocks install lua-resty-openidc
-RUN git clone -b v${PLUGIN_OIDC_VERSION} https://github.com/revomatico/kong-oidc.git
-RUN (cd kong-oidc && luarocks make && luarocks pack kong-oidc ${PLUGIN_OIDC_VERSION})
-
-RUN git clone https://github.com/ikethecoder/kong-oidc-consumer.git
-RUN (cd kong-oidc-consumer && luarocks make && luarocks pack kong-oidc-consumer ${PLUGIN_OIDC_CONSUMER_VERSION})
-
-RUN git clone -b kong28 https://github.com/ikethecoder/kong-plugin-jwt-keycloak.git
-RUN (cd kong-plugin-jwt-keycloak && luarocks make && luarocks pack kong-plugin-jwt-keycloak ${PLUGIN_VERSION})
-
-RUN git clone -b feature/kong-2.0-upgrade https://github.com/bcgov/gwa-kong-endpoint.git
+RUN git clone https://github.com/bcgov/gwa-kong-endpoint.git
 RUN (cd gwa-kong-endpoint && ./devBuild.sh)
 
-RUN git clone -b hotfix/ips-not-always-string https://github.com/bcgov/gwa-ip-anonymity.git
+RUN git clone https://github.com/bcgov/gwa-ip-anonymity.git
 RUN (cd gwa-ip-anonymity && ./devBuild.sh)
 
-RUN luarocks install lua-resty-jwt 0.2.2-0 \
-  && luarocks install lua-resty-session 2.26-1 \
-  && luarocks install lua-resty-openidc 1.7.5-1 \
-  && luarocks install kong-spec-expose \
-  && luarocks install kong-upstream-jwt \
-  && luarocks install kong-plugin-referer \
-  && luarocks install kong-upstream-jwt \
-  && luarocks install kong-oidc/kong-oidc-${PLUGIN_OIDC_VERSION}.all.rock \
-  && luarocks install kong-plugin-upstream-auth-basic/kong-plugin-upstream-auth-basic-${PLUGIN_UPSTREAM_BASIC_VERSION}.all.rock \
-  && luarocks install kong-oidc-consumer/kong-oidc-consumer-${PLUGIN_OIDC_CONSUMER_VERSION}.all.rock \
-  && luarocks install kong-plugin-jwt-keycloak/kong-plugin-jwt-keycloak-${PLUGIN_VERSION}.all.rock
+RUN luarocks install kong-spec-expose \
+ && luarocks install kong-plugin-referer \
+ && luarocks install kong-upstream-jwt
 
 RUN git clone https://github.com/Kong/priority-updater.git
 RUN (cd priority-updater/template/plugin && KONG_PRIORITY=902 KONG_PRIORITY_NAME=rate-limiting /usr/local/openresty/luajit/bin/luajit ../priority.lua)
 RUN (cd priority-updater/template/plugin && KONG_PRIORITY=1010 KONG_PRIORITY_NAME=jwt-keycloak /usr/local/openresty/luajit/bin/luajit ../priority.lua)
+RUN (cd priority-updater/template/plugin && KONG_PRIORITY=200 KONG_PRIORITY_NAME=post-function /usr/local/openresty/luajit/bin/luajit ../priority.lua)
 
 USER kong
 
-ENV KONG_PLUGINS="bundled, jwt-keycloak_1010, rate-limiting_902, oidc, oidc-consumer, bcgov-gwa-endpoint, gwa-ip-anonymity, kong-spec-expose, kong-upstream-jwt, referer, jwt-keycloak, kong-upstream-jwt, upstream-auth-basic"
+ENV KONG_PLUGINS="bundled, jwt-keycloak_1010, rate-limiting_902, post-function_200, oidc, oidc-consumer, bcgov-gwa-endpoint, gwa-ip-anonymity, kong-spec-expose, kong-upstream-jwt, referer, jwt-keycloak, kong-upstream-jwt"

src/nextapp/components/products-list/dataset-input.tsx

@@ -52,7 +52,9 @@ const DatasetInput: React.FC<DatasetInputProps> = ({ dataset }) => {
     [setSearch]
   );
   const handleBlur = () => {
-    if (search.trim()) {
+    if (selected) {
+      setSelected(selected);
+    } else if (search.trim()) {
       const result = data?.allDatasets.find((d) => {
         if (search.trim()) {
           return d.title.toLowerCase() === search.toLowerCase();
@@ -76,6 +78,11 @@ const DatasetInput: React.FC<DatasetInputProps> = ({ dataset }) => {
 
   const isInvalid = search.length > 0 && !selected;
 
+  const titleCounts = (results || []).reduce((acc, d) => {
+    acc[d.title] = (acc[d.title] || 0) + 1;
+    return acc;
+  }, {} as Record<string, number>);
+
   return (
     <>
       <FormControl id="dataset" position="relative" isInvalid={isInvalid}>
@@ -158,7 +165,9 @@ const DatasetInput: React.FC<DatasetInputProps> = ({ dataset }) => {
                         },
                       })}
                     >
-                      <Text fontSize="md">{d.title}</Text>
+                      <Text fontSize="md">
+                        {d.title} {titleCounts[d.title] > 1 ? `(${d.name})` : ''}
+                      </Text>
                     </Box>
                   ))}
                 {isOpen && isSuccess && !results.length && (

src/services/keystone/batch-service.ts

@@ -61,6 +61,65 @@ export class BatchService {
     return result['data'][query].length == 0 ? [] : result['data'][query];
   }
 
+  public async listAllPages(
+    query: any,
+    fields: string[],
+    where: BatchWhereClause = undefined
+  ) {
+    const records: any[] = [];
+
+    const pageSize = 50;
+    const first = pageSize;
+    let skip = 0;
+    let more = true;
+
+    logger.debug('[listAllPages] : %s', query);
+    do {
+      logger.debug('[listAllPages] : %d', skip);
+      let queryString;
+      if (where) {
+        queryString = `query(${where.query}) {
+        ${query}(where: ${where.clause}, first: ${first}, skip: ${skip}) {
+          id, ${fields.join(',')}
+        }
+      }`;
+      } else {
+        queryString = `query {
+        ${query}(first: ${first}, skip: ${skip}) {
+          id, ${fields.join(',')}
+        }
+      }`;
+      }
+      logger.debug('[listAllPages] %s', queryString);
+
+      const result = await this.context.executeGraphQL({
+        query: queryString,
+        variables: where ? where.variables : {},
+      });
+
+      if ('errors' in result) {
+        logger.error('[listAll] RESULT %j', result);
+        return null;
+      }
+
+      more = result['data'][query].length > 0;
+
+      skip += pageSize;
+
+      logger.debug(
+        '[listAllPages] RESULT COUNT %d',
+        result['data'][query].length
+      );
+      records.push(
+        ...(result['data'][query].length == 0 ? [] : result['data'][query])
+      );
+    } while (more);
+
+    logger.info('[listAllPages] (%s) TOTAL COUNT %d', query, records.length);
+
+    return records;
+  }
+
   public async list(
     query: any,
     refKey: string,

src/services/report/ops-metrics.ts

@@ -524,22 +524,15 @@ async function getAllRoutes(ctx: any) {
 async function getAllConsumers(ctx: any) {
   const batch = new BatchService(ctx);
 
-  // Limiting to 1000 is not great!  We should really recurse until we get to the end!
-  const allConsumers = await batch.listAll(
-    'allServiceAccesses',
-    [
-      'namespace',
-      'active',
-      'consumerType',
-      'consumer { username }',
-      'application { name, owner { name }}',
-      'productEnvironment { namespace, name, flow, product { name, namespace, dataset { title } } }',
-      'createdAt',
-    ],
-    undefined,
-    0,
-    1000
-  );
+  const allConsumers = await batch.listAllPages('allServiceAccesses', [
+    'namespace',
+    'active',
+    'consumerType',
+    'consumer { username }',
+    'application { name, owner { name }}',
+    'productEnvironment { namespace, name, flow, product { name, namespace, dataset { title } } }',
+    'createdAt',
+  ]);
 
   return allConsumers;
 }

src/test/integrated/batchworker/paging.ts

@@ -0,0 +1,91 @@
+/*
+Wire up directly with Keycloak and use the Services
+To run:
+npm run ts-build
+npm run ts-watch
+node dist/test/integrated/batchworker/paging.js
+*/
+
+import InitKeystone from '../keystonejs/init';
+import {
+  getRecords,
+  parseJsonString,
+  transformAllRefID,
+  removeEmpty,
+  removeKeys,
+  syncRecords,
+} from '../../../batch/feed-worker';
+import { o } from '../util';
+import { BatchService } from '../../../services/keystone/batch-service';
+import { newEnvironmentID, newProductID } from '../../../services/identifiers';
+
+(async () => {
+  const keystone = await InitKeystone();
+  console.log('K = ' + keystone);
+
+  const ns = 'platform';
+  const skipAccessControl = false;
+
+  const identity = {
+    id: null,
+    username: 'sample_username',
+    namespace: ns,
+    roles: JSON.stringify(['api-owner']),
+    scopes: [],
+    userId: null,
+  } as any;
+
+  const ctx = keystone.createContext({
+    skipAccessControl,
+    authentication: { item: identity },
+  });
+
+  if (false) {
+    const json = {
+      name: 'Refactor Time Test2',
+      namespace: ns,
+      environments: [
+        {
+          name: 'stage',
+          appId: '0A021EB0',
+          //services: [] as any,
+          //services: ['a-service-for-refactortime'],
+          // services: ['a-service-for-refactortime', 'a-service-for-aps-moh-proto'],
+        },
+      ] as any,
+    };
+    const res = await syncRecords(ctx, 'Product', null, json);
+    o(res);
+  }
+  if (false) {
+    for (let i = 0; i < 1000; i++) {
+      const appId = newProductID();
+      console.log(appId);
+      const json = {
+        name: 'Refactor Time Test 4',
+        appId: appId,
+        namespace: ns,
+        environments: [
+          {
+            name: 'stage',
+            appId: newEnvironmentID(),
+            services: [] as any,
+          },
+        ],
+      };
+      const res = await syncRecords(ctx, 'Product', appId, json);
+      o(res);
+    }
+  }
+
+  const batchService = new BatchService(ctx);
+
+  const res = await batchService.listAllPages('allProducts', [
+    'name',
+    'namespace',
+    'dataset { title }',
+    'createdAt',
+  ]);
+  //console.log(res);
+  await keystone.disconnect();
+})();