diff --git a/.github/workflows/aps-cypress-e2e.yaml b/.github/workflows/aps-cypress-e2e.yaml index 5245af7bf..f0ce8c604 100644 --- a/.github/workflows/aps-cypress-e2e.yaml +++ b/.github/workflows/aps-cypress-e2e.yaml @@ -28,7 +28,9 @@ jobs: docker build -t gwa-api:e2e . - name: Checkout Portal - uses: actions/checkout@v2 + uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Build Docker Images run: | @@ -85,6 +87,32 @@ jobs: name: code-coverage path: ${{ github.workspace }}/e2e/coverage + - name: Instrument the code for coverage analysis + if: always() + run: | + # Rewrite the paths as the coverage starts with '../app'! + sed -e 's/..\/app/./g' ./e2e/coverage/lcov.info > lcov.info + + #cd src + #npm install --legacy-peer-deps + #npx nyc instrument --compact=false . --in-place + + - name: SonarCloud Scan + if: always() + uses: sonarsource/sonarqube-scan-action@v3.1.0 + with: + args: > + -Dsonar.organization=bcgov-oss + -Dsonar.projectKey=aps-portal-e2e + -Dsonar.host.url=https://sonarcloud.io + -Dsonar.projectBaseDir=src + -Dsonar.sources=. + -Dsonar.exclusions=nextapp/**,mocks/**,test/**,tools/**,*.json,*.js + -Dsonar.javascript.lcov.reportPaths=/github/workspace/lcov.info + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + - name: Upload Astra scan results uses: actions/upload-artifact@v4 with: @@ -114,30 +142,6 @@ jobs: exit 1 fi - - name: Instrument the code for coverage analysis - run: | - # Rewrite the paths as the coverage starts with '../app'! - sed -e 's/..\/app/./g' ./e2e/coverage/lcov.info > lcov.info - - #cd src - #npm install --legacy-peer-deps - #npx nyc instrument --compact=false . --in-place - - - name: SonarCloud Scan - uses: sonarsource/sonarqube-scan-action@master - with: - args: > - -Dsonar.organization=bcgov-oss - -Dsonar.projectKey=aps-portal-e2e - -Dsonar.host.url=https://sonarcloud.io - -Dsonar.projectBaseDir=src - -Dsonar.sources=. - -Dsonar.exclusions=nextapp/**,mocks/**,test/**,tools/**,*.json,*.js - -Dsonar.javascript.lcov.reportPaths=/github/workspace/lcov.info - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - - name: Set up Python 3.9 if: failure() uses: actions/setup-python@v2 @@ -157,4 +161,4 @@ jobs: env: JIRA_EMAIL: ${{ secrets.JIRA_EMAIL }} JIRA_API_KEY: ${{ secrets.JIRA_API_KEY }} - ASTRA_SCAN_RESULTS: ${{ github.workspace }}/e2e/cypress/fixtures/state/scanResult.json + ASTRA_SCAN_RESULTS: ${{ github.workspace }}/e2e/cypress/fixtures/state/scanResult.json \ No newline at end of file diff --git a/.github/workflows/ci-feat-sonar.yaml b/.github/workflows/ci-feat-sonar.yaml index 6c2644dec..851649d02 100644 --- a/.github/workflows/ci-feat-sonar.yaml +++ b/.github/workflows/ci-feat-sonar.yaml @@ -15,7 +15,9 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 + with: + fetch-depth: 0 - uses: actions/setup-node@v2 with: diff --git a/gwa-api b/gwa-api new file mode 160000 index 000000000..eed2c5196 --- /dev/null +++ b/gwa-api @@ -0,0 +1 @@ +Subproject commit eed2c5196cbc8b5a3aafe4d549cdd26d4247e947 diff --git a/local/kong/Dockerfile b/local/kong/Dockerfile index 397e07dff..296fad529 100644 --- a/local/kong/Dockerfile +++ b/local/kong/Dockerfile @@ -1,50 +1,30 @@ -FROM kong:2.8.3 +FROM docker.io/kong:2.8.5 USER root -RUN apk add git ethtool strace +RUN apk add git ethtool strace unzip -ARG PLUGIN_VERSION=1.1.1-1 -ARG PLUGIN_OIDC_VERSION=1.2.4-2 -ARG PLUGIN_OIDC_CONSUMER_VERSION=0.0.1-0 -ARG PLUGIN_UPSTREAM_BASIC_VERSION=1.0.0-1 -ARG FORCE_BUILD=8 +RUN git clone -b v1.5.0-1 https://github.com/bcgov/kong-oss-plugins.git \ + && cd kong-oss-plugins/plugins \ + && (cd jwt-keycloak && luarocks make) \ + && (cd oidc && luarocks make) \ + && (cd oidc-consumer && luarocks make) -RUN git clone https://github.com/ikethecoder/kong-plugin-upstream-auth-basic.git -RUN (cd kong-plugin-upstream-auth-basic && luarocks make && luarocks pack kong-plugin-upstream-auth-basic ${PLUGIN_UPSTREAM_BASIC_VERSION}) - -RUN luarocks install lua-resty-openidc -RUN git clone -b v${PLUGIN_OIDC_VERSION} https://github.com/revomatico/kong-oidc.git -RUN (cd kong-oidc && luarocks make && luarocks pack kong-oidc ${PLUGIN_OIDC_VERSION}) - -RUN git clone https://github.com/ikethecoder/kong-oidc-consumer.git -RUN (cd kong-oidc-consumer && luarocks make && luarocks pack kong-oidc-consumer ${PLUGIN_OIDC_CONSUMER_VERSION}) - -RUN git clone -b kong28 https://github.com/ikethecoder/kong-plugin-jwt-keycloak.git -RUN (cd kong-plugin-jwt-keycloak && luarocks make && luarocks pack kong-plugin-jwt-keycloak ${PLUGIN_VERSION}) - -RUN git clone -b feature/kong-2.0-upgrade https://github.com/bcgov/gwa-kong-endpoint.git +RUN git clone https://github.com/bcgov/gwa-kong-endpoint.git RUN (cd gwa-kong-endpoint && ./devBuild.sh) -RUN git clone -b hotfix/ips-not-always-string https://github.com/bcgov/gwa-ip-anonymity.git +RUN git clone https://github.com/bcgov/gwa-ip-anonymity.git RUN (cd gwa-ip-anonymity && ./devBuild.sh) -RUN luarocks install lua-resty-jwt 0.2.2-0 \ - && luarocks install lua-resty-session 2.26-1 \ - && luarocks install lua-resty-openidc 1.7.5-1 \ - && luarocks install kong-spec-expose \ - && luarocks install kong-upstream-jwt \ - && luarocks install kong-plugin-referer \ - && luarocks install kong-upstream-jwt \ - && luarocks install kong-oidc/kong-oidc-${PLUGIN_OIDC_VERSION}.all.rock \ - && luarocks install kong-plugin-upstream-auth-basic/kong-plugin-upstream-auth-basic-${PLUGIN_UPSTREAM_BASIC_VERSION}.all.rock \ - && luarocks install kong-oidc-consumer/kong-oidc-consumer-${PLUGIN_OIDC_CONSUMER_VERSION}.all.rock \ - && luarocks install kong-plugin-jwt-keycloak/kong-plugin-jwt-keycloak-${PLUGIN_VERSION}.all.rock +RUN luarocks install kong-spec-expose \ + && luarocks install kong-plugin-referer \ + && luarocks install kong-upstream-jwt RUN git clone https://github.com/Kong/priority-updater.git RUN (cd priority-updater/template/plugin && KONG_PRIORITY=902 KONG_PRIORITY_NAME=rate-limiting /usr/local/openresty/luajit/bin/luajit ../priority.lua) RUN (cd priority-updater/template/plugin && KONG_PRIORITY=1010 KONG_PRIORITY_NAME=jwt-keycloak /usr/local/openresty/luajit/bin/luajit ../priority.lua) +RUN (cd priority-updater/template/plugin && KONG_PRIORITY=200 KONG_PRIORITY_NAME=post-function /usr/local/openresty/luajit/bin/luajit ../priority.lua) USER kong -ENV KONG_PLUGINS="bundled, jwt-keycloak_1010, rate-limiting_902, oidc, oidc-consumer, bcgov-gwa-endpoint, gwa-ip-anonymity, kong-spec-expose, kong-upstream-jwt, referer, jwt-keycloak, kong-upstream-jwt, upstream-auth-basic" \ No newline at end of file +ENV KONG_PLUGINS="bundled, jwt-keycloak_1010, rate-limiting_902, post-function_200, oidc, oidc-consumer, bcgov-gwa-endpoint, gwa-ip-anonymity, kong-spec-expose, kong-upstream-jwt, referer, jwt-keycloak, kong-upstream-jwt" \ No newline at end of file diff --git a/src/nextapp/components/products-list/dataset-input.tsx b/src/nextapp/components/products-list/dataset-input.tsx index 7a8cbfa9c..e0677ad71 100644 --- a/src/nextapp/components/products-list/dataset-input.tsx +++ b/src/nextapp/components/products-list/dataset-input.tsx @@ -52,7 +52,9 @@ const DatasetInput: React.FC = ({ dataset }) => { [setSearch] ); const handleBlur = () => { - if (search.trim()) { + if (selected) { + setSelected(selected); + } else if (search.trim()) { const result = data?.allDatasets.find((d) => { if (search.trim()) { return d.title.toLowerCase() === search.toLowerCase(); @@ -76,6 +78,11 @@ const DatasetInput: React.FC = ({ dataset }) => { const isInvalid = search.length > 0 && !selected; + const titleCounts = (results || []).reduce((acc, d) => { + acc[d.title] = (acc[d.title] || 0) + 1; + return acc; + }, {} as Record); + return ( <> @@ -158,7 +165,9 @@ const DatasetInput: React.FC = ({ dataset }) => { }, })} > - {d.title} + + {d.title} {titleCounts[d.title] > 1 ? `(${d.name})` : ''} + ))} {isOpen && isSuccess && !results.length && ( diff --git a/src/services/keystone/batch-service.ts b/src/services/keystone/batch-service.ts index c75090228..db1d9c335 100644 --- a/src/services/keystone/batch-service.ts +++ b/src/services/keystone/batch-service.ts @@ -61,6 +61,65 @@ export class BatchService { return result['data'][query].length == 0 ? [] : result['data'][query]; } + public async listAllPages( + query: any, + fields: string[], + where: BatchWhereClause = undefined + ) { + const records: any[] = []; + + const pageSize = 50; + const first = pageSize; + let skip = 0; + let more = true; + + logger.debug('[listAllPages] : %s', query); + do { + logger.debug('[listAllPages] : %d', skip); + let queryString; + if (where) { + queryString = `query(${where.query}) { + ${query}(where: ${where.clause}, first: ${first}, skip: ${skip}) { + id, ${fields.join(',')} + } + }`; + } else { + queryString = `query { + ${query}(first: ${first}, skip: ${skip}) { + id, ${fields.join(',')} + } + }`; + } + logger.debug('[listAllPages] %s', queryString); + + const result = await this.context.executeGraphQL({ + query: queryString, + variables: where ? where.variables : {}, + }); + + if ('errors' in result) { + logger.error('[listAll] RESULT %j', result); + return null; + } + + more = result['data'][query].length > 0; + + skip += pageSize; + + logger.debug( + '[listAllPages] RESULT COUNT %d', + result['data'][query].length + ); + records.push( + ...(result['data'][query].length == 0 ? [] : result['data'][query]) + ); + } while (more); + + logger.info('[listAllPages] (%s) TOTAL COUNT %d', query, records.length); + + return records; + } + public async list( query: any, refKey: string, diff --git a/src/services/report/ops-metrics.ts b/src/services/report/ops-metrics.ts index 30b2bec02..dfc2f5963 100644 --- a/src/services/report/ops-metrics.ts +++ b/src/services/report/ops-metrics.ts @@ -524,22 +524,15 @@ async function getAllRoutes(ctx: any) { async function getAllConsumers(ctx: any) { const batch = new BatchService(ctx); - // Limiting to 1000 is not great! We should really recurse until we get to the end! - const allConsumers = await batch.listAll( - 'allServiceAccesses', - [ - 'namespace', - 'active', - 'consumerType', - 'consumer { username }', - 'application { name, owner { name }}', - 'productEnvironment { namespace, name, flow, product { name, namespace, dataset { title } } }', - 'createdAt', - ], - undefined, - 0, - 1000 - ); + const allConsumers = await batch.listAllPages('allServiceAccesses', [ + 'namespace', + 'active', + 'consumerType', + 'consumer { username }', + 'application { name, owner { name }}', + 'productEnvironment { namespace, name, flow, product { name, namespace, dataset { title } } }', + 'createdAt', + ]); return allConsumers; } diff --git a/src/test/integrated/batchworker/paging.ts b/src/test/integrated/batchworker/paging.ts new file mode 100644 index 000000000..4ab8dc90b --- /dev/null +++ b/src/test/integrated/batchworker/paging.ts @@ -0,0 +1,91 @@ +/* +Wire up directly with Keycloak and use the Services +To run: +npm run ts-build +npm run ts-watch +node dist/test/integrated/batchworker/paging.js +*/ + +import InitKeystone from '../keystonejs/init'; +import { + getRecords, + parseJsonString, + transformAllRefID, + removeEmpty, + removeKeys, + syncRecords, +} from '../../../batch/feed-worker'; +import { o } from '../util'; +import { BatchService } from '../../../services/keystone/batch-service'; +import { newEnvironmentID, newProductID } from '../../../services/identifiers'; + +(async () => { + const keystone = await InitKeystone(); + console.log('K = ' + keystone); + + const ns = 'platform'; + const skipAccessControl = false; + + const identity = { + id: null, + username: 'sample_username', + namespace: ns, + roles: JSON.stringify(['api-owner']), + scopes: [], + userId: null, + } as any; + + const ctx = keystone.createContext({ + skipAccessControl, + authentication: { item: identity }, + }); + + if (false) { + const json = { + name: 'Refactor Time Test2', + namespace: ns, + environments: [ + { + name: 'stage', + appId: '0A021EB0', + //services: [] as any, + //services: ['a-service-for-refactortime'], + // services: ['a-service-for-refactortime', 'a-service-for-aps-moh-proto'], + }, + ] as any, + }; + const res = await syncRecords(ctx, 'Product', null, json); + o(res); + } + if (false) { + for (let i = 0; i < 1000; i++) { + const appId = newProductID(); + console.log(appId); + const json = { + name: 'Refactor Time Test 4', + appId: appId, + namespace: ns, + environments: [ + { + name: 'stage', + appId: newEnvironmentID(), + services: [] as any, + }, + ], + }; + const res = await syncRecords(ctx, 'Product', appId, json); + o(res); + } + } + + const batchService = new BatchService(ctx); + + const res = await batchService.listAllPages('allProducts', [ + 'name', + 'namespace', + 'dataset { title }', + 'createdAt', + ]); + //console.log(res); + await keystone.disconnect(); +})();