Skip to content

Commit 5610595

Browse files
ikethecoderikethecoder
authored
Merge pull request #1230 from bcgov/dev
dataset selection and ops metrics fix
2 parents 5959497 + 951d551 commit 5610595

File tree

8 files changed

+218
-79
lines changed

8 files changed

+218
-79
lines changed

‎.github/workflows/aps-cypress-e2e.yaml‎

Lines changed: 30 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,9 @@ jobs:
2828
docker build -t gwa-api:e2e .
2929
3030
- name: Checkout Portal
31-
uses: actions/checkout@v2
31+
uses: actions/checkout@v4
32+
with:
33+
fetch-depth: 0
3234

3335
- name: Build Docker Images
3436
run: |
@@ -85,6 +87,32 @@ jobs:
8587
name: code-coverage
8688
path: ${{ github.workspace }}/e2e/coverage
8789

90+
- name: Instrument the code for coverage analysis
91+
if: always()
92+
run: |
93+
# Rewrite the paths as the coverage starts with '../app'!
94+
sed -e 's/..\/app/./g' ./e2e/coverage/lcov.info > lcov.info
95+
96+
#cd src
97+
#npm install --legacy-peer-deps
98+
#npx nyc instrument --compact=false . --in-place
99+
100+
- name: SonarCloud Scan
101+
if: always()
102+
uses: sonarsource/sonarqube-scan-action@v3.1.0
103+
with:
104+
args: >
105+
-Dsonar.organization=bcgov-oss
106+
-Dsonar.projectKey=aps-portal-e2e
107+
-Dsonar.host.url=https://sonarcloud.io
108+
-Dsonar.projectBaseDir=src
109+
-Dsonar.sources=.
110+
-Dsonar.exclusions=nextapp/**,mocks/**,test/**,tools/**,*.json,*.js
111+
-Dsonar.javascript.lcov.reportPaths=/github/workspace/lcov.info
112+
env:
113+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
114+
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
115+
88116
- name: Upload Astra scan results
89117
uses: actions/upload-artifact@v4
90118
with:
@@ -114,30 +142,6 @@ jobs:
114142
exit 1
115143
fi
116144
117-
- name: Instrument the code for coverage analysis
118-
run: |
119-
# Rewrite the paths as the coverage starts with '../app'!
120-
sed -e 's/..\/app/./g' ./e2e/coverage/lcov.info > lcov.info
121-
122-
#cd src
123-
#npm install --legacy-peer-deps
124-
#npx nyc instrument --compact=false . --in-place
125-
126-
- name: SonarCloud Scan
127-
uses: sonarsource/sonarqube-scan-action@master
128-
with:
129-
args: >
130-
-Dsonar.organization=bcgov-oss
131-
-Dsonar.projectKey=aps-portal-e2e
132-
-Dsonar.host.url=https://sonarcloud.io
133-
-Dsonar.projectBaseDir=src
134-
-Dsonar.sources=.
135-
-Dsonar.exclusions=nextapp/**,mocks/**,test/**,tools/**,*.json,*.js
136-
-Dsonar.javascript.lcov.reportPaths=/github/workspace/lcov.info
137-
env:
138-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
139-
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
140-
141145
- name: Set up Python 3.9
142146
if: failure()
143147
uses: actions/setup-python@v2
@@ -157,4 +161,4 @@ jobs:
157161
env:
158162
JIRA_EMAIL: ${{ secrets.JIRA_EMAIL }}
159163
JIRA_API_KEY: ${{ secrets.JIRA_API_KEY }}
160-
ASTRA_SCAN_RESULTS: ${{ github.workspace }}/e2e/cypress/fixtures/state/scanResult.json
164+
ASTRA_SCAN_RESULTS: ${{ github.workspace }}/e2e/cypress/fixtures/state/scanResult.json

‎.github/workflows/ci-feat-sonar.yaml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,9 @@ jobs:
1515
env:
1616
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
1717
steps:
18-
- uses: actions/checkout@v2
18+
- uses: actions/checkout@v4
19+
with:
20+
fetch-depth: 0
1921

2022
- uses: actions/setup-node@v2
2123
with:

‎gwa-api‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Subproject commit eed2c5196cbc8b5a3aafe4d549cdd26d4247e947

‎local/kong/Dockerfile‎

Lines changed: 14 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,50 +1,30 @@
1-
FROM kong:2.8.3
1+
FROM docker.io/kong:2.8.5
22

33
USER root
44

5-
RUN apk add git ethtool strace
5+
RUN apk add git ethtool strace unzip
66

7-
ARG PLUGIN_VERSION=1.1.1-1
8-
ARG PLUGIN_OIDC_VERSION=1.2.4-2
9-
ARG PLUGIN_OIDC_CONSUMER_VERSION=0.0.1-0
10-
ARG PLUGIN_UPSTREAM_BASIC_VERSION=1.0.0-1
11-
ARG FORCE_BUILD=8
7+
RUN git clone -b v1.5.0-1 https://github.com/bcgov/kong-oss-plugins.git \
8+
&& cd kong-oss-plugins/plugins \
9+
&& (cd jwt-keycloak && luarocks make) \
10+
&& (cd oidc && luarocks make) \
11+
&& (cd oidc-consumer && luarocks make)
1212

13-
RUN git clone https://github.com/ikethecoder/kong-plugin-upstream-auth-basic.git
14-
RUN (cd kong-plugin-upstream-auth-basic && luarocks make && luarocks pack kong-plugin-upstream-auth-basic ${PLUGIN_UPSTREAM_BASIC_VERSION})
15-
16-
RUN luarocks install lua-resty-openidc
17-
RUN git clone -b v${PLUGIN_OIDC_VERSION} https://github.com/revomatico/kong-oidc.git
18-
RUN (cd kong-oidc && luarocks make && luarocks pack kong-oidc ${PLUGIN_OIDC_VERSION})
19-
20-
RUN git clone https://github.com/ikethecoder/kong-oidc-consumer.git
21-
RUN (cd kong-oidc-consumer && luarocks make && luarocks pack kong-oidc-consumer ${PLUGIN_OIDC_CONSUMER_VERSION})
22-
23-
RUN git clone -b kong28 https://github.com/ikethecoder/kong-plugin-jwt-keycloak.git
24-
RUN (cd kong-plugin-jwt-keycloak && luarocks make && luarocks pack kong-plugin-jwt-keycloak ${PLUGIN_VERSION})
25-
26-
RUN git clone -b feature/kong-2.0-upgrade https://github.com/bcgov/gwa-kong-endpoint.git
13+
RUN git clone https://github.com/bcgov/gwa-kong-endpoint.git
2714
RUN (cd gwa-kong-endpoint && ./devBuild.sh)
2815

29-
RUN git clone -b hotfix/ips-not-always-string https://github.com/bcgov/gwa-ip-anonymity.git
16+
RUN git clone https://github.com/bcgov/gwa-ip-anonymity.git
3017
RUN (cd gwa-ip-anonymity && ./devBuild.sh)
3118

32-
RUN luarocks install lua-resty-jwt 0.2.2-0 \
33-
&& luarocks install lua-resty-session 2.26-1 \
34-
&& luarocks install lua-resty-openidc 1.7.5-1 \
35-
&& luarocks install kong-spec-expose \
36-
&& luarocks install kong-upstream-jwt \
37-
&& luarocks install kong-plugin-referer \
38-
&& luarocks install kong-upstream-jwt \
39-
&& luarocks install kong-oidc/kong-oidc-${PLUGIN_OIDC_VERSION}.all.rock \
40-
&& luarocks install kong-plugin-upstream-auth-basic/kong-plugin-upstream-auth-basic-${PLUGIN_UPSTREAM_BASIC_VERSION}.all.rock \
41-
&& luarocks install kong-oidc-consumer/kong-oidc-consumer-${PLUGIN_OIDC_CONSUMER_VERSION}.all.rock \
42-
&& luarocks install kong-plugin-jwt-keycloak/kong-plugin-jwt-keycloak-${PLUGIN_VERSION}.all.rock
19+
RUN luarocks install kong-spec-expose \
20+
&& luarocks install kong-plugin-referer \
21+
&& luarocks install kong-upstream-jwt
4322

4423
RUN git clone https://github.com/Kong/priority-updater.git
4524
RUN (cd priority-updater/template/plugin && KONG_PRIORITY=902 KONG_PRIORITY_NAME=rate-limiting /usr/local/openresty/luajit/bin/luajit ../priority.lua)
4625
RUN (cd priority-updater/template/plugin && KONG_PRIORITY=1010 KONG_PRIORITY_NAME=jwt-keycloak /usr/local/openresty/luajit/bin/luajit ../priority.lua)
26+
RUN (cd priority-updater/template/plugin && KONG_PRIORITY=200 KONG_PRIORITY_NAME=post-function /usr/local/openresty/luajit/bin/luajit ../priority.lua)
4727

4828
USER kong
4929

50-
ENV KONG_PLUGINS="bundled, jwt-keycloak_1010, rate-limiting_902, oidc, oidc-consumer, bcgov-gwa-endpoint, gwa-ip-anonymity, kong-spec-expose, kong-upstream-jwt, referer, jwt-keycloak, kong-upstream-jwt, upstream-auth-basic"
30+
ENV KONG_PLUGINS="bundled, jwt-keycloak_1010, rate-limiting_902, post-function_200, oidc, oidc-consumer, bcgov-gwa-endpoint, gwa-ip-anonymity, kong-spec-expose, kong-upstream-jwt, referer, jwt-keycloak, kong-upstream-jwt"

‎src/nextapp/components/products-list/dataset-input.tsx‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,9 @@ const DatasetInput: React.FC<DatasetInputProps> = ({ dataset }) => {
5252
[setSearch]
5353
);
5454
const handleBlur = () => {
55-
if (search.trim()) {
55+
if (selected) {
56+
setSelected(selected);
57+
} else if (search.trim()) {
5658
const result = data?.allDatasets.find((d) => {
5759
if (search.trim()) {
5860
return d.title.toLowerCase() === search.toLowerCase();
@@ -76,6 +78,11 @@ const DatasetInput: React.FC<DatasetInputProps> = ({ dataset }) => {
7678

7779
const isInvalid = search.length > 0 && !selected;
7880

81+
const titleCounts = (results || []).reduce((acc, d) => {
82+
acc[d.title] = (acc[d.title] || 0) + 1;
83+
return acc;
84+
}, {} as Record<string, number>);
85+
7986
return (
8087
<>
8188
<FormControl id="dataset" position="relative" isInvalid={isInvalid}>
@@ -158,7 +165,9 @@ const DatasetInput: React.FC<DatasetInputProps> = ({ dataset }) => {
158165
},
159166
})}
160167
>
161-
<Text fontSize="md">{d.title}</Text>
168+
<Text fontSize="md">
169+
{d.title} {titleCounts[d.title] > 1 ? `(${d.name})` : ''}
170+
</Text>
162171
</Box>
163172
))}
164173
{isOpen && isSuccess && !results.length && (

‎src/services/keystone/batch-service.ts‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,65 @@ export class BatchService {
6161
return result['data'][query].length == 0 ? [] : result['data'][query];
6262
}
6363

64+
public async listAllPages(
65+
query: any,
66+
fields: string[],
67+
where: BatchWhereClause = undefined
68+
) {
69+
const records: any[] = [];
70+
71+
const pageSize = 50;
72+
const first = pageSize;
73+
let skip = 0;
74+
let more = true;
75+
76+
logger.debug('[listAllPages] : %s', query);
77+
do {
78+
logger.debug('[listAllPages] : %d', skip);
79+
let queryString;
80+
if (where) {
81+
queryString = `query(${where.query}) {
82+
${query}(where: ${where.clause}, first: ${first}, skip: ${skip}) {
83+
id, ${fields.join(',')}
84+
}
85+
}`;
86+
} else {
87+
queryString = `query {
88+
${query}(first: ${first}, skip: ${skip}) {
89+
id, ${fields.join(',')}
90+
}
91+
}`;
92+
}
93+
logger.debug('[listAllPages] %s', queryString);
94+
95+
const result = await this.context.executeGraphQL({
96+
query: queryString,
97+
variables: where ? where.variables : {},
98+
});
99+
100+
if ('errors' in result) {
101+
logger.error('[listAll] RESULT %j', result);
102+
return null;
103+
}
104+
105+
more = result['data'][query].length > 0;
106+
107+
skip += pageSize;
108+
109+
logger.debug(
110+
'[listAllPages] RESULT COUNT %d',
111+
result['data'][query].length
112+
);
113+
records.push(
114+
...(result['data'][query].length == 0 ? [] : result['data'][query])
115+
);
116+
} while (more);
117+
118+
logger.info('[listAllPages] (%s) TOTAL COUNT %d', query, records.length);
119+
120+
return records;
121+
}
122+
64123
public async list(
65124
query: any,
66125
refKey: string,

‎src/services/report/ops-metrics.ts‎

Lines changed: 9 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -524,22 +524,15 @@ async function getAllRoutes(ctx: any) {
524524
async function getAllConsumers(ctx: any) {
525525
const batch = new BatchService(ctx);
526526

527-
// Limiting to 1000 is not great! We should really recurse until we get to the end!
528-
const allConsumers = await batch.listAll(
529-
'allServiceAccesses',
530-
[
531-
'namespace',
532-
'active',
533-
'consumerType',
534-
'consumer { username }',
535-
'application { name, owner { name }}',
536-
'productEnvironment { namespace, name, flow, product { name, namespace, dataset { title } } }',
537-
'createdAt',
538-
],
539-
undefined,
540-
0,
541-
1000
542-
);
527+
const allConsumers = await batch.listAllPages('allServiceAccesses', [
528+
'namespace',
529+
'active',
530+
'consumerType',
531+
'consumer { username }',
532+
'application { name, owner { name }}',
533+
'productEnvironment { namespace, name, flow, product { name, namespace, dataset { title } } }',
534+
'createdAt',
535+
]);
543536

544537
return allConsumers;
545538
}

‎src/test/integrated/batchworker/paging.ts‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
/*
2+
Wire up directly with Keycloak and use the Services
3+
To run:
4+
npm run ts-build
5+
npm run ts-watch
6+
node dist/test/integrated/batchworker/paging.js
7+
*/
8+
9+
import InitKeystone from '../keystonejs/init';
10+
import {
11+
getRecords,
12+
parseJsonString,
13+
transformAllRefID,
14+
removeEmpty,
15+
removeKeys,
16+
syncRecords,
17+
} from '../../../batch/feed-worker';
18+
import { o } from '../util';
19+
import { BatchService } from '../../../services/keystone/batch-service';
20+
import { newEnvironmentID, newProductID } from '../../../services/identifiers';
21+
22+
(async () => {
23+
const keystone = await InitKeystone();
24+
console.log('K = ' + keystone);
25+
26+
const ns = 'platform';
27+
const skipAccessControl = false;
28+
29+
const identity = {
30+
id: null,
31+
username: 'sample_username',
32+
namespace: ns,
33+
roles: JSON.stringify(['api-owner']),
34+
scopes: [],
35+
userId: null,
36+
} as any;
37+
38+
const ctx = keystone.createContext({
39+
skipAccessControl,
40+
authentication: { item: identity },
41+
});
42+
43+
if (false) {
44+
const json = {
45+
name: 'Refactor Time Test2',
46+
namespace: ns,
47+
environments: [
48+
{
49+
name: 'stage',
50+
appId: '0A021EB0',
51+
//services: [] as any,
52+
//services: ['a-service-for-refactortime'],
53+
// services: ['a-service-for-refactortime', 'a-service-for-aps-moh-proto'],
54+
},
55+
] as any,
56+
};
57+
const res = await syncRecords(ctx, 'Product', null, json);
58+
o(res);
59+
}
60+
if (false) {
61+
for (let i = 0; i < 1000; i++) {
62+
const appId = newProductID();
63+
console.log(appId);
64+
const json = {
65+
name: 'Refactor Time Test 4',
66+
appId: appId,
67+
namespace: ns,
68+
environments: [
69+
{
70+
name: 'stage',
71+
appId: newEnvironmentID(),
72+
services: [] as any,
73+
},
74+
],
75+
};
76+
const res = await syncRecords(ctx, 'Product', appId, json);
77+
o(res);
78+
}
79+
}
80+
81+
const batchService = new BatchService(ctx);
82+
83+
const res = await batchService.listAllPages('allProducts', [
84+
'name',
85+
'namespace',
86+
'dataset { title }',
87+
'createdAt',
88+
]);
89+
//console.log(res);
90+
await keystone.disconnect();
91+
})();

0 commit comments

Comments
 (0)