Skip to content

Commit 669cc11

Browse files
committed
test: add AKS integration test
1 parent fd74c82 commit 669cc11

File tree

9 files changed

+176
-2
lines changed

9 files changed

+176
-2
lines changed

.circleci/config.yml

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,35 @@ commands:
3232
echo '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"' >> $BASH_ENV
3333
echo 'nvm use 16' >> $BASH_ENV
3434
jobs:
35+
aks_integration_tests:
36+
machine:
37+
docker_layer_caching: true
38+
enabled: true
39+
image: ubuntu-2004:202010-01
40+
steps:
41+
- checkout
42+
- install_python_requests
43+
- azure-cli/install
44+
- run:
45+
command: mkdir -p /tmp/logs/test/integration/aks
46+
name: Create temp dir for logs
47+
- run:
48+
command: |
49+
export NVM_DIR="/opt/circleci/.nvm"
50+
[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
51+
nvm install v16
52+
npm install
53+
export KUBERNETES_MONITOR_IMAGE_NAME_AND_TAG=$(./scripts/circleci-jobs/setup-integration-tests.py)
54+
.circleci/do-exclusively --branch staging --job ${CIRCLE_JOB} npm run test:integration:aks:yaml
55+
name: Integration tests AKS
56+
- run:
57+
command: |
58+
./scripts/slack/notify_failure_on_branch.py "${CIRCLE_BRANCH}" "${CIRCLE_JOB}" "${CIRCLE_BUILD_URL}" "${CIRCLE_PULL_REQUEST}" "${SLACK_WEBHOOK}"
59+
name: Notify Slack on failure
60+
when: on_fail
61+
- store_artifacts:
62+
path: /tmp/logs/test/integration/aks
63+
working_directory: ~/kubernetes-monitor
3564
build_and_upload_operator:
3665
docker:
3766
- auth:
@@ -849,6 +878,7 @@ master_branch_only_filter:
849878
only:
850879
- master
851880
orbs:
881+
azure-cli: circleci/[email protected]
852882
redhat-openshift: circleci/[email protected]
853883
staging_branch_only_filter:
854884
filters:
@@ -950,6 +980,13 @@ workflows:
950980
- staging
951981
requires:
952982
- build_image
983+
- aks_integration_tests:
984+
filters:
985+
branches:
986+
only:
987+
- staging
988+
requires:
989+
- build_image
953990
- openshift3_integration_tests:
954991
filters:
955992
branches:

.circleci/config/@config.yml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,10 @@ workflows:
6969
requires:
7070
- build_image
7171
<<: *staging_branch_only_filter
72+
- aks_integration_tests:
73+
requires:
74+
- build_image
75+
<<: *staging_branch_only_filter
7276
- openshift3_integration_tests:
7377
requires:
7478
- build_image

.circleci/config/jobs/@jobs.yml

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -264,6 +264,37 @@ eks_integration_tests:
264264
- store_artifacts:
265265
path: /tmp/logs/test/integration/eks
266266

267+
aks_integration_tests:
268+
machine:
269+
image: ubuntu-2004:202010-01
270+
enabled: true
271+
docker_layer_caching: true
272+
working_directory: ~/kubernetes-monitor
273+
steps:
274+
- checkout
275+
- install_python_requests
276+
- azure-cli/install
277+
- run:
278+
name: Create temp dir for logs
279+
command: mkdir -p /tmp/logs/test/integration/aks
280+
- run:
281+
name: Integration tests AKS
282+
# WARNING! Do not use the step "setup_node16" here - the call to "nvm use 16" breaks the tests!
283+
command: |
284+
export NVM_DIR="/opt/circleci/.nvm"
285+
[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
286+
nvm install v16
287+
npm install
288+
export KUBERNETES_MONITOR_IMAGE_NAME_AND_TAG=$(./scripts/circleci-jobs/setup-integration-tests.py)
289+
.circleci/do-exclusively --branch staging --job ${CIRCLE_JOB} npm run test:integration:aks:yaml
290+
- run:
291+
name: Notify Slack on failure
292+
command: |
293+
./scripts/slack/notify_failure_on_branch.py "${CIRCLE_BRANCH}" "${CIRCLE_JOB}" "${CIRCLE_BUILD_URL}" "${CIRCLE_PULL_REQUEST}" "${SLACK_WEBHOOK}"
294+
when: on_fail
295+
- store_artifacts:
296+
path: /tmp/logs/test/integration/aks
297+
267298
openshift3_integration_tests:
268299
machine:
269300
image: ubuntu-2004:202010-01
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
azure-cli: circleci/[email protected]

package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
"test:integration:kind:proxy": "DEPLOYMENT_TYPE=Proxy TEST_PLATFORM=kind CREATE_CLUSTER=true jest --logHeapUsage --ci --maxWorkers=1 test/integration/kubernetes.spec.ts",
2121
"test:integration:kindolm:operator": "DEPLOYMENT_TYPE=OperatorOLM TEST_PLATFORM=kindolm CREATE_CLUSTER=true jest --logHeapUsage --ci --maxWorkers=1 test/integration/kubernetes.spec.ts",
2222
"test:integration:eks:yaml": "DEPLOYMENT_TYPE=YAML TEST_PLATFORM=eks CREATE_CLUSTER=false jest --logHeapUsage --ci --maxWorkers=1 test/integration/kubernetes.spec.ts",
23+
"test:integration:aks:yaml": "DEPLOYMENT_TYPE=YAML TEST_PLATFORM=aks CREATE_CLUSTER=false jest --logHeapUsage --ci --maxWorkers=1 test/integration/kubernetes.spec.ts",
2324
"test:integration:openshift3:yaml": "DEPLOYMENT_TYPE=YAML TEST_PLATFORM=openshift3 CREATE_CLUSTER=true jest --logHeapUsage --ci --maxWorkers=1 test/integration/kubernetes.spec.ts",
2425
"test:integration:openshift4:operator": "scripts/test-openshift4.sh",
2526
"test:coverage": "npm run test:unit -- --coverage",

test/README.md

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
- [Integration tests](#integration-tests)
66
- [KinD](#kind)
77
- [EKS](#eks)
8+
- [AKS](#aks)
89
- [OpenShift 4](#openshift-4)
910
- [KinD with OLM](#kind-with-olm)
1011
- [Debugging with Tilt](#debugging-with-tilt)
@@ -49,7 +50,7 @@ All integration tests require the Kubernetes-Monitor to be built into an image o
4950
The easiest way to achieve it is by running the `scripts/docker/build-image.sh` script.
5051
Please note that `docker` needs to be installed in order for this script to succeed.
5152

52-
As part of these tests, we attempt pulling and scanning an image hosted on a private GCR registry. For this test case to work, one has to define the following environment variables: `GCR_IO_SERVICE_ACCOUNT`, `GCR_IO_DOCKERCFG`, `DOCKER_HUB_RO_USERNAME`, `DOCKER_HUB_RO_PASSWORD`.
53+
As part of these tests, we attempt pulling and scanning an image hosted on a private GCR registry. For this test case to work, one has to define the following environment variables: `GCR_IO_SERVICE_ACCOUNT`, `GCR_IO_DOCKERCFG`, `PRIVATE_REGISTRIES_DOCKERCFG`, `DOCKER_HUB_RO_USERNAME`, `DOCKER_HUB_RO_PASSWORD`.
5354

5455
Our integration tests may use different Kubernetes platforms to host the Kubernetes-Monitor. These platforms may use an existing cluster, or create a new one. Both decisions are based on the environment variables:
5556
* `TEST_PLATFORM` (`kind`, `kindolm`, `eks`)
@@ -84,6 +85,19 @@ This test runs whenever we commit to our `staging` branch, and at the moment may
8485

8586
Run with `npm run test:integration:eks`.
8687

88+
### AKS ###
89+
90+
AKS is Azure's Kubernetes platform and helps us ensure we support not only the generic Kubernetes API, but also specifically AKS.
91+
92+
This test uses an existing Azure account with an existing AKS cluster, and as such has a few more prerequisites:
93+
- `pip` is used to ensure the `az` CLI is installed and up to date. `az` is then used to generate a `kubeconfig` file to access the AKS cluster, as well as credentials to ACR.
94+
- AZ environment variables: `AZ_SP_APP_ID`, `AZ_SP_TENANT`, `AZ_SP_PASSWORD`, `AZ_ACR_REGISTRY`, `AZ_SUBSCRIPTION`, `AZ_RESOURCE_NAME`, `AZ_RESOURCE_GROUP` are used to authenticate against the Azure account.
95+
- `docker` is used to push the Kubernetes-Monitor's image to ACR.
96+
97+
This test runs whenever we commit to our `staging` branch, and at the moment may only run once concurrently since it uses the same cluster.
98+
99+
Run with `npm run test:integration:aks:yaml`.
100+
87101
### OpenShift 4 ###
88102

89103
See the [OpenShift 4 README](README.md) for setup instructions.

test/setup/index.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ async function predeploy(
9090
} catch (error) {
9191
console.log(`Namespace ${namespace} already exist`);
9292
}
93-
const gcrDockercfg = process.env['GCR_IO_DOCKERCFG'] || '{}';
93+
const gcrDockercfg = process.env['PRIVATE_REGISTRIES_DOCKERCFG'] || '{}';
9494
await kubectl.createSecret(secretName, namespace, {
9595
'dockercfg.json': gcrDockercfg,
9696
integrationId,

test/setup/platforms/aks.ts

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
import { throwIfEnvironmentVariableUnset } from './helpers';
2+
import * as kubectl from '../../helpers/kubectl';
3+
import { execWrapper as exec } from '../../helpers/exec';
4+
5+
export async function validateRequiredEnvironment(): Promise<void> {
6+
console.log(
7+
'Checking for the required environment variables: AZ_SP_APP_ID, AZ_SP_TENANT, AZ_SP_PASSWORD, AZ_ACR_REGISTRY, AZ_SUBSCRIPTION, AZ_RESOURCE_NAME, AZ_RESOURCE_GROUP',
8+
);
9+
throwIfEnvironmentVariableUnset('AZ_SP_APP_ID');
10+
throwIfEnvironmentVariableUnset('AZ_SP_TENANT');
11+
throwIfEnvironmentVariableUnset('AZ_SP_PASSWORD');
12+
throwIfEnvironmentVariableUnset('AZ_ACR_REGISTRY');
13+
throwIfEnvironmentVariableUnset('AZ_SUBSCRIPTION');
14+
throwIfEnvironmentVariableUnset('AZ_RESOURCE_NAME');
15+
throwIfEnvironmentVariableUnset('AZ_RESOURCE_GROUP');
16+
throwIfEnvironmentVariableUnset('DOCKER_HUB_RO_USERNAME');
17+
throwIfEnvironmentVariableUnset('DOCKER_HUB_RO_PASSWORD');
18+
}
19+
20+
export async function setupTester(): Promise<void> {
21+
const {
22+
AZ_SP_APP_ID,
23+
AZ_SP_PASSWORD,
24+
AZ_SP_TENANT,
25+
AZ_RESOURCE_GROUP,
26+
AZ_RESOURCE_NAME,
27+
AZ_SUBSCRIPTION,
28+
} = process.env;
29+
await exec(
30+
`az login --service-principal --username ${AZ_SP_APP_ID} --password ${AZ_SP_PASSWORD} --tenant ${AZ_SP_TENANT}`,
31+
);
32+
33+
await exec(`az account set --subscription ${AZ_SUBSCRIPTION}`);
34+
await exec(
35+
`az aks get-credentials --resource-group ${AZ_RESOURCE_GROUP} --name ${AZ_RESOURCE_NAME}`,
36+
);
37+
}
38+
39+
export async function createCluster(): Promise<void> {
40+
throw new Error('Not implemented');
41+
}
42+
43+
export async function deleteCluster(): Promise<void> {
44+
throw new Error('Not implemented');
45+
}
46+
47+
export async function exportKubeConfig(): Promise<void> {
48+
const { HOME } = process.env;
49+
process.env.KUBECONFIG = `${HOME}/.kube/config`;
50+
}
51+
52+
export async function loadImageInCluster(
53+
imageNameAndTag: string,
54+
): Promise<string> {
55+
const { AZ_ACR_REGISTRY } = process.env;
56+
console.log(`Loading image ${imageNameAndTag} in ACR...`);
57+
58+
await exec(`az acr login --name ${AZ_ACR_REGISTRY}`);
59+
60+
const targetImage = `${AZ_ACR_REGISTRY}.azurecr.io/${imageNameAndTag}`;
61+
62+
await exec(`docker tag ${imageNameAndTag} ${targetImage}`);
63+
await exec(`docker push ${targetImage}`);
64+
65+
console.log(`Loaded image in ACR`);
66+
return targetImage;
67+
}
68+
69+
export async function clean(): Promise<void> {
70+
await Promise.all([
71+
kubectl.deleteNamespace('services'),
72+
kubectl.deleteNamespace('snyk-monitor'),
73+
]);
74+
}

test/setup/platforms/index.ts

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import * as kind from './kind';
22
import * as kindOlm from './kind-olm';
33
import * as eks from './eks';
4+
import * as aks from './aks';
45
import * as openshift3 from './openshift3';
56
import * as openshift4 from './openshift4';
67

@@ -51,6 +52,16 @@ const eksSetup: IPlatformSetup = {
5152
validateRequiredEnvironment: eks.validateRequiredEnvironment,
5253
};
5354

55+
const aksSetup: IPlatformSetup = {
56+
create: aks.createCluster,
57+
loadImage: aks.loadImageInCluster,
58+
delete: aks.deleteCluster,
59+
config: aks.exportKubeConfig,
60+
clean: aks.clean,
61+
setupTester: aks.setupTester,
62+
validateRequiredEnvironment: aks.validateRequiredEnvironment,
63+
};
64+
5465
// Use a kind cluster pinned to a specific Kubernetes version to mimic OS3.
5566
const openshift3Setup: IPlatformSetup = {
5667
create: kind.createCluster,
@@ -85,6 +96,7 @@ export default {
8596
kind: kindSetup,
8697
kindolm: kindOlmSetup,
8798
eks: eksSetup,
99+
aks: aksSetup,
88100
openshift3: openshift3Setup,
89101
openshift4: openshift4Setup,
90102
} as {

0 commit comments

Comments
 (0)