Skip to content

Commit 06c3d9e

Browse files
ivanstanevivanstanev
authored
Merge pull request #370 from snyk/chore/e2e-operator
chore/e2e operator
2 parents 23bfc5c + bfaf35b commit 06c3d9e

File tree

13 files changed

+212
-65
lines changed

13 files changed

+212
-65
lines changed

‎.circleci/config.yml‎

Lines changed: 70 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,11 @@ default_container_config: &default_container_config
1111
- image: circleci/node:10
1212
working_directory: ~/kubernetes-monitor
1313

14+
python_container_config: &python_container_config
15+
docker:
16+
- image: circleci/python:3
17+
working_directory: ~/kubernetes-monitor
18+
1419
staging_branch_only_filter: &staging_branch_only_filter
1520
filters:
1621
branches:
@@ -164,9 +169,17 @@ jobs:
164169
command: |
165170
export IMAGE_TAG=$([[ "$CIRCLE_BRANCH" == "staging" ]] && echo "staging-candidate" || echo "discardable")
166171
export SNYK_MONITOR_IMAGE_TAG="${IMAGE_TAG}-${CIRCLE_SHA1}"
167-
export SNYK_OPERATOR_VERSION="0.0.1"
172+
export SNYK_OPERATOR_VERSION="0.0.1-${CIRCLE_SHA1}"
168173
export SNYK_OPERATOR_IMAGE_TAG="${SNYK_MONITOR_IMAGE_TAG}"
169174
./scripts/package-operator.sh "${SNYK_OPERATOR_VERSION}" "${SNYK_OPERATOR_IMAGE_TAG}" "${SNYK_MONITOR_IMAGE_TAG}"
175+
- run:
176+
name: Remove templated Operator before persisting to workspace
177+
command: |
178+
rm -rf snyk-operator/deploy/olm-catalog/snyk-operator/0.0.0
179+
- persist_to_workspace:
180+
root: snyk-operator
181+
paths:
182+
- deploy/olm-catalog/snyk-operator
170183
- run:
171184
name: Notify Slack on failure
172185
command: |
@@ -177,6 +190,24 @@ jobs:
177190
fi
178191
when: on_fail
179192

193+
upload_operator:
194+
<<: *python_container_config
195+
steps:
196+
- attach_workspace:
197+
at: snyk-operator
198+
- run:
199+
name: Install operator-courier
200+
command: pip3 install operator-courier
201+
- run:
202+
name: Upload Operator to Quay
203+
command: |
204+
export QUAY_TOKEN=$(curl -H "Content-Type: application/json" -XPOST https://quay.io/cnr/api/v1/users/login -d "{\"user\": {\"username\": \"${QUAY_USERNAME}\", \"password\": \"${QUAY_PASSWORD}\"}}" | jq -r .token)
205+
export OPERATOR_DIR=./snyk-operator/deploy/olm-catalog/snyk-operator/
206+
export QUAY_NAMESPACE=snyk-runtime
207+
export PACKAGE_NAME=snyk-operator
208+
export PACKAGE_VERSION="0.0.1-${CIRCLE_SHA1}"
209+
operator-courier push "${OPERATOR_DIR}" "${QUAY_NAMESPACE}" "${PACKAGE_NAME}" "${PACKAGE_VERSION}" "${QUAY_TOKEN}"
210+
180211
unit_tests:
181212
<<: *default_machine_config
182213
steps:
@@ -308,7 +339,7 @@ jobs:
308339
export IMAGE_TAG=$([[ "$CIRCLE_BRANCH" == "staging" ]] && echo "staging-candidate" || echo "discardable") &&
309340
export KUBERNETES_MONITOR_IMAGE_NAME_AND_TAG=snyk/kubernetes-monitor:${IMAGE_TAG}-${CIRCLE_SHA1} &&
310341
docker pull ${KUBERNETES_MONITOR_IMAGE_NAME_AND_TAG} &&
311-
.circleci/do-exclusively --branch staging --job ${CIRCLE_JOB} npm run test:integration:openshift4
342+
.circleci/do-exclusively --branch chore/e2e-operator --job ${CIRCLE_JOB} npm run test:integration:openshift4
312343
- run:
313344
name: Notify Slack on failure
314345
command: |
@@ -440,6 +471,37 @@ jobs:
440471

441472
workflows:
442473
version: 2
474+
# TODO: REMOVE THIS, ONLY TESTING :)
475+
TEST:
476+
jobs:
477+
- build_image:
478+
filters:
479+
branches:
480+
only:
481+
- chore/e2e-operator
482+
- build_operator:
483+
filters:
484+
branches:
485+
only:
486+
- chore/e2e-operator
487+
- upload_operator:
488+
filters:
489+
branches:
490+
only:
491+
- chore/e2e-operator
492+
requires:
493+
- build_image
494+
- build_operator
495+
- openshift4_integration_tests:
496+
filters:
497+
branches:
498+
only:
499+
- chore/e2e-operator
500+
requires:
501+
- build_image
502+
- build_operator
503+
- upload_operator
504+
443505
PR_TO_STAGING:
444506
jobs:
445507
- build_image:
@@ -473,6 +535,10 @@ workflows:
473535
<<: *staging_branch_only_filter
474536
- build_operator:
475537
<<: *staging_branch_only_filter
538+
- upload_operator:
539+
requires:
540+
- build_operator
541+
<<: *staging_branch_only_filter
476542
- unit_tests:
477543
<<: *staging_branch_only_filter
478544
- system_tests:
@@ -488,6 +554,8 @@ workflows:
488554
- openshift4_integration_tests:
489555
requires:
490556
- build_image
557+
- build_operator
558+
- upload_operator
491559
<<: *staging_branch_only_filter
492560
- package_manager_test_apk:
493561
requires:

‎package.json‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,15 @@
77
"test": "npm run lint && npm run build && npm run test:unit && npm run test:integration",
88
"test:unit": "NODE_ENV=test tap test/unit",
99
"test:system": "tap test/system --timeout=600",
10-
"test:integration": "TEST_PLATFORM=kind CREATE_CLUSTER=true tap test/integration/kubernetes.test.ts --timeout=900",
11-
"test:integration:kind": "TEST_PLATFORM=kind CREATE_CLUSTER=true tap test/integration/kubernetes.test.ts --timeout=900",
12-
"test:integration:eks": "TEST_PLATFORM=eks CREATE_CLUSTER=false tap test/integration/kubernetes.test.ts --timeout=900",
13-
"test:integration:openshift4": "TEST_PLATFORM=openshift4 CREATE_CLUSTER=false tap test/integration/kubernetes.test.ts --timeout=900",
10+
"test:integration": "DEPLOYMENT_TYPE=YAML TEST_PLATFORM=kind CREATE_CLUSTER=true tap test/integration/kubernetes.test.ts --timeout=900",
11+
"test:integration:kind": "DEPLOYMENT_TYPE=YAML TEST_PLATFORM=kind CREATE_CLUSTER=true tap test/integration/kubernetes.test.ts --timeout=900",
12+
"test:integration:eks": "DEPLOYMENT_TYPE=YAML TEST_PLATFORM=eks CREATE_CLUSTER=false tap test/integration/kubernetes.test.ts --timeout=900",
13+
"test:integration:openshift4": "DEPLOYMENT_TYPE=Operator TEST_PLATFORM=openshift4 CREATE_CLUSTER=false tap test/integration/kubernetes.test.ts --timeout=900",
1414
"test:coverage": "npm run test:unit -- --coverage",
1515
"test:watch": "tsc-watch --onSuccess 'npm run test:unit'",
16-
"test:apk": "TEST_PLATFORM=kind CREATE_CLUSTER=true PACKAGE_MANAGER=apk tap test/integration/package-manager.test.ts --timeout=900",
17-
"test:apt": "TEST_PLATFORM=kind CREATE_CLUSTER=true PACKAGE_MANAGER=apt tap test/integration/package-manager.test.ts --timeout=900",
18-
"test:rpm": "TEST_PLATFORM=kind CREATE_CLUSTER=true PACKAGE_MANAGER=rpm tap test/integration/package-manager.test.ts --timeout=900",
16+
"test:apk": "DEPLOYMENT_TYPE=YAML TEST_PLATFORM=kind CREATE_CLUSTER=true PACKAGE_MANAGER=apk tap test/integration/package-manager.test.ts --timeout=900",
17+
"test:apt": "DEPLOYMENT_TYPE=YAML TEST_PLATFORM=kind CREATE_CLUSTER=true PACKAGE_MANAGER=apt tap test/integration/package-manager.test.ts --timeout=900",
18+
"test:rpm": "DEPLOYMENT_TYPE=YAML TEST_PLATFORM=kind CREATE_CLUSTER=true PACKAGE_MANAGER=rpm tap test/integration/package-manager.test.ts --timeout=900",
1919
"start": "bin/start",
2020
"prepare": "npm run build",
2121
"build": "tsc",

‎test/README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,9 @@ Our integration tests may use different Kubernetes platforms to host the Kuberne
4343
* `TEST_PLATFORM` (`kind`, `eks`)
4444
* `CREATE_CLUSTER` (`true`, `false`).
4545

46+
Additionally, the deployment of the Kubernetes-Monitor can be configured through an environment variable:
47+
* `DEPLOYMENT_TYPE` (`YAML`, `Operator`)
48+
4649
All integration tests determine the image to be tested based on the environment variable called `KUBERNETES_MONITOR_IMAGE_NAME_AND_TAG`, and fallback to `snyk/kubernetes-monitor:local`, meaning one has to make sure the image desired to be tested is properly named and tagged.
4750

4851
### KinD ###

‎test/fixtures/operator/custom-resource.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
apiVersion: charts.helm.k8s.io/v1alpha1
2+
kind: SnykMonitor
3+
metadata:
4+
name: snyk-monitor
5+
namespace: snyk-monitor
6+
spec:
7+
integrationApi: https://kubernetes-upstream.dev.snyk.io

‎test/fixtures/operator/installation.yaml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
apiVersion: operators.coreos.com/v1
2+
kind: OperatorGroup
3+
metadata:
4+
name: snyk-operator
5+
namespace: snyk-monitor
6+
spec:
7+
targetNamespaces:
8+
- snyk-monitor
9+
---
10+
apiVersion: operators.coreos.com/v1alpha1
11+
kind: Subscription
12+
metadata:
13+
name: snyk-operator
14+
namespace: snyk-monitor
15+
spec:
16+
channel: stable
17+
name: snyk-operator
18+
source: snyk-operator
19+
sourceNamespace: openshift-marketplace

‎test/fixtures/operator/operator-source.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
apiVersion: operators.coreos.com/v1
2+
kind: OperatorSource
3+
metadata:
4+
name: snyk-operator
5+
namespace: openshift-marketplace
6+
spec:
7+
type: appregistry
8+
endpoint: https://quay.io/cnr
9+
registryNamespace: snyk-runtime
10+
displayName: "Snyk Operator"
11+
publisher: "Snyk Ltd."

‎test/helpers/kubectl.ts‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,18 @@ export async function createDeploymentFromImage(name: string, image: string, nam
6767
console.log(`Done Letting Kubernetes decide how to manage image ${image} with name ${name}`);
6868
}
6969

70+
export async function patchResourceFinalizers(kind: string, name: string, namespace: string) {
71+
console.log(`Patching resource finalizers for ${kind} ${name} in namespace ${namespace}...`);
72+
await exec(`./kubectl patch ${kind} ${name} -p '{"metadata":{"finalizers":[]}}' --type=merge -n ${namespace}`);
73+
console.log(`Patched resources finalizers for ${kind} ${name}`);
74+
}
75+
76+
export async function deleteResource(kind: string, name: string, namespace: string) {
77+
console.log(`Deleting ${kind} ${name} in namespace ${namespace}...`);
78+
await exec(`./kubectl delete ${kind} ${name} -n ${namespace}`);
79+
console.log(`Deleted ${kind} ${name}`);
80+
}
81+
7082
export async function deleteDeployment(deploymentName: string, namespace: string) {
7183
console.log(`Deleting deployment ${deploymentName} in namespace ${namespace}...`);
7284
await exec(`./kubectl delete deployment ${deploymentName} -n ${namespace}`);
@@ -96,6 +108,22 @@ export async function getPodLogs(podName: string, namespace: string): Promise<an
96108
return logsOutput.stdout;
97109
}
98110

111+
export async function waitForDeployment(name: string, namespace: string): Promise<void> {
112+
console.log(`Trying to find deployment ${name} in namespace ${namespace}`);
113+
for (let attempt = 0; attempt < 60; attempt++) {
114+
try {
115+
await exec(`./kubectl get deployment.apps/${name} -n ${namespace}`);
116+
} catch (error) {
117+
await sleep(1000);
118+
}
119+
}
120+
console.log(`Found deployment ${name} in namespace ${namespace}`);
121+
122+
console.log(`Begin waiting for deployment ${name} in namespace ${namespace}`);
123+
await exec(`./kubectl wait --for=condition=available deployment.apps/${name} -n ${namespace} --timeout=60s`);
124+
console.log(`Deployment ${name} in namespace ${namespace} is available`);
125+
}
126+
99127
export async function waitForServiceAccount(name: string, namespace: string): Promise<void> {
100128
// TODO: add some timeout
101129
while (true) {

‎test/setup/deployers/index.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
1-
import { DeploymentType } from './types';
21
import { yamlDeployer } from './yaml';
2+
import { operatorDeployer } from './operator';
33

44
export default {
5-
[DeploymentType.YAML]: yamlDeployer,
5+
YAML: yamlDeployer,
6+
Operator: operatorDeployer,
67
};

‎test/setup/deployers/operator.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
import { IDeployer } from './types';
2+
import * as kubectl from '../../helpers/kubectl';
3+
4+
export const operatorDeployer: IDeployer = {
5+
deploy: deployKubernetesMonitor,
6+
};
7+
8+
async function deployKubernetesMonitor(
9+
integrationId: string,
10+
_imageOpts: {
11+
imageNameAndTag: string;
12+
imagePullPolicy: string;
13+
},
14+
): Promise<void> {
15+
const namespace = 'snyk-monitor';
16+
await kubectl.createNamespace(namespace);
17+
18+
const secretName = 'snyk-monitor';
19+
const gcrDockercfg = process.env['GCR_IO_DOCKERCFG'] || '{}';
20+
await kubectl.createSecret(secretName, namespace, {
21+
'dockercfg.json': gcrDockercfg,
22+
integrationId,
23+
});
24+
25+
await kubectl.applyK8sYaml('./test/fixtures/operator/operator-source.yaml');
26+
await kubectl.applyK8sYaml('./test/fixtures/operator/installation.yaml');
27+
28+
// Await for the Operator to become available, only then
29+
// the Operator can start processing the custom resource.
30+
await kubectl.waitForDeployment('snyk-operator', 'snyk-monitor');
31+
32+
await kubectl.applyK8sYaml('./test/fixtures/operator/custom-resource.yaml');
33+
}

‎test/setup/deployers/types.ts‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,3 @@
1-
export enum DeploymentType {
2-
YAML,
3-
Helm,
4-
Operator,
5-
}
6-
71
export interface IDeployer {
82
deploy: (
93
integrationId: string,

0 commit comments

Comments
 (0)