test: operator on kind

Author: shaimendel

.circleci/config/@config.yml

@@ -38,6 +38,10 @@ workflows:
           requires:
             - build_image
           <<: *main_branches_filter
+      - integration_tests_proxy:
+          requires:
+            - build_image
+          <<: *main_branches_filter
 
   MERGE_TO_STAGING:
     jobs:
@@ -73,11 +77,18 @@ workflows:
           requires:
             - build_image
           <<: *staging_branch_only_filter
+      - integration_tests_operator_on_k8s:
+          requires:
+            - build_image
+            - build_operator
+            - upload_operator
+          <<: *staging_branch_only_filter
       - openshift4_integration_tests:
           requires:
             - build_image
             - build_operator
             - upload_operator
+            - integration_tests_operator_on_k8s
           <<: *staging_branch_only_filter
       - tag_and_push:
           requires:

.circleci/config/jobs/@jobs.yml

@@ -174,6 +174,36 @@ integration_tests_helm:
     - store_artifacts:
         path: /tmp/logs/test/integration/kind-helm
 
+integration_tests_operator_on_k8s:
+    machine:
+      docker_layer_caching: true
+      enabled: true
+    steps:
+    - checkout
+    - setup_node12
+    - install_python_requests
+    - run:
+        command: mkdir -p /tmp/logs/test/integration/kind-olm-operator
+        name: Create temporary directory for logs
+    - run:
+        command: |
+          export KUBERNETES_MONITOR_IMAGE_NAME_AND_TAG=$(./scripts/circleci-jobs/setup-integration-tests.py)
+          npm run test:integration:kindolm:operator
+        name: Operator integration tests on plain k8s
+    - run:
+        name: Delete Operator from Quay
+        command: |
+          ./scripts/operator/delete-operator-from-quay.sh
+        when: on_fail # it will be deleted on success in the openshift4 tests
+    - run:
+        command: |
+          ./scripts/slack/notify_failure_on_branch.py "staging-integration-operator-k8s-tests-${CIRCLE_SHA1}"
+        name: Notify Slack on failure
+        when: on_fail
+    - store_artifacts:
+        path: /tmp/logs/test/integration/kind-olm-operator
+    working_directory: ~/kubernetes-monitor
+
 integration_tests_proxy:
   machine:
     enabled: true

package.json

@@ -10,6 +10,7 @@
     "test:integration:kind:yaml": "DEPLOYMENT_TYPE=YAML TEST_PLATFORM=kind CREATE_CLUSTER=true tap test/integration/kubernetes.test.ts --timeout=900",
     "test:integration:kind:helm": "DEPLOYMENT_TYPE=Helm TEST_PLATFORM=kind CREATE_CLUSTER=true tap test/integration/kubernetes.test.ts --timeout=900",
     "test:integration:kind:proxy": "DEPLOYMENT_TYPE=Proxy TEST_PLATFORM=kind CREATE_CLUSTER=true tap test/integration/kubernetes.test.ts --timeout=900",
+    "test:integration:kindolm:operator": "DEPLOYMENT_TYPE=Operator TEST_PLATFORM=kindolm CREATE_CLUSTER=true tap test/integration/kubernetes.test.ts --timeout=900",
     "test:integration:eks:yaml": "DEPLOYMENT_TYPE=YAML TEST_PLATFORM=eks CREATE_CLUSTER=false tap test/integration/kubernetes.test.ts --timeout=900",
     "test:integration:openshift3:yaml": "DEPLOYMENT_TYPE=YAML TEST_PLATFORM=openshift3 CREATE_CLUSTER=true tap test/integration/kubernetes.test.ts --timeout=900",
     "test:integration:openshift4:operator": "DEPLOYMENT_TYPE=Operator TEST_PLATFORM=openshift4 CREATE_CLUSTER=false tap test/integration/kubernetes.test.ts --timeout=900",

test/README.md

@@ -40,7 +40,7 @@ Please note that `docker` needs to be installed in order for this script to succ
 As part of these tests, we attempt pulling and scanning an image hosted on a private GCR registry. For this test case to work, one has to define the following environment variables: `GCR_IO_SERVICE_ACCOUNT`, `GCR_IO_DOCKERCFG`.
 
 Our integration tests may use different Kubernetes platforms to host the Kubernetes-Monitor. These platforms may use an existing cluster, or create a new one. Both decisions are based on the environment variables:
-* `TEST_PLATFORM` (`kind`, `eks`)
+* `TEST_PLATFORM` (`kind`, `kindolm`, `eks`)
 * `CREATE_CLUSTER` (`true`, `false`).
 
 Additionally, the deployment of the Kubernetes-Monitor can be configured through an environment variable:
@@ -83,6 +83,14 @@ This test runs whenever we commit to our `staging` branch, and at the moment may
 
 Run with `npm run test:integration:openshift4`.
 
+### KinD with OLM ###
+
+This test helps us ensure our operator is installable and functioning on a plain k8s cluster.
+
+This test runs whenever we commit to any branch.
+
+Run with `npm run test:integration:kindolm:operator`.
+
 ### Package Managers ###
 
 These tests attempt to provide some more thorough coverage for our scans of specific package manager: APK, APT and RPM.

test/fixtures/operator/installation-k8s.yaml

@@ -0,0 +1,20 @@
+apiVersion: operators.coreos.com/v1
+kind: OperatorGroup
+metadata:
+  name: snyk-operator
+  namespace: snyk-monitor
+spec:
+  targetNamespaces:
+  - snyk-monitor
+---
+apiVersion: operators.coreos.com/v1alpha1
+kind: Subscription
+metadata:
+  name: snyk-operator
+  namespace: snyk-monitor
+spec:
+  channel: stable
+  name: snyk-operator
+  installPlanApproval: Automatic
+  source: snyk-operator
+  sourceNamespace: marketplace

test/fixtures/operator/operator-source-k8s.yaml

@@ -0,0 +1,11 @@
+apiVersion: operators.coreos.com/v1
+kind: OperatorSource
+metadata:
+  name: snyk-operator
+  namespace: marketplace
+spec:
+  type: appregistry
+  endpoint: https://quay.io/cnr
+  registryNamespace: snyk-runtime
+  displayName: "Snyk Operator"
+  publisher: "Snyk Ltd."

test/helpers/kubectl.ts

@@ -74,7 +74,14 @@ export async function createConfigMap(
   console.log(`Created config map ${configMapName}`);
 }
 
-export async function applyK8sYaml(pathToYamlDeployment: string): Promise<void> {
+export async function applyK8sYaml(pathToYamlDeployment: string, namespace?: string): Promise<void> {
+  if (namespace) {
+    console.log(`Applying ${pathToYamlDeployment} to namespace ${namespace}...`);
+    await exec(`./kubectl apply -f ${pathToYamlDeployment} -n ${namespace}`);
+    console.log(`Applied ${pathToYamlDeployment} to namespace ${namespace}`);
+    return;
+  }
+
   console.log(`Applying ${pathToYamlDeployment}...`);
   await exec(`./kubectl apply -f ${pathToYamlDeployment}`);
   console.log(`Applied ${pathToYamlDeployment}`);
@@ -122,6 +129,13 @@ export async function getPodNames(namespace: string): Promise<string[]> {
   return podsOutput.stdout.split('\n');
 }
 
+export async function getNamespaces(): Promise<string[]> {
+  const commandPrefix = `./kubectl get ns`;
+  const onlyNames = ' --template \'{{range .items}}{{.metadata.name}}{{"\\n"}}{{end}}\'';
+  const nsOutput = await exec(commandPrefix+onlyNames);
+  return nsOutput.stdout.split('\n');
+}
+
 export async function getPodLogs(podName: string, namespace: string): Promise<any> {
   const logsOutput = await exec(`./kubectl -n ${namespace} logs ${podName}`);
   return logsOutput.stdout;
@@ -155,6 +169,18 @@ export async function waitForServiceAccount(name: string, namespace: string): Pr
   }
 }
 
+export async function waitForCRD(name: string): Promise<void> {
+  // TODO: add some timeout
+  while (true) {
+    try {
+      await exec(`./kubectl get crd ${name}`);
+      break;
+    } catch (err) {
+      await sleep(500);
+    }
+  }
+}
+
 export async function waitForJob(name: string, namespace: string): Promise<void> {
   console.log(`Trying to find job ${name} in namespace ${namespace}`);
   for (let attempt = 0; attempt < 60; attempt++) {

test/setup/deployers/operator.ts

@@ -8,12 +8,21 @@ export const operatorDeployer: IDeployer = {
 async function deployKubernetesMonitor(
   _imageOptions: IImageOptions,
 ): Promise<void> {
-  await kubectl.applyK8sYaml('./test/fixtures/operator/operator-source.yaml');
-  await kubectl.applyK8sYaml('./test/fixtures/operator/installation.yaml');
-
-  // Await for the Operator to become available, only then
-  // the Operator can start processing the custom resource.
-  await kubectl.waitForDeployment('snyk-operator', 'snyk-monitor');
+  const namespaces = await kubectl.getNamespaces();
+  if (namespaces.includes('openshift-marketplace')) {
+    await kubectl.applyK8sYaml('./test/fixtures/operator/operator-source.yaml');
+    await kubectl.applyK8sYaml('./test/fixtures/operator/installation.yaml');
+    // Await for the Operator to become available, only then
+    // the Operator can start processing the custom resource.
+    await kubectl.waitForDeployment('snyk-operator', 'openshift-marketplace');
+  } else {
+    await kubectl.applyK8sYaml('./test/fixtures/operator/operator-source-k8s.yaml');
+    await kubectl.applyK8sYaml('./test/fixtures/operator/installation-k8s.yaml');
+    // Await for the Operator to become available, only then
+    // the Operator can start processing the custom resource.
+    await kubectl.waitForDeployment('snyk-operator', 'marketplace');
+    await kubectl.waitForCRD('snykmonitors.charts.helm.k8s.io');
+  }
 
   await kubectl.applyK8sYaml('./test/fixtures/operator/custom-resource.yaml');
 }

test/setup/index.ts

@@ -133,7 +133,7 @@ export async function deployMonitor(): Promise<string> {
     await predeploy(integrationId);
 
     // TODO: hack, rewrite this
-    const imagePullPolicy = testPlatform === 'kind' || testPlatform === 'openshift3' ? 'Never' : 'Always';
+    const imagePullPolicy = testPlatform === 'kind' || testPlatform === 'kindolm' || testPlatform === 'openshift3' ? 'Never' : 'Always';
     const deploymentImageOptions: IImageOptions = {
       nameAndTag: remoteImageName,
       pullPolicy: imagePullPolicy,

test/setup/platforms/index.ts

@@ -1,4 +1,5 @@
 import * as kind from './kind';
+import * as kindOlm from './kind-olm';
 import * as eks from './eks';
 import * as openshift3 from './openshift3';
 import * as openshift4 from './openshift4';
@@ -27,6 +28,15 @@ const kindSetup: IPlatformSetup = {
   setupTester: kind.setupTester,
 };
 
+const kindOlmSetup: IPlatformSetup = {
+  create: kindOlm.createCluster,
+  loadImage: kind.loadImageInCluster,
+  delete: kind.deleteCluster,
+  config: kind.exportKubeConfig,
+  clean: kind.clean,
+  setupTester: kind.setupTester,
+};
+
 const eksSetup: IPlatformSetup = {
   create: eks.createCluster,
   loadImage: eks.loadImageInCluster,
@@ -66,6 +76,7 @@ export function getKubernetesVersionForPlatform(testPlatform: string): string {
 
 export default {
   kind: kindSetup,
+  kindolm: kindOlmSetup,
   eks: eksSetup,
   openshift3: openshift3Setup,
   openshift4: openshift4Setup,