chore: eks/ecr integration testing preparations continued

- decoupling cluster creation from making the image accessible to the cluster
EKS clusters, even if existing, will need access to the image being tested.
this is most easily achieved by pushing the image to ECR, just like in KinD we load the image to KinD.

- loadImageInCluster may decide on an updated image name for testing
this is because when we push the image to the remote cluster, its name/tag may change to include the registry's prefix.

- clean an environment instead of deleting it
for clusters we don't want to delete (EKS), just clean up.

- more generic decoupling from KinD

- adding the EKS test to CircleCI config

Author: Shesekino

.circleci/config.yml

@@ -173,6 +173,32 @@ jobs:
             fi
           when: on_fail
 
+  eks_integration_tests:
+    <<: *default_machine_config
+    steps:
+      - checkout
+      - run:
+          name: INTEGRATION TESTS EKS
+          command: |
+            export NVM_DIR="/opt/circleci/.nvm" &&
+            [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" &&
+            nvm install v10 &&
+            npm install &&
+            docker login --username ${DOCKERHUB_USER} --password ${DOCKERHUB_PASSWORD} &&
+            export IMAGE_TAG=$([[ "$CIRCLE_BRANCH" == "staging" ]] && echo "staging-candidate" || echo "discardable") &&
+            export KUBERNETES_MONITOR_IMAGE_NAME_AND_TAG=snyk/kubernetes-monitor:${IMAGE_TAG}-${CIRCLE_SHA1} &&
+            docker pull ${KUBERNETES_MONITOR_IMAGE_NAME_AND_TAG} &&
+            npm run test:integration:eks
+      - run:
+          name: Notify Slack on failure
+          command: |
+            if [[ "$CIRCLE_BRANCH" == "staging" ]]; then
+              ./scripts/slack-notify-failure.sh "staging-eks-integration-tests-${CIRCLE_SHA1}"
+            else
+              echo "Current branch is $CIRCLE_BRANCH so skipping notifying Slack"
+            fi
+          when: on_fail
+
 ######################## MERGE TO STAGING ########################
   tag_and_push:
     <<: *default_container_config
@@ -293,6 +319,10 @@ workflows:
           requires:
             - build_image
           <<: *staging_branch_only_filter
+      - eks_integration_tests:
+          requires:
+            - build_image
+          <<: *staging_branch_only_filter
       - package_manager_test_apk:
           requires:
             - build_image

test/helpers/kubectl.ts

@@ -34,6 +34,12 @@ export async function createNamespace(namespace: string): Promise<void> {
   console.log(`Created namespace ${namespace}`);
 }
 
+export async function deleteNamespace(namespace: string): Promise<void> {
+  console.log(`Deleting namespace ${namespace}...`);
+  await exec(`./kubectl delete namespace ${namespace}`);
+  console.log(`Deleted namespace ${namespace}`);
+}
+
 export async function createSecret(
   secretName: string,
   namespace: string,

test/integration/kubernetes.test.ts

@@ -37,7 +37,15 @@ tap.test('deploy snyk-monitor', async (t) => {
 
 // Next we apply some sample workloads
 tap.test('deploy sample workloads', async (t) => {
-  await setup.createSampleDeployments();
+  const servicesNamespace = 'services';
+  const someImageWithSha = 'alpine@sha256:7746df395af22f04212cd25a92c1d6dbc5a06a0ca9579a229ef43008d4d1302a';
+  await Promise.all([
+    kubectl.applyK8sYaml('./test/fixtures/alpine-pod.yaml'),
+    kubectl.applyK8sYaml('./test/fixtures/nginx-replicationcontroller.yaml'),
+    kubectl.applyK8sYaml('./test/fixtures/redis-deployment.yaml'),
+    kubectl.applyK8sYaml('./test/fixtures/centos-deployment.yaml'),
+    kubectl.createDeploymentFromImage('alpine-from-sha', someImageWithSha, servicesNamespace),
+  ]);
   t.pass('successfully deployed sample workloads');
 });
 

test/setup/index.ts

@@ -6,6 +6,9 @@ import platforms from './platforms';
 import * as kubectl from '../helpers/kubectl';
 import * as waiters from './waiters';
 
+const testPlatform = process.env['TEST_PLATFORM'] || 'kind';
+const createCluster = process.env['CREATE_CLUSTER'] === 'true';
+
 function getIntegrationId(): string {
   const integrationId = uuidv4();
   console.log(`Generated new integration ID ${integrationId}`);
@@ -55,9 +58,13 @@ function createTestYamlDeployment(
 
 export async function removeMonitor(): Promise<void> {
   try {
-    await platforms.kind.delete();
+    if (createCluster) {
+      await platforms[testPlatform].delete();
+    } else {
+      await platforms[testPlatform].clean();
+    }
   } catch (error) {
-    console.log(`Could not delete kind cluster: ${error.message}`);
+    console.log(`Could not remove the Kubernetes-Monitor: ${error.message}`);
   }
 
   console.log('Removing KUBECONFIG environment variable...');
@@ -73,20 +80,20 @@ export async function removeMonitor(): Promise<void> {
 
 async function createEnvironment(): Promise<void> {
   // TODO: we probably want to use k8s-api for that, not kubectl
-  const servicesNamespace = 'services';
-  await kubectl.createNamespace(servicesNamespace);
+  await kubectl.createNamespace('services');
   // Small hack to prevent timing problems in CircleCI...
+  // TODO: should be replaced by actively waiting for the namespace to be created
   await sleep(5000);
+}
 
-  // Create imagePullSecrets for pulling private images from gcr.io.
-  // This is needed for deploying gcr.io images in KinD (this is _not_ used by snyk-monitor).
+async function createSecretForGcrIoAccess(): Promise<void> {
   const gcrSecretName = 'gcr-io';
   const gcrKubectlSecretsKeyPrefix = '--';
   const gcrSecretType = 'docker-registry';
   const gcrToken = getEnvVariableOrDefault('GCR_IO_SERVICE_ACCOUNT', '{}');
   await kubectl.createSecret(
     gcrSecretName,
-    servicesNamespace,
+    'services',
     {
       'docker-server': 'https://gcr.io',
       'docker-username': '_json_key',
@@ -129,17 +136,17 @@ export async function deployMonitor(): Promise<string> {
       'snyk/kubernetes-monitor:local',
     );
 
-    const testPlatform = process.env['TEST_PLATFORM'] || 'kind';
-    const createCluster = process.env['CREATE_CLUSTER'] === 'true';
     console.log(`platform chosen is ${testPlatform}, createCluster===${createCluster}`);
 
     await kubectl.downloadKubectl();
     if (createCluster) {
-      await platforms[testPlatform].create(imageNameAndTag);
+      await platforms[testPlatform].create();
     }
+    const remoteImageName = await platforms[testPlatform].loadImage(imageNameAndTag);
     await platforms[testPlatform].config();
     await createEnvironment();
-    const integrationId = await installKubernetesMonitor(imageNameAndTag);
+    await createSecretForGcrIoAccess();
+    const integrationId = await installKubernetesMonitor(remoteImageName);
     await waiters.waitForMonitorToBeReady();
     console.log(`Deployed the snyk-monitor with integration ID ${integrationId}`);
     return integrationId;
@@ -157,15 +164,3 @@ export async function deployMonitor(): Promise<string> {
     throw err;
   }
 }
-
-export async function createSampleDeployments(): Promise<void> {
-  const servicesNamespace = 'services';
-  const someImageWithSha = 'alpine@sha256:7746df395af22f04212cd25a92c1d6dbc5a06a0ca9579a229ef43008d4d1302a';
-  await Promise.all([
-    kubectl.applyK8sYaml('./test/fixtures/alpine-pod.yaml'),
-    kubectl.applyK8sYaml('./test/fixtures/nginx-replicationcontroller.yaml'),
-    kubectl.applyK8sYaml('./test/fixtures/redis-deployment.yaml'),
-    kubectl.applyK8sYaml('./test/fixtures/centos-deployment.yaml'),
-    kubectl.createDeploymentFromImage('alpine-from-sha', someImageWithSha, servicesNamespace),
-  ]);
-}

test/setup/platforms/eks.ts

@@ -1,13 +1,62 @@
-export async function createCluster(imageNameAndTag: string): Promise<void> {
-  exportKubeConfig();
+import { exec } from 'child-process-promise';
+import * as kubectl from '../../helpers/kubectl';
+
+export async function createCluster(): Promise<void> {
   throw new Error('Not implemented');
-  // process.env.KUBECONFIG = 'path-to-/kubeconfig-aws';
 }
 
 export async function deleteCluster(): Promise<void> {
   throw new Error('Not implemented');
 }
 
 export async function exportKubeConfig(): Promise<void> {
-  throw new Error('Not implemented');
+  await exec('aws eks update-kubeconfig --name runtime-experiments --kubeconfig ./kubeconfig');
+  process.env.KUBECONFIG = './kubeconfig';
+}
+
+export async function loadImageInCluster(imageNameAndTag: string): Promise<string> {
+  console.log(`Loading image ${imageNameAndTag} in ECR...`);
+
+  // update the `aws` CLI, the one in CircleCI's default image is outdated and doens't support eks
+  await exec('pip install awscli --ignore-installed six');
+
+  // TODO: assert all the vars are present before starting the setup?
+  // TODO: wipe out the data during teardown?
+  await exec(`aws configure set aws_access_key_id ${process.env['AWS_ACCESS_KEY_ID']}`);
+  await exec(`aws configure set aws_secret_access_key ${process.env['AWS_SECRET_ACCESS_KEY']}`);
+  await exec(`aws configure set region ${process.env['AWS_REGION']}`);
+
+  const ecrLogin = await exec('aws ecr get-login --region us-east-2 --no-include-email');
+
+  // aws ecr get-login returns something that looks like:
+  // docker login -U AWS -p <secret> https://the-address-of-ecr-we-should-use.com
+  // `docker tag` wants just the last part without https://
+  // `docker login` wants everything
+
+  // validate output so we don't execute malicious stuff
+  if (ecrLogin.stdout.indexOf('docker login -u AWS -p') !== 0) {
+    throw new Error('aws ecr get-login returned an unexpected output');
+  }
+
+  const targetImage = targetImageFromLoginDetails(ecrLogin.stdout);
+
+  await exec(`docker tag ${imageNameAndTag} ${targetImage}`);
+  await exec(ecrLogin.stdout);
+  await exec(`docker push ${targetImage}`);
+
+  console.log(`Loaded image ${targetImage} in ECR`);
+  return targetImage;
+}
+
+export async function clean(): Promise<void> {
+  await Promise.all([
+    kubectl.deleteNamespace('services'),
+    kubectl.deleteNamespace('snyk-monitor'),
+  ]);
+}
+
+function targetImageFromLoginDetails(ecrLoginOutput: string): string {
+  const split = ecrLoginOutput.split(' ');
+  const targetImagePrefix = split[split.length - 1].replace('https://', '').trim();
+  return `${targetImagePrefix}/snyk/kubernetes-monitor:local`;
 }

test/setup/platforms/index.ts

@@ -3,23 +3,31 @@ import * as eks from './eks';
 
 interface IPlatformSetup {
   // create a Kubernetes cluster
-  create: (imageNameAndTag: string) => Promise<void>;
+  create: () => Promise<void>;
+  // loads the image so Kubernetes may run it, return the name of the image in its registry's format
+  loadImage: (imageNameAndTag: string) => Promise<string>;
   // delete a Kubernetes cluster
   delete: () => Promise<void>;
   // set KUBECONFIG to point at the tested cluster
   config: () => Promise<void>;
+  // clean up whatever we littered an existing cluster with
+  clean: () => Promise<void>;
 }
 
 const kindSetup: IPlatformSetup = {
   create: kind.createCluster,
+  loadImage: kind.loadImageInCluster,
   delete: kind.deleteCluster,
   config: kind.exportKubeConfig,
+  clean: kind.clean,
 };
 
 const eksSetup: IPlatformSetup = {
   create: eks.createCluster,
+  loadImage: eks.loadImageInCluster,
   delete: eks.deleteCluster,
   config: eks.exportKubeConfig,
+  clean: eks.clean,
 };
 
 export default {

test/setup/platforms/kind.ts

@@ -6,11 +6,10 @@ import * as needle from 'needle';
 
 const clusterName = 'kind';
 
-export async function createCluster(imageNameAndTag: string): Promise<void> {
+export async function createCluster(): Promise<void> {
   const osDistro = platform();
   await download(osDistro);
   await createKindCluster(clusterName);
-  await loadImageInCluster(imageNameAndTag);
 }
 
 export async function deleteCluster(): Promise<void> {
@@ -27,6 +26,18 @@ export async function exportKubeConfig(): Promise<void> {
   console.log('Exported K8s config!');
 }
 
+export async function loadImageInCluster(imageNameAndTag: string): Promise<string> {
+  console.log(`Loading image ${imageNameAndTag} in KinD cluster...`);
+  await exec(`./kind load docker-image ${imageNameAndTag}`);
+  console.log(`Loaded image ${imageNameAndTag}`);
+  return imageNameAndTag;
+}
+
+export async function clean(): Promise<void> {
+  // just delete the cluster instead
+  throw new Error('Not implemented');
+}
+
 async function download(osDistro: string): Promise<void> {
   try {
     accessSync(resolve(process.cwd(), 'kind'), constants.R_OK);
@@ -62,9 +73,3 @@ async function createKindCluster(clusterName, kindImageTag = 'latest'): Promise<
   await exec(`./kind create cluster --name="${clusterName}" ${kindImageArgument}`);
   console.log(`Created cluster ${clusterName}!`);
 }
-
-async function loadImageInCluster(imageNameAndTag): Promise<void> {
-  console.log(`Loading image ${imageNameAndTag} in cluster...`);
-  await exec(`./kind load docker-image ${imageNameAndTag}`);
-  console.log(`Loaded image ${imageNameAndTag}`);
-}