Merge pull request #292 from snyk/test/system

Test/system

Author: Amir Moualem

.circleci/config.yml

@@ -146,6 +146,29 @@ jobs:
               echo "Current branch is $CIRCLE_BRANCH so skipping notifying Slack"
             fi
           when: on_fail
+  
+  system_tests:
+    <<: *default_machine_config
+    steps:
+      - checkout
+      - run:
+          name: SYSTEM TESTS
+          command: |
+            export NVM_DIR="/opt/circleci/.nvm" &&
+            [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" &&
+            nvm install v10 &&
+            npm install &&
+            npm run build &&
+            npm run test:system
+      - run:
+          name: Notify Slack on failure
+          command: |
+            if [[ "$CIRCLE_BRANCH" == "staging" ]]; then
+              ./scripts/slack-notify-failure.sh "staging-system-tests-${CIRCLE_SHA1}"
+            else
+              echo "Current branch is $CIRCLE_BRANCH so skipping notifying Slack"
+            fi
+          when: on_fail
 
   integration_tests:
     <<: *default_machine_config
@@ -292,6 +315,8 @@ workflows:
           <<: *main_branches_filter
       - unit_tests:
           <<: *main_branches_filter
+      - system_tests:
+          <<: *main_branches_filter
       - integration_tests:
           requires:
             - build_image
@@ -315,6 +340,8 @@ workflows:
           <<: *staging_branch_only_filter
       - unit_tests:
           <<: *staging_branch_only_filter
+      - system_tests:
+          <<: *staging_branch_only_filter
       - integration_tests:
           requires:
             - build_image
@@ -339,6 +366,7 @@ workflows:
           requires:
             - build_image
             - unit_tests
+            - system_tests
             - integration_tests
             - package_manager_test_apk
             - package_manager_test_apt

package-lock.json

@@ -6,6 +6,7 @@
     "pretest": "./scripts/build-image.sh",
     "test": "npm run lint && npm run build && npm run test:unit && npm run test:integration",
     "test:unit": "NODE_ENV=test tap test/unit",
+    "test:system": "tap test/system --timeout=600",
     "test:integration": "TEST_PLATFORM=kind CREATE_CLUSTER=true tap test/integration/kubernetes.test.ts --timeout=1200",
     "test:integration:kind": "TEST_PLATFORM=kind CREATE_CLUSTER=true tap test/integration/kubernetes.test.ts --timeout=1200",
     "test:integration:eks": "TEST_PLATFORM=eks CREATE_CLUSTER=false tap test/integration/kubernetes.test.ts --timeout=1200",
@@ -51,6 +52,7 @@
     "@typescript-eslint/parser": "^2.6.1",
     "eslint": "^6.6.0",
     "eslint-config-prettier": "^6.5.0",
+    "nock": "^11.7.2",
     "sinon": "^8.0.1",
     "tap": "^14.10.5",
     "ts-node": "^8.1.0",

package.json

@@ -0,0 +1,22 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: java
+  namespace: services
+  labels:
+    app.kubernetes.io/name: java
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: java
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: java
+    spec:
+      containers:
+      - image: java:latest
+        imagePullPolicy: Always
+        name: java
+      securityContext: {}

test/fixtures/java-deployment.yaml

@@ -3,6 +3,7 @@ import { accessSync, chmodSync, constants, writeFileSync } from 'fs';
 import { platform } from 'os';
 import { resolve } from 'path';
 import * as needle from 'needle';
+import * as sleep from 'sleep-promise';
 
 export async function downloadKubectl(): Promise<void> {
   try {
@@ -83,6 +84,18 @@ export async function getDeploymentJson(deploymentName: string, namespace: strin
   return JSON.parse(getDeploymentResult.stdout);
 }
 
+export async function waitForServiceAccount(name: string, namespace: string): Promise<void> {
+  // TODO: add some timeout
+  while (true) {
+    try {
+      await exec(`./kubectl get serviceaccount ${name} -n ${namespace}`);
+      break;
+    } catch (err) {
+      await sleep(500);
+    }
+  }
+}
+
 async function getLatestStableK8sRelease(): Promise<string> {
   const k8sRelease = await needle('get',
     'https://storage.googleapis.com/kubernetes-release/release/stable.txt',

test/helpers/kubectl.ts

@@ -168,7 +168,5 @@ export async function deployMonitor(): Promise<string> {
       // ... but make sure the test suite doesn't proceed if the setup failed
       process.exit(-1);
     }
-
-    throw err;
   }
 }

test/setup/index.ts

@@ -0,0 +1,99 @@
+import * as tap from 'tap';
+import * as nock from 'nock';
+import * as sleep from 'sleep-promise';
+import { exec } from 'child-process-promise';
+
+import * as kubectl from '../helpers/kubectl';
+import * as kind from '../setup/platforms/kind';
+
+// let integrationId: string;
+
+async function tearDown() {
+  console.log('Begin removing the snyk-monitor...');
+  await kind.deleteCluster();
+  console.log('Removed the snyk-monitor!');
+}
+
+tap.tearDown(tearDown);
+
+tap.test('Kubernetes-Monitor with KinD', async (t) => {
+
+  // Start fresh
+  try {
+    await tearDown();
+  } catch (error) {
+    console.log(`could not start with a clean environment: ${error.message}`);
+  }
+
+  // install Skopeo
+  // TODO: this thing should probably be in a setup test environment script
+  // not in this file
+  try {
+    await exec('which skopeo');
+    console.log('Skopeo already installed :tada:');
+  } catch (err) {
+    // linux-oriented, not mac
+    // for mac, install skopeo with brew
+    console.log('installing Skopeo');
+    await exec('git clone https://github.com/containers/skopeo');
+    await exec('(cd skopeo && make binary-static DISABLE_CGO=1)');
+    await exec('sudo mkdir -p /etc/containers');
+    await exec('sudo chown circleci:circleci /etc/containers');
+    await exec('cp ./skopeo/default-policy.json /etc/containers/policy.json');
+
+    process.env['PATH'] = process.env['PATH'] + ':./skopeo';
+  }
+
+  // kubectl
+  await kubectl.downloadKubectl();
+
+  // KinD
+  await kind.createCluster();
+  await kind.exportKubeConfig();
+  Promise.all([
+    kubectl.createNamespace('snyk-monitor'),
+    kubectl.createNamespace('services'),
+  ]);
+  // wait for default service account
+  await kubectl.waitForServiceAccount('default', 'default');
+
+  // Services
+  await Promise.all([
+    kubectl.applyK8sYaml('./test/fixtures/java-deployment.yaml'),
+  ]);
+
+  // TODO: wait for the services to start?
+
+  // Setup nocks
+  nock('https://kubernetes-upstream.snyk.io')
+    .post('/api/v1/workload')
+    .times(1)
+    .reply(200, (uri, requestBody) => {
+      // TODO assert POST payload
+    });
+
+  nock('https://kubernetes-upstream.snyk.io')
+    .post('/api/v1/dependency-graph')
+    .times(1)
+    .reply(200, (uri, requestBody) => {
+      // TODO assert POST payload
+    });
+
+  // Start the monitor
+  require('../../src');
+
+  // TODO: replace with being event driven?
+  // will still need SOME timeout 
+  while (true) {
+    if (nock.isDone()) {
+      break;
+    } else {
+      await sleep(5 * 1000);
+    }
+  }
+
+  // additional asserts?
+  t.ok(nock.isDone(), 'all outgoing calls were made');
+
+  // TODO cleanup the images we saved to /var/tmp?
+});