Merge pull request #499 from snyk/test/add-local-registry-coverage

Test/add local registry coverage

Author: agatakrajewska

test/fixtures/insecure-registries/push-dockerhub-image-to-local-registry.yaml

@@ -0,0 +1,22 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: push-to-local-registry
+spec:
+  template:
+    spec:
+      containers:
+        - name: my-container
+          image: golang:1.13.1-alpine3.10
+          command: 
+            - "sh"
+          args:
+            - "-c"
+            - "apk --no-cache add git make gcc musl-dev ostree-dev go-md2man &&
+            git clone --depth 1 -b 'v0.2.0' https://github.com/containers/skopeo $GOPATH/src/github.com/containers/skopeo &&
+            cd $GOPATH/src/github.com/containers/skopeo &&
+            make binary-local-static DISABLE_CGO=1 &&
+            make install &&
+            skopeo copy --dest-tls-verify=false docker://python:rc-buster docker://kind-registry:5000/python:rc-buster"
+      restartPolicy: Never
+  backoffLimit: 4

test/fixtures/insecure-registries/python-local-deployment.yaml

@@ -0,0 +1,24 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: python-local
+  namespace: services
+  labels:
+    app.kubernetes.io/name: python-local
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: python-local
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: python-local
+    spec:
+      containers:
+      - image: kind-registry:5000/python:rc-buster
+        imagePullPolicy: Always
+        name: python-local
+        command: ['/bin/sleep']
+        args: ['9999999']
+      securityContext: {}

test/fixtures/insecure-registries/registries.conf

@@ -0,0 +1,3 @@
+[[registry]]
+location = "kind-registry:5000"
+insecure = true

test/helpers/kubectl.ts

@@ -64,6 +64,16 @@ export async function createSecret(
   console.log(`Created secret ${secretName}`);
 }
 
+export async function createConfigMap(
+  configMapName: string,
+  namespace: string,
+  filePath: string
+): Promise<void> {
+  console.log(`Creating config map ${configMapName} in namespace ${namespace}...`);
+  await exec(`./kubectl create configmap ${configMapName} -n ${namespace} --from-file=${filePath}`);
+  console.log(`Created config map ${configMapName}`);
+}
+
 export async function applyK8sYaml(pathToYamlDeployment: string): Promise<void> {
   console.log(`Applying ${pathToYamlDeployment}...`);
   await exec(`./kubectl apply -f ${pathToYamlDeployment}`);
@@ -145,6 +155,22 @@ export async function waitForServiceAccount(name: string, namespace: string): Pr
   }
 }
 
+export async function waitForJob(name: string, namespace: string): Promise<void> {
+  console.log(`Trying to find job ${name} in namespace ${namespace}`);
+  for (let attempt = 0; attempt < 60; attempt++) {
+    try {
+      await exec(`./kubectl get jobs/${name} -n ${namespace}`);
+    } catch (error) {
+      await sleep(1000);
+    }
+  }
+  console.log(`Found job ${name} in namespace ${namespace}`);
+
+  console.log(`Begin waiting for job ${name} in namespace ${namespace} to complete`);
+  await exec(`./kubectl wait --for=condition=complete jobs/${name} -n ${namespace} --timeout=240s`);
+  console.log(`Job ${name} in namespace ${namespace} is complete`);
+}
+
 async function getLatestStableK8sRelease(): Promise<string> {
   const k8sRelease = await needle('get',
     'https://storage.googleapis.com/kubernetes-release/release/stable.txt',

test/integration/kubernetes.test.ts

@@ -1,4 +1,5 @@
 import { CoreV1Api, KubeConfig, AppsV1Api } from '@kubernetes/client-node';
+import { exec } from 'child-process-promise';
 import setup = require('../setup');
 import * as tap from 'tap';
 import { WorkloadKind } from '../../src/supervisor/types';
@@ -21,6 +22,12 @@ tap.tearDown(async() => {
   console.log('Begin removing the snyk-monitor...');
   await setup.removeMonitor();
   console.log('Removed the snyk-monitor!');
+  console.log('Begin removing local container registry...');
+  await setup.removeLocalContainerRegistry();
+  console.log('Removed local container registry');
+  console.log('Begin removing "kind" network...');
+  await setup.removeUnusedKindNetwork();
+  console.log('Removed "kind" network');
 });
 
 // Make sure this runs first -- deploying the monitor for the next tests
@@ -65,6 +72,19 @@ tap.test('snyk-monitor container started', async (t) => {
   console.log('Done -- snyk-monitor exists!');
 });
 
+tap.test('create local container registry and push an image', async (t) => {
+  console.log('Creating local container registry...');
+  await exec('docker run -d --restart=always -p "5000:5000" --name "kind-registry" registry:2');
+  await exec('docker network connect "kind" "kind-registry"');
+  
+  console.log('Pushing python:rc-buster image to the local registry');
+  //Note: this job takes a while and waitForJob() should be called before trying to access local registry image,
+  //to make sure it completed
+  await kubectl.applyK8sYaml('./test/fixtures/insecure-registries/push-dockerhub-image-to-local-registry.yaml');
+
+  t.pass('successfully started a job to push image to a local registry');
+});
+
 tap.test('snyk-monitor sends data to kubernetes-upstream', async (t) => {
   t.plan(7);
 
@@ -226,6 +246,46 @@ tap.test('snyk-monitor pulls images from a private ECR and sends data to kuberne
     'snyk-monitor sent expected data to upstream in the expected timeframe');
 });
 
+tap.test('snyk-monitor pulls images from a local registry and sends data to kubernetes-upstream', async (t) => {
+  t.plan(4);
+
+  const deploymentName = 'python-local';
+  const namespace = 'services';
+  const clusterName = 'Default cluster';
+  const deploymentType = WorkloadKind.Deployment;
+  const imageName = 'kind-registry:5000/python:rc-buster';
+
+  await kubectl.waitForJob('push-to-local-registry', 'default');
+  
+  console.log('Applying local registry workload...');
+  await kubectl.applyK8sYaml('./test/fixtures/insecure-registries/python-local-deployment.yaml');
+
+  console.log(`Begin polling upstream for the expected kind-registry:5000 image with integration ID ${integrationId}...`);
+
+  const validatorFn: WorkloadLocatorValidator = (workloads) => {
+    return workloads !== undefined &&
+      workloads.find((workload) => workload.name === deploymentName &&
+        workload.type === WorkloadKind.Deployment) !== undefined;
+  };
+
+  const testResult = await validateUpstreamStoredData(
+    validatorFn, `api/v2/workloads/${integrationId}/${clusterName}/${namespace}`);
+  t.ok(testResult, 'snyk-monitor sent expected data to upstream in the expected timeframe');
+
+  const depGraphResult = await getUpstreamResponseBody(
+    `api/v1/dependency-graphs/${integrationId}/${clusterName}/${namespace}/${deploymentType}/${deploymentName}`);
+  
+  t.ok('dependencyGraphResults' in depGraphResult,
+  'expected dependencyGraphResults field to exist in /dependency-graphs response');
+
+  /* Because of a bug in removeTagFromImage() func in src/scanner/images/index.ts, 
+  which chops off everything after ':' from the image name, we store a wrong image name 
+  and the result does not exist in the object referred below */
+  t.same(depGraphResult.dependencyGraphResults[imageName], null,
+  'expected result for image kind-registry:5000/python:rc-buster does not exist');
+  t.ok('kind-registry' in depGraphResult.dependencyGraphResults, 'BUG: the full image name is not stored in kubernetes-upstream');
+});
+
 tap.test('snyk-monitor sends deleted workload to kubernetes-upstream', async (t) => {
   // First ensure the deployment exists from the previous test
   const deploymentValidatorFn: WorkloadLocatorValidator = (workloads) => {

test/setup/index.ts

@@ -1,6 +1,7 @@
 import * as fs from 'fs';
 import * as sleep from 'sleep-promise';
 import * as uuidv4 from 'uuid/v4';
+import { exec } from 'child-process-promise';
 
 import platforms, { getKubernetesVersionForPlatform } from './platforms';
 import deployers from './deployers';
@@ -37,6 +38,22 @@ export async function removeMonitor(): Promise<void> {
   }
 }
 
+export async function removeLocalContainerRegistry(): Promise<void> {
+  try {
+    await exec('docker rm kind-registry --force');
+  } catch (error) {
+    console.log(`Could not remove container registry, it probably did not exist: ${error.message}`);
+  }
+}
+
+export async function removeUnusedKindNetwork(): Promise<void> {
+  try {
+    await exec('docker network rm kind');
+  } catch (error) {
+    console.log(`Could not remove "kind" network: ${error.message}`);
+  }
+}
+
 async function createEnvironment(): Promise<void> {
   // TODO: we probably want to use k8s-api for that, not kubectl
   await kubectl.createNamespace('services');
@@ -56,6 +73,7 @@ async function predeploy(integrationId: string): Promise<void> {
       'dockercfg.json': gcrDockercfg,
       integrationId,
     });
+    await createRegistriesConfigMap();
   } catch (error) {
     console.log('Could not create namespace and secret, they probably already exist');
   }
@@ -80,6 +98,10 @@ async function createSecretForGcrIoAccess(): Promise<void> {
   );
 }
 
+async function createRegistriesConfigMap(): Promise<void> {
+  await kubectl.createConfigMap('snyk-monitor-registries-conf', 'snyk-monitor', './test/fixtures/insecure-registries/registries.conf');
+}
+
 export async function deployMonitor(): Promise<string> {
   console.log('Begin deploying the snyk-monitor...');
 

test/setup/platforms/cluster-config.yaml

@@ -0,0 +1,6 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+containerdConfigPatches:
+- |-
+  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."kind-registry:5000"]
+    endpoint = ["http://kind-registry:5000"]

test/setup/platforms/kind.ts

@@ -21,7 +21,9 @@ export async function createCluster(version: string): Promise<void> {
     // which does not necessarily have the "latest" tag
     kindImageArgument = `--image="kindest/node:${kindImageTag}"`;
   }
-  await exec(`./kind create cluster --name="${clusterName}" ${kindImageArgument}`);
+  const clusterConfigPath = "test/setup/platforms/cluster-config.yaml";
+
+  await exec(`./kind create cluster --name="${clusterName}" ${kindImageArgument} --config="${clusterConfigPath}"`);
   console.log(`Created cluster ${clusterName}!`);
 }
 
@@ -59,7 +61,7 @@ async function download(osDistro: string): Promise<void> {
   } catch (error) {
     console.log('Downloading KinD...');
 
-    const url = `https://github.com/kubernetes-sigs/kind/releases/download/v0.7.0/kind-${osDistro}-amd64`;
+    const url = `https://github.com/kubernetes-sigs/kind/releases/download/v0.8.1/kind-${osDistro}-amd64`;
     await exec(`curl -Lo ./kind ${url}`);
     chmodSync('kind', 0o755); // rwxr-xr-x
 

test/system/kind.test.ts

@@ -36,7 +36,7 @@ tap.test('Kubernetes-Monitor with KinD', async (t) => {
     // linux-oriented, not mac
     // for mac, install skopeo with brew
     console.log('installing Skopeo');
-    await exec('git clone https://github.com/containers/skopeo');
+    await exec('git clone --depth 1 -b "v0.2.0" https://github.com/containers/skopeo');
     await exec('(cd skopeo && make binary-static DISABLE_CGO=1)');
     await exec('sudo mkdir -p /etc/containers');
     await exec('sudo chown circleci:circleci /etc/containers');