chore: use imagePullSecrets in tests to avoid Docker Hub rate limiting

ivanstanev · ivanstanev · commit 4510198cd909 · 2021-04-09T10:51:48.000+01:00
This adds a new requirement of having extra environment variables. They are used to create imagePullSecrets to use for our fixtures, thus avoiding getting rate limited by Docker Hub when these images need to be pulled by KinD.

This does not solve the rate limiting from inside snyk-monitor using skopeo, which we've seen only very rarely. If that happens then the workaround is to set the dockercfg.json of snyk-monitor with auth details for Docker Hub.
diff --git a/test/README.md b/test/README.md
@@ -49,7 +49,7 @@ All integration tests require the Kubernetes-Monitor to be built into an image o
 The easiest way to achieve it is by running the `scripts/docker/build-image.sh` script.
 Please note that `docker` needs to be installed in order for this script to succeed.
 
-As part of these tests, we attempt pulling and scanning an image hosted on a private GCR registry. For this test case to work, one has to define the following environment variables: `GCR_IO_SERVICE_ACCOUNT`, `GCR_IO_DOCKERCFG`.
+As part of these tests, we attempt pulling and scanning an image hosted on a private GCR registry. For this test case to work, one has to define the following environment variables: `GCR_IO_SERVICE_ACCOUNT`, `GCR_IO_DOCKERCFG`, `DOCKER_HUB_RO_USERNAME`, `DOCKER_HUB_RO_PASSWORD`.
 
 Our integration tests may use different Kubernetes platforms to host the Kubernetes-Monitor. These platforms may use an existing cluster, or create a new one. Both decisions are based on the environment variables:
 * `TEST_PLATFORM` (`kind`, `kindolm`, `eks`)
diff --git a/test/fixtures/alpine-pod.yaml b/test/fixtures/alpine-pod.yaml
@@ -6,6 +6,8 @@ metadata:
   labels:
     app: alpine
 spec:
+  imagePullSecrets:
+  - name: docker-io
   containers:
   - name: alpine
     image: alpine
diff --git a/test/fixtures/binaries-deployment.yaml b/test/fixtures/binaries-deployment.yaml
@@ -15,6 +15,8 @@ spec:
       labels:
         app: binaries
     spec:
+      imagePullSecrets:
+      - name: docker-io
       containers:
       - name: node
         image: node@sha256:215a9fbef4df2c1ceb7c79481d3cfd94ad8f1f0105bade39f3be907bf386c5e1
diff --git a/test/fixtures/centos-deployment.yaml b/test/fixtures/centos-deployment.yaml
@@ -14,6 +14,8 @@ spec:
       labels:
         app.kubernetes.io/name: centos
     spec:
+      imagePullSecrets:
+      - name: docker-io
       containers:
       - image: centos:7
         imagePullPolicy: Always
diff --git a/test/fixtures/consul-deployment.yaml b/test/fixtures/consul-deployment.yaml
@@ -15,6 +15,8 @@ spec:
       labels:
         app.kubernetes.io/name: consul
     spec:
+      imagePullSecrets:
+      - name: docker-io
       containers:
       - image: snyk/runtime-fixtures:consul
         imagePullPolicy: Always
diff --git a/test/fixtures/insecure-registries/push-dockerhub-image-to-local-registry.yaml b/test/fixtures/insecure-registries/push-dockerhub-image-to-local-registry.yaml
@@ -5,6 +5,8 @@ metadata:
 spec:
   template:
     spec:
+      imagePullSecrets:
+        - name: docker-io
       containers:
         - name: my-container
           image: golang:1.13.1-alpine3.10
diff --git a/test/fixtures/java-deployment.yaml b/test/fixtures/java-deployment.yaml
@@ -15,6 +15,8 @@ spec:
       labels:
         app.kubernetes.io/name: java
     spec:
+      imagePullSecrets:
+      - name: docker-io
       containers:
       - image: java:latest
         imagePullPolicy: Always
diff --git a/test/fixtures/nginx-replicationcontroller.yaml b/test/fixtures/nginx-replicationcontroller.yaml
@@ -13,6 +13,8 @@ spec:
       labels:
         app: nginx
     spec:
+      imagePullSecrets:
+      - name: docker-io
       containers:
       - name: nginx
         image: nginx
diff --git a/test/fixtures/proxying/tinyproxy-deployment.yaml b/test/fixtures/proxying/tinyproxy-deployment.yaml
@@ -12,6 +12,8 @@ spec:
       labels:
         app: forwarding-proxy
     spec:
+      imagePullSecrets:
+      - name: docker-io
       containers:
       - name: forwarding-proxy
         image: snyk/runtime-fixtures:tinyproxy
diff --git a/test/fixtures/redis-deployment.yaml b/test/fixtures/redis-deployment.yaml
@@ -15,6 +15,8 @@ spec:
       labels:
         app.kubernetes.io/name: redis
     spec:
+      imagePullSecrets:
+      - name: docker-io
       containers:
       - image: redis:latest
         imagePullPolicy: Always
diff --git a/test/fixtures/scratch-deployment.yaml b/test/fixtures/scratch-deployment.yaml
@@ -15,6 +15,8 @@ spec:
       labels:
         app.kubernetes.io/name: busybox
     spec:
+      imagePullSecrets:
+      - name: docker-io
       containers:
       - image: busybox:1.31.1
         imagePullPolicy: Always
diff --git a/test/setup/index.ts b/test/setup/index.ts
@@ -96,6 +96,25 @@ async function predeploy(integrationId: string, namespace: string): Promise<void
   }
 }
 
+/** This is used in order to avoid Docker Hub rate limiting on our integration tests. */
+async function createSecretForDockerHubAccess(): Promise<void> {
+  const secretName = 'docker-io';
+  const secretsKeyPrefix = '--';
+  const secretType = 'docker-registry';
+  await kubectl.createSecret(
+    secretName,
+    'services',
+    {
+      'docker-server': 'https://docker.io',
+      'docker-username': getEnvVariableOrDefault('DOCKER_HUB_RO_USERNAME', ''),
+      'docker-email': 'runtime@snyk.io',
+      'docker-password': getEnvVariableOrDefault('DOCKER_HUB_RO_PASSWORD', ''),
+    },
+    secretsKeyPrefix,
+    secretType,
+  );
+}
+
 async function createSecretForGcrIoAccess(): Promise<void> {
   const gcrSecretName = 'gcr-io';
   const gcrKubectlSecretsKeyPrefix = '--';
@@ -147,6 +166,7 @@ export async function deployMonitor(): Promise<string> {
     const remoteImageName = await platforms[testPlatform].loadImage(imageNameAndTag);
     await createEnvironment();
     await createSecretForGcrIoAccess();
+    await createSecretForDockerHubAccess();
 
     const integrationId = getIntegrationId();
     await predeploy(integrationId, namespace);
diff --git a/test/setup/platforms/eks.ts b/test/setup/platforms/eks.ts
@@ -9,6 +9,8 @@ export async function validateRequiredEnvironment(): Promise<void> {
   throwIfEnvironmentVariableUnset('AWS_ACCESS_KEY_ID');
   throwIfEnvironmentVariableUnset('AWS_SECRET_ACCESS_KEY');
   throwIfEnvironmentVariableUnset('AWS_REGION');
+  throwIfEnvironmentVariableUnset('DOCKER_HUB_RO_USERNAME');
+  throwIfEnvironmentVariableUnset('DOCKER_HUB_RO_PASSWORD');
 }
 
 export async function setupTester(): Promise<void> {
diff --git a/test/setup/platforms/index.ts b/test/setup/platforms/index.ts
@@ -28,7 +28,7 @@ const kindSetup: IPlatformSetup = {
   config: kind.exportKubeConfig,
   clean: kind.clean,
   setupTester: kind.setupTester,
-  validateRequiredEnvironment: () => Promise.resolve(),
+  validateRequiredEnvironment: kind.validateRequiredEnvironment,
 };
 
 const kindOlmSetup: IPlatformSetup = {
@@ -38,7 +38,7 @@ const kindOlmSetup: IPlatformSetup = {
   config: kind.exportKubeConfig,
   clean: kind.clean,
   setupTester: kind.setupTester,
-  validateRequiredEnvironment: () => Promise.resolve(),
+  validateRequiredEnvironment: kindOlm.validateRequiredEnvironment,
 };
 
 const eksSetup: IPlatformSetup = {
@@ -59,7 +59,7 @@ const openshift3Setup: IPlatformSetup = {
   config: kind.exportKubeConfig,
   clean: kind.clean,
   setupTester: openshift3.setupTester,
-  validateRequiredEnvironment: () => Promise.resolve(),
+  validateRequiredEnvironment: openshift3.validateRequiredEnvironment,
 };
 
 const openshift4Setup: IPlatformSetup = {
diff --git a/test/setup/platforms/kind-olm.ts b/test/setup/platforms/kind-olm.ts
@@ -1,6 +1,7 @@
 import { createCluster as kindCreateCluster } from './kind';
 import * as kubectl from '../../helpers/kubectl';
 import * as sleep from 'sleep-promise';
+import { throwIfEnvironmentVariableUnset } from './helpers';
 
 export async function createCluster(version: string): Promise<void> {
   await kindCreateCluster(version);
@@ -25,3 +26,10 @@ export async function createCluster(version: string): Promise<void> {
   await kubectl.applyK8sYaml('./test/fixtures/operator/marketplace-operator.yaml');
 }
 
+export async function validateRequiredEnvironment(): Promise<void> {
+  console.log(
+    'Checking for the required environment variables: DOCKER_HUB_RO_USERNAME, DOCKER_HUB_RO_PASSWORD',
+  );
+  throwIfEnvironmentVariableUnset('DOCKER_HUB_RO_USERNAME');
+  throwIfEnvironmentVariableUnset('DOCKER_HUB_RO_PASSWORD');
+}
diff --git a/test/setup/platforms/kind.ts b/test/setup/platforms/kind.ts
@@ -2,6 +2,7 @@ import { accessSync, chmodSync, constants, writeFileSync } from 'fs';
 import { platform } from 'os';
 import { resolve } from 'path';
 import { execWrapper as exec } from '../../helpers/exec';
+import { throwIfEnvironmentVariableUnset } from './helpers';
 
 const clusterName = 'kind';
 
@@ -68,3 +69,11 @@ export async function download(osDistro: string, kindVersion: string): Promise<v
     console.log('KinD downloaded!');
   }
 }
+
+export async function validateRequiredEnvironment(): Promise<void> {
+  console.log(
+    'Checking for the required environment variables: DOCKER_HUB_RO_USERNAME, DOCKER_HUB_RO_PASSWORD',
+  );
+  throwIfEnvironmentVariableUnset('DOCKER_HUB_RO_USERNAME');
+  throwIfEnvironmentVariableUnset('DOCKER_HUB_RO_PASSWORD');
+}
diff --git a/test/setup/platforms/openshift3.ts b/test/setup/platforms/openshift3.ts
@@ -1,7 +1,16 @@
 import { platform } from 'os';
-import { download } from '../platforms/kind'; 
+import { download } from '../platforms/kind';
+import { throwIfEnvironmentVariableUnset } from './helpers';
 
 export async function setupTester(): Promise<void> {
   const osDistro = platform();
   await download(osDistro, 'v0.7.0');
 }
+
+export async function validateRequiredEnvironment(): Promise<void> {
+  console.log(
+    'Checking for the required environment variables: DOCKER_HUB_RO_USERNAME, DOCKER_HUB_RO_PASSWORD',
+  );
+  throwIfEnvironmentVariableUnset('DOCKER_HUB_RO_USERNAME');
+  throwIfEnvironmentVariableUnset('DOCKER_HUB_RO_PASSWORD');
+}
diff --git a/test/setup/platforms/openshift4.ts b/test/setup/platforms/openshift4.ts
@@ -12,11 +12,13 @@ const OPENSHIFT_CLI_VERSION = '4.7.0';
 
 export async function validateRequiredEnvironment(): Promise<void> {
   console.log(
-    'Checking for the required environment variables: OPENSHIFT4_USER, OPENSHIFT4_PASSWORD, OPENSHIFT4_CLUSTER_URL',
+    'Checking for the required environment variables: OPENSHIFT4_USER, OPENSHIFT4_PASSWORD, OPENSHIFT4_CLUSTER_URL, DOCKER_HUB_RO_USERNAME, DOCKER_HUB_RO_PASSWORD',
   );
   throwIfEnvironmentVariableUnset('OPENSHIFT4_USER');
   throwIfEnvironmentVariableUnset('OPENSHIFT4_PASSWORD');
   throwIfEnvironmentVariableUnset('OPENSHIFT4_CLUSTER_URL');
+  throwIfEnvironmentVariableUnset('DOCKER_HUB_RO_USERNAME');
+  throwIfEnvironmentVariableUnset('DOCKER_HUB_RO_PASSWORD');
 }
 
 export async function setupTester(): Promise<void> {
