Merge pull request #992 from snyk/feat/configure-workers-compression

[RUN-2038] feat: allow configuring skopeo compression and image scan worker count

Author: ivanstanev

config.default.json

@@ -11,7 +11,7 @@
     "MAX_SIZE": 10000,
     "MAX_AGE_MS": 60000
   },
-  "WORKLOADS_TO_SCAN_QUEUE_WORKER_COUNT": 10,
+  "WORKERS_COUNT": 10,
   "REQUEST_QUEUE_LENGTH": 2,
   "QUEUE_LENGTH_LOG_FREQUENCY_MINUTES": 15,
   "INTEGRATION_ID": "",

snyk-monitor/templates/deployment.yaml

@@ -128,6 +128,10 @@ spec:
             value: {{ .Values.log_level }}
           - name: SKIP_K8S_JOBS
             value: {{ quote .Values.skip_k8s_jobs }}
+          - name: SNYK_SKOPEO_COMPRESSION_LEVEL
+            value: {{ quote .Values.skopeo.compression.level }}
+          - name: SNYK_WORKERS_COUNT
+            value: {{ quote .Values.workers.count }}
           {{- with .Values.envs }}
           {{- toYaml . | trim | nindent 10 -}}
           {{- end }}

snyk-monitor/values.yaml

@@ -136,3 +136,10 @@ tolerations: []
 volumes:
   projected:
     serviceAccountToken: false
+
+skopeo:
+  compression:
+    level: 6
+
+workers:
+  count: 10

src/common/config.ts

@@ -42,8 +42,8 @@ config.CLUSTER_NAME = getClusterName();
 config.IMAGE_STORAGE_ROOT = '/var/tmp';
 config.POLICIES_STORAGE_ROOT = '/tmp/policies';
 config.EXCLUDED_NAMESPACES = loadExcludedNamespaces();
-config.WORKLOADS_TO_SCAN_QUEUE_WORKER_COUNT =
-  Number(config.WORKLOADS_TO_SCAN_QUEUE_WORKER_COUNT) || 10;
+config.WORKERS_COUNT = Number(config.WORKERS_COUNT) || 10;
+config.SKOPEO_COMPRESSION_LEVEL = Number(config.SKOPEO_COMPRESSION_LEVEL) || 6;
 
 /**
  * Important: we delete the following env vars because we don't want to proxy requests to the Kubernetes API server.

src/scanner/images/skopeo.ts

@@ -44,6 +44,8 @@ export async function pull(
 
   const args: Array<processWrapper.IProcessArgument> = [];
   args.push({ body: 'copy', sanitise: false });
+  args.push({ body: '--dest-compress-level', sanitise: false });
+  args.push({ body: `${config.SKOPEO_COMPRESSION_LEVEL}`, sanitise: false });
   args.push(...credentialsParameters);
   args.push(...certificatesParameters);
   args.push({

src/supervisor/watchers/handlers/pod.ts

@@ -61,7 +61,7 @@ async function queueWorkerWorkloadScan(
 
 const workloadsToScanQueue = async.queue<ImagesToScanQueueData>(
   queueWorkerWorkloadScan,
-  config.WORKLOADS_TO_SCAN_QUEUE_WORKER_COUNT,
+  config.WORKERS_COUNT,
 );
 
 workloadsToScanQueue.error(function (err, task) {

test/common/config.spec.ts

@@ -71,6 +71,7 @@ describe('extractNamespaceName()', () => {
     expect(config.HTTP_PROXY).toBeUndefined();
     expect(config.NO_PROXY).toBeUndefined();
     expect(config.SKIP_K8S_JOBS).toEqual(false);
-    expect(config.WORKLOADS_TO_SCAN_QUEUE_WORKER_COUNT).toEqual(10);
+    expect(config.WORKERS_COUNT).toEqual(10);
+    expect(config.SKOPEO_COMPRESSION_LEVEL).toEqual(6);
   });
 });

test/setup/deployers/helm.ts

@@ -37,7 +37,9 @@ async function deployKubernetesMonitor(
       '--set log_level="INFO" ' +
       '--set rbac.serviceAccount.annotations."foo"="bar" ' +
       '--set volumes.projected.serviceAccountToken=true ' +
-      '--set securityContext.fsGroup=65534 ',
+      '--set securityContext.fsGroup=65534 ' +
+      '--set skopeo.compression.level=1 ' +
+      '--set workers.count=5 ',
   );
   console.log(
     `Deployed ${imageOptions.nameAndTag} with pull policy ${imageOptions.pullPolicy}`,