feat: allow configuring skopeo compression and image scan worker count

Allowing to experiment with these values to see if it helps alleviate performance issues.
We see that in some cases the CPU is pegged to 100% and causes lots of throttling by Kubernetes.

We suspect that part of the problem is the heavy compression done by skopeo which is used to pull an image from the container registry and then store it to disk.
Additionally, allow to configure the number of workers that run in parallel. This would allow to further reduce the CPU load.

Author: ivanstanev

config.default.json

@@ -11,7 +11,7 @@
     "MAX_SIZE": 10000,
     "MAX_AGE_MS": 60000
   },
-  "WORKLOADS_TO_SCAN_QUEUE_WORKER_COUNT": 10,
+  "WORKERS_COUNT": 10,
   "REQUEST_QUEUE_LENGTH": 2,
   "QUEUE_LENGTH_LOG_FREQUENCY_MINUTES": 15,
   "INTEGRATION_ID": "",

snyk-monitor/templates/deployment.yaml

@@ -128,6 +128,10 @@ spec:
             value: {{ .Values.log_level }}
           - name: SKIP_K8S_JOBS
             value: {{ quote .Values.skip_k8s_jobs }}
+          - name: SNYK_SKOPEO_COMPRESSION_LEVEL
+            value: {{ quote .Values.skopeo.compression.level }}
+          - name: SNYK_WORKERS_COUNT
+            value: {{ quote .Values.workers.count }}
           {{- with .Values.envs }}
           {{- toYaml . | trim | nindent 10 -}}
           {{- end }}

snyk-monitor/values.yaml

@@ -136,3 +136,10 @@ tolerations: []
 volumes:
   projected:
     serviceAccountToken: false
+
+skopeo:
+  compression:
+    level: 6
+
+workers:
+  count: 10

src/common/config.ts

@@ -42,8 +42,8 @@ config.CLUSTER_NAME = getClusterName();
 config.IMAGE_STORAGE_ROOT = '/var/tmp';
 config.POLICIES_STORAGE_ROOT = '/tmp/policies';
 config.EXCLUDED_NAMESPACES = loadExcludedNamespaces();
-config.WORKLOADS_TO_SCAN_QUEUE_WORKER_COUNT =
-  Number(config.WORKLOADS_TO_SCAN_QUEUE_WORKER_COUNT) || 10;
+config.WORKERS_COUNT = Number(config.WORKERS_COUNT) || 10;
+config.SKOPEO_COMPRESSION_LEVEL = Number(config.SKOPEO_COMPRESSION_LEVEL) || 6;
 
 /**
  * Important: we delete the following env vars because we don't want to proxy requests to the Kubernetes API server.

src/scanner/images/skopeo.ts

@@ -44,6 +44,8 @@ export async function pull(
 
   const args: Array<processWrapper.IProcessArgument> = [];
   args.push({ body: 'copy', sanitise: false });
+  args.push({ body: '--dest-compress-level', sanitise: false });
+  args.push({ body: `${config.SKOPEO_COMPRESSION_LEVEL}`, sanitise: false });
   args.push(...credentialsParameters);
   args.push(...certificatesParameters);
   args.push({

src/supervisor/watchers/handlers/pod.ts

@@ -61,7 +61,7 @@ async function queueWorkerWorkloadScan(
 
 const workloadsToScanQueue = async.queue<ImagesToScanQueueData>(
   queueWorkerWorkloadScan,
-  config.WORKLOADS_TO_SCAN_QUEUE_WORKER_COUNT,
+  config.WORKERS_COUNT,
 );
 
 workloadsToScanQueue.error(function (err, task) {

test/common/config.spec.ts

@@ -71,6 +71,7 @@ describe('extractNamespaceName()', () => {
     expect(config.HTTP_PROXY).toBeUndefined();
     expect(config.NO_PROXY).toBeUndefined();
     expect(config.SKIP_K8S_JOBS).toEqual(false);
-    expect(config.WORKLOADS_TO_SCAN_QUEUE_WORKER_COUNT).toEqual(10);
+    expect(config.WORKERS_COUNT).toEqual(10);
+    expect(config.SKOPEO_COMPRESSION_LEVEL).toEqual(6);
   });
 });

test/setup/deployers/helm.ts

@@ -37,7 +37,9 @@ async function deployKubernetesMonitor(
       '--set log_level="INFO" ' +
       '--set rbac.serviceAccount.annotations."foo"="bar" ' +
       '--set volumes.projected.serviceAccountToken=true ' +
-      '--set securityContext.fsGroup=65534 ',
+      '--set securityContext.fsGroup=65534 ' +
+      '--set skopeo.compression.level=1 ' +
+      '--set workers.count=5 ',
   );
   console.log(
     `Deployed ${imageOptions.nameAndTag} with pull policy ${imageOptions.pullPolicy}`,