Merge pull request #663 from snyk/feat/queue-size-log

feat: log the size of the different work queues every 15 minutes

Author: Amir Moualem

config.default.json

@@ -13,6 +13,7 @@
   },
   "WORKLOADS_TO_SCAN_QUEUE_WORKER_COUNT": 10,
   "METADATA_TO_SEND_QUEUE_WORKER_COUNT": 10,
+  "QUEUE_LENGTH_LOG_FREQUENCY_MINUTES": 15,
   "INTEGRATION_ID": "",
   "DEFAULT_KUBERNETES_UPSTREAM_URL": "https://kubernetes-upstream.snyk.io"
 }

src/supervisor/watchers/handlers/pod.ts

@@ -54,6 +54,17 @@ metadataToSendQueue.error(function(err, task) {
   logger.error({err, task}, 'error processing a workload metadata send task');
 });
 
+setInterval(() => {
+  try {
+    const queueDataToReport: {[key: string]: any} = {};
+    queueDataToReport.workloadsToScanLength = workloadsToScanQueue.length();
+    queueDataToReport.metadataToSendLength = metadataToSendQueue.length();
+    logger.info(queueDataToReport, 'queue sizes report');
+  } catch (err) {
+    logger.warn({err}, 'failed logging queue sizes');
+  }
+}, config.QUEUE_LENGTH_LOG_FREQUENCY_MINUTES * 60 * 1000).unref();
+
 function handleReadyPod(workloadMetadata: IWorkload[]): void {
   const imagesToScan: IWorkload[] = [];
   const imageKeys: string[] = [];