Merge pull request #848 from snyk/fix/stabilise-agentId

fix: stabilise the monitor agent id

Author: jmickey

config.default.json

@@ -15,5 +15,6 @@
   "REQUEST_QUEUE_LENGTH": 2,
   "QUEUE_LENGTH_LOG_FREQUENCY_MINUTES": 15,
   "INTEGRATION_ID": "",
-  "DEFAULT_KUBERNETES_UPSTREAM_URL": "https://kubernetes-upstream.snyk.io"
+  "DEFAULT_KUBERNETES_UPSTREAM_URL": "https://kubernetes-upstream.snyk.io",
+  "MAX_RETRY_BACKOFF_DURATION_SECONDS": 300
 }

src/supervisor/agent.ts

@@ -1,7 +1,7 @@
 import { config } from '../common/config';
 import { logger } from '../common/logger';
 import { k8sApi } from './cluster';
-import { retryKubernetesApiRequest } from './kuberenetes-api-wrappers';
+import { retryKubernetesApiRequestIndefinitely } from './kuberenetes-api-wrappers';
 
 export async function setSnykMonitorAgentId(): Promise<void> {
   const name = config.DEPLOYMENT_NAME;
@@ -20,8 +20,9 @@ async function getSnykMonitorDeploymentUid(
   namespace: string,
 ): Promise<string | undefined> {
   try {
-    const attemptedApiCall = await retryKubernetesApiRequest(() =>
-      k8sApi.appsClient.readNamespacedDeployment(name, namespace),
+    const attemptedApiCall = await retryKubernetesApiRequestIndefinitely(
+      () => k8sApi.appsClient.readNamespacedDeployment(name, namespace),
+      config.MAX_RETRY_BACKOFF_DURATION_SECONDS,
     );
     return attemptedApiCall.body.metadata?.uid;
   } catch (error) {

src/supervisor/kuberenetes-api-wrappers.ts

@@ -1,5 +1,6 @@
 import * as http from 'http';
 import * as sleep from 'sleep-promise';
+import { logger } from '../common/logger';
 import { IRequestError } from './types';
 
 export const ATTEMPTS_MAX = 3;
@@ -26,6 +27,42 @@ export async function retryKubernetesApiRequest<ResponseType>(
   throw new Error('Could not receive a response from the Kubernetes API');
 }
 
+/**
+ * This function retries requests to the Kubernetes API indefinitely. We use this
+ * function when starting the Kubernetes Monitor to ensure the agentId is correctly
+ * set to the deployment ID.
+ *
+ * @param func function to retry
+ * @param maxSleepDuration maximum sleep duration in seconds (e.g. 300)
+ * @returns Promise<ResponseType>
+ */
+export async function retryKubernetesApiRequestIndefinitely<ResponseType>(
+  func: IKubernetesApiFunction<ResponseType>,
+  maxSleepDuration: number,
+): Promise<ResponseType> {
+  let attempts: number = 1;
+
+  while (true) {
+    try {
+      return await func();
+    } catch (err) {
+      if (!shouldRetryRequest(err, 1)) {
+        throw err;
+      }
+
+      const backoff = Math.pow(2, attempts);
+      const sleepSeconds = Math.min(backoff, maxSleepDuration);
+      logger.error(
+        { error: err },
+        `connection to kubernetes API failed, retrying in ${sleepSeconds} seconds`,
+      );
+
+      await sleep(sleepSeconds * 1000);
+      attempts++;
+    }
+  }
+}
+
 export function calculateSleepSeconds(
   httpResponse: http.IncomingMessage,
 ): number {

test/system/kind.spec.ts

@@ -58,7 +58,7 @@ test('Kubernetes-Monitor with KinD', async (jestDoneCallback) => {
 
   const agentId = randomUUID();
   const retryKubernetesApiRequestMock = jest
-    .spyOn(kubernetesApiWrappers, 'retryKubernetesApiRequest')
+    .spyOn(kubernetesApiWrappers, 'retryKubernetesApiRequestIndefinitely')
     .mockResolvedValueOnce({
       body: {
         metadata: {