fix: stabilise the monitor agent id

Rather than retrying the request at most 3 times, and then simply setting the agentId to an automatically generated UUID, we instead keep retrying the request indefinitely until we're able to retrieve the Kubernetes Deployment ID.

The retry logic will backoff exponentially up to a maximum configured value in seconds.

Author: jmickey

config.default.json

@@ -15,5 +15,6 @@
   "REQUEST_QUEUE_LENGTH": 2,
   "QUEUE_LENGTH_LOG_FREQUENCY_MINUTES": 15,
   "INTEGRATION_ID": "",
-  "DEFAULT_KUBERNETES_UPSTREAM_URL": "https://kubernetes-upstream.snyk.io"
+  "DEFAULT_KUBERNETES_UPSTREAM_URL": "https://kubernetes-upstream.snyk.io",
+  "MAX_RETRY_BACKOFF_DURATION_SECONDS": 300
 }

src/supervisor/agent.ts

@@ -1,7 +1,7 @@
 import { config } from '../common/config';
 import { logger } from '../common/logger';
 import { k8sApi } from './cluster';
-import { retryKubernetesApiRequest } from './kuberenetes-api-wrappers';
+import { retryKubernetesApiRequestIndefinitely } from './kuberenetes-api-wrappers';
 
 export async function setSnykMonitorAgentId(): Promise<void> {
   const name = config.DEPLOYMENT_NAME;
@@ -20,8 +20,9 @@ async function getSnykMonitorDeploymentUid(
   namespace: string,
 ): Promise<string | undefined> {
   try {
-    const attemptedApiCall = await retryKubernetesApiRequest(() =>
-      k8sApi.appsClient.readNamespacedDeployment(name, namespace),
+    const attemptedApiCall = await retryKubernetesApiRequestIndefinitely(
+      () => k8sApi.appsClient.readNamespacedDeployment(name, namespace),
+      config.MAX_RETRY_BACKOFF_DURATION_SECONDS,
     );
     return attemptedApiCall.body.metadata?.uid;
   } catch (error) {

src/supervisor/kuberenetes-api-wrappers.ts

@@ -1,5 +1,6 @@
 import * as http from 'http';
 import * as sleep from 'sleep-promise';
+import { logger } from '../common/logger';
 import { IRequestError } from './types';
 
 export const ATTEMPTS_MAX = 3;
@@ -26,6 +27,42 @@ export async function retryKubernetesApiRequest<ResponseType>(
   throw new Error('Could not receive a response from the Kubernetes API');
 }
 
+/**
+ * This function retries requests to the Kubernetes API indefinitely. We use this
+ * function when starting the Kubernetes Monitor to ensure the agentId is correctly
+ * set to the deployment ID.
+ *
+ * @param func function to retry
+ * @param maxSleepDuration maximum sleep duration in seconds (e.g. 300)
+ * @returns Promise<ResponseType>
+ */
+export async function retryKubernetesApiRequestIndefinitely<ResponseType>(
+  func: IKubernetesApiFunction<ResponseType>,
+  maxSleepDuration: number,
+): Promise<ResponseType> {
+  let attempts: number = 1;
+
+  while (true) {
+    try {
+      return await func();
+    } catch (err) {
+      if (!shouldRetryRequest(err, 1)) {
+        throw err;
+      }
+
+      const backoff = Math.pow(2, attempts);
+      const sleepSeconds = Math.min(backoff, maxSleepDuration);
+      logger.error(
+        { error: err },
+        `connection to kubernetes API failed, retrying in ${sleepSeconds} seconds`,
+      );
+
+      await sleep(sleepSeconds * 1000);
+      attempts++;
+    }
+  }
+}
+
 export function calculateSleepSeconds(
   httpResponse: http.IncomingMessage,
 ): number {

test/system/kind.spec.ts

@@ -58,7 +58,7 @@ test('Kubernetes-Monitor with KinD', async (jestDoneCallback) => {
 
   const agentId = randomUUID();
   const retryKubernetesApiRequestMock = jest
-    .spyOn(kubernetesApiWrappers, 'retryKubernetesApiRequest')
+    .spyOn(kubernetesApiWrappers, 'retryKubernetesApiRequestIndefinitely')
     .mockResolvedValueOnce({
       body: {
         metadata: {