fix: retry on additional errors for Kubernetes API calls

 Some requests to the Kubernetes API server can fail due to network errors which we don't handle and can cause the container to crash. We think the errors may be temporary and have added a retry mechanism that repeats the request a few times. If the error does not resolve after a few retries then we allow the container to crash like it did previously.

Author: Arthur Granado

src/supervisor/watchers/handlers/index.ts

@@ -10,6 +10,7 @@ import { replicaSetWatchHandler } from './replica-set';
 import { replicationControllerWatchHandler } from './replication-controller';
 import { statefulSetWatchHandler } from './stateful-set';
 import { k8sApi, kubeConfig } from '../../cluster';
+import * as kubernetesApiWrappers from '../../kuberenetes-api-wrappers';
 import { IWorkloadWatchMetadata, FALSY_WORKLOAD_NAME_MARKER } from './types';
 
 /**
@@ -98,7 +99,8 @@ export function setupInformer(namespace: string, workloadKind: WorkloadKind) {
   const listMethod = workloadMetadata.listFactory(namespace);
   const loggedListMethod = async () => {
     try {
-      return await listMethod();
+      return await kubernetesApiWrappers.retryKubernetesApiRequest(
+        () => listMethod());
     } catch (err) {
       logger.error({err, namespace, workloadKind}, 'error while listing entities on namespace');
       throw err;

src/supervisor/watchers/index.ts

@@ -6,6 +6,7 @@ import logger = require('../../common/logger');
 import { WorkloadKind } from '../types';
 import { setupInformer } from './handlers';
 import { kubeConfig, k8sApi } from '../cluster';
+import * as kubernetesApiWrappers from '../kuberenetes-api-wrappers';
 import { kubernetesInternalNamespaces } from './internal-namespaces';
 
 /**
@@ -52,7 +53,8 @@ function setupWatchesForCluster(): void {
     '/api/v1/namespaces',
     async () => {
       try {
-        return await k8sApi.coreClient.listNamespace();
+        return await kubernetesApiWrappers.retryKubernetesApiRequest(
+          () => k8sApi.coreClient.listNamespace());
       } catch (err) {
         logger.error({err}, 'error while listing namespaces');
         throw err;

test/system/kind.test.ts

@@ -69,6 +69,30 @@ tap.test('Kubernetes-Monitor with KinD', async (t) => {
   ]);
 
   // Setup nocks
+  nock(/https\:\/\/127\.0\.0\.1\:\d+/, { allowUnmocked: true})
+    .get('/api/v1/namespaces')
+    .times(1)
+    .replyWithError({
+      code: 'ECONNREFUSED'
+    })
+    .get('/api/v1/namespaces')
+    .times(1)
+    .replyWithError({
+      code: 'ETIMEDOUT'
+    });
+
+  nock(/https\:\/\/127\.0\.0\.1\:\d+/, { allowUnmocked: true})
+    .get('/apis/apps/v1/namespaces/snyk-monitor/deployments')
+    .times(1)
+    .replyWithError({
+      code: 'ECONNREFUSED'
+    })
+    .get('/apis/apps/v1/namespaces/snyk-monitor/deployments')
+    .times(1)
+    .replyWithError({
+      code: 'ETIMEDOUT'
+    });
+
   nock('https://kubernetes-upstream.snyk.io')
     .post('/api/v1/workload')
     .times(1)