Merge pull request #303 from snyk/fix/things

Fix/things

Author: Amir Moualem

src/index.ts

@@ -1,9 +1,15 @@
 import * as SourceMapSupport from 'source-map-support';
+
+import * as state from './state';
 import logger = require('./common/logger');
 import { currentClusterName } from './supervisor/cluster';
 import { beginWatchingWorkloads } from './supervisor/watchers';
 
 process.on('uncaughtException', (err) => {
+  if (state.shutdownInProgress) {
+    return;
+  }
+
   try {
     logger.error({err}, 'UNCAUGHT EXCEPTION!');
   } catch (ignore) {
@@ -13,8 +19,18 @@ process.on('uncaughtException', (err) => {
   }
 });
 
-process.on('unhandledRejection', (reason, promise) => {
-  logger.error({reason, promise}, 'unhandled rejection');
+process.on('unhandledRejection', (reason) => {
+  if (state.shutdownInProgress) {
+    return;
+  }
+
+  try {
+    logger.error({reason}, 'UNHANDLED REJECTION!');
+  } catch (ignore) {
+    console.log('UNHANDLED REJECTION!', reason);
+  } finally {
+    process.exit(1);
+  }
 });
 
 function monitor(): void {

src/state.ts

@@ -21,6 +21,7 @@ const workloadsLruCacheOptions = {
 };
 
 const state = {
+  shutdownInProgress: false,
   imagesAlreadyScanned: new lruCache<string, string>(imagesLruCacheOptions),
   workloadsAlreadyScanned: new lruCache<string, string>(workloadsLruCacheOptions),
 };

src/supervisor/watchers/handlers/index.ts

@@ -95,8 +95,17 @@ export function setupInformer(namespace: string, workloadKind: WorkloadKind) {
   const workloadMetadata = workloadWatchMetadata[workloadKind];
   const namespacedEndpoint = workloadMetadata.endpoint.replace('{namespace}', namespace);
 
-  const informer = makeInformer<KubernetesObject>(kubeConfig, namespacedEndpoint,
-    workloadMetadata.listFactory(namespace));
+  const listMethod = workloadMetadata.listFactory(namespace);
+  const loggedListMethod = async () => {
+    try {
+      return await listMethod();
+    } catch (err) {
+      logger.error({err, namespace, workloadKind}, 'error while listing entities on namespace');
+      throw err;
+    }
+  };
+
+  const informer = makeInformer<KubernetesObject>(kubeConfig, namespacedEndpoint, loggedListMethod);
 
   for (const informerVerb of Object.keys(workloadMetadata.handlers)) {
     informer.on(informerVerb, async (watchedWorkload) => {

src/supervisor/watchers/index.ts

@@ -1,5 +1,6 @@
 import { makeInformer, ADD } from '@kubernetes/client-node';
 import { V1Namespace } from '@kubernetes/client-node';
+
 import config = require('../../common/config');
 import logger = require('../../common/logger');
 import { WorkloadKind } from '../types';
@@ -51,7 +52,18 @@ export function isKubernetesInternalNamespace(namespace: string): boolean {
 }
 
 function setupWatchesForCluster(): void {
-  const informer = makeInformer(kubeConfig, '/api/v1/namespaces', () => k8sApi.coreClient.listNamespace());
+  const informer = makeInformer(
+    kubeConfig,
+    '/api/v1/namespaces',
+    async () => {
+      try {
+        return await k8sApi.coreClient.listNamespace();
+      } catch (err) {
+        logger.error({err}, 'error while listing namespaces');
+        throw err;
+      }
+    },
+  );
 
   informer.on(ADD, (namespace: V1Namespace) => {
     try {

test/system/kind.test.ts

@@ -97,5 +97,10 @@ tap.test('Kubernetes-Monitor with KinD', async (t) => {
   // additional asserts?
   t.ok(nock.isDone(), 'all outgoing calls were made');
 
+  // instruct the Monitor to ignore errors from this point
+  // eslint-disable-next-line @typescript-eslint/no-var-requires
+  const kubernetesMonitorState = require('../../src/state');
+  kubernetesMonitorState.shutdownInProgress = true;
+
   // TODO cleanup the images we saved to /var/tmp?
 });