Merge pull request #121 from snyk/feat/workload-metadata

Tal Kaptsan · web-flow · commit 0eb93bbf575b · 2019-09-08T19:00:00.000+03:00
Feat/workload metadata
diff --git a/src/kube-scanner/index.ts b/src/kube-scanner/index.ts
@@ -10,7 +10,7 @@ import { pullImages } from '../images';
 import { scanImages, IScanResult } from './image-scanner';
 import { deleteHomebaseWorkload, sendDepGraph } from '../transmitter';
 import { constructHomebaseDeleteWorkloadPayload, constructHomebaseWorkloadPayloads } from '../transmitter/payload';
-import { IDepGraphPayload, IKubeImage, ILocalWorkloadLocator } from '../transmitter/types';
+import { IDepGraphPayload, IWorkload, ILocalWorkloadLocator } from '../transmitter/types';
 
 export = class WorkloadWorker {
   private readonly name: string;
@@ -19,7 +19,7 @@ export = class WorkloadWorker {
     this.name = name;
   }
 
-  public async process(workloadMetadata: IKubeImage[]) {
+  public async process(workloadMetadata: IWorkload[]) {
     const workloadName = this.name;
     const allImages = workloadMetadata.map((meta) => meta.imageName);
     logger.info({workloadName, imageCount: allImages.length}, 'Queried workloads');
diff --git a/src/kube-scanner/metadata-extractor.ts b/src/kube-scanner/metadata-extractor.ts
@@ -1,5 +1,5 @@
 import { V1OwnerReference, V1Pod, V1Container, V1ContainerStatus } from '@kubernetes/client-node';
-import { IKubeImage, ILocalWorkloadLocator } from '../transmitter/types';
+import { IWorkload, ILocalWorkloadLocator } from '../transmitter/types';
 import { currentClusterName } from './cluster';
 import { KubeObjectMetadata } from './types';
 import { getSupportedWorkload, getWorkloadReader } from './workload-reader';
@@ -12,8 +12,8 @@ const loopingThreshold = 20;
 export function buildImageMetadata(
   workloadMeta: KubeObjectMetadata,
   containerStatuses: V1ContainerStatus[],
-  ): IKubeImage[] {
-  const { kind, objectMeta, specMeta, containers } = workloadMeta;
+  ): IWorkload[] {
+  const { kind, objectMeta, specMeta, containers, revision } = workloadMeta;
   const { name, namespace, labels, annotations, uid } = objectMeta;
 
   const containerNameToSpec: {[key: string]: V1Container} = {};
@@ -39,7 +39,8 @@ export function buildImageMetadata(
       imageName: containerNameToSpec[containerName].image,
       imageId: containerNameToStatus[containerName].imageID,
       cluster: currentClusterName,
-    } as IKubeImage),
+      revision,
+    } as IWorkload),
   );
   return images;
 }
@@ -88,7 +89,7 @@ export function buildWorkloadMetadata(kubernetesMetadata: KubeObjectMetadata): I
   };
 }
 
-export async function buildMetadataForWorkload(pod: V1Pod): Promise<IKubeImage[] | undefined> {
+export async function buildMetadataForWorkload(pod: V1Pod): Promise<IWorkload[] | undefined> {
   const isAssociatedWithParent = isPodAssociatedWithParent(pod);
 
   if (!pod.metadata || pod.metadata.namespace === undefined || !pod.spec) {
diff --git a/src/kube-scanner/types.ts b/src/kube-scanner/types.ts
@@ -18,6 +18,7 @@ export interface KubeObjectMetadata {
   specMeta: V1ObjectMeta;
   containers: V1Container[];
   ownerRefs: V1OwnerReference[] | undefined;
+  revision?: number;
 }
 
 export interface IK8sClients {
diff --git a/src/kube-scanner/watchers/handlers/daemon-set.ts b/src/kube-scanner/watchers/handlers/daemon-set.ts
@@ -5,7 +5,7 @@ import { FALSY_WORKLOAD_NAME_MARKER } from './types';
 
 export async function daemonSetWatchHandler(daemonSet: V1DaemonSet) {
   if (!daemonSet.metadata || !daemonSet.spec || !daemonSet.spec.template.metadata ||
-      !daemonSet.spec.template.spec) {
+      !daemonSet.spec.template.spec || !daemonSet.status) {
     // TODO(ivanstanev): possibly log this. It shouldn't happen but we should track it!
     return;
   }
@@ -18,5 +18,6 @@ export async function daemonSetWatchHandler(daemonSet: V1DaemonSet) {
     specMeta: daemonSet.spec.template.metadata,
     containers: daemonSet.spec.template.spec.containers,
     ownerRefs: daemonSet.metadata.ownerReferences,
+    revision: daemonSet.status.observedGeneration,
   }, workloadName);
 }
diff --git a/src/kube-scanner/watchers/handlers/deployment.ts b/src/kube-scanner/watchers/handlers/deployment.ts
@@ -5,7 +5,7 @@ import { FALSY_WORKLOAD_NAME_MARKER } from './types';
 
 export async function deploymentWatchHandler(deployment: V1Deployment) {
   if (!deployment.metadata || !deployment.spec || !deployment.spec.template.metadata ||
-      !deployment.spec.template.spec) {
+      !deployment.spec.template.spec || !deployment.status) {
     // TODO(ivanstanev): possibly log this. It shouldn't happen but we should track it!
     return;
   }
@@ -18,5 +18,6 @@ export async function deploymentWatchHandler(deployment: V1Deployment) {
     specMeta: deployment.spec.template.metadata,
     containers: deployment.spec.template.spec.containers,
     ownerRefs: deployment.metadata.ownerReferences,
+    revision: deployment.status.observedGeneration,
   }, workloadName);
 }
diff --git a/src/kube-scanner/watchers/handlers/pod.ts b/src/kube-scanner/watchers/handlers/pod.ts
@@ -3,7 +3,7 @@ import async = require('async');
 import config = require('../../../common/config');
 import logger = require('../../../common/logger');
 import WorkloadWorker = require('../../../kube-scanner');
-import { IKubeImage } from '../../../transmitter/types';
+import { IWorkload } from '../../../transmitter/types';
 import { buildMetadataForWorkload } from '../../metadata-extractor';
 import { PodPhase } from '../types';
 import state = require('../../../state');
@@ -25,8 +25,8 @@ workloadsToScanQueue.error(function(err, task) {
   logger.error({err, task}, 'error processing a workload in the pod handler 1');
 });
 
-async function handleReadyPod(workloadWorker: WorkloadWorker, workloadMetadata: IKubeImage[]) {
-  const imagesToScan: IKubeImage[] = [];
+async function handleReadyPod(workloadWorker: WorkloadWorker, workloadMetadata: IWorkload[]) {
+  const imagesToScan: IWorkload[] = [];
   const imageKeys: string[] = [];
   for (const image of workloadMetadata) {
     const imageKey = `${image.namespace}/${image.type}/${image.name}/${image.imageId}`;
diff --git a/src/kube-scanner/watchers/handlers/replica-set.ts b/src/kube-scanner/watchers/handlers/replica-set.ts
@@ -5,7 +5,7 @@ import { FALSY_WORKLOAD_NAME_MARKER } from './types';
 
 export async function replicaSetWatchHandler(replicaSet: V1ReplicaSet) {
   if (!replicaSet.metadata || !replicaSet.spec || !replicaSet.spec.template ||
-      !replicaSet.spec.template.metadata || !replicaSet.spec.template.spec) {
+      !replicaSet.spec.template.metadata || !replicaSet.spec.template.spec || !replicaSet.status) {
     // TODO(ivanstanev): possibly log this. It shouldn't happen but we should track it!
     return;
   }
@@ -18,5 +18,6 @@ export async function replicaSetWatchHandler(replicaSet: V1ReplicaSet) {
     specMeta: replicaSet.spec.template.metadata,
     containers: replicaSet.spec.template.spec.containers,
     ownerRefs: replicaSet.metadata.ownerReferences,
+    revision: replicaSet.status.observedGeneration,
   }, workloadName);
 }
diff --git a/src/kube-scanner/watchers/handlers/replication-controller.ts b/src/kube-scanner/watchers/handlers/replication-controller.ts
@@ -5,7 +5,8 @@ import { FALSY_WORKLOAD_NAME_MARKER } from './types';
 
 export async function replicationControllerWatchHandler(replicationController: V1ReplicationController) {
   if (!replicationController.metadata || !replicationController.spec || !replicationController.spec.template ||
-      !replicationController.spec.template.metadata || !replicationController.spec.template.spec) {
+      !replicationController.spec.template.metadata || !replicationController.spec.template.spec ||
+      !replicationController.status) {
     // TODO(ivanstanev): possibly log this. It shouldn't happen but we should track it!
     return;
   }
@@ -18,5 +19,6 @@ export async function replicationControllerWatchHandler(replicationController: V
     specMeta: replicationController.spec.template.metadata,
     containers: replicationController.spec.template.spec.containers,
     ownerRefs: replicationController.metadata.ownerReferences,
+    revision: replicationController.status.observedGeneration,
   }, workloadName);
 }
diff --git a/src/kube-scanner/watchers/handlers/stateful-set.ts b/src/kube-scanner/watchers/handlers/stateful-set.ts
@@ -5,7 +5,7 @@ import { FALSY_WORKLOAD_NAME_MARKER } from './types';
 
 export async function statefulSetWatchHandler(statefulSet: V1StatefulSet) {
   if (!statefulSet.metadata || !statefulSet.spec || !statefulSet.spec.template.metadata ||
-      !statefulSet.spec.template.spec) {
+      !statefulSet.spec.template.spec || !statefulSet.status) {
     // TODO(ivanstanev): possibly log this. It shouldn't happen but we should track it!
     return;
   }
@@ -18,5 +18,6 @@ export async function statefulSetWatchHandler(statefulSet: V1StatefulSet) {
     specMeta: statefulSet.spec.template.metadata,
     containers: statefulSet.spec.template.spec.containers,
     ownerRefs: statefulSet.metadata.ownerReferences,
+    revision: statefulSet.status.observedGeneration,
   }, workloadName);
 }
diff --git a/src/kube-scanner/workload-reader.ts b/src/kube-scanner/workload-reader.ts
@@ -13,7 +13,7 @@ const deploymentReader: IWorkloadReaderFunc = async (workloadName, namespace) =>
   const deployment = deploymentResult.body;
 
   if (!deployment.metadata || !deployment.spec || !deployment.spec.template.metadata ||
-      !deployment.spec.template.spec) {
+      !deployment.spec.template.spec || !deployment.status) {
     // TODO(ivanstanev): add logging to know when/if it happens!
     return undefined;
   }
@@ -24,6 +24,7 @@ const deploymentReader: IWorkloadReaderFunc = async (workloadName, namespace) =>
     specMeta: deployment.spec.template.metadata,
     containers: deployment.spec.template.spec.containers,
     ownerRefs: deployment.metadata.ownerReferences,
+    revision: deployment.status.observedGeneration,
   };
 };
 
@@ -33,7 +34,7 @@ const replicaSetReader: IWorkloadReaderFunc = async (workloadName, namespace) =>
   const replicaSet = replicaSetResult.body;
 
   if (!replicaSet.metadata || !replicaSet.spec || !replicaSet.spec.template ||
-      !replicaSet.spec.template.metadata || !replicaSet.spec.template.spec) {
+      !replicaSet.spec.template.metadata || !replicaSet.spec.template.spec || !replicaSet.status) {
     // TODO(ivanstanev): add logging to know when/if it happens!
     return undefined;
   }
@@ -44,6 +45,7 @@ const replicaSetReader: IWorkloadReaderFunc = async (workloadName, namespace) =>
     specMeta: replicaSet.spec.template.metadata,
     containers: replicaSet.spec.template.spec.containers,
     ownerRefs: replicaSet.metadata.ownerReferences,
+    revision: replicaSet.status.observedGeneration,
   };
 };
 
@@ -53,7 +55,7 @@ const statefulSetReader: IWorkloadReaderFunc = async (workloadName, namespace) =
   const statefulSet = statefulSetResult.body;
 
   if (!statefulSet.metadata || !statefulSet.spec || !statefulSet.spec.template.metadata ||
-      !statefulSet.spec.template.spec) {
+      !statefulSet.spec.template.spec || !statefulSet.status) {
     // TODO(ivanstanev): add logging to know when/if it happens!
     return undefined;
   }
@@ -64,6 +66,7 @@ const statefulSetReader: IWorkloadReaderFunc = async (workloadName, namespace) =
     specMeta: statefulSet.spec.template.metadata,
     containers: statefulSet.spec.template.spec.containers,
     ownerRefs: statefulSet.metadata.ownerReferences,
+    revision: statefulSet.status.observedGeneration,
   };
 };
 
@@ -73,7 +76,7 @@ const daemonSetReader: IWorkloadReaderFunc = async (workloadName, namespace) =>
   const daemonSet = daemonSetResult.body;
 
   if (!daemonSet.metadata || !daemonSet.spec || !daemonSet.spec.template.spec ||
-      !daemonSet.spec.template.metadata) {
+      !daemonSet.spec.template.metadata || !daemonSet.status) {
     // TODO(ivanstanev): add logging to know when/if it happens!
     return undefined;
   }
@@ -84,6 +87,7 @@ const daemonSetReader: IWorkloadReaderFunc = async (workloadName, namespace) =>
     specMeta: daemonSet.spec.template.metadata,
     containers: daemonSet.spec.template.spec.containers,
     ownerRefs: daemonSet.metadata.ownerReferences,
+    revision: daemonSet.status.observedGeneration,
   };
 };
 
@@ -135,7 +139,8 @@ const replicationControllerReader: IWorkloadReaderFunc = async (workloadName, na
   const replicationController = replicationControllerResult.body;
 
   if (!replicationController.metadata || !replicationController.spec || !replicationController.spec.template ||
-      !replicationController.spec.template.metadata || !replicationController.spec.template.spec) {
+      !replicationController.spec.template.metadata || !replicationController.spec.template.spec ||
+      !replicationController.status) {
     // TODO(ivanstanev): add logging to know when/if it happens!
     return undefined;
   }
@@ -146,6 +151,7 @@ const replicationControllerReader: IWorkloadReaderFunc = async (workloadName, na
     specMeta: replicationController.spec.template.metadata,
     containers: replicationController.spec.template.spec.containers,
     ownerRefs: replicationController.metadata.ownerReferences,
+    revision: replicationController.status.observedGeneration,
   };
 };
 
diff --git a/src/transmitter/payload.ts b/src/transmitter/payload.ts
@@ -1,16 +1,16 @@
 import config = require('../common/config');
 import { currentClusterName } from '../kube-scanner/cluster';
 import { IScanResult } from '../kube-scanner/image-scanner';
-import { IDeleteWorkloadPayload, IDepGraphPayload, IKubeImage, ILocalWorkloadLocator, IImageLocator } from './types';
+import { IDeleteWorkloadPayload, IDepGraphPayload, IWorkload, ILocalWorkloadLocator, IImageLocator } from './types';
 
 export function constructHomebaseWorkloadPayloads(
     scannedImages: IScanResult[],
-    workloadMetadata: IKubeImage[],
+    workloadMetadata: IWorkload[],
 ): IDepGraphPayload[] {
   const results = scannedImages.map((scannedImage) => {
-    const kubeImage: IKubeImage = workloadMetadata.find((meta) => meta.imageName === scannedImage.imageWithTag)!;
+    const kubeWorkload: IWorkload = workloadMetadata.find((meta) => meta.imageName === scannedImage.imageWithTag)!;
 
-    const { cluster, namespace, type, name } = kubeImage;
+    const { cluster, namespace, type, name } = kubeWorkload;
 
     const imageLocator: IImageLocator = {
       userLocator: config.INTEGRATION_ID,
diff --git a/src/transmitter/types.ts b/src/transmitter/types.ts
@@ -32,15 +32,16 @@ export interface IDeleteWorkloadPayload {
   agentId: string;
 }
 
-export interface IKubeImage {
+export interface IWorkload {
   type: string;
   name: string;
   namespace: string;
   labels: StringMap | undefined;
-  annotations: StringMap  | undefined;
+  annotations: StringMap | undefined;
   uid: string;
-  specLabels: StringMap  | undefined;
-  specAnnotations: StringMap  | undefined;
+  revision: number | undefined;
+  specLabels: StringMap | undefined;
+  specAnnotations: StringMap | undefined;
   containerName: string;
   imageName: string;
   imageId: string;
diff --git a/test/unit/transmitter-payload.test.ts b/test/unit/transmitter-payload.test.ts
@@ -18,7 +18,7 @@ tap.test('constructHomebaseWorkloadPayloads breaks when workloadMetadata is miss
     },
   ];
 
-  const workloadMetadata: transmitterTypes.IKubeImage[] = [
+  const workloadMetadata: transmitterTypes.IWorkload[] = [
     {
       type: 'type',
       name: 'workloadName',
@@ -32,6 +32,7 @@ tap.test('constructHomebaseWorkloadPayloads breaks when workloadMetadata is miss
       imageName: 'myImage',
       imageId: 'does this matter?',
       cluster: 'grapefruit',
+      revision: undefined,
     },
   ];
 
@@ -48,7 +49,7 @@ tap.test('constructHomebaseWorkloadPayloads happy flow', async (t) => {
     },
   ];
 
-  const workloadMetadata: transmitterTypes.IKubeImage[] = [
+  const workloadMetadata: transmitterTypes.IWorkload[] = [
     {
       type: 'type',
       name: 'workloadName',
@@ -62,6 +63,7 @@ tap.test('constructHomebaseWorkloadPayloads happy flow', async (t) => {
       imageName: 'myImage:tag',
       imageId: 'does this matter?',
       cluster: 'grapefruit',
+      revision: 1,
     },
   ];
 

Original file line number	Diff line number	Diff line change
`@@ -18,6 +18,7 @@ export interface KubeObjectMetadata {`
`18`	`18`	`specMeta: V1ObjectMeta;`
`19`	`19`	`containers: V1Container[];`
`20`	`20`	`ownerRefs: V1OwnerReference[] \| undefined;`
	`21`	`+ revision?: number;`
`21`	`22`	`}`
`22`	`23`
`23`	`24`	`export interface IK8sClients {`
Original file line number	Diff line number	Diff line change
`@@ -5,7 +5,7 @@ import { FALSY_WORKLOAD_NAME_MARKER } from './types';`
`5`	`5`
`6`	`6`	`export async function daemonSetWatchHandler(daemonSet: V1DaemonSet) {`
`7`	`7`	`if (!daemonSet.metadata \|\| !daemonSet.spec \|\| !daemonSet.spec.template.metadata \|\|`
`8`		`- !daemonSet.spec.template.spec) {`
	`8`	`+ !daemonSet.spec.template.spec \|\| !daemonSet.status) {`
`9`	`9`	`// TODO(ivanstanev): possibly log this. It shouldn't happen but we should track it!`
`10`	`10`	`return;`
`11`	`11`	`}`
`@@ -18,5 +18,6 @@ export async function daemonSetWatchHandler(daemonSet: V1DaemonSet) {`
`18`	`18`	`specMeta: daemonSet.spec.template.metadata,`
`19`	`19`	`containers: daemonSet.spec.template.spec.containers,`
`20`	`20`	`ownerRefs: daemonSet.metadata.ownerReferences,`
	`21`	`+ revision: daemonSet.status.observedGeneration,`
`21`	`22`	`}, workloadName);`
`22`	`23`	`}`
Original file line number	Diff line number	Diff line change
`@@ -5,7 +5,7 @@ import { FALSY_WORKLOAD_NAME_MARKER } from './types';`
`5`	`5`
`6`	`6`	`export async function deploymentWatchHandler(deployment: V1Deployment) {`
`7`	`7`	`if (!deployment.metadata \|\| !deployment.spec \|\| !deployment.spec.template.metadata \|\|`
`8`		`- !deployment.spec.template.spec) {`
	`8`	`+ !deployment.spec.template.spec \|\| !deployment.status) {`
`9`	`9`	`// TODO(ivanstanev): possibly log this. It shouldn't happen but we should track it!`
`10`	`10`	`return;`
`11`	`11`	`}`
`@@ -18,5 +18,6 @@ export async function deploymentWatchHandler(deployment: V1Deployment) {`
`18`	`18`	`specMeta: deployment.spec.template.metadata,`
`19`	`19`	`containers: deployment.spec.template.spec.containers,`
`20`	`20`	`ownerRefs: deployment.metadata.ownerReferences,`
	`21`	`+ revision: deployment.status.observedGeneration,`
`21`	`22`	`}, workloadName);`
`22`	`23`	`}`
Original file line number	Diff line number	Diff line change
`@@ -5,7 +5,7 @@ import { FALSY_WORKLOAD_NAME_MARKER } from './types';`
`5`	`5`
`6`	`6`	`export async function replicaSetWatchHandler(replicaSet: V1ReplicaSet) {`
`7`	`7`	`if (!replicaSet.metadata \|\| !replicaSet.spec \|\| !replicaSet.spec.template \|\|`
`8`		`- !replicaSet.spec.template.metadata \|\| !replicaSet.spec.template.spec) {`
	`8`	`+ !replicaSet.spec.template.metadata \|\| !replicaSet.spec.template.spec \|\| !replicaSet.status) {`
`9`	`9`	`// TODO(ivanstanev): possibly log this. It shouldn't happen but we should track it!`
`10`	`10`	`return;`
`11`	`11`	`}`
`@@ -18,5 +18,6 @@ export async function replicaSetWatchHandler(replicaSet: V1ReplicaSet) {`
`18`	`18`	`specMeta: replicaSet.spec.template.metadata,`
`19`	`19`	`containers: replicaSet.spec.template.spec.containers,`
`20`	`20`	`ownerRefs: replicaSet.metadata.ownerReferences,`
	`21`	`+ revision: replicaSet.status.observedGeneration,`
`21`	`22`	`}, workloadName);`
`22`	`23`	`}`
Original file line number	Diff line number	Diff line change
`@@ -5,7 +5,7 @@ import { FALSY_WORKLOAD_NAME_MARKER } from './types';`
`5`	`5`
`6`	`6`	`export async function statefulSetWatchHandler(statefulSet: V1StatefulSet) {`
`7`	`7`	`if (!statefulSet.metadata \|\| !statefulSet.spec \|\| !statefulSet.spec.template.metadata \|\|`
`8`		`- !statefulSet.spec.template.spec) {`
	`8`	`+ !statefulSet.spec.template.spec \|\| !statefulSet.status) {`
`9`	`9`	`// TODO(ivanstanev): possibly log this. It shouldn't happen but we should track it!`
`10`	`10`	`return;`
`11`	`11`	`}`
`@@ -18,5 +18,6 @@ export async function statefulSetWatchHandler(statefulSet: V1StatefulSet) {`
`18`	`18`	`specMeta: statefulSet.spec.template.metadata,`
`19`	`19`	`containers: statefulSet.spec.template.spec.containers,`
`20`	`20`	`ownerRefs: statefulSet.metadata.ownerReferences,`
	`21`	`+ revision: statefulSet.status.observedGeneration,`
`21`	`22`	`}, workloadName);`
`22`	`23`	`}`