Merge pull request #144 from snyk/feat/workload-metadata

Feat/send workload metadata

Author: Tal Kaptsan

src/kube-scanner/index.ts

@@ -9,7 +9,7 @@ import logger = require('../common/logger');
 import { pullImages } from '../images';
 import { scanImages, IScanResult } from './image-scanner';
 import { deleteHomebaseWorkload, sendDepGraph } from '../transmitter';
-import { constructHomebaseDeleteWorkloadPayload, constructHomebaseWorkloadPayloads } from '../transmitter/payload';
+import { constructHomebaseDeleteWorkloadPayload, constructHomebaseDepGraphPayloads } from '../transmitter/payload';
 import { IDepGraphPayload, IWorkload, ILocalWorkloadLocator } from '../transmitter/types';
 
 export = class WorkloadWorker {
@@ -40,8 +40,8 @@ export = class WorkloadWorker {
       return;
     }
 
-    const homebasePayloads: IDepGraphPayload[] = constructHomebaseWorkloadPayloads(scannedImages, workloadMetadata);
-    await sendDepGraph(...homebasePayloads);
+    const depGraphPayloads: IDepGraphPayload[] = constructHomebaseDepGraphPayloads(scannedImages, workloadMetadata);
+    await sendDepGraph(...depGraphPayloads);
 
     const pulledImageMetadata = workloadMetadata.filter((meta) =>
       pulledImages.includes(meta.imageName));

src/kube-scanner/watchers/handlers/pod.ts

@@ -3,7 +3,9 @@ import async = require('async');
 import config = require('../../../common/config');
 import logger = require('../../../common/logger');
 import WorkloadWorker = require('../../../kube-scanner');
+import { sendWorkloadMetadata } from '../../../transmitter';
 import { IWorkload } from '../../../transmitter/types';
+import { constructHomebaseWorkloadMetadataPayload } from '../../../transmitter/payload';
 import { buildMetadataForWorkload } from '../../metadata-extractor';
 import { PodPhase } from '../types';
 import state = require('../../../state');
@@ -59,7 +61,11 @@ export async function podWatchHandler(pod: V1Pod) {
       return;
     }
 
-    const workloadName = workloadMetadata[0].name;
+    // every element contains the workload information, so we can get it from the first one
+    const workloadMember = workloadMetadata[0];
+    const workloadMetadataPayload = constructHomebaseWorkloadMetadataPayload(workloadMember);
+    sendWorkloadMetadata(workloadMetadataPayload);
+    const workloadName = workloadMember.name;
     const workloadWorker = new WorkloadWorker(workloadName);
     await handleReadyPod(workloadWorker, workloadMetadata);
   } catch (error) {

src/transmitter/index.ts

@@ -1,7 +1,7 @@
 import needle = require('needle');
 import * as config from '../common/config';
 import logger = require('../common/logger');
-import { IDeleteWorkloadPayload, IDepGraphPayload } from './types';
+import { IDeleteWorkloadPayload, IDepGraphPayload, IWorkloadMetadataPayload } from './types';
 
 const homebaseUrl = config.INTEGRATION_API || config.DEFAULT_HOMEBASE_URL;
 
@@ -27,6 +27,22 @@ export async function sendDepGraph(...payloads: IDepGraphPayload[]) {
   }
 }
 
+export async function sendWorkloadMetadata(payload: IWorkloadMetadataPayload) {
+    try {
+      const result = await needle('post', `${homebaseUrl}/api/v1/workload`, payload, {
+          json: true,
+          compressed: true,
+        },
+      );
+
+      if (!isSuccessStatusCode(result.statusCode)) {
+        throw new Error(`${result.statusCode} ${result.statusMessage}`);
+      }
+    } catch (error) {
+      logger.error({error}, 'Could not send workload metadata to Homebase');
+    }
+}
+
 export async function deleteHomebaseWorkload(payload: IDeleteWorkloadPayload) {
   try {
     const result = await needle('delete', `${homebaseUrl}/api/v1/workload`, payload, {

src/transmitter/payload.ts

@@ -1,9 +1,18 @@
 import config = require('../common/config');
 import { currentClusterName } from '../kube-scanner/cluster';
 import { IScanResult } from '../kube-scanner/image-scanner';
-import { IDeleteWorkloadPayload, IDepGraphPayload, IWorkload, ILocalWorkloadLocator, IImageLocator } from './types';
+import {
+  IDeleteWorkloadPayload,
+  IDepGraphPayload,
+  IWorkload,
+  ILocalWorkloadLocator,
+  IImageLocator,
+  IWorkloadMetadataPayload,
+  IWorkloadMetadata,
+  IWorkloadLocator,
+} from './types';
 
-export function constructHomebaseWorkloadPayloads(
+export function constructHomebaseDepGraphPayloads(
     scannedImages: IScanResult[],
     workloadMetadata: IWorkload[],
 ): IDepGraphPayload[] {
@@ -31,6 +40,28 @@ export function constructHomebaseWorkloadPayloads(
   return results;
 }
 
+export function constructHomebaseWorkloadMetadataPayload(workload: IWorkload): IWorkloadMetadataPayload {
+  if (!workload) {
+    throw new Error('can\'t build workload metadata payload for undefined workload');
+  }
+
+  const workloadLocator: IWorkloadLocator = {
+    userLocator: config.INTEGRATION_ID,
+    cluster: workload.cluster,
+    namespace: workload.namespace,
+    type: workload.type,
+    name: workload.name,
+  };
+  const workloadMetadata: IWorkloadMetadata = {
+    labels: workload.labels,
+    specLabels: workload.specLabels,
+    annotations: workload.annotations,
+    specAnnotations: workload.specAnnotations,
+    revision: workload.revision,
+  };
+  return { workloadLocator, agentId: config.AGENT_ID, workloadMetadata };
+}
+
 export function constructHomebaseDeleteWorkloadPayload(
   localWorkloadLocator: ILocalWorkloadLocator,
 ): IDeleteWorkloadPayload {

src/transmitter/types.ts

@@ -11,10 +11,12 @@ export interface IWorkloadLocator extends ILocalWorkloadLocator {
   cluster: string;
 }
 
-export interface IWorkloadMetadata extends IWorkloadLocator {
+export interface IWorkloadMetadata {
   labels: StringMap | undefined;
+  specLabels: StringMap | undefined;
   annotations: StringMap | undefined;
-  uid: string;
+  specAnnotations: StringMap | undefined;
+  revision: number | undefined;
 }
 
 export interface IImageLocator extends IWorkloadLocator {
@@ -27,6 +29,12 @@ export interface IDepGraphPayload {
   dependencyGraph?: any;
 }
 
+export interface IWorkloadMetadataPayload {
+  workloadLocator: IWorkloadLocator;
+  agentId: string;
+  workloadMetadata: IWorkloadMetadata;
+}
+
 export interface IDeleteWorkloadPayload {
   workloadLocator: IWorkloadLocator;
   agentId: string;

test/integration/kubernetes.test.ts

@@ -5,7 +5,7 @@ import setup = require('../setup'); // Must be located before 'tap' import
 // tslint:disable-next-line: ordered-imports
 import * as tap from 'tap';
 import * as config from '../../src/common/config';
-import { IWorkloadLocator } from '../../src/transmitter/types';
+import { IWorkloadLocator, IWorkloadMetadata } from '../../src/transmitter/types';
 import { getKindConfigPath } from '../helpers/kind';
 import { WorkloadKind } from '../../src/kube-scanner/types';
 
@@ -14,6 +14,7 @@ const toneDownFactor = 5;
 const maxPodChecks = setup.KUBERNETES_MONITOR_MAX_WAIT_TIME_SECONDS / toneDownFactor;
 
 type WorkloadLocatorValidator = (workloads: IWorkloadLocator[] | undefined) => boolean;
+type WorkloadMetadataValidator = (workloadInfo: IWorkloadMetadata | undefined) => boolean;
 
 async function tearDown() {
   console.log('Begin removing the snyk-monitor...');
@@ -75,7 +76,6 @@ tap.test('snyk-monitor container started', async (t) => {
 async function validateHomebaseStoredData(
   validatorFn: WorkloadLocatorValidator, relativeUrl: string, remainingChecks: number = maxPodChecks,
 ): Promise<boolean> {
-  // TODO: consider if we're OK to expose this publicly?
   while (remainingChecks > 0) {
     const responseBody = await getHomebaseResponseBody(relativeUrl);
     const workloads: IWorkloadLocator[] | undefined = responseBody.workloads;
@@ -89,6 +89,22 @@ async function validateHomebaseStoredData(
   return false;
 }
 
+async function validateHomebaseStoredMetadata(
+  validatorFn: WorkloadMetadataValidator, relativeUrl: string, remainingChecks: number = maxPodChecks,
+): Promise<boolean> {
+  while (remainingChecks > 0) {
+    const responseBody = await getHomebaseResponseBody(relativeUrl);
+    const workloadInfo: IWorkloadMetadata | undefined = responseBody.workloadInfo;
+    const result = validatorFn(workloadInfo);
+    if (result) {
+      return true;
+    }
+    await sleep(1000 * toneDownFactor);
+    remainingChecks--;
+  }
+  return false;
+}
+
 async function getHomebaseResponseBody(relativeUrl: string): Promise<any> {
   const url = `https://${config.INTERNAL_PROXY_CREDENTIALS}@homebase-int.dev.snyk.io/${relativeUrl}`;
   const homebaseResponse = await needle('get', url, null);
@@ -97,7 +113,7 @@ async function getHomebaseResponseBody(relativeUrl: string): Promise<any> {
 }
 
 tap.test('snyk-monitor sends data to homebase', async (t) => {
-  t.plan(1);
+  t.plan(2);
 
   console.log(`Begin polling Homebase for the expected workloads with integration ID ${integrationId}...`);
 
@@ -113,10 +129,18 @@ tap.test('snyk-monitor sends data to homebase', async (t) => {
         workload.type === WorkloadKind.Deployment) !== undefined;
   };
 
+  const metaValidator: WorkloadMetadataValidator = (workloadInfo) => {
+    return workloadInfo !== undefined && 'revision' in workloadInfo && 'labels' in workloadInfo &&
+      'specLabels' in workloadInfo && 'annotations' in workloadInfo && 'specAnnotations' in workloadInfo;
+  };
+
   // We don't want to spam Homebase with requests; do it infrequently
-  const homebaseTestResult = await validateHomebaseStoredData(
+  const homebaseDepGraphTestResult = await validateHomebaseStoredData(
     validatorFn, `api/v2/workloads/${integrationId}/Default cluster/services`);
-  t.ok(homebaseTestResult, 'snyk-monitor sent expected data to homebase in the expected timeframe');
+  t.ok(homebaseDepGraphTestResult, 'snyk-monitor sent expected data to homebase in the expected timeframe');
+  const homebaseWorkloadMetadataResult = await validateHomebaseStoredMetadata(metaValidator,
+    `api/v1/workload/${integrationId}/Default cluster/services/Deployment/redis`);
+  t.ok(homebaseWorkloadMetadataResult, 'snyk-monitor sent expected metadata in the expected timeframe');
 });
 
 tap.test('snyk-monitor sends correct data to homebase after adding another deployment', async (t) => {

test/unit/transmitter-payload.test.ts

@@ -4,7 +4,7 @@ import imageScanner = require('../../src/kube-scanner/image-scanner');
 import payload = require('../../src/transmitter/payload');
 import transmitterTypes = require('../../src/transmitter/types');
 
-tap.test('constructHomebaseWorkloadPayloads breaks when workloadMetadata is missing items', async (t) => {
+tap.test('constructHomebaseDepGraphPayloads breaks when workloadMetadata is missing items', async (t) => {
   const scannedImages: imageScanner.IScanResult[] = [
     {
       image: 'myImage',
@@ -36,11 +36,11 @@ tap.test('constructHomebaseWorkloadPayloads breaks when workloadMetadata is miss
     },
   ];
 
-  t.throws(() => payload.constructHomebaseWorkloadPayloads(scannedImages, workloadMetadata),
-    'constructHomebaseWorkloadPayloads throws when workloadMetadata is missing items from scannedImages');
+  t.throws(() => payload.constructHomebaseDepGraphPayloads(scannedImages, workloadMetadata),
+    'constructHomebaseDepGraphPayloads throws when workloadMetadata is missing items from scannedImages');
 });
 
-tap.test('constructHomebaseWorkloadPayloads happy flow', async (t) => {
+tap.test('constructHomebaseDepGraphPayloads happy flow', async (t) => {
   const scannedImages: imageScanner.IScanResult[] = [
     {
       image: 'myImage',
@@ -67,7 +67,7 @@ tap.test('constructHomebaseWorkloadPayloads happy flow', async (t) => {
     },
   ];
 
-  const payloads = payload.constructHomebaseWorkloadPayloads(scannedImages, workloadMetadata);
+  const payloads = payload.constructHomebaseDepGraphPayloads(scannedImages, workloadMetadata);
 
   t.equals(payloads.length, 1, 'one payload to send to Homebase');
   t.equals(payloads[0].dependencyGraph, JSON.stringify('whatever1'), 'dependency graph present in payload');
@@ -76,3 +76,33 @@ tap.test('constructHomebaseWorkloadPayloads happy flow', async (t) => {
   t.equals(payloads[0].imageLocator.name, 'workloadName', 'workload name present in payload');
   t.equals(payloads[0].imageLocator.type, 'type', 'workload type present in payload');
 });
+
+tap.test('constructHomebaseWorkloadMetadataPayload happy flow', async (t) => {
+  const workloadWithImages: transmitterTypes.IWorkload = {
+    type: 'type',
+    name: 'workloadName',
+    namespace: 'spacename',
+    labels: undefined,
+    annotations: undefined,
+    uid: 'udi',
+    specLabels: undefined,
+    specAnnotations: undefined,
+    containerName: 'contener',
+    imageName: 'myImage:tag',
+    imageId: 'does this matter?',
+    cluster: 'grapefruit',
+    revision: 1,
+  };
+
+  const workloadMetadataPayload = payload.constructHomebaseWorkloadMetadataPayload(workloadWithImages);
+
+  t.equals(workloadMetadataPayload.workloadLocator.cluster, 'grapefruit', 'cluster present in payload');
+  t.equals(workloadMetadataPayload.workloadLocator.namespace, 'spacename', 'image ID present in payload');
+  t.equals(workloadMetadataPayload.workloadLocator.name, 'workloadName', 'workload name present in payload');
+  t.equals(workloadMetadataPayload.workloadLocator.type, 'type', 'workload type present in payload');
+  t.equals(workloadMetadataPayload.workloadMetadata.revision, 1, 'revision present in metadata');
+  t.ok('annotations' in workloadMetadataPayload.workloadMetadata, 'annotations present in metadata');
+  t.ok('specAnnotations' in workloadMetadataPayload.workloadMetadata, 'specAnnotations present in metadata');
+  t.ok('labels' in workloadMetadataPayload.workloadMetadata, 'labels present in metadata');
+  t.ok('specLabels' in workloadMetadataPayload.workloadMetadata, 'specLabels present in metadata');
+});