Merge pull request #320 from snyk/chore/ci-improvements

Chore/ci improvements

Author: ivanstanev

package.json

@@ -7,14 +7,14 @@
     "test": "npm run lint && npm run build && npm run test:unit && npm run test:integration",
     "test:unit": "NODE_ENV=test tap test/unit",
     "test:system": "tap test/system --timeout=600",
-    "test:integration": "TEST_PLATFORM=kind CREATE_CLUSTER=true tap test/integration/kubernetes.test.ts --timeout=1200",
-    "test:integration:kind": "TEST_PLATFORM=kind CREATE_CLUSTER=true tap test/integration/kubernetes.test.ts --timeout=1200",
-    "test:integration:eks": "TEST_PLATFORM=eks CREATE_CLUSTER=false tap test/integration/kubernetes.test.ts --timeout=1200",
+    "test:integration": "TEST_PLATFORM=kind CREATE_CLUSTER=true tap test/integration/kubernetes.test.ts --timeout=900",
+    "test:integration:kind": "TEST_PLATFORM=kind CREATE_CLUSTER=true tap test/integration/kubernetes.test.ts --timeout=900",
+    "test:integration:eks": "TEST_PLATFORM=eks CREATE_CLUSTER=false tap test/integration/kubernetes.test.ts --timeout=900",
     "test:coverage": "npm run test:unit -- --coverage",
     "test:watch": "tsc-watch --onSuccess 'npm run test:unit'",
-    "test:apk": "TEST_PLATFORM=kind CREATE_CLUSTER=true PACKAGE_MANAGER=apk tap test/integration/package-manager.test.ts --timeout=7200",
-    "test:apt": "TEST_PLATFORM=kind CREATE_CLUSTER=true PACKAGE_MANAGER=apt tap test/integration/package-manager.test.ts --timeout=7200",
-    "test:rpm": "TEST_PLATFORM=kind CREATE_CLUSTER=true PACKAGE_MANAGER=rpm tap test/integration/package-manager.test.ts --timeout=7200",
+    "test:apk": "TEST_PLATFORM=kind CREATE_CLUSTER=true PACKAGE_MANAGER=apk tap test/integration/package-manager.test.ts --timeout=900",
+    "test:apt": "TEST_PLATFORM=kind CREATE_CLUSTER=true PACKAGE_MANAGER=apt tap test/integration/package-manager.test.ts --timeout=900",
+    "test:rpm": "TEST_PLATFORM=kind CREATE_CLUSTER=true PACKAGE_MANAGER=rpm tap test/integration/package-manager.test.ts --timeout=900",
     "start": "bin/start",
     "prepare": "npm run build",
     "build": "tsc",

src/supervisor/kuberenetes-api-wrappers.ts

@@ -4,10 +4,11 @@ import * as sleep from 'sleep-promise';
 export const ATTEMPTS_MAX = 3;
 export const DEFAULT_SLEEP_SEC = 1;
 export const MAX_SLEEP_SEC = 5;
-type IKubernetesApiFunction = () => Promise<any>;
-
-export async function retryKubernetesApiRequest(func: IKubernetesApiFunction) {
+type IKubernetesApiFunction<ResponseType> = () => Promise<ResponseType>;
 
+export async function retryKubernetesApiRequest<ResponseType>(
+  func: IKubernetesApiFunction<ResponseType>
+): Promise<ResponseType> {
   for (let attempt = 1; attempt <= ATTEMPTS_MAX; attempt++) {
     try {
       return await func();
@@ -29,6 +30,8 @@ export async function retryKubernetesApiRequest(func: IKubernetesApiFunction) {
       await sleep(sleepSeconds * 1000);
     }
   }
+
+  throw new Error('Could not receive a response from the Kubernetes API');
 }
 
 export function calculateSleepSeconds(httpResponse: http.IncomingMessage): number {

src/transmitter/types.ts

@@ -36,7 +36,7 @@ export interface IKubernetesMonitorMetadata {
 export interface IDepGraphPayload {
   imageLocator: IImageLocator;
   agentId: string;
-  dependencyGraph?: any;
+  dependencyGraph?: string;
   metadata: IKubernetesMonitorMetadata;
 }
 

test/helpers/kubernetes-upstream.ts

@@ -7,8 +7,10 @@ import {
 import { WorkloadLocatorValidator, WorkloadMetadataValidator } from './types';
 import config = require('../../src/common/config');
 
-const toneDownFactor = 5;
-const maxPodChecks = 600 / toneDownFactor;
+const UPSTREAM_POLLING_CONFIGURATION = {
+  WAIT_BETWEEN_REQUESTS_MS: 5000,
+  MAXIMUM_REQUESTS: 120,
+};
 
 export async function getUpstreamResponseBody(
   relativeUrl: string,
@@ -22,16 +24,17 @@ export async function getUpstreamResponseBody(
 export async function validateUpstreamStoredData(
   validatorFn: WorkloadLocatorValidator,
   relativeUrl: string,
-  remainingChecks: number = maxPodChecks,
+  remainingChecks: number = UPSTREAM_POLLING_CONFIGURATION.MAXIMUM_REQUESTS,
 ): Promise<boolean> {
   while (remainingChecks > 0) {
+    console.log(`Pinging upstream for existing data (${remainingChecks} checks remaining)...`);
     const responseBody = await getUpstreamResponseBody(relativeUrl);
     const workloads: IWorkloadLocator[] | undefined = responseBody.workloads;
     const result = validatorFn(workloads);
     if (result) {
       return true;
     }
-    await sleep(1000 * toneDownFactor);
+    await sleep(UPSTREAM_POLLING_CONFIGURATION.WAIT_BETWEEN_REQUESTS_MS);
     remainingChecks--;
   }
   return false;
@@ -40,17 +43,18 @@ export async function validateUpstreamStoredData(
 export async function validateUpstreamStoredMetadata(
   validatorFn: WorkloadMetadataValidator,
   relativeUrl: string,
-  remainingChecks: number = maxPodChecks,
+  remainingChecks: number = UPSTREAM_POLLING_CONFIGURATION.MAXIMUM_REQUESTS,
 ): Promise<boolean> {
   while (remainingChecks > 0) {
+    console.log(`Pinging upstream for existing metadata (${remainingChecks} checks remaining)...`);
     const responseBody = await getUpstreamResponseBody(relativeUrl);
     const workloadInfo: IWorkloadMetadata | undefined =
       responseBody.workloadInfo;
     const result = validatorFn(workloadInfo);
     if (result) {
       return true;
     }
-    await sleep(1000 * toneDownFactor);
+    await sleep(UPSTREAM_POLLING_CONFIGURATION.WAIT_BETWEEN_REQUESTS_MS);
     remainingChecks--;
   }
   return false;