feat: support scanning images from ECR

- try to identify the source registry of the images we're pulling
- for ECR, attempt to generate an authentication token using aws's sdk. pass the resulting credentials to Skopeo's copy command.
- for any other registry, don't change existing behaviour (keep searching the credentials in the default dir of ~/.docker/config.json)

Author: Shesekino

package-lock.json

@@ -33,6 +33,7 @@
     "@types/node": "^10.12.5",
     "@types/sinon": "^7.5.1",
     "async": "^2.6.2",
+    "aws-sdk": "^2.596.0",
     "bunyan": "^1.8.12",
     "child-process-promise": "^2.2.1",
     "lru-cache": "^5.1.1",

package.json

@@ -16,6 +16,7 @@ export function exec(bin: string, ...args: string[]):
   return spawn(bin, args, { env, capture: [ 'stdout', 'stderr' ] })
     .catch((error) => {
       const message = (error && error.stderr) || 'Unknown reason';
+      // TODO: sanitise args for secrets
       logger.warn({message, bin, args}, 'could not spawn the process');
       throw error;
     });

src/common/process.ts

@@ -0,0 +1,40 @@
+import * as aws from 'aws-sdk';
+
+export async function getSourceCredentials(imageSource: string): Promise<string | undefined> {
+  // TODO is this the best way we can determine the image's source?
+  if (imageSource.indexOf('.ecr.') !== -1) {
+    return getEcrCredentials();
+  }
+  return undefined;
+}
+
+function getEcrCredentials(): Promise<string> {
+  return new Promise(async (resolve, reject) => {
+    // TODO grab region... from...? ask users to provide it?
+    const ecr = new aws.ECR({region: 'us-east-2'});
+    return ecr.getAuthorizationToken({}, (err, data) => {
+      if (err) {
+        return reject(err);
+      }
+
+      if (!(
+        data &&
+        data.authorizationData &&
+        Array.isArray(data.authorizationData) &&
+        data.authorizationData.length > 0
+      )) {
+        return reject('unexpected data format from ecr.getAuthorizationToken');
+      }
+
+      const authorizationTokenBase64 = data.authorizationData[0].authorizationToken;
+
+      if (!authorizationTokenBase64) {
+        return reject('empty authorization token from ecr.getAuthorizationToken');
+      }
+
+      const buff = new Buffer(authorizationTokenBase64, 'base64');
+      const userColonPassword = buff.toString('utf-8');
+      return resolve(userColonPassword);
+    });
+  });
+}

src/images/credentials.ts

@@ -2,6 +2,7 @@ import { SkopeoRepositoryType } from '../kube-scanner/types';
 import { SpawnPromiseResult } from 'child-process-promise';
 import { exec } from '../common/process';
 import config = require('../common/config');
+import * as credentials from './credentials';
 
 function getUniqueIdentifier(): string {
   const [seconds, nanoseconds] = process.hrtime();
@@ -28,12 +29,24 @@ function prefixRespository(target: string, type: SkopeoRepositoryType): string {
   }
 }
 
-export function pull(
+export async function pull(
   image: string,
   destination: string,
 ): Promise<SpawnPromiseResult> {
-  return exec('skopeo', 'copy',
+  const creds = await credentials.getSourceCredentials(image);
+  const credentialsParameters = getCredentialParameters(creds);
+
+  return exec('skopeo', 'copy', ...credentialsParameters,
     prefixRespository(image, SkopeoRepositoryType.ImageRegistry),
     prefixRespository(destination, SkopeoRepositoryType.DockerArchive),
   );
 }
+
+export function getCredentialParameters(credentials: string | undefined): Array<string> {
+  const credentialsParameters: Array<string> = [];
+  if (credentials) {
+    credentialsParameters.push('--src-creds');
+    credentialsParameters.push(credentials);
+  }
+  return credentialsParameters;
+}

src/images/skopeo.ts

@@ -0,0 +1,20 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: debian-ecr
+  namespace: services
+spec:
+  selector:
+    matchLabels:
+      app: debian-ecr
+  template:
+    metadata:
+      labels:
+        app: debian-ecr
+    spec:
+      containers:
+      - name: debian-ecr
+        image: 291964488713.dkr.ecr.us-east-2.amazonaws.com/snyk/debian:10
+        imagePullPolicy: Always
+        securityContext: {}
+        command: ['sh', '-c', 'echo Hello from ECR alpine pod! && sleep 360000']

test/fixtures/private-registries/debian-deployment-ecr.yaml

@@ -164,6 +164,41 @@ tap.test('snyk-monitor pulls images from a private gcr.io registry and sends dat
     'snyk-monitor sent expected data to upstream in the expected timeframe');
 });
 
+tap.test('snyk-monitor pulls images from a private ECR and sends data to homebase', async (t) => {
+  if (process.env['TEST_PLATFORM'] !== 'eks') {
+    t.pass('Not testing private ECR images because we\'re not running in EKS');
+    return;
+  }
+
+  t.plan(3);
+
+  const deploymentName = 'debian-ecr';
+  const namespace = 'services';
+  const clusterName = 'Default cluster';
+  const deploymentType = WorkloadKind.Deployment;
+  const imageName = '291964488713.dkr.ecr.us-east-2.amazonaws.com/snyk/debian';
+
+  await kubectl.applyK8sYaml('./test/fixtures/private-registries/debian-deployment-ecr.yaml');
+  console.log(`Begin polling upstream for the expected private ECR image with integration ID ${integrationId}...`);
+
+  const validatorFn: WorkloadLocatorValidator = (workloads) => {
+    return workloads !== undefined &&
+      workloads.find((workload) => workload.name === deploymentName &&
+        workload.type === WorkloadKind.Deployment) !== undefined;
+  };
+
+  const homebaseTestResult = await validateHomebaseStoredData(
+    validatorFn, `api/v2/workloads/${integrationId}/${clusterName}/${namespace}`);
+  t.ok(homebaseTestResult, 'snyk-monitor sent expected data to upstream in the expected timeframe');
+
+  const depGraphResult = await getHomebaseResponseBody(
+    `api/v1/dependency-graphs/${integrationId}/${clusterName}/${namespace}/${deploymentType}/${deploymentName}`);
+  t.ok('dependencyGraphResults' in depGraphResult,
+    'expected dependencyGraphResults field to exist in /dependency-graphs response');
+  t.ok('imageMetadata' in JSON.parse(depGraphResult.dependencyGraphResults[imageName]),
+    'snyk-monitor sent expected data to upstream in the expected timeframe');
+});
+
 tap.test('snyk-monitor sends deleted workload to homebase', async (t) => {
   // First ensure the deployment exists from the previous test
   const deploymentValidatorFn: WorkloadLocatorValidator = (workloads) => {

test/integration/kubernetes.test.ts

@@ -0,0 +1,21 @@
+import * as tap from 'tap';
+
+import * as skopeo from '../../src/images/skopeo';
+
+tap.test('getCredentialParameters()', async (t) => {
+  const noCredentials = undefined;
+  const credentialParametersForNoCredentials = skopeo.getCredentialParameters(noCredentials);
+  t.same(credentialParametersForNoCredentials, [], 'returns an empty array for no credentials');
+
+  const emptyCredentials = '';
+  const credentialParametersForEmptyCredentials = skopeo.getCredentialParameters(emptyCredentials);
+  t.same(credentialParametersForEmptyCredentials, [], 'returns an empty array for empty credentials');
+
+  const someCredentials = 'secret-things-happening';
+  const credentialParametersForSomeCredentials = skopeo.getCredentialParameters(someCredentials);
+  t.same(
+    credentialParametersForSomeCredentials,
+    ['--src-creds', someCredentials],
+    'returns Skopeo\'s args for source credentials',
+  );
+});