Merge pull request #587 from snyk/fix/pulling-by-sha

fix: fallback to tag if digest missing

Author: dkontorovskyy

src/scanner/images/index.ts

@@ -3,31 +3,33 @@ import * as plugin from 'snyk-docker-plugin';
 
 import logger = require('../../common/logger');
 import { pull as skopeoCopy, getDestinationForImage } from './skopeo';
-import { IPullableImage } from './types';
+import { IPullableImage, IScanImage } from './types';
 import { IStaticAnalysisOptions, StaticAnalysisImageType, IScanResult, IPluginOptions } from '../types';
 
 export async function pullImages(images: IPullableImage[]): Promise<IPullableImage[]> {
   const pulledImages: IPullableImage[] = [];
 
   for (const image of images) {
-    const {imageName, fileSystemPath} = image;
+    const { imageName, imageWithDigest, fileSystemPath } = image;
     if (!fileSystemPath) {
       continue;
     }
 
     try {
-      await skopeoCopy(imageName, fileSystemPath);
+      // Scan image by digest if exists, other way fallback tag
+      const scanId = imageWithDigest ?? imageName;
+      await skopeoCopy(scanId, fileSystemPath);
       pulledImages.push(image);
     } catch (error) {
-      logger.error({error, image: imageName}, 'failed to pull image');
+      logger.error({error, image: imageWithDigest}, 'failed to pull image');
     }
   }
 
   return pulledImages;
 }
 
-export function getImagesWithFileSystemPath(images: string[]): IPullableImage[] {
-  return images.map((image) => ({ imageName: image, fileSystemPath: getDestinationForImage(image) }));
+export function getImagesWithFileSystemPath(images: IScanImage[]): IPullableImage[] {
+  return images.map((image) => ({ ...image, fileSystemPath: getDestinationForImage(image.imageName) }));
 }
 
 export async function removePulledImages(images: IPullableImage[]): Promise<void> {
@@ -41,15 +43,15 @@ export async function removePulledImages(images: IPullableImage[]): Promise<void
 }
 
 // Exported for testing
-export function getImageParts(imageWithTag: string) : {imageName: string, imageTag: string} {
+export function getImageParts(imageWithTag: string) : {imageName: string, imageTag: string, imageDigest: string} {
   // we're matching pattern: <registry:port_number>(optional)/<image_name>(mandatory):<image_tag>(optional)@<tag_identifier>(optional)
   // extracted from https://github.com/docker/distribution/blob/master/reference/regexp.go
   const regex = /^((?:(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])(?:(?:\.(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9]))+)?(?::[0-9]+)?\/)?[a-z0-9]+(?:(?:(?:[._]|__|[-]*)[a-z0-9]+)+)?(?:(?:\/[a-z0-9]+(?:(?:(?:[._]|__|[-]*)[a-z0-9]+)+)?)+)?)(?::([\w][\w.-]{0,127}))?(?:@([A-Za-z][A-Za-z0-9]*(?:[-_+.][A-Za-z][A-Za-z0-9]*)*[:][A-Fa-f0-9]{32,}))?$/ig;
   const groups  = regex.exec(imageWithTag);
-  
+
   if(!groups){
     logger.error({image: imageWithTag}, 'Image with tag is malformed, cannot extract valid parts');
-    return {imageName: imageWithTag, imageTag: ''};
+    return { imageName: imageWithTag, imageTag: '', imageDigest: '' };
   }
 
   const IMAGE_NAME_GROUP = 1;
@@ -58,8 +60,8 @@ export function getImageParts(imageWithTag: string) : {imageName: string, imageT
 
   return {
     imageName: groups[IMAGE_NAME_GROUP],
-    // prefer tag over digest
-    imageTag: groups[IMAGE_TAG_GROUP] || groups[IMAGE_DIGEST_GROUP] || '',
+    imageTag: groups[IMAGE_TAG_GROUP] || '',
+    imageDigest: groups[IMAGE_DIGEST_GROUP] || '',
   };
 }
 
@@ -78,7 +80,7 @@ export async function scanImages(images: IPullableImage[]): Promise<IScanResult[
 
   const dockerfile = undefined;
 
-  for (const {imageName, fileSystemPath} of images) {
+  for (const { imageName, fileSystemPath, imageWithDigest } of images) {
     try {
       const staticAnalysisOptions = constructStaticAnalysisOptions(fileSystemPath);
       const options: IPluginOptions = {
@@ -92,16 +94,19 @@ export async function scanImages(images: IPullableImage[]): Promise<IScanResult[
         throw Error('Unexpected empty result from docker-plugin');
       }
 
-      const imageParts: {imageName: string, imageTag: string} = getImageParts(imageName);
+      const imageParts = getImageParts(imageName);
+      const imageDigest = imageWithDigest && getImageParts(imageWithDigest).imageDigest;
 
       result.imageMetadata = {
         image: imageParts.imageName,
         imageTag: imageParts.imageTag,
+        imageDigest,
       };
 
       scannedImages.push({
         image: imageParts.imageName,
         imageWithTag: imageName,
+        imageWithDigest: imageWithDigest,
         pluginResult: result,
       });
     } catch (error) {

src/scanner/images/types.ts

@@ -1,5 +1,11 @@
+export interface IScanImage {
+  imageName: string;
+  imageWithDigest?: string;
+}
+
 export interface IPullableImage {
   imageName: string;
+  imageWithDigest?: string;
   fileSystemPath: string;
 }
 

src/scanner/index.ts

@@ -1,22 +1,20 @@
 import logger = require('../common/logger');
-import { pullImages, removePulledImages, getImagesWithFileSystemPath, scanImages } from './images';
+import { pullImages, removePulledImages, getImagesWithFileSystemPath, scanImages, getImageParts } from './images';
 import { deleteWorkload, sendDepGraph } from '../transmitter';
 import { constructDeleteWorkload, constructDepGraph } from '../transmitter/payload';
 import { IWorkload, ILocalWorkloadLocator } from '../transmitter/types';
-import { IPullableImage } from './images/types';
+import { IPullableImage, IScanImage } from './images/types';
 
 export async function processWorkload(workloadMetadata: IWorkload[]): Promise<void> {
   // every workload metadata references the same workload name, grab it from the first one
   const workloadName = workloadMetadata[0].name;
-  const allImages = workloadMetadata.map((meta) => meta.imageName);
-  logger.info({workloadName, imageCount: allImages.length}, 'queried workloads');
-  const uniqueImages = [...new Set<string>(allImages)];
+  const uniqueImages: IScanImage[] = getUniqueImages(workloadMetadata);
 
-  logger.info({workloadName, imageCount: uniqueImages.length}, 'pulling unique images');
+  logger.info({ workloadName, imageCount: uniqueImages.length }, 'pulling unique images');
   const imagesWithFileSystemPath = getImagesWithFileSystemPath(uniqueImages);
   const pulledImages = await pullImages(imagesWithFileSystemPath);
   if (pulledImages.length === 0) {
-    logger.info({workloadName}, 'no images were pulled, halting scanner process.');
+    logger.info({ workloadName }, 'no images were pulled, halting scanner process.');
     return;
   }
 
@@ -35,6 +33,38 @@ export async function sendDeleteWorkloadRequest(workloadName: string, localWorkl
   await deleteWorkload(deletePayload);
 }
 
+export function getUniqueImages(workloadMetadata: IWorkload[]): IScanImage[] {
+  const uniqueImages: { [key: string]: IScanImage } = workloadMetadata.reduce((accum, meta) => {
+    logger.info({
+      workloadName: workloadMetadata[0].name,
+      name: meta.imageName,
+      id: meta.imageId
+    }, 'image metadata');
+    // example: For DCR "redis:latest"
+    // example: For GCR "gcr.io/test-dummy/redis:latest"
+    // example: For ECR "291964488713.dkr.ecr.us-east-2.amazonaws.com/snyk/redis:latest"
+    // meta.imageName can be different depends on CR
+    const { imageName } = getImageParts(meta.imageName);
+    // meta.imageId can be different depends on CR
+    // example: For DCR "docker.io/library/redis@sha256:8e9f8546050da8aae393a41d65ad37166b4f0d8131d627a520c0f0451742e9d6"
+    // example: For GCR "sha256:8e9f8546050da8aae393a41d65ad37166b4f0d8131d627a520c0f0451742e9d6"
+    // example: For ECR "sha256:8e9f8546050da8aae393a41d65ad37166b4f0d8131d627a520c0f0451742e9d6"
+    let digest: string | undefined = undefined;
+    if (meta.imageId.lastIndexOf('@') > -1 || meta.imageId.startsWith('sha')) {
+      digest = meta.imageId.substring(meta.imageId.lastIndexOf('@') + 1);
+    }
+
+    accum[meta.imageName] = {
+      imageWithDigest: digest && `${imageName}@${digest}`,
+      imageName: meta.imageName, // Image name with tag
+    };
+
+    return accum;
+  }, {});
+
+  return Object.values(uniqueImages);
+}
+
 async function scanImagesAndSendResults(
   workloadName: string,
   pulledImages: IPullableImage[],

src/scanner/types.ts

@@ -1,5 +1,6 @@
 export interface IScanResult {
   image: string;
+  imageWithDigest?: string;
   imageWithTag: string;
   pluginResult: any;
 }

src/transmitter/payload.ts

@@ -27,6 +27,7 @@ export function constructDepGraph(
     const imageLocator: IImageLocator = {
       userLocator: config.INTEGRATION_ID,
       imageId: scannedImage.image,
+      imageWithDigest: scannedImage.imageWithDigest,
       cluster,
       namespace,
       type,

src/transmitter/types.ts

@@ -25,6 +25,7 @@ export interface IWorkloadMetadata {
 
 export interface IImageLocator extends IWorkloadLocator {
   imageId: string;
+  imageWithDigest?: string;
 }
 
 export interface IKubernetesMonitorMetadata {

test/unit/scanner/images.test.ts

@@ -1,16 +1,18 @@
 import * as tap from 'tap';
 
-import { IPullableImage } from '../../../src/scanner/images/types';
+import {IPullableImage, IScanImage} from '../../../src/scanner/images/types';
 import config = require('../../../src/common/config');
 import * as scannerImages from '../../../src/scanner/images';
 
-
 tap.test('getImagesWithFileSystemPath()', async (t) => {
-  const noImages: string[] = [];
+  const noImages: IScanImage[] = [];
   const noImagesResult = scannerImages.getImagesWithFileSystemPath(noImages);
   t.same(noImagesResult, [], 'correctly maps an empty array');
 
-  const image = ['nginx:latest'];
+  const image: IScanImage[] = [{
+    imageName: 'nginx:latest',
+    imageWithDigest: 'nginx@sha256:4949aa7259aa6f827450207db5ad94cabaa9248277c6d736d5e1975d200c7e43',
+  }];
   const imageResult = scannerImages.getImagesWithFileSystemPath(image);
   t.same(imageResult.length, 1, 'expected 1 item');
 
@@ -28,7 +30,10 @@ tap.test('getImagesWithFileSystemPath()', async (t) => {
   t.ok(expectedPattern, 'the file system path starts with an expected pattern');
 
   // Ensure that two consecutive calls do not return the same file system path
-  const someImage = ['centos:latest'];
+  const someImage = [{
+    imageName: 'centos:latest',
+    imageWithDigest: 'centos@sha256:fc4a234b91cc4b542bac8a6ad23b2ddcee60ae68fc4dbd4a52efb5f1b0baad71',
+  }];
   const firstCallResult = scannerImages.getImagesWithFileSystemPath(someImage)[0];
   const secondCallResult = scannerImages.getImagesWithFileSystemPath(someImage)[0];
   t.ok(
@@ -44,8 +49,6 @@ tap.test('pullImages() skips on missing file system path', async (t) => {
 });
 
 tap.test('constructStaticAnalysisOptions() tests', async (t) => {
-  t.plan(1);
-
   const somePath = '/var/tmp/file.tar';
   const options = scannerImages.constructStaticAnalysisOptions(somePath);
   const expectedResult = {
@@ -57,11 +60,10 @@ tap.test('constructStaticAnalysisOptions() tests', async (t) => {
 });
 
 tap.test('extracted image tag tests', async (t) => {
-  t.plan(6);
-
   const imageWithSha = 'nginx@sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2';
   const imageWithShaResult = scannerImages.getImageParts(imageWithSha);
-  t.same(imageWithShaResult.imageTag, 'sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2', 'image sha is returned');
+  t.same(imageWithShaResult.imageTag, '', 'image tag is empty');
+  t.same(imageWithShaResult.imageDigest, 'sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2', 'image digest is returned');
 
   const imageWithTag = 'nginx:latest';
   const imageWithTagResult = scannerImages.getImageParts(imageWithTag);
@@ -85,8 +87,6 @@ tap.test('extracted image tag tests', async (t) => {
 });
 
 tap.test('extracted image name tests', async (t) => {
-  t.plan(5);
-
   t.same(scannerImages.getImageParts('nginx:latest').imageName, 'nginx', 'removed image:tag');
   t.same(scannerImages.getImageParts('node@sha256:215a9fbef4df2c1ceb7c79481d3cfd94ad8f1f0105bade39f3be907bf386c5e1').imageName, 'node', 'removed image@sha:hex');
   t.same(scannerImages.getImageParts('kind-registry:5000/python:rc-buster').imageName, 'kind-registry:5000/python', 'removed repository/image:tag');

test/unit/scanner/index.test.ts

@@ -0,0 +1,76 @@
+import * as tap from 'tap';
+
+import * as scanner from '../../../src/scanner';
+import { IWorkload } from '../../../src/transmitter/types';
+
+tap.test('getUniqueImages()', async (t) => {
+    const workload: Partial<IWorkload>[] = [
+        // 1.DCR
+        {
+            imageName: 'redis:latest',
+            imageId: 'docker.io/library/redis@sha256:8e9f8546050da8aae393a41d65ad37166b4f0d8131d627a520c0f0451742e9d6'
+        },
+        // 2.Duplicate to verify uniqueness
+        {
+            imageName: 'redis:latest',
+            imageId: 'docker.io/library/redis@sha256:8e9f8546050da8aae393a41d65ad37166b4f0d8131d627a520c0f0451742e9d6'
+        },
+        // 3. With SHA instead of tag
+        {
+            imageName: 'redis@sha256:8e9f8546050da8aae393a41d65ad37166b4f0d8131d627a520c0f0451742e9d6',
+            imageId: 'docker.io/library/redis@sha256:8e9f8546050da8aae393a41d65ad37166b4f0d8131d627a520c0f0451742e9d6'
+        },
+        // 4. With SHA missing in imageId
+        {
+            imageName: 'redis:prod',
+            imageId: 'docker.io/library/redis:eaa6f054e4a140bc3a1696cc7b1e84529e7e9567',
+        },
+        // 5. GCR
+        {
+            imageName: 'gcr.io/test-dummy/redis:latest',
+            imageId: 'sha256:8e9f8546050da8aae393a41d65ad37166b4f0d8131d627a520c0f0451742e9d6'
+        },
+        // 6. ECR
+        {
+            imageName: '291964488713.dkr.ecr.us-east-2.amazonaws.com/snyk/redis:latest',
+            imageId: 'sha256:8e9f8546050da8aae393a41d65ad37166b4f0d8131d627a520c0f0451742e9d6'
+        },
+        // 7. With docker-pullable as protocol in imageId
+        {
+            imageName: 'redis:some-tag',
+            imageId: 'docker-pullable://name@sha256:8e9f8546050da8aae393a41d65ad37166b4f0d8131d627a520c0f0451742e9d6',
+        },
+        // 8. With docker as protocol in imageId
+        {
+            imageName: 'redis:another-tag',
+            imageId: 'docker://name@sha256:8e9f8546050da8aae393a41d65ad37166b4f0d8131d627a520c0f0451742e9d6'
+        }
+    ];
+
+    const result = scanner.getUniqueImages(workload as any);
+
+    t.strictEqual(result.length, 7, 'removed duplicate image');
+    const resultWithDigest = result.filter(({imageWithDigest}) => imageWithDigest);
+    const resultWithoutDigest = result.filter(({imageWithDigest}) => !imageWithDigest);
+
+    t.strictEqual(resultWithDigest.length, 6, 'correct amount with digest');
+    t.strictEqual(resultWithoutDigest.length, 1, 'correct amount without digest');
+
+    resultWithDigest.map((metaData) => {
+        t.ok(metaData.imageWithDigest!.includes('redis'), 'has name in imageWithDigest');
+        t.ok(metaData.imageWithDigest!.includes('sha256:8e9f8546050da8aae393a41d65ad37166b4f0d8131d627a520c0f0451742e9d6'), 'has digest');
+
+        if (metaData.imageWithDigest!.includes('gcr')) {
+            t.ok(metaData.imageWithDigest!.includes('/'), 'contains / in GCR imageWithDigest');
+        }
+
+        if (metaData.imageWithDigest!.includes('ecr')) {
+            t.ok(metaData.imageWithDigest!.includes('/'), 'contains / in ECR imageWithDigest');
+        }
+    });
+
+    resultWithoutDigest.map((metadata) => {
+        t.ok(metadata.imageName.includes('redis'));
+        t.notOk(metadata.imageWithDigest);
+    });
+});

test/unit/transmitter-payload.test.ts

@@ -11,11 +11,13 @@ tap.test('constructDepGraph breaks when workloadMetadata is missing items', asyn
     {
       image: 'myImage',
       imageWithTag: 'myImage:tag',
+      imageWithDigest: 'myImage@sha256:idontcarewhatissha',
       pluginResult: 'whatever1',
     },
     {
       image: 'anotherImage',
       imageWithTag: 'anotherImage:1.2.3-alpha',
+      imageWithDigest: 'myImage@sha256:somuchdifferentsha256',
       pluginResult: 'whatever3',
     },
   ];
@@ -48,6 +50,7 @@ tap.test('constructDepGraph happy flow', async (t) => {
     {
       image: 'myImage',
       imageWithTag: 'myImage:tag',
+      imageWithDigest: 'myImage@sha256:idontcarewhatissha',
       pluginResult: 'whatever1',
     },
   ];