Merge pull request #580 from snyk/feat/pull-image-by-sha

feat: pull image by SHA instead of tag

Author: dkontorovskyy

src/scanner/images/index.ts

@@ -3,31 +3,31 @@ import * as plugin from 'snyk-docker-plugin';
 
 import logger = require('../../common/logger');
 import { pull as skopeoCopy, getDestinationForImage } from './skopeo';
-import { IPullableImage } from './types';
+import { IPullableImage, IScanImage } from './types';
 import { IStaticAnalysisOptions, StaticAnalysisImageType, IScanResult, IPluginOptions } from '../types';
 
 export async function pullImages(images: IPullableImage[]): Promise<IPullableImage[]> {
   const pulledImages: IPullableImage[] = [];
 
   for (const image of images) {
-    const {imageName, fileSystemPath} = image;
+    const {imageWithDigest, fileSystemPath} = image;
     if (!fileSystemPath) {
       continue;
     }
 
     try {
-      await skopeoCopy(imageName, fileSystemPath);
+      await skopeoCopy(imageWithDigest, fileSystemPath);
       pulledImages.push(image);
     } catch (error) {
-      logger.error({error, image: imageName}, 'failed to pull image');
+      logger.error({error, image: imageWithDigest}, 'failed to pull image');
     }
   }
 
   return pulledImages;
 }
 
-export function getImagesWithFileSystemPath(images: string[]): IPullableImage[] {
-  return images.map((image) => ({ imageName: image, fileSystemPath: getDestinationForImage(image) }));
+export function getImagesWithFileSystemPath(images: IScanImage[]): IPullableImage[] {
+  return images.map((image) => ({ ...image, fileSystemPath: getDestinationForImage(image.imageName) }));
 }
 
 export async function removePulledImages(images: IPullableImage[]): Promise<void> {
@@ -41,15 +41,15 @@ export async function removePulledImages(images: IPullableImage[]): Promise<void
 }
 
 // Exported for testing
-export function getImageParts(imageWithTag: string) : {imageName: string, imageTag: string} {
+export function getImageParts(imageWithTag: string) : {imageName: string, imageTag: string, imageDigest: string} {
   // we're matching pattern: <registry:port_number>(optional)/<image_name>(mandatory):<image_tag>(optional)@<tag_identifier>(optional)
   // extracted from https://github.com/docker/distribution/blob/master/reference/regexp.go
   const regex = /^((?:(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])(?:(?:\.(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9]))+)?(?::[0-9]+)?\/)?[a-z0-9]+(?:(?:(?:[._]|__|[-]*)[a-z0-9]+)+)?(?:(?:\/[a-z0-9]+(?:(?:(?:[._]|__|[-]*)[a-z0-9]+)+)?)+)?)(?::([\w][\w.-]{0,127}))?(?:@([A-Za-z][A-Za-z0-9]*(?:[-_+.][A-Za-z][A-Za-z0-9]*)*[:][A-Fa-f0-9]{32,}))?$/ig;
   const groups  = regex.exec(imageWithTag);
-  
+
   if(!groups){
     logger.error({image: imageWithTag}, 'Image with tag is malformed, cannot extract valid parts');
-    return {imageName: imageWithTag, imageTag: ''};
+    return { imageName: imageWithTag, imageTag: '', imageDigest: '' };
   }
 
   const IMAGE_NAME_GROUP = 1;
@@ -58,8 +58,8 @@ export function getImageParts(imageWithTag: string) : {imageName: string, imageT
 
   return {
     imageName: groups[IMAGE_NAME_GROUP],
-    // prefer tag over digest
-    imageTag: groups[IMAGE_TAG_GROUP] || groups[IMAGE_DIGEST_GROUP] || '',
+    imageTag: groups[IMAGE_TAG_GROUP] || '',
+    imageDigest: groups[IMAGE_DIGEST_GROUP] || '',
   };
 }
 
@@ -78,7 +78,7 @@ export async function scanImages(images: IPullableImage[]): Promise<IScanResult[
 
   const dockerfile = undefined;
 
-  for (const {imageName, fileSystemPath} of images) {
+  for (const { imageName, fileSystemPath, imageWithDigest } of images) {
     try {
       const staticAnalysisOptions = constructStaticAnalysisOptions(fileSystemPath);
       const options: IPluginOptions = {
@@ -92,16 +92,18 @@ export async function scanImages(images: IPullableImage[]): Promise<IScanResult[
         throw Error('Unexpected empty result from docker-plugin');
       }
 
-      const imageParts: {imageName: string, imageTag: string} = getImageParts(imageName);
+      const imageParts = getImageParts(imageName);
 
       result.imageMetadata = {
         image: imageParts.imageName,
         imageTag: imageParts.imageTag,
+        imageDigest: getImageParts(imageWithDigest).imageDigest,
       };
 
       scannedImages.push({
         image: imageParts.imageName,
         imageWithTag: imageName,
+        imageWithDigest: imageWithDigest,
         pluginResult: result,
       });
     } catch (error) {

src/scanner/images/types.ts

@@ -1,5 +1,11 @@
+export interface IScanImage {
+  imageName: string;
+  imageWithDigest: string;
+}
+
 export interface IPullableImage {
   imageName: string;
+  imageWithDigest: string;
   fileSystemPath: string;
 }
 

src/scanner/index.ts

@@ -3,17 +3,25 @@ import { pullImages, removePulledImages, getImagesWithFileSystemPath, scanImages
 import { deleteWorkload, sendDepGraph } from '../transmitter';
 import { constructDeleteWorkload, constructDepGraph } from '../transmitter/payload';
 import { IWorkload, ILocalWorkloadLocator } from '../transmitter/types';
-import { IPullableImage } from './images/types';
+import { IPullableImage, IScanImage } from './images/types';
 
 export async function processWorkload(workloadMetadata: IWorkload[]): Promise<void> {
   // every workload metadata references the same workload name, grab it from the first one
   const workloadName = workloadMetadata[0].name;
-  const allImages = workloadMetadata.map((meta) => meta.imageName);
-  logger.info({workloadName, imageCount: allImages.length}, 'queried workloads');
-  const uniqueImages = [...new Set<string>(allImages)];
+  const uniqueImages: { [key: string]: IScanImage } = workloadMetadata.reduce((accum, meta) => {
+    // example: "docker.io/library/redis@sha256:33ca074e6019b451235735772a9c3e7216f014aae8eb0580d7e94834fe23efb3"
+    const imageWithDigest = meta.imageId.substring(meta.imageId.lastIndexOf('/') + 1);
 
-  logger.info({workloadName, imageCount: uniqueImages.length}, 'pulling unique images');
-  const imagesWithFileSystemPath = getImagesWithFileSystemPath(uniqueImages);
+    accum[meta.imageName] = {
+      imageName: meta.imageName,
+      imageWithDigest,
+    };
+
+    return accum;
+  }, {});
+
+  logger.info({workloadName, imageCount: Object.values(uniqueImages).length}, 'pulling unique images');
+  const imagesWithFileSystemPath = getImagesWithFileSystemPath(Object.values(uniqueImages));
   const pulledImages = await pullImages(imagesWithFileSystemPath);
   if (pulledImages.length === 0) {
     logger.info({workloadName}, 'no images were pulled, halting scanner process.');

src/scanner/types.ts

@@ -1,5 +1,6 @@
 export interface IScanResult {
   image: string;
+  imageWithDigest: string;
   imageWithTag: string;
   pluginResult: any;
 }

src/transmitter/payload.ts

@@ -27,6 +27,7 @@ export function constructDepGraph(
     const imageLocator: IImageLocator = {
       userLocator: config.INTEGRATION_ID,
       imageId: scannedImage.image,
+      imageWithDigest: scannedImage.imageWithDigest,
       cluster,
       namespace,
       type,

src/transmitter/types.ts

@@ -25,6 +25,7 @@ export interface IWorkloadMetadata {
 
 export interface IImageLocator extends IWorkloadLocator {
   imageId: string;
+  imageWithDigest: string;
 }
 
 export interface IKubernetesMonitorMetadata {

test/unit/scanner/images.test.ts

@@ -1,16 +1,18 @@
 import * as tap from 'tap';
 
-import { IPullableImage } from '../../../src/scanner/images/types';
+import {IPullableImage, IScanImage} from '../../../src/scanner/images/types';
 import config = require('../../../src/common/config');
 import * as scannerImages from '../../../src/scanner/images';
 
-
 tap.test('getImagesWithFileSystemPath()', async (t) => {
-  const noImages: string[] = [];
+  const noImages: IScanImage[] = [];
   const noImagesResult = scannerImages.getImagesWithFileSystemPath(noImages);
   t.same(noImagesResult, [], 'correctly maps an empty array');
 
-  const image = ['nginx:latest'];
+  const image: IScanImage[] = [{
+    imageName: 'nginx:latest',
+    imageWithDigest: 'nginx@sha256:4949aa7259aa6f827450207db5ad94cabaa9248277c6d736d5e1975d200c7e43',
+  }];
   const imageResult = scannerImages.getImagesWithFileSystemPath(image);
   t.same(imageResult.length, 1, 'expected 1 item');
 
@@ -28,7 +30,10 @@ tap.test('getImagesWithFileSystemPath()', async (t) => {
   t.ok(expectedPattern, 'the file system path starts with an expected pattern');
 
   // Ensure that two consecutive calls do not return the same file system path
-  const someImage = ['centos:latest'];
+  const someImage = [{
+    imageName: 'centos:latest',
+    imageWithDigest: 'centos@sha256:fc4a234b91cc4b542bac8a6ad23b2ddcee60ae68fc4dbd4a52efb5f1b0baad71',
+  }];
   const firstCallResult = scannerImages.getImagesWithFileSystemPath(someImage)[0];
   const secondCallResult = scannerImages.getImagesWithFileSystemPath(someImage)[0];
   t.ok(
@@ -44,8 +49,6 @@ tap.test('pullImages() skips on missing file system path', async (t) => {
 });
 
 tap.test('constructStaticAnalysisOptions() tests', async (t) => {
-  t.plan(1);
-
   const somePath = '/var/tmp/file.tar';
   const options = scannerImages.constructStaticAnalysisOptions(somePath);
   const expectedResult = {
@@ -57,11 +60,10 @@ tap.test('constructStaticAnalysisOptions() tests', async (t) => {
 });
 
 tap.test('extracted image tag tests', async (t) => {
-  t.plan(6);
-
   const imageWithSha = 'nginx@sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2';
   const imageWithShaResult = scannerImages.getImageParts(imageWithSha);
-  t.same(imageWithShaResult.imageTag, 'sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2', 'image sha is returned');
+  t.same(imageWithShaResult.imageTag, '', 'image tag is empty');
+  t.same(imageWithShaResult.imageDigest, 'sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2', 'image digest is returned');
 
   const imageWithTag = 'nginx:latest';
   const imageWithTagResult = scannerImages.getImageParts(imageWithTag);
@@ -85,8 +87,6 @@ tap.test('extracted image tag tests', async (t) => {
 });
 
 tap.test('extracted image name tests', async (t) => {
-  t.plan(5);
-
   t.same(scannerImages.getImageParts('nginx:latest').imageName, 'nginx', 'removed image:tag');
   t.same(scannerImages.getImageParts('node@sha256:215a9fbef4df2c1ceb7c79481d3cfd94ad8f1f0105bade39f3be907bf386c5e1').imageName, 'node', 'removed image@sha:hex');
   t.same(scannerImages.getImageParts('kind-registry:5000/python:rc-buster').imageName, 'kind-registry:5000/python', 'removed repository/image:tag');

test/unit/transmitter-payload.test.ts

@@ -11,11 +11,13 @@ tap.test('constructDepGraph breaks when workloadMetadata is missing items', asyn
     {
       image: 'myImage',
       imageWithTag: 'myImage:tag',
+      imageWithDigest: 'myImage@sha256:idontcarewhatissha',
       pluginResult: 'whatever1',
     },
     {
       image: 'anotherImage',
       imageWithTag: 'anotherImage:1.2.3-alpha',
+      imageWithDigest: 'myImage@sha256:somuchdifferentsha256',
       pluginResult: 'whatever3',
     },
   ];
@@ -48,6 +50,7 @@ tap.test('constructDepGraph happy flow', async (t) => {
     {
       image: 'myImage',
       imageWithTag: 'myImage:tag',
+      imageWithDigest: 'myImage@sha256:idontcarewhatissha',
       pluginResult: 'whatever1',
     },
   ];