Merge pull request #585 from snyk/fix/revert-to-pulling-by-tag

feat: revert to pulling by tag

Author: ivanstanev

src/scanner/images/index.ts

@@ -3,31 +3,31 @@ import * as plugin from 'snyk-docker-plugin';
 
 import logger = require('../../common/logger');
 import { pull as skopeoCopy, getDestinationForImage } from './skopeo';
-import { IPullableImage, IScanImage } from './types';
+import { IPullableImage } from './types';
 import { IStaticAnalysisOptions, StaticAnalysisImageType, IScanResult, IPluginOptions } from '../types';
 
 export async function pullImages(images: IPullableImage[]): Promise<IPullableImage[]> {
   const pulledImages: IPullableImage[] = [];
 
   for (const image of images) {
-    const {imageWithDigest, fileSystemPath} = image;
+    const {imageName, fileSystemPath} = image;
     if (!fileSystemPath) {
       continue;
     }
 
     try {
-      await skopeoCopy(imageWithDigest, fileSystemPath);
+      await skopeoCopy(imageName, fileSystemPath);
       pulledImages.push(image);
     } catch (error) {
-      logger.error({error, image: imageWithDigest}, 'failed to pull image');
+      logger.error({error, image: imageName}, 'failed to pull image');
     }
   }
 
   return pulledImages;
 }
 
-export function getImagesWithFileSystemPath(images: IScanImage[]): IPullableImage[] {
-  return images.map((image) => ({ ...image, fileSystemPath: getDestinationForImage(image.imageName) }));
+export function getImagesWithFileSystemPath(images: string[]): IPullableImage[] {
+  return images.map((image) => ({ imageName: image, fileSystemPath: getDestinationForImage(image) }));
 }
 
 export async function removePulledImages(images: IPullableImage[]): Promise<void> {
@@ -41,15 +41,15 @@ export async function removePulledImages(images: IPullableImage[]): Promise<void
 }
 
 // Exported for testing
-export function getImageParts(imageWithTag: string) : {imageName: string, imageTag: string, imageDigest: string} {
+export function getImageParts(imageWithTag: string) : {imageName: string, imageTag: string} {
   // we're matching pattern: <registry:port_number>(optional)/<image_name>(mandatory):<image_tag>(optional)@<tag_identifier>(optional)
   // extracted from https://github.com/docker/distribution/blob/master/reference/regexp.go
   const regex = /^((?:(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])(?:(?:\.(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9]))+)?(?::[0-9]+)?\/)?[a-z0-9]+(?:(?:(?:[._]|__|[-]*)[a-z0-9]+)+)?(?:(?:\/[a-z0-9]+(?:(?:(?:[._]|__|[-]*)[a-z0-9]+)+)?)+)?)(?::([\w][\w.-]{0,127}))?(?:@([A-Za-z][A-Za-z0-9]*(?:[-_+.][A-Za-z][A-Za-z0-9]*)*[:][A-Fa-f0-9]{32,}))?$/ig;
   const groups  = regex.exec(imageWithTag);
-
+  
   if(!groups){
     logger.error({image: imageWithTag}, 'Image with tag is malformed, cannot extract valid parts');
-    return { imageName: imageWithTag, imageTag: '', imageDigest: '' };
+    return {imageName: imageWithTag, imageTag: ''};
   }
 
   const IMAGE_NAME_GROUP = 1;
@@ -58,8 +58,8 @@ export function getImageParts(imageWithTag: string) : {imageName: string, imageT
 
   return {
     imageName: groups[IMAGE_NAME_GROUP],
-    imageTag: groups[IMAGE_TAG_GROUP] || '',
-    imageDigest: groups[IMAGE_DIGEST_GROUP] || '',
+    // prefer tag over digest
+    imageTag: groups[IMAGE_TAG_GROUP] || groups[IMAGE_DIGEST_GROUP] || '',
   };
 }
 
@@ -78,7 +78,7 @@ export async function scanImages(images: IPullableImage[]): Promise<IScanResult[
 
   const dockerfile = undefined;
 
-  for (const { imageName, fileSystemPath, imageWithDigest } of images) {
+  for (const {imageName, fileSystemPath} of images) {
     try {
       const staticAnalysisOptions = constructStaticAnalysisOptions(fileSystemPath);
       const options: IPluginOptions = {
@@ -92,18 +92,16 @@ export async function scanImages(images: IPullableImage[]): Promise<IScanResult[
         throw Error('Unexpected empty result from docker-plugin');
       }
 
-      const imageParts = getImageParts(imageName);
+      const imageParts: {imageName: string, imageTag: string} = getImageParts(imageName);
 
       result.imageMetadata = {
         image: imageParts.imageName,
         imageTag: imageParts.imageTag,
-        imageDigest: getImageParts(imageWithDigest).imageDigest,
       };
 
       scannedImages.push({
         image: imageParts.imageName,
         imageWithTag: imageName,
-        imageWithDigest: imageWithDigest,
         pluginResult: result,
       });
     } catch (error) {

src/scanner/images/types.ts

@@ -1,11 +1,5 @@
-export interface IScanImage {
-  imageName: string;
-  imageWithDigest: string;
-}
-
 export interface IPullableImage {
   imageName: string;
-  imageWithDigest: string;
   fileSystemPath: string;
 }
 

src/scanner/index.ts

@@ -1,20 +1,22 @@
 import logger = require('../common/logger');
-import { pullImages, removePulledImages, getImagesWithFileSystemPath, scanImages, getImageParts } from './images';
+import { pullImages, removePulledImages, getImagesWithFileSystemPath, scanImages } from './images';
 import { deleteWorkload, sendDepGraph } from '../transmitter';
 import { constructDeleteWorkload, constructDepGraph } from '../transmitter/payload';
 import { IWorkload, ILocalWorkloadLocator } from '../transmitter/types';
-import { IPullableImage, IScanImage } from './images/types';
+import { IPullableImage } from './images/types';
 
 export async function processWorkload(workloadMetadata: IWorkload[]): Promise<void> {
   // every workload metadata references the same workload name, grab it from the first one
   const workloadName = workloadMetadata[0].name;
-  const uniqueImages = getUniqueImages(workloadMetadata);
+  const allImages = workloadMetadata.map((meta) => meta.imageName);
+  logger.info({workloadName, imageCount: allImages.length}, 'queried workloads');
+  const uniqueImages = [...new Set<string>(allImages)];
 
-  logger.info({ workloadName, imageCount: uniqueImages.length }, 'pulling unique images');
+  logger.info({workloadName, imageCount: uniqueImages.length}, 'pulling unique images');
   const imagesWithFileSystemPath = getImagesWithFileSystemPath(uniqueImages);
   const pulledImages = await pullImages(imagesWithFileSystemPath);
   if (pulledImages.length === 0) {
-    logger.info({ workloadName }, 'no images were pulled, halting scanner process.');
+    logger.info({workloadName}, 'no images were pulled, halting scanner process.');
     return;
   }
 
@@ -33,31 +35,6 @@ export async function sendDeleteWorkloadRequest(workloadName: string, localWorkl
   await deleteWorkload(deletePayload);
 }
 
-export function getUniqueImages(workloadMetadata: IWorkload[]): IScanImage[] {
-  const uniqueImages: { [key: string]: IScanImage } = workloadMetadata.reduce((accum, meta) => {
-    // example: For DCR "redis:latest"
-    // example: For GCR "gcr.io/test-dummy/redis:latest"
-    // example: For ECR "291964488713.dkr.ecr.us-east-2.amazonaws.com/snyk/redis:latest"
-    // meta.imageName can be different depends on CR
-    const { imageName } = getImageParts(meta.imageName);
-    // meta.imageId can be different depends on CR
-    // example: For DCR "docker.io/library/redis@sha256:8e9f8546050da8aae393a41d65ad37166b4f0d8131d627a520c0f0451742e9d6"
-    // example: For GCR "sha256:8e9f8546050da8aae393a41d65ad37166b4f0d8131d627a520c0f0451742e9d6"
-    // example: For ECR "sha256:8e9f8546050da8aae393a41d65ad37166b4f0d8131d627a520c0f0451742e9d6"
-    const digest = meta.imageId.substring(meta.imageId.lastIndexOf('@') + 1);
-    const imageWithDigest = `${imageName}@${digest}`;
-
-    accum[imageWithDigest] = {
-      imageWithDigest,
-      imageName: meta.imageName, // Image name with tag
-    };
-
-    return accum;
-  }, {});
-
-  return Object.values(uniqueImages);
-}
-
 async function scanImagesAndSendResults(
   workloadName: string,
   pulledImages: IPullableImage[],

src/scanner/types.ts

@@ -1,6 +1,5 @@
 export interface IScanResult {
   image: string;
-  imageWithDigest: string;
   imageWithTag: string;
   pluginResult: any;
 }

src/transmitter/payload.ts

@@ -27,7 +27,6 @@ export function constructDepGraph(
     const imageLocator: IImageLocator = {
       userLocator: config.INTEGRATION_ID,
       imageId: scannedImage.image,
-      imageWithDigest: scannedImage.imageWithDigest,
       cluster,
       namespace,
       type,

src/transmitter/types.ts

@@ -25,7 +25,6 @@ export interface IWorkloadMetadata {
 
 export interface IImageLocator extends IWorkloadLocator {
   imageId: string;
-  imageWithDigest: string;
 }
 
 export interface IKubernetesMonitorMetadata {

test/unit/scanner/images.test.ts

@@ -1,18 +1,16 @@
 import * as tap from 'tap';
 
-import {IPullableImage, IScanImage} from '../../../src/scanner/images/types';
+import { IPullableImage } from '../../../src/scanner/images/types';
 import config = require('../../../src/common/config');
 import * as scannerImages from '../../../src/scanner/images';
 
+
 tap.test('getImagesWithFileSystemPath()', async (t) => {
-  const noImages: IScanImage[] = [];
+  const noImages: string[] = [];
   const noImagesResult = scannerImages.getImagesWithFileSystemPath(noImages);
   t.same(noImagesResult, [], 'correctly maps an empty array');
 
-  const image: IScanImage[] = [{
-    imageName: 'nginx:latest',
-    imageWithDigest: 'nginx@sha256:4949aa7259aa6f827450207db5ad94cabaa9248277c6d736d5e1975d200c7e43',
-  }];
+  const image = ['nginx:latest'];
   const imageResult = scannerImages.getImagesWithFileSystemPath(image);
   t.same(imageResult.length, 1, 'expected 1 item');
 
@@ -30,10 +28,7 @@ tap.test('getImagesWithFileSystemPath()', async (t) => {
   t.ok(expectedPattern, 'the file system path starts with an expected pattern');
 
   // Ensure that two consecutive calls do not return the same file system path
-  const someImage = [{
-    imageName: 'centos:latest',
-    imageWithDigest: 'centos@sha256:fc4a234b91cc4b542bac8a6ad23b2ddcee60ae68fc4dbd4a52efb5f1b0baad71',
-  }];
+  const someImage = ['centos:latest'];
   const firstCallResult = scannerImages.getImagesWithFileSystemPath(someImage)[0];
   const secondCallResult = scannerImages.getImagesWithFileSystemPath(someImage)[0];
   t.ok(
@@ -49,6 +44,8 @@ tap.test('pullImages() skips on missing file system path', async (t) => {
 });
 
 tap.test('constructStaticAnalysisOptions() tests', async (t) => {
+  t.plan(1);
+
   const somePath = '/var/tmp/file.tar';
   const options = scannerImages.constructStaticAnalysisOptions(somePath);
   const expectedResult = {
@@ -60,10 +57,11 @@ tap.test('constructStaticAnalysisOptions() tests', async (t) => {
 });
 
 tap.test('extracted image tag tests', async (t) => {
+  t.plan(6);
+
   const imageWithSha = 'nginx@sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2';
   const imageWithShaResult = scannerImages.getImageParts(imageWithSha);
-  t.same(imageWithShaResult.imageTag, '', 'image tag is empty');
-  t.same(imageWithShaResult.imageDigest, 'sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2', 'image digest is returned');
+  t.same(imageWithShaResult.imageTag, 'sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2', 'image sha is returned');
 
   const imageWithTag = 'nginx:latest';
   const imageWithTagResult = scannerImages.getImageParts(imageWithTag);
@@ -87,6 +85,8 @@ tap.test('extracted image tag tests', async (t) => {
 });
 
 tap.test('extracted image name tests', async (t) => {
+  t.plan(5);
+
   t.same(scannerImages.getImageParts('nginx:latest').imageName, 'nginx', 'removed image:tag');
   t.same(scannerImages.getImageParts('node@sha256:215a9fbef4df2c1ceb7c79481d3cfd94ad8f1f0105bade39f3be907bf386c5e1').imageName, 'node', 'removed image@sha:hex');
   t.same(scannerImages.getImageParts('kind-registry:5000/python:rc-buster').imageName, 'kind-registry:5000/python', 'removed repository/image:tag');

test/unit/scanner/index.test.ts

@@ -11,13 +11,11 @@ tap.test('constructDepGraph breaks when workloadMetadata is missing items', asyn
     {
       image: 'myImage',
       imageWithTag: 'myImage:tag',
-      imageWithDigest: 'myImage@sha256:idontcarewhatissha',
       pluginResult: 'whatever1',
     },
     {
       image: 'anotherImage',
       imageWithTag: 'anotherImage:1.2.3-alpha',
-      imageWithDigest: 'myImage@sha256:somuchdifferentsha256',
       pluginResult: 'whatever3',
     },
   ];
@@ -50,7 +48,6 @@ tap.test('constructDepGraph happy flow', async (t) => {
     {
       image: 'myImage',
       imageWithTag: 'myImage:tag',
-      imageWithDigest: 'myImage@sha256:idontcarewhatissha',
       pluginResult: 'whatever1',
     },
   ];