fix: pulling by digest from private registries

Author: dkontorovskyy

src/scanner/index.ts

@@ -1,5 +1,5 @@
 import logger = require('../common/logger');
-import { pullImages, removePulledImages, getImagesWithFileSystemPath, scanImages } from './images';
+import { pullImages, removePulledImages, getImagesWithFileSystemPath, scanImages, getImageParts } from './images';
 import { deleteWorkload, sendDepGraph } from '../transmitter';
 import { constructDeleteWorkload, constructDepGraph } from '../transmitter/payload';
 import { IWorkload, ILocalWorkloadLocator } from '../transmitter/types';
@@ -8,23 +8,13 @@ import { IPullableImage, IScanImage } from './images/types';
 export async function processWorkload(workloadMetadata: IWorkload[]): Promise<void> {
   // every workload metadata references the same workload name, grab it from the first one
   const workloadName = workloadMetadata[0].name;
-  const uniqueImages: { [key: string]: IScanImage } = workloadMetadata.reduce((accum, meta) => {
-    // example: "docker.io/library/redis@sha256:33ca074e6019b451235735772a9c3e7216f014aae8eb0580d7e94834fe23efb3"
-    const imageWithDigest = meta.imageId.substring(meta.imageId.lastIndexOf('/') + 1);
+  const uniqueImages = getUniqueImages(workloadMetadata);
 
-    accum[meta.imageName] = {
-      imageName: meta.imageName,
-      imageWithDigest,
-    };
-
-    return accum;
-  }, {});
-
-  logger.info({workloadName, imageCount: Object.values(uniqueImages).length}, 'pulling unique images');
-  const imagesWithFileSystemPath = getImagesWithFileSystemPath(Object.values(uniqueImages));
+  logger.info({ workloadName, imageCount: uniqueImages.length }, 'pulling unique images');
+  const imagesWithFileSystemPath = getImagesWithFileSystemPath(uniqueImages);
   const pulledImages = await pullImages(imagesWithFileSystemPath);
   if (pulledImages.length === 0) {
-    logger.info({workloadName}, 'no images were pulled, halting scanner process.');
+    logger.info({ workloadName }, 'no images were pulled, halting scanner process.');
     return;
   }
 
@@ -43,6 +33,31 @@ export async function sendDeleteWorkloadRequest(workloadName: string, localWorkl
   await deleteWorkload(deletePayload);
 }
 
+export function getUniqueImages(workloadMetadata: IWorkload[]): IScanImage[] {
+  const uniqueImages: { [key: string]: IScanImage } = workloadMetadata.reduce((accum, meta) => {
+    // example: For DCR "redis:latest"
+    // example: For GCR "gcr.io/test-dummy/redis:latest"
+    // example: For ECR "291964488713.dkr.ecr.us-east-2.amazonaws.com/snyk/redis:latest"
+    // meta.imageName can be different depends on CR
+    const { imageName } = getImageParts(meta.imageName);
+    // meta.imageId can be different depends on CR
+    // example: For DCR "docker.io/library/redis@sha256:8e9f8546050da8aae393a41d65ad37166b4f0d8131d627a520c0f0451742e9d6"
+    // example: For GCR "sha256:8e9f8546050da8aae393a41d65ad37166b4f0d8131d627a520c0f0451742e9d6"
+    // example: For ECR "sha256:8e9f8546050da8aae393a41d65ad37166b4f0d8131d627a520c0f0451742e9d6"
+    const digest = meta.imageId.substring(meta.imageId.lastIndexOf('@') + 1);
+    const imageWithDigest = `${imageName}@${digest}`;
+
+    accum[imageWithDigest] = {
+      imageWithDigest,
+      imageName: meta.imageName, // Image name with tag
+    };
+
+    return accum;
+  }, {});
+
+  return Object.values(uniqueImages);
+}
+
 async function scanImagesAndSendResults(
   workloadName: string,
   pulledImages: IPullableImage[],

test/unit/scanner/index.test.ts

@@ -0,0 +1,55 @@
+import * as tap from 'tap';
+
+import * as scanner from '../../../src/scanner';
+import { IWorkload } from '../../../src/transmitter/types';
+
+tap.test('getUniqueImages()', async (t) => {
+    const workload: Partial<IWorkload>[] = [
+        // 1.DCR
+        {
+            imageName: 'redis:latest',
+            imageId: 'docker.io/library/redis@sha256:8e9f8546050da8aae393a41d65ad37166b4f0d8131d627a520c0f0451742e9d6'
+        },
+        // 2.Duplicate to verify uniqueness
+        {
+            imageName: 'redis:latest',
+            imageId: 'docker.io/library/redis@sha256:8e9f8546050da8aae393a41d65ad37166b4f0d8131d627a520c0f0451742e9d6'
+        },
+        // 3. Duplicate without tag
+        {
+            imageName: 'redis',
+            imageId: 'docker.io/library/redis@sha256:8e9f8546050da8aae393a41d65ad37166b4f0d8131d627a520c0f0451742e9d6'
+        },
+        // 4. Duplicate with SHA instead of tag
+        {
+            imageName: 'redis@sha256:8e9f8546050da8aae393a41d65ad37166b4f0d8131d627a520c0f0451742e9d6',
+            imageId: 'docker.io/library/redis@sha256:8e9f8546050da8aae393a41d65ad37166b4f0d8131d627a520c0f0451742e9d6'
+        },
+        // 5. GCR
+        {
+            imageName: 'gcr.io/test-dummy/redis:latest',
+            imageId: 'sha256:8e9f8546050da8aae393a41d65ad37166b4f0d8131d627a520c0f0451742e9d6'
+        },
+        // 6. ECR
+        {
+            imageName: '291964488713.dkr.ecr.us-east-2.amazonaws.com/snyk/redis:latest',
+            imageId: 'sha256:8e9f8546050da8aae393a41d65ad37166b4f0d8131d627a520c0f0451742e9d6'
+        },
+    ];
+
+    const result = scanner.getUniqueImages(workload as any);
+
+    t.strictEqual(result.length, 3, 'removed duplicate image');
+    result.map((metaData) => {
+        t.ok(metaData.imageWithDigest.includes('redis'), 'has name in imageWithDigest');
+        t.ok(metaData.imageWithDigest.includes('sha256:8e9f8546050da8aae393a41d65ad37166b4f0d8131d627a520c0f0451742e9d6'), 'has digest');
+
+        if (metaData.imageWithDigest.includes('gcr')) {
+            t.ok(metaData.imageWithDigest.includes('/'), 'contains / in GCR imageWithDigest');
+        }
+
+        if (metaData.imageWithDigest.includes('ecr')) {
+            t.ok(metaData.imageWithDigest.includes('/'), 'contains / in ECR imageWithDigest');
+        }
+    });
+});