Merge pull request #993 from snyk/fix/pull-oci-archive

fix: pull OCI archives instead of Docker archives

Author: ivanstanev

package-lock.json

@@ -47,7 +47,7 @@
     "needle": "^3.0.0",
     "sleep-promise": "^9.1.0",
     "snyk-config": "5.0.0",
-    "snyk-docker-plugin": "^4.33.2",
+    "snyk-docker-plugin": "^4.34.2",
     "source-map-support": "^0.5.21",
     "tunnel": "0.0.6",
     "typescript": "^4.5.2",

package.json

@@ -1,17 +1,21 @@
-import { unlink } from 'fs';
+import { unlink, stat } from 'fs';
+import { promisify } from 'util';
 import { PluginResponse, scan } from 'snyk-docker-plugin';
 import { DepGraph, legacy } from '@snyk/dep-graph';
 
 import { logger } from '../../common/logger';
 import { pull as skopeoCopy, getDestinationForImage } from './skopeo';
-import { IPullableImage, IScanImage, SkopeoRepositoryType } from './types';
+import { IPullableImage, IScanImage } from './types';
 import { IScanResult } from '../types';
 import {
   buildDockerPropertiesOnDepTree,
   DependencyTree,
   extractFactsFromDockerPluginResponse,
   LegacyPluginResponse,
 } from './docker-plugin-shim';
+import type { Telemetry } from '../../transmitter/types';
+
+const statAsync = promisify(stat);
 
 /*
  pulled images by skopeo archive repo type:
@@ -22,23 +26,11 @@ async function pullImageBySkopeoRepo(
 ): Promise<IPullableImage> {
   // Scan image by digest if exists, other way fallback tag
   const scanId = imageToPull.imageWithDigest ?? imageToPull.imageName;
-  imageToPull.skopeoRepoType = SkopeoRepositoryType.DockerArchive;
-  try {
-    // copy docker archive image
-    await skopeoCopy(
-      scanId,
-      imageToPull.fileSystemPath,
-      imageToPull.skopeoRepoType,
-    );
-  } catch (dockerError) {
-    imageToPull.skopeoRepoType = SkopeoRepositoryType.OciArchive;
-    // copy oci archive image
-    await skopeoCopy(
-      scanId,
-      imageToPull.fileSystemPath,
-      imageToPull.skopeoRepoType,
-    );
-  }
+  await skopeoCopy(
+    scanId,
+    imageToPull.fileSystemPath,
+    imageToPull.skopeoRepoType,
+  );
   return imageToPull;
 }
 
@@ -117,25 +109,17 @@ export function getImageParts(imageWithTag: string): {
 
 export async function scanImages(
   images: IPullableImage[],
+  telemetry: Partial<Telemetry>,
 ): Promise<IScanResult[]> {
   const scannedImages: IScanResult[] = [];
 
-  for (const {
-    imageName,
-    fileSystemPath,
-    imageWithDigest,
-    skopeoRepoType,
-  } of images) {
+  for (const { imageName, fileSystemPath, imageWithDigest } of images) {
     try {
       const shouldIncludeAppVulns = true;
-      const archiveType =
-        skopeoRepoType == SkopeoRepositoryType.DockerArchive
-          ? 'docker-archive'
-          : 'oci-archive';
-      const dockerArchivePath = `${archiveType}:${fileSystemPath}`;
+      const archivePath = `docker-archive:${fileSystemPath}`;
 
       const pluginResponse = await scan({
-        path: dockerArchivePath,
+        path: archivePath,
         imageNameAndTag: imageName,
         'app-vulns': shouldIncludeAppVulns,
       });
@@ -148,6 +132,19 @@ export async function scanImages(
         throw Error('Unexpected empty result from docker-plugin');
       }
 
+      try {
+        const fileStats = await statAsync(fileSystemPath);
+        if (!telemetry.imageSizeBytes) {
+          telemetry.imageSizeBytes = 0;
+        }
+        telemetry.imageSizeBytes += fileStats.size;
+      } catch (err) {
+        logger.warn(
+          { error: err, imageName, imageWithDigest, fileSystemPath },
+          'could not determine archive size',
+        );
+      }
+
       const depTree = await getDependencyTreeFromPluginResponse(
         pluginResponse,
         imageName,

src/scanner/images/index.ts

@@ -26,7 +26,6 @@ function prefixRespository(target: string, type: SkopeoRepositoryType): string {
     case SkopeoRepositoryType.ImageRegistry:
       return `${type}://${target}`;
     case SkopeoRepositoryType.DockerArchive:
-    case SkopeoRepositoryType.OciArchive:
       return `${type}:${target}`;
     default:
       throw new Error(`Unhandled Skopeo repository type ${type}`);

src/scanner/images/skopeo.ts

@@ -16,7 +16,5 @@ export interface IPullableImage {
  */
 export enum SkopeoRepositoryType {
   DockerArchive = 'docker-archive',
-  OciArchive = 'oci',
   ImageRegistry = 'docker',
-  Directory = 'dir', // Note, skopeo marks this as a non-standard format
 }

src/scanner/images/types.ts

@@ -17,7 +17,11 @@ import {
   ILocalWorkloadLocator,
   Telemetry,
 } from '../transmitter/types';
-import { IPullableImage, IScanImage } from './images/types';
+import {
+  IPullableImage,
+  IScanImage,
+  SkopeoRepositoryType,
+} from './images/types';
 import {
   getWorkloadAlreadyScanned,
   getWorkloadImageAlreadyScanned,
@@ -29,7 +33,7 @@ export async function processWorkload(
 ): Promise<void> {
   // every workload metadata references the same workload name, grab it from the first one
   const workloadName = workloadMetadata[0].name;
-  const uniqueImages: IScanImage[] = getUniqueImages(workloadMetadata);
+  const uniqueImages = getUniqueImages(workloadMetadata);
 
   logger.info(
     { workloadName, imageCount: uniqueImages.length },
@@ -74,42 +78,37 @@ export async function sendDeleteWorkloadRequest(
 }
 
 export function getUniqueImages(workloadMetadata: IWorkload[]): IScanImage[] {
-  const uniqueImages: { [key: string]: IScanImage } = workloadMetadata.reduce(
-    (accum, meta) => {
-      logger.info(
-        {
-          workloadName: workloadMetadata[0].name,
-          name: meta.imageName,
-          id: meta.imageId,
-        },
-        'image metadata',
-      );
-      // example: For DCR "redis:latest"
-      // example: For GCR "gcr.io/test-dummy/redis:latest"
-      // example: For ECR "291964488713.dkr.ecr.us-east-2.amazonaws.com/snyk/redis:latest"
-      // meta.imageName can be different depends on CR
-      const { imageName } = getImageParts(meta.imageName);
-      // meta.imageId can be different depends on CR
-      // example: For DCR "docker.io/library/redis@sha256:8e9f8546050da8aae393a41d65ad37166b4f0d8131d627a520c0f0451742e9d6"
-      // example: For GCR "sha256:8e9f8546050da8aae393a41d65ad37166b4f0d8131d627a520c0f0451742e9d6"
-      // example: For ECR "sha256:8e9f8546050da8aae393a41d65ad37166b4f0d8131d627a520c0f0451742e9d6"
-      let digest: string | undefined = undefined;
-      if (
-        meta.imageId.lastIndexOf('@') > -1 ||
-        meta.imageId.startsWith('sha')
-      ) {
-        digest = meta.imageId.substring(meta.imageId.lastIndexOf('@') + 1);
-      }
-
-      accum[meta.imageName] = {
-        imageWithDigest: digest && `${imageName}@${digest}`,
-        imageName: meta.imageName, // Image name with tag
-      };
-
-      return accum;
-    },
-    {},
-  );
+  const uniqueImages = workloadMetadata.reduce((accum, meta) => {
+    logger.info(
+      {
+        workloadName: workloadMetadata[0].name,
+        name: meta.imageName,
+        id: meta.imageId,
+      },
+      'image metadata',
+    );
+    // example: For DCR "redis:latest"
+    // example: For GCR "gcr.io/test-dummy/redis:latest"
+    // example: For ECR "291964488713.dkr.ecr.us-east-2.amazonaws.com/snyk/redis:latest"
+    // meta.imageName can be different depends on CR
+    const { imageName } = getImageParts(meta.imageName);
+    // meta.imageId can be different depends on CR
+    // example: For DCR "docker.io/library/redis@sha256:8e9f8546050da8aae393a41d65ad37166b4f0d8131d627a520c0f0451742e9d6"
+    // example: For GCR "sha256:8e9f8546050da8aae393a41d65ad37166b4f0d8131d627a520c0f0451742e9d6"
+    // example: For ECR "sha256:8e9f8546050da8aae393a41d65ad37166b4f0d8131d627a520c0f0451742e9d6"
+    let digest: string | undefined = undefined;
+    if (meta.imageId.lastIndexOf('@') > -1 || meta.imageId.startsWith('sha')) {
+      digest = meta.imageId.substring(meta.imageId.lastIndexOf('@') + 1);
+    }
+
+    accum[meta.imageName] = {
+      imageWithDigest: digest && `${imageName}@${digest}`,
+      imageName: meta.imageName, // Image name with tag
+      skopeoRepoType: SkopeoRepositoryType.DockerArchive,
+    };
+
+    return accum;
+  }, {} as Record<string, IScanImage>);
 
   return Object.values(uniqueImages);
 }
@@ -121,7 +120,7 @@ async function scanImagesAndSendResults(
   telemetry: Partial<Telemetry>,
 ): Promise<void> {
   const imageScanStartTimestampMs = Date.now();
-  const scannedImages = await scanImages(pulledImages);
+  const scannedImages = await scanImages(pulledImages, telemetry);
   const imageScanDurationMs = Date.now() - imageScanStartTimestampMs;
 
   if (scannedImages.length === 0) {

src/scanner/index.ts

@@ -161,7 +161,7 @@ async function isSupportedWorkload(
       attemptedApiCall.response.statusCode < 300
     );
   } catch (error) {
-    logger.info(
+    logger.debug(
       { error, workloadKind },
       'Failed on Kubernetes API call to list DeploymentConfig',
     );
@@ -176,7 +176,7 @@ export async function setupInformer(
   const logContext: Record<string, unknown> = { namespace, workloadKind };
   const isSupported = await isSupportedWorkload(namespace, workloadKind);
   if (!isSupported) {
-    logger.info(
+    logger.debug(
       logContext,
       'The Kubernetes cluster does not support this workload',
     );

src/supervisor/watchers/handlers/index.ts

@@ -101,6 +101,10 @@ export interface IRequestError {
 export interface Telemetry {
   enqueueDurationMs: number;
   queueSize: number;
+  /** This metric captures the total duration to pull all images of a workload. */
   imagePullDurationMs: number;
+  /** This metric captures the total duration to scan all images of a workload. */
   imageScanDurationMs: number;
+  /** This metric captures the combined size of all images of a workload. */
+  imageSizeBytes: number;
 }

src/transmitter/types.ts

@@ -254,6 +254,7 @@ test('Kubernetes-Monitor with KinD', async (jestDoneCallback) => {
             enqueueDurationMs: expect.any(Number),
             imagePullDurationMs: expect.any(Number),
             imageScanDurationMs: expect.any(Number),
+            imageSizeBytes: expect.any(Number),
             queueSize: expect.any(Number),
           },
           imageLocator: expect.objectContaining({