Skip to content

Commit a2a1b69

Browse files
agouilagouil
authored
Merge pull request #1452 from snyk/feat/produce-oci-distribution-metadata
feat: produce oci distribution metadata [OI-61]
2 parents c42bbbb + ac948bd commit a2a1b69

File tree

8 files changed

+204
-28
lines changed

8 files changed

+204
-28
lines changed

‎package-lock.json

Lines changed: 10 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@
4949
"packageurl-js": "^1.2.1",
5050
"sleep-promise": "^9.1.0",
5151
"snyk-config": "5.1.0",
52-
"snyk-docker-plugin": "^6.8.3",
52+
"snyk-docker-plugin": "^6.10.0",
5353
"source-map-support": "^0.5.21",
5454
"tunnel": "0.0.6",
5555
"typescript": "4.7.4",

‎src/scanner/images/index.ts

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,12 +27,14 @@ async function pullImageBySkopeoRepo(
2727
): Promise<IPullableImage> {
2828
// Scan image by digest if exists, other way fallback tag
2929
const scanId = imageToPull.imageWithDigest ?? imageToPull.imageName;
30-
await skopeoCopy(
30+
const { manifestDigest, indexDigest } = await skopeoCopy(
3131
scanId,
3232
imageToPull.fileSystemPath,
3333
imageToPull.skopeoRepoType,
3434
workloadName,
3535
);
36+
imageToPull.manifestDigest = manifestDigest;
37+
imageToPull.indexDigest = indexDigest;
3638
return imageToPull;
3739
}
3840

@@ -126,14 +128,23 @@ export async function scanImages(
126128
): Promise<IScanResult[]> {
127129
const scannedImages: IScanResult[] = [];
128130

129-
for (const { imageName, fileSystemPath, imageWithDigest } of images) {
131+
for (const {
132+
imageName,
133+
fileSystemPath,
134+
imageWithDigest,
135+
manifestDigest,
136+
indexDigest,
137+
} of images) {
130138
try {
131139
const archivePath = `docker-archive:${fileSystemPath}`;
132140

133141
const pluginResponse = await scan({
134142
path: archivePath,
135143
imageNameAndTag: imageName,
136-
imageNameAndDigest: imageWithDigest,
144+
digests: {
145+
manifest: manifestDigest,
146+
index: indexDigest,
147+
},
137148
});
138149

139150
if (

‎src/scanner/images/skopeo.ts

Lines changed: 98 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,15 @@
11
import * as fs from 'fs';
22
import sleep from 'sleep-promise';
3+
import crypto from 'crypto';
34

45
import { logger } from '../../common/logger';
56
import { config } from '../../common/config';
67
import * as processWrapper from '../../common/process';
78
import * as credentials from './credentials';
8-
import { SkopeoRepositoryType } from './types';
9+
import { ImageDigests, ImageManifest, SkopeoRepositoryType } from './types';
10+
11+
const DEFAULT_PLATFORM_OS = 'linux';
12+
const DEFAULT_PLATFORM_ARCH = 'amd64';
913

1014
function getUniqueIdentifier(): string {
1115
const [seconds, nanoseconds] = process.hrtime();
@@ -37,10 +41,10 @@ export async function pull(
3741
destination: string,
3842
skopeoRepoType: SkopeoRepositoryType,
3943
workloadName: string,
40-
): Promise<void> {
44+
): Promise<ImageDigests> {
4145
const creds = await credentials.getSourceCredentials(image);
42-
const credentialsParameters = getCredentialParameters(creds);
43-
const certificatesParameters = getCertificatesParameters();
46+
const credentialsParameters = getCopyCredentialParameters(creds);
47+
const certificatesParameters = getCopyCertificatesParameters();
4448

4549
const args: Array<processWrapper.IProcessArgument> = [];
4650
args.push({ body: 'copy', sanitise: false });
@@ -57,11 +61,26 @@ export async function pull(
5761
sanitise: false,
5862
});
5963

60-
await pullWithRetry(args, destination, workloadName);
64+
const env: Record<string, string | undefined> = {
65+
// The Azure CR credentials helper requires these env vars:
66+
AZURE_CLIENT_ID: process.env.AZURE_CLIENT_ID,
67+
AZURE_TENANT_ID: process.env.AZURE_TENANT_ID,
68+
AZURE_FEDERATED_TOKEN_FILE: process.env.AZURE_FEDERATED_TOKEN_FILE,
69+
AZURE_FEDERATED_TOKEN: process.env.AZURE_FEDERATED_TOKEN,
70+
AZURE_AUTHORITY_HOST: process.env.AZURE_AUTHORITY_HOST,
71+
};
72+
await pullWithRetry(args, env, destination, workloadName);
73+
74+
return await extractImageDigests(
75+
prefixRespository(image, SkopeoRepositoryType.ImageRegistry),
76+
env,
77+
creds,
78+
);
6179
}
6280

6381
async function pullWithRetry(
6482
args: Array<processWrapper.IProcessArgument>,
83+
env: Record<string, string | undefined>,
6584
destination: string,
6685
workloadName: string,
6786
): Promise<void> {
@@ -70,17 +89,9 @@ async function pullWithRetry(
7089

7190
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
7291
try {
73-
const env: Record<string, string | undefined> = {
74-
// The Azure CR credentials helper requires these env vars:
75-
AZURE_CLIENT_ID: process.env.AZURE_CLIENT_ID,
76-
AZURE_TENANT_ID: process.env.AZURE_TENANT_ID,
77-
AZURE_FEDERATED_TOKEN_FILE: process.env.AZURE_FEDERATED_TOKEN_FILE,
78-
AZURE_FEDERATED_TOKEN: process.env.AZURE_FEDERATED_TOKEN,
79-
AZURE_AUTHORITY_HOST: process.env.AZURE_AUTHORITY_HOST,
80-
};
8192
await processWrapper.exec('skopeo', env, ...args);
8293
return;
83-
} catch (err) {
94+
} catch (err: unknown) {
8495
try {
8596
if (fs.existsSync(destination)) {
8697
fs.unlinkSync(destination);
@@ -99,7 +110,60 @@ async function pullWithRetry(
99110
}
100111
}
101112

102-
export function getCredentialParameters(
113+
export async function extractImageDigests(
114+
image: string,
115+
env: Record<string, string | undefined> = {},
116+
creds?: string,
117+
): Promise<ImageDigests> {
118+
let indexDigest: string | undefined = undefined;
119+
let manifestDigest: string | undefined = undefined;
120+
121+
const args: Array<processWrapper.IProcessArgument> = [
122+
{ body: 'inspect', sanitise: false },
123+
{ body: '--raw', sanitise: false },
124+
{ body: image, sanitise: false },
125+
...getInspectCredentialParameters(creds),
126+
...getInspectCertificatesParameters(),
127+
];
128+
129+
try {
130+
const { stdout } = await processWrapper.exec('skopeo', env, ...args);
131+
const manifest = JSON.parse(stdout) as ImageManifest;
132+
if (isIndex(manifest)) {
133+
manifestDigest = manifest.manifests?.find(
134+
(m) =>
135+
m.platform.os === DEFAULT_PLATFORM_OS &&
136+
m.platform.architecture === DEFAULT_PLATFORM_ARCH,
137+
)?.digest;
138+
indexDigest = manifestDigest ? calculateDigest(stdout) : undefined;
139+
} else {
140+
manifestDigest = calculateDigest(stdout);
141+
}
142+
} catch (error) {
143+
logger.warn(
144+
{ error },
145+
'could not get the image digests through Skopeo inspect-raw',
146+
);
147+
}
148+
return { indexDigest, manifestDigest };
149+
}
150+
151+
function isIndex(manifest: ImageManifest): boolean {
152+
return (
153+
manifest.mediaType.includes('vnd.oci.image.index') ||
154+
manifest.mediaType.includes('vnd.docker.distribution.manifest.list')
155+
);
156+
}
157+
158+
function calculateDigest(manifest: string): string {
159+
return `sha256:${crypto
160+
.createHash('sha256')
161+
.update(manifest)
162+
.digest('hex')
163+
.toString()}`;
164+
}
165+
166+
export function getCopyCredentialParameters(
103167
credentials: string | undefined,
104168
): Array<processWrapper.IProcessArgument> {
105169
const credentialsParameters: Array<processWrapper.IProcessArgument> = [];
@@ -110,9 +174,27 @@ export function getCredentialParameters(
110174
return credentialsParameters;
111175
}
112176

113-
export function getCertificatesParameters(): Array<processWrapper.IProcessArgument> {
177+
export function getInspectCredentialParameters(
178+
credentials?: string,
179+
): Array<processWrapper.IProcessArgument> {
180+
const credentialsParameters: Array<processWrapper.IProcessArgument> = [];
181+
if (credentials) {
182+
credentialsParameters.push({ body: '--creds', sanitise: true });
183+
credentialsParameters.push({ body: credentials, sanitise: true });
184+
}
185+
return credentialsParameters;
186+
}
187+
188+
export function getCopyCertificatesParameters(): Array<processWrapper.IProcessArgument> {
114189
const certificatesParameters: Array<processWrapper.IProcessArgument> = [];
115190
certificatesParameters.push({ body: '--src-cert-dir', sanitise: true });
116191
certificatesParameters.push({ body: '/srv/app/certs', sanitise: true });
117192
return certificatesParameters;
118193
}
194+
195+
export function getInspectCertificatesParameters(): Array<processWrapper.IProcessArgument> {
196+
const certificatesParameters: Array<processWrapper.IProcessArgument> = [];
197+
certificatesParameters.push({ body: '--cert-dir', sanitise: true });
198+
certificatesParameters.push({ body: '/srv/app/certs', sanitise: true });
199+
return certificatesParameters;
200+
}

‎src/scanner/images/types.ts

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ export interface IScanImage {
77
export interface IPullableImage {
88
imageName: string;
99
imageWithDigest?: string;
10+
manifestDigest?: string;
11+
indexDigest?: string;
1012
fileSystemPath: string;
1113
skopeoRepoType: SkopeoRepositoryType;
1214
}
@@ -18,3 +20,20 @@ export enum SkopeoRepositoryType {
1820
DockerArchive = 'docker-archive',
1921
ImageRegistry = 'docker',
2022
}
23+
24+
export type ImageDigests = {
25+
manifestDigest?: string;
26+
indexDigest?: string;
27+
};
28+
29+
export type ImageManifest = {
30+
mediaType: string;
31+
manifests?: Array<{
32+
digest: string;
33+
platform: {
34+
architecture: string;
35+
os: string;
36+
variant: string;
37+
};
38+
}>;
39+
};

‎test/system/kind.spec.ts

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,9 +249,22 @@ test('Kubernetes-Monitor with KinD', async () => {
249249
expect.stringContaining(
250250
'docker.io/library/openjdk@sha256:',
251251
),
252+
expect.stringContaining(
253+
'docker.io/library/openjdk@sha256:',
254+
),
252255
],
253256
},
254257
},
258+
{
259+
type: 'ociDistributionMetadata',
260+
data: {
261+
imageTag: 'latest',
262+
indexDigest: expect.stringContaining('sha256:'),
263+
manifestDigest: expect.stringContaining('sha256:'),
264+
registryHost: 'docker.io',
265+
repository: 'library/openjdk',
266+
},
267+
},
255268
]),
256269
target: { image: 'docker-image|docker.io/library/openjdk' },
257270
identity: { type: 'rpm', args: { platform: 'linux/amd64' } },

‎test/system/skopeo.spec.ts

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
import * as skopeo from '../../src/scanner/images/skopeo';
2+
3+
describe('extract index & manifest digests', () => {
4+
test('single arch image', async () => {
5+
const imageName =
6+
'docker://docker.io/snyk/container-registry-agent:69f4e8ef61b0f7380101e71d48dd2c0e348fbe83';
7+
8+
const { indexDigest, manifestDigest } = await skopeo.extractImageDigests(
9+
imageName,
10+
);
11+
12+
expect(indexDigest).toBe(undefined);
13+
expect(manifestDigest).toBe(
14+
'sha256:4b954a219312a87a3d119cbfaa015350be94175fb9b87d2e203cad68f42c50a8',
15+
);
16+
});
17+
18+
test('multi-arch image', async () => {
19+
const imageName =
20+
'docker://docker.io/library/ubuntu@sha256:36fa0c7153804946e17ee951fdeffa6a1c67e5088438e5b90de077de5c600d4c';
21+
22+
const { indexDigest, manifestDigest } = await skopeo.extractImageDigests(
23+
imageName,
24+
);
25+
26+
expect(indexDigest).toBe(
27+
'sha256:36fa0c7153804946e17ee951fdeffa6a1c67e5088438e5b90de077de5c600d4c',
28+
);
29+
expect(manifestDigest).toBe(
30+
'sha256:bce129bec07bab56ada102d312ebcfe70463885bdf68fb32182974bd994816e0',
31+
);
32+
});
33+
34+
test('multi-arch image with no linux/amd64 manifest', async () => {
35+
const imageName =
36+
'docker://docker.io/carlosedp/arm_exporter@sha256:c2510142e3824686cba8af75826737a8158b25648e29867e262d26f553de5211';
37+
38+
const { indexDigest, manifestDigest } = await skopeo.extractImageDigests(
39+
imageName,
40+
);
41+
42+
expect(indexDigest).toBe(undefined);
43+
expect(manifestDigest).toBe(undefined);
44+
});
45+
});

‎test/unit/scanner/skopeo.spec.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,22 +4,22 @@ describe('skopeo module tests', () => {
44
test.concurrent('getCredentialParameters()', async () => {
55
const noCredentials = undefined;
66
const credentialParametersForNoCredentials =
7-
skopeo.getCredentialParameters(noCredentials);
7+
skopeo.getCopyCredentialParameters(noCredentials);
88
expect(credentialParametersForNoCredentials).toEqual([]);
99

1010
const emptyCredentials = '';
1111
const credentialParametersForEmptyCredentials =
12-
skopeo.getCredentialParameters(emptyCredentials);
12+
skopeo.getCopyCredentialParameters(emptyCredentials);
1313
expect(credentialParametersForEmptyCredentials).toEqual([]);
1414

1515
const someCredentials = 'secret-things-happening';
1616
const credentialParametersForSomeCredentials =
17-
skopeo.getCredentialParameters(someCredentials);
17+
skopeo.getCopyCredentialParameters(someCredentials);
1818
expect(credentialParametersForSomeCredentials).toEqual([
1919
{ body: '--src-creds', sanitise: true },
2020
{ body: someCredentials, sanitise: true },
2121
]);
22-
const certificatesParameters = skopeo.getCertificatesParameters();
22+
const certificatesParameters = skopeo.getCopyCertificatesParameters();
2323
expect(certificatesParameters).toEqual([
2424
{ body: '--src-cert-dir', sanitise: true },
2525
{ body: '/srv/app/certs', sanitise: true },

0 commit comments

Comments
 (0)