Skip to content

Commit cc39718

Browse files
surayya-MSsurayya-MS
authored
Fix publishing OCI image as tarball (#44693)
2 parents 172a471 + ee3029b commit cc39718

File tree

22 files changed

+816
-466
lines changed

22 files changed

+816
-466
lines changed

‎src/Containers/Microsoft.NET.Build.Containers/LocalDaemons/DockerCli.cs

Lines changed: 159 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ internal sealed class DockerCli
3131
private string? _fullCommandPath;
3232
#endif
3333

34+
private const string _blobsPath = "blobs/sha256";
35+
3436
public DockerCli(string? command, ILoggerFactory loggerFactory)
3537
{
3638
if (!(command == null ||
@@ -100,8 +102,8 @@ public async Task LoadAsync(BuiltImage image, SourceImageReference sourceReferen
100102
}
101103

102104
// Create new stream tarball
103-
104-
await WriteImageToStreamAsync(image, sourceReference, destinationReference, loadProcess.StandardInput.BaseStream, cancellationToken).ConfigureAwait(false);
105+
// We want to be able to export to docker, even oci images.
106+
await WriteDockerImageToStreamAsync(image, sourceReference, destinationReference, loadProcess.StandardInput.BaseStream, cancellationToken).ConfigureAwait(false);
105107

106108
cancellationToken.ThrowIfCancellationRequested();
107109

@@ -266,13 +268,53 @@ public static bool IsInsecureRegistry(string registryDomain)
266268

267269
#if NET
268270
public static async Task WriteImageToStreamAsync(BuiltImage image, SourceImageReference sourceReference, DestinationImageReference destinationReference, Stream imageStream, CancellationToken cancellationToken)
271+
{
272+
if (image.ManifestMediaType == SchemaTypes.DockerManifestV2)
273+
{
274+
await WriteDockerImageToStreamAsync(image, sourceReference, destinationReference, imageStream, cancellationToken);
275+
}
276+
else if (image.ManifestMediaType == SchemaTypes.OciManifestV1)
277+
{
278+
await WriteOciImageToStreamAsync(image, sourceReference, destinationReference, imageStream, cancellationToken);
279+
}
280+
else
281+
{
282+
throw new ArgumentException(Resource.FormatString(nameof(Strings.UnsupportedMediaTypeForTarball), image.Manifest.MediaType));
283+
}
284+
}
285+
286+
private static async Task WriteDockerImageToStreamAsync(
287+
BuiltImage image,
288+
SourceImageReference sourceReference,
289+
DestinationImageReference destinationReference,
290+
Stream imageStream,
291+
CancellationToken cancellationToken)
269292
{
270293
cancellationToken.ThrowIfCancellationRequested();
271294
using TarWriter writer = new(imageStream, TarEntryFormat.Pax, leaveOpen: true);
272295

273-
274-
// Feed each layer tarball into the stream
275296
JsonArray layerTarballPaths = new JsonArray();
297+
await WriteImageLayers(writer, image, sourceReference, d => $"{d.Substring("sha256:".Length)}/layer.tar", cancellationToken, layerTarballPaths)
298+
.ConfigureAwait(false);
299+
300+
string configTarballPath = $"{image.ImageSha}.json";
301+
await WriteImageConfig(writer, image, configTarballPath, cancellationToken)
302+
.ConfigureAwait(false);
303+
304+
// Add manifest
305+
await WriteManifestForDockerImage(writer, destinationReference, configTarballPath, layerTarballPaths, cancellationToken)
306+
.ConfigureAwait(false);
307+
}
308+
309+
private static async Task WriteImageLayers(
310+
TarWriter writer,
311+
BuiltImage image,
312+
SourceImageReference sourceReference,
313+
Func<string, string> layerPathFunc,
314+
CancellationToken cancellationToken,
315+
JsonArray? layerTarballPaths = null)
316+
{
317+
cancellationToken.ThrowIfCancellationRequested();
276318

277319
foreach (var d in image.LayerDescriptors)
278320
{
@@ -283,9 +325,9 @@ public static async Task WriteImageToStreamAsync(BuiltImage image, SourceImageRe
283325

284326
// Stuff that (uncompressed) tarball into the image tar stream
285327
// TODO uncompress!!
286-
string layerTarballPath = $"{d.Digest.Substring("sha256:".Length)}/layer.tar";
328+
string layerTarballPath = layerPathFunc(d.Digest);
287329
await writer.WriteEntryAsync(localPath, layerTarballPath, cancellationToken).ConfigureAwait(false);
288-
layerTarballPaths.Add(layerTarballPath);
330+
layerTarballPaths?.Add(layerTarballPath);
289331
}
290332
else
291333
{
@@ -295,21 +337,33 @@ public static async Task WriteImageToStreamAsync(BuiltImage image, SourceImageRe
295337
sourceReference.Registry?.ToString() ?? "<null>"));
296338
}
297339
}
340+
}
298341

299-
// add config
300-
string configTarballPath = $"{image.ImageSha}.json";
342+
private static async Task WriteImageConfig(
343+
TarWriter writer,
344+
BuiltImage image,
345+
string configPath,
346+
CancellationToken cancellationToken)
347+
{
301348
cancellationToken.ThrowIfCancellationRequested();
349+
302350
using (MemoryStream configStream = new MemoryStream(Encoding.UTF8.GetBytes(image.Config)))
303351
{
304-
PaxTarEntry configEntry = new(TarEntryType.RegularFile, configTarballPath)
352+
PaxTarEntry configEntry = new(TarEntryType.RegularFile, configPath)
305353
{
306354
DataStream = configStream
307355
};
308-
309356
await writer.WriteEntryAsync(configEntry, cancellationToken).ConfigureAwait(false);
310357
}
358+
}
311359

312-
// Add manifest
360+
private static async Task WriteManifestForDockerImage(
361+
TarWriter writer,
362+
DestinationImageReference destinationReference,
363+
string configTarballPath,
364+
JsonArray layerTarballPaths,
365+
CancellationToken cancellationToken)
366+
{
313367
JsonArray tagsNode = new();
314368
foreach (string tag in destinationReference.Tags)
315369
{
@@ -335,6 +389,100 @@ public static async Task WriteImageToStreamAsync(BuiltImage image, SourceImageRe
335389
}
336390
}
337391

392+
private static async Task WriteOciImageToStreamAsync(
393+
BuiltImage image,
394+
SourceImageReference sourceReference,
395+
DestinationImageReference destinationReference,
396+
Stream imageStream,
397+
CancellationToken cancellationToken)
398+
{
399+
if (destinationReference.Tags.Length > 1)
400+
{
401+
throw new ArgumentException(Resource.FormatString(nameof(Strings.OciImageMultipleTagsNotSupported)));
402+
}
403+
404+
cancellationToken.ThrowIfCancellationRequested();
405+
using TarWriter writer = new(imageStream, TarEntryFormat.Pax, leaveOpen: true);
406+
407+
await WriteOciLayout(writer, cancellationToken)
408+
.ConfigureAwait(false);
409+
410+
await WriteImageLayers(writer, image, sourceReference, d => $"{_blobsPath}/{d.Substring("sha256:".Length)}", cancellationToken)
411+
.ConfigureAwait(false);
412+
413+
await WriteImageConfig(writer, image, $"{_blobsPath}/{image.ImageSha}", cancellationToken)
414+
.ConfigureAwait(false);
415+
416+
await WriteManifestForOciImage(writer, image, destinationReference, cancellationToken)
417+
.ConfigureAwait(false);
418+
}
419+
420+
private static async Task WriteOciLayout(TarWriter writer, CancellationToken cancellationToken)
421+
{
422+
cancellationToken.ThrowIfCancellationRequested();
423+
424+
string ociLayoutPath = "oci-layout";
425+
var ociLayoutContent = "{\"imageLayoutVersion\": \"1.0.0\"}";
426+
using (MemoryStream ociLayoutStream = new MemoryStream(Encoding.UTF8.GetBytes(ociLayoutContent)))
427+
{
428+
PaxTarEntry layoutEntry = new(TarEntryType.RegularFile, ociLayoutPath)
429+
{
430+
DataStream = ociLayoutStream
431+
};
432+
await writer.WriteEntryAsync(layoutEntry, cancellationToken).ConfigureAwait(false);
433+
}
434+
}
435+
436+
private static async Task WriteManifestForOciImage(
437+
TarWriter writer,
438+
BuiltImage image,
439+
DestinationImageReference destinationReference,
440+
CancellationToken cancellationToken)
441+
{
442+
cancellationToken.ThrowIfCancellationRequested();
443+
444+
string manifestContent = JsonSerializer.SerializeToNode(image.Manifest)!.ToJsonString();
445+
string manifestDigest = image.Manifest.GetDigest();
446+
447+
// 1. add manifest to blobs
448+
string manifestPath = $"{_blobsPath}/{manifestDigest.Substring("sha256:".Length)}";
449+
using (MemoryStream manifestStream = new MemoryStream(Encoding.UTF8.GetBytes(manifestContent)))
450+
{
451+
PaxTarEntry manifestEntry = new(TarEntryType.RegularFile, manifestPath)
452+
{
453+
DataStream = manifestStream
454+
};
455+
await writer.WriteEntryAsync(manifestEntry, cancellationToken).ConfigureAwait(false);
456+
}
457+
458+
cancellationToken.ThrowIfCancellationRequested();
459+
460+
// 2. add index.json
461+
var index = new ImageIndexV1
462+
{
463+
schemaVersion = 2,
464+
mediaType = SchemaTypes.OciImageIndexV1,
465+
manifests =
466+
[
467+
new PlatformSpecificOciManifest
468+
{
469+
mediaType = SchemaTypes.OciManifestV1,
470+
size = manifestContent.Length,
471+
digest = manifestDigest,
472+
annotations = new Dictionary<string, string> { { "org.opencontainers.image.ref.name", $"{destinationReference.Repository}:{destinationReference.Tags[0]}" } }
473+
}
474+
]
475+
};
476+
using (MemoryStream indexStream = new MemoryStream(Encoding.UTF8.GetBytes(JsonSerializer.SerializeToNode(index)!.ToJsonString())))
477+
{
478+
PaxTarEntry indexEntry = new(TarEntryType.RegularFile, "index.json")
479+
{
480+
DataStream = indexStream
481+
};
482+
await writer.WriteEntryAsync(indexEntry, cancellationToken).ConfigureAwait(false);
483+
}
484+
}
485+
338486
private async ValueTask<string?> GetCommandAsync(CancellationToken cancellationToken)
339487
{
340488
if (_command != null)

‎src/Containers/Microsoft.NET.Build.Containers/ManifestListV2.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,6 @@ public record struct ManifestListV2(int schemaVersion, string mediaType, Platfor
1010
public record struct PlatformInformation(string architecture, string os, string? variant, string[] features, [property: JsonPropertyName("os.version")][field: JsonPropertyName("os.version")] string? version);
1111

1212
public record struct PlatformSpecificManifest(string mediaType, long size, string digest, PlatformInformation platform);
13+
public record struct ImageIndexV1(int schemaVersion, string mediaType, PlatformSpecificOciManifest[] manifests);
1314

14-
public record struct ImageIndexV1(int schemaVersion, string mediaType, PlatformSpecificManifest[] manifests);
15+
public record struct PlatformSpecificOciManifest(string mediaType, long size, string digest, PlatformInformation platform, Dictionary<string, string> annotations);

‎src/Containers/Microsoft.NET.Build.Containers/PublicAPI/net8.0/PublicAPI.Unshipped.txt

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -74,8 +74,8 @@ Microsoft.NET.Build.Containers.ManifestListV2.schemaVersion.get -> int
7474
Microsoft.NET.Build.Containers.ManifestListV2.schemaVersion.set -> void
7575
Microsoft.NET.Build.Containers.ImageIndexV1
7676
Microsoft.NET.Build.Containers.ImageIndexV1.ImageIndexV1() -> void
77-
Microsoft.NET.Build.Containers.ImageIndexV1.ImageIndexV1(int schemaVersion, string! mediaType, Microsoft.NET.Build.Containers.PlatformSpecificManifest[]! manifests) -> void
78-
Microsoft.NET.Build.Containers.ImageIndexV1.manifests.get -> Microsoft.NET.Build.Containers.PlatformSpecificManifest[]!
77+
Microsoft.NET.Build.Containers.ImageIndexV1.ImageIndexV1(int schemaVersion, string! mediaType, Microsoft.NET.Build.Containers.PlatformSpecificOciManifest[]! manifests) -> void
78+
Microsoft.NET.Build.Containers.ImageIndexV1.manifests.get -> Microsoft.NET.Build.Containers.PlatformSpecificOciManifest[]!
7979
Microsoft.NET.Build.Containers.ImageIndexV1.manifests.set -> void
8080
Microsoft.NET.Build.Containers.ImageIndexV1.mediaType.get -> string!
8181
Microsoft.NET.Build.Containers.ImageIndexV1.mediaType.set -> void
@@ -118,6 +118,19 @@ Microsoft.NET.Build.Containers.PlatformSpecificManifest.PlatformSpecificManifest
118118
Microsoft.NET.Build.Containers.PlatformSpecificManifest.PlatformSpecificManifest(string! mediaType, long size, string! digest, Microsoft.NET.Build.Containers.PlatformInformation platform) -> void
119119
Microsoft.NET.Build.Containers.PlatformSpecificManifest.size.get -> long
120120
Microsoft.NET.Build.Containers.PlatformSpecificManifest.size.set -> void
121+
Microsoft.NET.Build.Containers.PlatformSpecificOciManifest
122+
Microsoft.NET.Build.Containers.PlatformSpecificOciManifest.digest.get -> string!
123+
Microsoft.NET.Build.Containers.PlatformSpecificOciManifest.digest.set -> void
124+
Microsoft.NET.Build.Containers.PlatformSpecificOciManifest.mediaType.get -> string!
125+
Microsoft.NET.Build.Containers.PlatformSpecificOciManifest.mediaType.set -> void
126+
Microsoft.NET.Build.Containers.PlatformSpecificOciManifest.platform.get -> Microsoft.NET.Build.Containers.PlatformInformation
127+
Microsoft.NET.Build.Containers.PlatformSpecificOciManifest.platform.set -> void
128+
Microsoft.NET.Build.Containers.PlatformSpecificOciManifest.size.get -> long
129+
Microsoft.NET.Build.Containers.PlatformSpecificOciManifest.size.set -> void
130+
Microsoft.NET.Build.Containers.PlatformSpecificOciManifest.PlatformSpecificOciManifest() -> void
131+
Microsoft.NET.Build.Containers.PlatformSpecificOciManifest.PlatformSpecificOciManifest(string! mediaType, long size, string! digest, Microsoft.NET.Build.Containers.PlatformInformation platform, System.Collections.Generic.Dictionary<string!, string!>! annotations) -> void
132+
Microsoft.NET.Build.Containers.PlatformSpecificOciManifest.annotations.get -> System.Collections.Generic.Dictionary<string!, string!>!
133+
Microsoft.NET.Build.Containers.PlatformSpecificOciManifest.annotations.set -> void
121134
Microsoft.NET.Build.Containers.Port
122135
Microsoft.NET.Build.Containers.Port.Deconstruct(out int Number, out Microsoft.NET.Build.Containers.PortType Type) -> void
123136
Microsoft.NET.Build.Containers.Port.Equals(Microsoft.NET.Build.Containers.Port other) -> bool
@@ -239,29 +252,36 @@ static Microsoft.NET.Build.Containers.ContainerHelpers.TryParsePort(string? port
239252
static readonly Microsoft.NET.Build.Containers.KnownLocalRegistryTypes.SupportedLocalRegistryTypes -> string![]!
240253
~override Microsoft.NET.Build.Containers.PlatformInformation.ToString() -> string
241254
~override Microsoft.NET.Build.Containers.PlatformSpecificManifest.ToString() -> string
255+
~override Microsoft.NET.Build.Containers.PlatformSpecificOciManifest.ToString() -> string
242256
static Microsoft.NET.Build.Containers.PlatformInformation.operator !=(Microsoft.NET.Build.Containers.PlatformInformation left, Microsoft.NET.Build.Containers.PlatformInformation right) -> bool
243257
static Microsoft.NET.Build.Containers.PlatformSpecificManifest.operator !=(Microsoft.NET.Build.Containers.PlatformSpecificManifest left, Microsoft.NET.Build.Containers.PlatformSpecificManifest right) -> bool
258+
static Microsoft.NET.Build.Containers.PlatformSpecificOciManifest.operator !=(Microsoft.NET.Build.Containers.PlatformSpecificOciManifest left, Microsoft.NET.Build.Containers.PlatformSpecificOciManifest right) -> bool
244259
static Microsoft.NET.Build.Containers.PlatformInformation.operator ==(Microsoft.NET.Build.Containers.PlatformInformation left, Microsoft.NET.Build.Containers.PlatformInformation right) -> bool
245260
~override Microsoft.NET.Build.Containers.ManifestConfig.ToString() -> string
246261
static Microsoft.NET.Build.Containers.PlatformSpecificManifest.operator ==(Microsoft.NET.Build.Containers.PlatformSpecificManifest left, Microsoft.NET.Build.Containers.PlatformSpecificManifest right) -> bool
262+
static Microsoft.NET.Build.Containers.PlatformSpecificOciManifest.operator ==(Microsoft.NET.Build.Containers.PlatformSpecificOciManifest left, Microsoft.NET.Build.Containers.PlatformSpecificOciManifest right) -> bool
247263
override Microsoft.NET.Build.Containers.PlatformInformation.GetHashCode() -> int
248264
static Microsoft.NET.Build.Containers.ManifestConfig.operator !=(Microsoft.NET.Build.Containers.ManifestConfig left, Microsoft.NET.Build.Containers.ManifestConfig right) -> bool
249265
override Microsoft.NET.Build.Containers.PlatformSpecificManifest.GetHashCode() -> int
266+
override Microsoft.NET.Build.Containers.PlatformSpecificOciManifest.GetHashCode() -> int
250267
static Microsoft.NET.Build.Containers.ManifestConfig.operator ==(Microsoft.NET.Build.Containers.ManifestConfig left, Microsoft.NET.Build.Containers.ManifestConfig right) -> bool
251268
~override Microsoft.NET.Build.Containers.PlatformInformation.Equals(object obj) -> bool
252269
~override Microsoft.NET.Build.Containers.Descriptor.ToString() -> string
253270
~override Microsoft.NET.Build.Containers.ManifestLayer.ToString() -> string
254271
override Microsoft.NET.Build.Containers.ManifestConfig.GetHashCode() -> int
255272
~override Microsoft.NET.Build.Containers.PlatformSpecificManifest.Equals(object obj) -> bool
273+
~override Microsoft.NET.Build.Containers.PlatformSpecificOciManifest.Equals(object obj) -> bool
256274
Microsoft.NET.Build.Containers.PlatformInformation.Equals(Microsoft.NET.Build.Containers.PlatformInformation other) -> bool
257275
~override Microsoft.NET.Build.Containers.ManifestConfig.Equals(object obj) -> bool
258276
static Microsoft.NET.Build.Containers.ManifestLayer.operator !=(Microsoft.NET.Build.Containers.ManifestLayer left, Microsoft.NET.Build.Containers.ManifestLayer right) -> bool
259277
Microsoft.NET.Build.Containers.PlatformSpecificManifest.Equals(Microsoft.NET.Build.Containers.PlatformSpecificManifest other) -> bool
278+
Microsoft.NET.Build.Containers.PlatformSpecificOciManifest.Equals(Microsoft.NET.Build.Containers.PlatformSpecificOciManifest other) -> bool
260279
static Microsoft.NET.Build.Containers.Descriptor.operator !=(Microsoft.NET.Build.Containers.Descriptor left, Microsoft.NET.Build.Containers.Descriptor right) -> bool
261280
Microsoft.NET.Build.Containers.PlatformInformation.Deconstruct(out string! architecture, out string! os, out string? variant, out string![]! features, out string? version) -> void
262281
Microsoft.NET.Build.Containers.ManifestConfig.Equals(Microsoft.NET.Build.Containers.ManifestConfig other) -> bool
263282
static Microsoft.NET.Build.Containers.ManifestLayer.operator ==(Microsoft.NET.Build.Containers.ManifestLayer left, Microsoft.NET.Build.Containers.ManifestLayer right) -> bool
264283
Microsoft.NET.Build.Containers.PlatformSpecificManifest.Deconstruct(out string! mediaType, out long size, out string! digest, out Microsoft.NET.Build.Containers.PlatformInformation platform) -> void
284+
Microsoft.NET.Build.Containers.PlatformSpecificOciManifest.Deconstruct(out string! mediaType, out long size, out string! digest, out Microsoft.NET.Build.Containers.PlatformInformation platform, out System.Collections.Generic.Dictionary<string!, string!>! annotations) -> void
265285
Microsoft.NET.Build.Containers.ManifestConfig.Deconstruct(out string! mediaType, out long size, out string! digest) -> void
266286
static Microsoft.NET.Build.Containers.Descriptor.operator ==(Microsoft.NET.Build.Containers.Descriptor left, Microsoft.NET.Build.Containers.Descriptor right) -> bool
267287
override Microsoft.NET.Build.Containers.ManifestLayer.GetHashCode() -> int
@@ -284,4 +304,4 @@ static Microsoft.NET.Build.Containers.ImageIndexV1.operator ==(Microsoft.NET.Bui
284304
override Microsoft.NET.Build.Containers.ImageIndexV1.GetHashCode() -> int
285305
~override Microsoft.NET.Build.Containers.ImageIndexV1.Equals(object obj) -> bool
286306
Microsoft.NET.Build.Containers.ImageIndexV1.Equals(Microsoft.NET.Build.Containers.ImageIndexV1 other) -> bool
287-
Microsoft.NET.Build.Containers.ImageIndexV1.Deconstruct(out int schemaVersion, out string! mediaType, out Microsoft.NET.Build.Containers.PlatformSpecificManifest[]! manifests) -> void
307+
Microsoft.NET.Build.Containers.ImageIndexV1.Deconstruct(out int schemaVersion, out string! mediaType, out Microsoft.NET.Build.Containers.PlatformSpecificOciManifest[]! manifests) -> void

0 commit comments

Comments
 (0)