[release/8.0.4xx] Multi-arch OCI images export as tarballs (#46467)

Author: marcpopMSFT

src/Containers/Microsoft.NET.Build.Containers/BuiltImage.cs

@@ -16,36 +16,51 @@ internal readonly struct BuiltImage
     /// <summary>
     /// Gets image digest.
     /// </summary>
-    internal required string ImageDigest { get; init; }
+    internal string? ImageDigest { get; init; }
 
     /// <summary>
     /// Gets image SHA.
     /// </summary>
-    internal required string ImageSha { get; init; }
+    internal string? ImageSha { get; init; }
 
     /// <summary>
-    /// Gets image size.
+    /// Gets image manifest.
     /// </summary>
-    internal required long ImageSize { get; init; }
+    internal required string Manifest { get; init; } 
 
     /// <summary>
-    /// Gets image manifest.
+    /// Gets manifest digest.
     /// </summary>
-    internal required ManifestV2 Manifest { get; init; }
+    internal required string ManifestDigest { get; init; }
 
     /// <summary>
     /// Gets manifest mediaType.
     /// </summary>
     internal required string ManifestMediaType { get; init; }
 
+    /// <summary>
+    /// Gets image layers.
+    /// </summary>
+    internal List<ManifestLayer>? Layers { get; init; }
+
+    /// <summary>
+    /// Gets image OS.
+    /// </summary>
+    internal string? OS { get; init; }
+
+    /// <summary>
+    /// Gets image architecture.
+    /// </summary>
+    internal string? Architecture { get; init; }
+
     /// <summary>
     /// Gets layers descriptors.
     /// </summary>
     internal IEnumerable<Descriptor> LayerDescriptors
     {
         get
         {
-            List<ManifestLayer> layersNode = Manifest.Layers ?? throw new NotImplementedException("Tried to get layer information but there is no layer node?");
+            List<ManifestLayer> layersNode = Layers ?? throw new NotImplementedException("Tried to get layer information but there is no layer node?");
             foreach (ManifestLayer layer in layersNode)
             {
                 yield return new(layer.mediaType, layer.digest, layer.size);

src/Containers/Microsoft.NET.Build.Containers/DigestUtils.cs

@@ -17,6 +17,16 @@ internal sealed class DigestUtils
     /// </summary>
     internal static string GetDigestFromSha(string sha) => $"sha256:{sha}";
 
+    internal static string GetShaFromDigest(string digest)
+    {
+        if (!digest.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase))
+        {
+            throw new ArgumentException($"Invalid digest '{digest}'. Digest must start with 'sha256:'.");
+        }
+
+        return digest.Substring("sha256:".Length);
+    }
+
     /// <summary>
     /// Gets the SHA of <paramref name="str"/>.
     /// </summary>

src/Containers/Microsoft.NET.Build.Containers/ImageBuilder.cs

@@ -5,6 +5,7 @@
 using Microsoft.NET.Build.Containers.Resources;
 using Microsoft.Extensions.Logging;
 using System.Text.RegularExpressions;
+using System.Text.Json;
 
 namespace Microsoft.NET.Build.Containers;
 
@@ -86,9 +87,10 @@ internal BuiltImage Build()
             Config = imageJsonStr,
             ImageDigest = imageDigest,
             ImageSha = imageSha,
-            ImageSize = imageSize,
-            Manifest = newManifest,
-            ManifestMediaType = ManifestMediaType
+            Manifest = JsonSerializer.SerializeToNode(newManifest)?.ToJsonString() ?? "",
+            ManifestDigest = newManifest.GetDigest(),
+            ManifestMediaType = ManifestMediaType,
+            Layers = _manifest.Layers
         };
     }
 

src/Containers/Microsoft.NET.Build.Containers/ImageIndexGenerator.cs

@@ -1,100 +1,138 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Text.Encodings.Web;
 using System.Text.Json;
-using System.Text.Json.Nodes;
+using System.Text.Json.Serialization;
 using Microsoft.NET.Build.Containers.Resources;
-using Microsoft.NET.Build.Containers.Tasks;
 
 namespace Microsoft.NET.Build.Containers;
 
-internal readonly struct ImageInfo
-{
-    internal string Config { get; init; }
-    internal string ManifestDigest { get; init; }
-    internal string Manifest { get; init; }
-    internal string ManifestMediaType { get; init; }
-
-    public override string ToString() => ManifestDigest;
-}
-
 internal static class ImageIndexGenerator
 {
     /// <summary>
     /// Generates an image index from the given images.
     /// </summary>
-    /// <param name="imageInfos"></param>
+    /// <param name="images">Images to generate image index from.</param>
     /// <returns>Returns json string of image index and image index mediaType.</returns>
     /// <exception cref="ArgumentException"></exception>
     /// <exception cref="NotSupportedException"></exception>
-    internal static (string, string) GenerateImageIndex(ImageInfo[] imageInfos)
+    internal static (string, string) GenerateImageIndex(BuiltImage[] images)
     {
-        if (imageInfos.Length == 0)
+        if (images.Length == 0)
         {
-            throw new ArgumentException(string.Format(Strings.ImagesEmpty));
+            throw new ArgumentException(Strings.ImagesEmpty);
         }
 
-        string manifestMediaType = imageInfos[0].ManifestMediaType;
+        string manifestMediaType = images[0].ManifestMediaType;
 
-        if (!imageInfos.All(image => string.Equals(image.ManifestMediaType, manifestMediaType, StringComparison.OrdinalIgnoreCase)))
+        if (!images.All(image => string.Equals(image.ManifestMediaType, manifestMediaType, StringComparison.OrdinalIgnoreCase)))
         {
             throw new ArgumentException(Strings.MixedMediaTypes);
         }
 
         if (manifestMediaType == SchemaTypes.DockerManifestV2)
         {
-            return GenerateImageIndex(imageInfos, SchemaTypes.DockerManifestV2, SchemaTypes.DockerManifestListV2);
+            return (GenerateImageIndex(images, SchemaTypes.DockerManifestV2, SchemaTypes.DockerManifestListV2), SchemaTypes.DockerManifestListV2);
         }
         else if (manifestMediaType == SchemaTypes.OciManifestV1)
         {
-            return GenerateImageIndex(imageInfos, SchemaTypes.OciManifestV1, SchemaTypes.OciImageIndexV1);
+            return (GenerateImageIndex(images, SchemaTypes.OciManifestV1, SchemaTypes.OciImageIndexV1), SchemaTypes.OciImageIndexV1);
         }
         else
         {
             throw new NotSupportedException(string.Format(Strings.UnsupportedMediaType, manifestMediaType));
         }
     }
 
-    private static (string, string) GenerateImageIndex(ImageInfo[] images, string manifestMediaType, string imageIndexMediaType)
+    /// <summary>
+    /// Generates an image index from the given images.
+    /// </summary>
+    /// <param name="images">Images to generate image index from.</param>
+    /// <param name="manifestMediaType">Media type of the manifest.</param>
+    /// <param name="imageIndexMediaType">Media type of the produced image index.</param>
+    /// <returns>Returns json string of image index and image index mediaType.</returns>
+    /// <exception cref="ArgumentException"></exception>
+    /// <exception cref="NotSupportedException"></exception>
+    internal static string GenerateImageIndex(BuiltImage[] images, string manifestMediaType, string imageIndexMediaType)
     {
+        if (images.Length == 0)
+        {
+            throw new ArgumentException(Strings.ImagesEmpty);
+        }
+
         // Here we are using ManifestListV2 struct, but we could use ImageIndexV1 struct as well.
-        // We are filling the same fiels, so we can use the same struct.
+        // We are filling the same fields, so we can use the same struct.
         var manifests = new PlatformSpecificManifest[images.Length];
+        
         for (int i = 0; i < images.Length; i++)
         {
-            var image = images[i];
-
-            var manifest = new PlatformSpecificManifest
+            manifests[i] = new PlatformSpecificManifest
             {
                 mediaType = manifestMediaType,
-                size = image.Manifest.Length,
-                digest = image.ManifestDigest,
-                platform = GetArchitectureAndOsFromConfig(image)
+                size = images[i].Manifest.Length,
+                digest = images[i].ManifestDigest,
+                platform = new PlatformInformation
+                {
+                    architecture = images[i].Architecture!,
+                    os = images[i].OS!
+                }
             };
-            manifests[i] = manifest;
         }
 
-        var dockerManifestList = new ManifestListV2
+        var imageIndex = new ManifestListV2
         {
             schemaVersion = 2,
             mediaType = imageIndexMediaType,
             manifests = manifests
         };
 
-        return (JsonSerializer.SerializeToNode(dockerManifestList)?.ToJsonString() ?? "", dockerManifestList.mediaType);
+        return GetJsonStringFromImageIndex(imageIndex);
     }
 
-    private static PlatformInformation GetArchitectureAndOsFromConfig(ImageInfo image)
+    internal static string GenerateImageIndexWithAnnotations(string manifestMediaType, string manifestDigest, long manifestSize, string repository, string[] tags)
     {
-        var configJson = JsonNode.Parse(image.Config) as JsonObject ??
-            throw new ArgumentException($"{nameof(image.Config)} should be a JSON object.", nameof(image.Config));
+        string containerdImageNamePrefix = repository.Contains('/') ? "docker.io/" : "docker.io/library/";
+        
+        var manifests = new PlatformSpecificOciManifest[tags.Length];
+        for (int i = 0; i < tags.Length; i++)
+        {
+            var tag = tags[i];
+            manifests[i] = new PlatformSpecificOciManifest
+            {
+                mediaType = manifestMediaType,
+                size = manifestSize,
+                digest = manifestDigest,
+                annotations = new Dictionary<string, string> 
+                {
+                    { "io.containerd.image.name", $"{containerdImageNamePrefix}{repository}:{tag}" },
+                    { "org.opencontainers.image.ref.name", tag } 
+                }
+            };
+        }
 
-        var architecture = configJson["architecture"]?.ToString() ??
-            throw new ArgumentException($"{nameof(image.Config)} should contain 'architecture'.", nameof(image.Config));
+        var index = new ImageIndexV1
+        {
+            schemaVersion = 2,
+            mediaType = SchemaTypes.OciImageIndexV1,
+            manifests = manifests
+        };
 
-        var os = configJson["os"]?.ToString() ??
-            throw new ArgumentException($"{nameof(image.Config)} should contain 'os'.", nameof(image.Config));
+        return GetJsonStringFromImageIndex(index);
+    }
+
+    private static string GetJsonStringFromImageIndex<T>(T imageIndex)
+    {
+        var nullIgnoreOptions = new JsonSerializerOptions
+        {
+            DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
+        };
+        // To avoid things like \u002B for '+' especially in media types ("application/vnd.oci.image.manifest.v1\u002Bjson"), we use UnsafeRelaxedJsonEscaping.
+        var escapeOptions = new JsonSerializerOptions
+        {
+            Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping
+        };
 
-        return new PlatformInformation { architecture = architecture, os = os };
+        return JsonSerializer.SerializeToNode(imageIndex, nullIgnoreOptions)?.ToJsonString(escapeOptions) ?? "";
     }
 }