refactor to bring back GeneratedImageIndex; fix ImageIndexGeneratorTests

Author: Surayya Huseyn Zada

src/Containers/Microsoft.NET.Build.Containers/ImageIndexGenerator.cs

@@ -47,6 +47,11 @@ internal static (string, string) GenerateImageIndex(BuiltImage[] images)
 
     internal static string GenerateImageIndex(BuiltImage[] images, string manifestMediaType, string imageIndexMediaType)
     {
+        if (images.Length == 0)
+        {
+            throw new ArgumentException(string.Format(Strings.ImagesEmpty));
+        }
+        
         // Here we are using ManifestListV2 struct, but we could use ImageIndexV1 struct as well.
         // We are filling the same fields, so we can use the same struct.
         var manifests = new PlatformSpecificManifest[images.Length];

src/Containers/Microsoft.NET.Build.Containers/ImagePublisher.cs

@@ -51,7 +51,7 @@ await PushToRemoteRegistryAsync(
     }
 
     public static async Task PublishImageAsync(
-        BuiltImage[] multiArchImage,
+        MultiArchImage multiArchImage,
         SourceImageReference sourceImageReference,
         DestinationImageReference destinationImageReference,
         Microsoft.Build.Utilities.TaskLoggingHelper Log,
@@ -82,16 +82,7 @@ await PushToRemoteRegistryAsync(
                     Log,
                     BuildEngine,
                     cancellationToken,
-                    async (images, source, destination, token) =>
-                    {
-                        (string imageIndex, string mediaType) = ImageIndexGenerator.GenerateImageIndex(images);
-                        await destinationImageReference.RemoteRegistry!.PushManifestListAsync(
-                            destinationImageReference.Repository,
-                            destinationImageReference.Tags,
-                            imageIndex,
-                            mediaType,
-                            cancellationToken).ConfigureAwait(false);
-                    },
+                    destinationImageReference.RemoteRegistry!.PushManifestListAsync,
                     Strings.ImageIndexUploadedToRegistry).ConfigureAwait(false);
                 break;
             default:

src/Containers/Microsoft.NET.Build.Containers/LocalDaemons/ArchiveFileRegistry.cs

@@ -46,10 +46,10 @@ public async Task LoadAsync(BuiltImage image, SourceImageReference sourceReferen
         => await LoadAsync(image, sourceReference, destinationReference, cancellationToken,
             DockerCli.WriteImageToStreamAsync);
 
-    public async Task LoadAsync(BuiltImage[] images, SourceImageReference sourceReference,
+    public async Task LoadAsync(MultiArchImage multiArchImage, SourceImageReference sourceReference,
         DestinationImageReference destinationReference,
         CancellationToken cancellationToken) 
-        => await LoadAsync(images, sourceReference, destinationReference, cancellationToken,
+        => await LoadAsync(multiArchImage, sourceReference, destinationReference, cancellationToken,
             DockerCli.WriteMultiArchOciImageToStreamAsync);
 
     public Task<bool> IsAvailableAsync(CancellationToken cancellationToken) => Task.FromResult(true);

src/Containers/Microsoft.NET.Build.Containers/LocalDaemons/DockerCli.cs

@@ -130,8 +130,8 @@ public async Task LoadAsync(BuiltImage image, SourceImageReference sourceReferen
         // For loading to the local registry, we use the Docker format. Two reasons: one - compatibility with previous behavior before oci formatted publishing was available, two - Podman cannot load multi tag oci image tarball.
         => await LoadAsync(image, sourceReference, destinationReference, WriteDockerImageToStreamAsync, cancellationToken);
 
-    public async Task LoadAsync(BuiltImage[] images, SourceImageReference sourceReference, DestinationImageReference destinationReference, CancellationToken cancellationToken) 
-        => await LoadAsync(images, sourceReference, destinationReference, WriteMultiArchOciImageToStreamAsync, cancellationToken, checkContainerdStore: true);
+    public async Task LoadAsync(MultiArchImage multiArchImage, SourceImageReference sourceReference, DestinationImageReference destinationReference, CancellationToken cancellationToken) 
+        => await LoadAsync(multiArchImage, sourceReference, destinationReference, WriteMultiArchOciImageToStreamAsync, cancellationToken, checkContainerdStore: true);
 
     public async Task<bool> IsAvailableAsync(CancellationToken cancellationToken)
     {
@@ -500,7 +500,7 @@ await WriteManifestForOciImage(writer, image, cancellationToken)
     }
 
     public static async Task WriteMultiArchOciImageToStreamAsync(
-        BuiltImage[] images,
+        MultiArchImage multiArchImage,
         SourceImageReference sourceReference,
         DestinationImageReference destinationReference,
         Stream imageStream,
@@ -510,13 +510,13 @@ public static async Task WriteMultiArchOciImageToStreamAsync(
 
         using TarWriter writer = new(imageStream, TarEntryFormat.Pax, leaveOpen: true);
 
-        foreach (var image in images)
+        foreach (var image in multiArchImage.Images!)
         {
             await WriteOciImageToBlobs(writer, image, sourceReference, cancellationToken)
             .ConfigureAwait(false);
         }
 
-        await WriteIndexJsonForMultiArchOciImage(writer, images, destinationReference, cancellationToken)
+        await WriteIndexJsonForMultiArchOciImage(writer, multiArchImage, destinationReference, cancellationToken)
             .ConfigureAwait(false);
 
         await WriteOciLayout(writer, cancellationToken)
@@ -525,21 +525,18 @@ await WriteOciLayout(writer, cancellationToken)
 
     private static async Task WriteIndexJsonForMultiArchOciImage(
         TarWriter writer,
-        BuiltImage[] images,
+        MultiArchImage multiArchImage,
         DestinationImageReference destinationReference,
         CancellationToken cancellationToken)
     {
         // 1. create manifest list for the blobs
         cancellationToken.ThrowIfCancellationRequested();
 
-        // For multi-arch we publish only oci-formatted image tarballs.
-        string manifestListJson = ImageIndexGenerator.GenerateImageIndex(images, SchemaTypes.OciManifestV1, SchemaTypes.OciImageIndexV1);
-
-        var manifestListDigest = DigestUtils.GetDigest(manifestListJson);
+        var manifestListDigest = DigestUtils.GetDigest(multiArchImage.ImageIndex);
         var manifestListSha = DigestUtils.GetShaFromDigest(manifestListDigest);
         var manifestListPath = $"{_blobsPath}/{manifestListSha}";
 
-        using (MemoryStream indexStream = new(Encoding.UTF8.GetBytes(manifestListJson)))
+        using (MemoryStream indexStream = new(Encoding.UTF8.GetBytes(multiArchImage.ImageIndex)))
         {
             PaxTarEntry indexEntry = new(TarEntryType.RegularFile, manifestListPath)
             {
@@ -552,9 +549,9 @@ private static async Task WriteIndexJsonForMultiArchOciImage(
         cancellationToken.ThrowIfCancellationRequested();
 
         string indexJson = ImageIndexGenerator.GenerateImageIndexWithAnnotations(
-            SchemaTypes.OciImageIndexV1, 
+            multiArchImage.ImageIndexMediaType, 
             manifestListDigest, 
-            manifestListJson.Length, 
+            multiArchImage.ImageIndex.Length, 
             destinationReference.Repository, 
             destinationReference.Tags);
 

src/Containers/Microsoft.NET.Build.Containers/LocalDaemons/ILocalRegistry.cs

@@ -16,7 +16,7 @@ internal interface ILocalRegistry {
     /// <summary>
     /// Loads a multi-arch image (presumably from a tarball) into the local registry.
     /// </summary>
-    public Task LoadAsync(BuiltImage[] images, SourceImageReference sourceReference, DestinationImageReference destinationReference, CancellationToken cancellationToken);
+    public Task LoadAsync(MultiArchImage multiArchImage, SourceImageReference sourceReference, DestinationImageReference destinationReference, CancellationToken cancellationToken);
 
     /// <summary>
     /// Checks to see if the local registry is available. This is used to give nice errors to the user.

src/Containers/Microsoft.NET.Build.Containers/MultiArchImage.cs

@@ -0,0 +1,16 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace Microsoft.NET.Build.Containers;
+
+/// <summary>
+/// Represents constructed image ready for further processing.
+/// </summary>
+internal sealed class MultiArchImage
+{
+    internal required string ImageIndex { get; init; }
+
+    internal required string ImageIndexMediaType { get; init; }
+
+    internal BuiltImage[]? Images { get; init; }
+}

src/Containers/Microsoft.NET.Build.Containers/PublicAPI/net8.0/PublicAPI.Unshipped.txt

@@ -34,6 +34,8 @@ Microsoft.NET.Build.Containers.Tasks.CreateImageIndex.GeneratedArchiveOutputPath
 Microsoft.NET.Build.Containers.Tasks.CreateImageIndex.Cancel() -> void
 Microsoft.NET.Build.Containers.Tasks.CreateImageIndex.CreateImageIndex() -> void
 Microsoft.NET.Build.Containers.Tasks.CreateImageIndex.Dispose() -> void
+Microsoft.NET.Build.Containers.Tasks.CreateImageIndex.GeneratedImageIndex.get -> string!
+Microsoft.NET.Build.Containers.Tasks.CreateImageIndex.GeneratedImageIndex.set -> void
 Microsoft.NET.Build.Containers.Tasks.CreateImageIndex.ImageTags.get -> string![]!
 Microsoft.NET.Build.Containers.Tasks.CreateImageIndex.ImageTags.set -> void
 Microsoft.NET.Build.Containers.Tasks.CreateImageIndex.GeneratedContainers.get -> Microsoft.Build.Framework.ITaskItem![]!

src/Containers/Microsoft.NET.Build.Containers/Registry/Registry.cs

@@ -525,13 +525,17 @@ private async Task UploadBlobAsync(string repository, string digest, Stream cont
 
     }
 
-    public async Task PushManifestListAsync(string repositoryName, string[] tags, string manifestListJson, string mediaType, CancellationToken cancellationToken)
+    public async Task PushManifestListAsync(
+        MultiArchImage multiArchImage,
+        SourceImageReference sourceImageReference,
+        DestinationImageReference destinationImageReference,
+        CancellationToken cancellationToken)
     {
         cancellationToken.ThrowIfCancellationRequested();
-        foreach (var tag in tags)
+        foreach (var tag in destinationImageReference.Tags)
         {
             _logger.LogInformation(Strings.Registry_TagUploadStarted, tag, RegistryName);
-            await _registryAPI.Manifest.PutAsync(repositoryName, tag, manifestListJson, mediaType, cancellationToken).ConfigureAwait(false);
+            await _registryAPI.Manifest.PutAsync(destinationImageReference.Repository, tag, multiArchImage.ImageIndex, multiArchImage.ImageIndexMediaType, cancellationToken).ConfigureAwait(false);
             _logger.LogInformation(Strings.Registry_TagUploaded, tag, RegistryName);
         }          
     }

src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateImageIndex.Interface.cs

@@ -61,9 +61,18 @@ partial class CreateImageIndex
     [Required]
     public string[] ImageTags { get; set; }
 
+    /// <summary>
+    /// The generated archive output path.
+    /// </summary>
     [Output]
     public string GeneratedArchiveOutputPath { get; set; }
 
+    /// <summary>
+    /// The generated image index (manifest list) in JSON format.
+    /// </summary>
+    [Output]
+    public string GeneratedImageIndex { get; set; }
+
     public CreateImageIndex()
     {
         BaseRegistry = string.Empty;
@@ -76,5 +85,6 @@ public CreateImageIndex()
         Repository = string.Empty;
         ImageTags = Array.Empty<string>();
         GeneratedArchiveOutputPath = string.Empty;
+        GeneratedImageIndex = string.Empty;
     }
 } 

src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateImageIndex.cs

@@ -74,14 +74,17 @@ internal async Task<bool> ExecuteAsync(CancellationToken cancellationToken)
             return false;
         }
 
+        var multiArchImage = CreateMultiArchImage(images, destinationImageReference.Kind);
+
+        GeneratedImageIndex = multiArchImage.ImageIndex;
+        GeneratedArchiveOutputPath = ArchiveOutputPath;
+
         logger.LogInformation(Strings.BuildingImageIndex, destinationImageReference, string.Join(", ", images.Select(i => i.ManifestDigest)));
 
         var telemetry = new Telemetry(sourceImageReference, destinationImageReference, Log);
 
-        await ImagePublisher.PublishImageAsync(images, sourceImageReference, destinationImageReference, Log, BuildEngine, telemetry, cancellationToken)
-            .ConfigureAwait(false);
-
-        GeneratedArchiveOutputPath = ArchiveOutputPath;
+        await ImagePublisher.PublishImageAsync(multiArchImage, sourceImageReference, destinationImageReference, Log, BuildEngine, telemetry, cancellationToken)
+            .ConfigureAwait(false); 
 
         return !Log.HasLoggedErrors;
     }
@@ -157,4 +160,29 @@ private BuiltImage[] ParseImages(DestinationImageReferenceKind destinationKind)
         }
         return (architecture, os);
     }
+
+    private static MultiArchImage CreateMultiArchImage(BuiltImage[] images, DestinationImageReferenceKind destinationImageKind)
+    {
+        switch (destinationImageKind)
+        {
+            case DestinationImageReferenceKind.LocalRegistry:
+                return new MultiArchImage()
+                {
+                    // For multi-arch we publish only oci-formatted image tarballs.
+                    ImageIndex = ImageIndexGenerator.GenerateImageIndex(images, SchemaTypes.OciManifestV1, SchemaTypes.OciImageIndexV1),
+                    ImageIndexMediaType = SchemaTypes.OciImageIndexV1,
+                    Images = images
+                };
+            case DestinationImageReferenceKind.RemoteRegistry:
+                (string imageIndex, string mediaType) = ImageIndexGenerator.GenerateImageIndex(images);
+                return new MultiArchImage()
+                {
+                    ImageIndex = imageIndex,
+                    ImageIndexMediaType = mediaType,
+                    // For remote registry we don't need individual images, as they should be pushed already
+                };
+            default:
+                throw new ArgumentOutOfRangeException();
+        }
+    }
 }