Add an image digest label to generated containers (#39160)

Author: baronfel

src/Containers/Microsoft.NET.Build.Containers/ContainerBuilder.cs

@@ -6,9 +6,9 @@
 
 namespace Microsoft.NET.Build.Containers;
 
-public static class ContainerBuilder
+internal static class ContainerBuilder
 {
-    public static async Task<int> ContainerizeAsync(
+    internal static async Task<int> ContainerizeAsync(
         DirectoryInfo publishDirectory,
         string workingDir,
         string baseRegistry,
@@ -31,6 +31,8 @@ public static async Task<int> ContainerizeAsync(
         string localRegistry,
         string? containerUser,
         string? archiveOutputPath,
+        bool generateLabels,
+        bool generateDigestLabel,
         ILoggerFactory loggerFactory,
         CancellationToken cancellationToken)
     {
@@ -124,11 +126,20 @@ public static async Task<int> ContainerizeAsync(
         }
         imageBuilder.SetEntrypointAndCmd(imageEntrypoint, imageCmd);
 
-        foreach (KeyValuePair<string, string> label in labels)
+        if (generateLabels)
         {
-            // labels are validated by System.CommandLine API
-            imageBuilder.AddLabel(label.Key, label.Value);
+            foreach (KeyValuePair<string, string> label in labels)
+            {
+                // labels are validated by System.CommandLine API
+                imageBuilder.AddLabel(label.Key, label.Value);
+            }
+
+            if (generateDigestLabel)
+            {
+                imageBuilder.AddBaseImageDigestLabel();
+            }
         }
+
         foreach (KeyValuePair<string, string> envVar in envVars)
         {
             imageBuilder.AddEnvironmentVariable(envVar.Key, envVar.Value);

src/Containers/Microsoft.NET.Build.Containers/ImageBuilder.cs

@@ -14,6 +14,10 @@ namespace Microsoft.NET.Build.Containers;
 /// </summary>
 internal sealed class ImageBuilder
 {
+    // a snapshot of the manifest that this builder is based on
+    private readonly ManifestV2 _baseImageManifest;
+
+    // the mutable internal manifest that we're building by modifying the base and applying customizations
     private readonly ManifestV2 _manifest;
     private readonly ImageConfig _baseImageConfig;
     private readonly ILogger _logger;
@@ -33,7 +37,8 @@ internal sealed class ImageBuilder
 
     internal ImageBuilder(ManifestV2 manifest, ImageConfig baseImageConfig, ILogger logger)
     {
-        _manifest = manifest;
+        _baseImageManifest = manifest;
+        _manifest = new ManifestV2() { SchemaVersion = manifest.SchemaVersion, Config = manifest.Config, Layers = new(manifest.Layers), MediaType = manifest.MediaType };
         _baseImageConfig = baseImageConfig;
         _logger = logger;
     }
@@ -63,9 +68,12 @@ internal BuiltImage Build()
             size = imageSize
         };
 
-        ManifestV2 newManifest = _manifest with
+        ManifestV2 newManifest = new ManifestV2()
         {
-            Config = newManifestConfig
+            Config = newManifestConfig,
+            SchemaVersion = _manifest.SchemaVersion,
+            MediaType = _manifest.MediaType,
+            Layers = _manifest.Layers
         };
 
         return new BuiltImage()
@@ -87,6 +95,11 @@ internal void AddLayer(Layer l)
         _baseImageConfig.AddLayer(l);
     }
 
+    internal void AddBaseImageDigestLabel()
+    {
+        AddLabel("org.opencontainers.image.base.digest", _baseImageManifest.GetDigest());
+    }
+
     /// <summary>
     /// Adds a label to a base image.
     /// </summary>

src/Containers/Microsoft.NET.Build.Containers/ManifestV2.cs

@@ -12,8 +12,11 @@ namespace Microsoft.NET.Build.Containers;
 /// <remarks>
 /// https://github.com/opencontainers/image-spec/blob/main/manifest.md
 /// </remarks>
-public readonly record struct ManifestV2
+public class ManifestV2
 {
+    [JsonIgnore]
+    public string? KnownDigest { get; set; }
+
     /// <summary>
     /// This REQUIRED property specifies the image manifest schema version.
     /// For this version of the specification, this MUST be 2 to ensure backward compatibility with older versions of Docker.
@@ -47,9 +50,9 @@ public readonly record struct ManifestV2
     /// <summary>
     /// Gets the digest for this manifest.
     /// </summary>
-    public string GetDigest() => DigestUtils.GetDigest(JsonSerializer.SerializeToNode(this)?.ToJsonString() ?? string.Empty);
+    public string GetDigest() => KnownDigest ??= DigestUtils.GetDigest(JsonSerializer.SerializeToNode(this)?.ToJsonString() ?? string.Empty);
 }
 
 public record struct ManifestConfig(string mediaType, long size, string digest);
 
-public record struct ManifestLayer(string mediaType, long size, string digest, string[]? urls);
+public record struct ManifestLayer(string mediaType, long size, string digest, [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)][field: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string[]? urls);

src/Containers/Microsoft.NET.Build.Containers/PublicAPI/net472/PublicAPI.Unshipped.txt

@@ -77,6 +77,10 @@ Microsoft.NET.Build.Containers.Tasks.CreateNewImage.RuntimeIdentifierGraphPath.g
 Microsoft.NET.Build.Containers.Tasks.CreateNewImage.RuntimeIdentifierGraphPath.set -> void
 Microsoft.NET.Build.Containers.Tasks.CreateNewImage.WorkingDirectory.get -> string!
 Microsoft.NET.Build.Containers.Tasks.CreateNewImage.WorkingDirectory.set -> void
+Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateLabels.get -> bool
+Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateLabels.set -> void
+Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateDigestLabel.get -> bool
+Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateDigestLabel.set -> void
 override Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ToolName.get -> string!
 override Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateCommandLineCommands() -> string!
 override Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateFullPathToTool() -> string!

src/Containers/Microsoft.NET.Build.Containers/PublicAPI/net8.0/PublicAPI.Unshipped.txt

@@ -13,9 +13,7 @@ Microsoft.NET.Build.Containers.Tasks.ComputeDotnetBaseImageAndTag.TargetRuntimeI
 Microsoft.NET.Build.Containers.Tasks.ComputeDotnetBaseImageAndTag.TargetRuntimeIdentifier.set -> void
 Microsoft.NET.Build.Containers.Tasks.ComputeDotnetBaseImageAndTag.UsesInvariantGlobalization.get -> bool
 Microsoft.NET.Build.Containers.Tasks.ComputeDotnetBaseImageAndTag.UsesInvariantGlobalization.set -> void
-static Microsoft.NET.Build.Containers.ContainerBuilder.ContainerizeAsync(System.IO.DirectoryInfo! publishDirectory, string! workingDir, string! baseRegistry, string! baseImageName, string! baseImageTag, string![]! entrypoint, string![]! entrypointArgs, string![]! defaultArgs, string![]! appCommand, string![]! appCommandArgs, string! appCommandInstruction, string! imageName, string![]! imageTags, string? outputRegistry, System.Collections.Generic.Dictionary<string!, string!>! labels, Microsoft.NET.Build.Containers.Port[]? exposedPorts, System.Collections.Generic.Dictionary<string!, string!>! envVars, string! containerRuntimeIdentifier, string! ridGraphPath, string! localRegistry, string? containerUser, string? archiveOutputPath, Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task<int>!
 static readonly Microsoft.NET.Build.Containers.Constants.Version -> string!
-Microsoft.NET.Build.Containers.ContainerBuilder
 Microsoft.NET.Build.Containers.ContainerHelpers
 Microsoft.NET.Build.Containers.ContainerHelpers.ParsePortError
 Microsoft.NET.Build.Containers.ContainerHelpers.ParsePortError.InvalidPortNumber = 1 -> Microsoft.NET.Build.Containers.ContainerHelpers.ParsePortError
@@ -73,6 +71,8 @@ Microsoft.NET.Build.Containers.ManifestV2
 Microsoft.NET.Build.Containers.ManifestV2.Config.get -> Microsoft.NET.Build.Containers.ManifestConfig
 Microsoft.NET.Build.Containers.ManifestV2.Config.init -> void
 Microsoft.NET.Build.Containers.ManifestV2.GetDigest() -> string!
+Microsoft.NET.Build.Containers.ManifestV2.KnownDigest.get -> string?
+Microsoft.NET.Build.Containers.ManifestV2.KnownDigest.set -> void
 Microsoft.NET.Build.Containers.ManifestV2.Layers.get -> System.Collections.Generic.List<Microsoft.NET.Build.Containers.ManifestLayer>!
 Microsoft.NET.Build.Containers.ManifestV2.Layers.init -> void
 Microsoft.NET.Build.Containers.ManifestV2.ManifestV2() -> void
@@ -192,6 +192,10 @@ Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ToolPath.get -> string!
 Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ToolPath.set -> void
 Microsoft.NET.Build.Containers.Tasks.CreateNewImage.WorkingDirectory.get -> string!
 Microsoft.NET.Build.Containers.Tasks.CreateNewImage.WorkingDirectory.set -> void
+Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateLabels.get -> bool
+Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateLabels.set -> void
+Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateDigestLabel.get -> bool
+Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateDigestLabel.set -> void
 Microsoft.NET.Build.Containers.Tasks.ParseContainerProperties
 Microsoft.NET.Build.Containers.Tasks.ParseContainerProperties.ContainerEnvironmentVariables.get -> Microsoft.Build.Framework.ITaskItem![]!
 Microsoft.NET.Build.Containers.Tasks.ParseContainerProperties.ContainerEnvironmentVariables.set -> void
@@ -252,12 +256,6 @@ Microsoft.NET.Build.Containers.ManifestLayer.Equals(Microsoft.NET.Build.Containe
 ~override Microsoft.NET.Build.Containers.Descriptor.Equals(object obj) -> bool
 Microsoft.NET.Build.Containers.ManifestLayer.Deconstruct(out string! mediaType, out long size, out string! digest, out string![]? urls) -> void
 Microsoft.NET.Build.Containers.Descriptor.Equals(Microsoft.NET.Build.Containers.Descriptor other) -> bool
-~override Microsoft.NET.Build.Containers.ManifestV2.ToString() -> string
-static Microsoft.NET.Build.Containers.ManifestV2.operator !=(Microsoft.NET.Build.Containers.ManifestV2 left, Microsoft.NET.Build.Containers.ManifestV2 right) -> bool
-static Microsoft.NET.Build.Containers.ManifestV2.operator ==(Microsoft.NET.Build.Containers.ManifestV2 left, Microsoft.NET.Build.Containers.ManifestV2 right) -> bool
-override Microsoft.NET.Build.Containers.ManifestV2.GetHashCode() -> int
-~override Microsoft.NET.Build.Containers.ManifestV2.Equals(object obj) -> bool
-Microsoft.NET.Build.Containers.ManifestV2.Equals(Microsoft.NET.Build.Containers.ManifestV2 other) -> bool
 ~override Microsoft.NET.Build.Containers.ManifestListV2.ToString() -> string
 static Microsoft.NET.Build.Containers.ManifestListV2.operator !=(Microsoft.NET.Build.Containers.ManifestListV2 left, Microsoft.NET.Build.Containers.ManifestListV2 right) -> bool
 static Microsoft.NET.Build.Containers.ManifestListV2.operator ==(Microsoft.NET.Build.Containers.ManifestListV2 left, Microsoft.NET.Build.Containers.ManifestListV2 right) -> bool

src/Containers/Microsoft.NET.Build.Containers/Registry/Registry.cs

@@ -6,12 +6,14 @@
 using NuGet.RuntimeModel;
 using System.Diagnostics;
 using System.Net.Http.Json;
+using System.Text.Json;
 using System.Text.Json.Nodes;
 using System.Text.RegularExpressions;
 
 namespace Microsoft.NET.Build.Containers;
 
-internal interface IManifestPicker {
+internal interface IManifestPicker
+{
     public PlatformSpecificManifest? PickBestManifestForRid(IReadOnlyDictionary<string, PlatformSpecificManifest> manifestList, string runtimeIdentifier);
 }
 
@@ -26,7 +28,8 @@ public RidGraphManifestPicker(string runtimeIdentifierGraphPath)
     public PlatformSpecificManifest? PickBestManifestForRid(IReadOnlyDictionary<string, PlatformSpecificManifest> ridManifestDict, string runtimeIdentifier)
     {
         var bestManifestRid = GetBestMatchingRid(_runtimeGraph, runtimeIdentifier, ridManifestDict.Keys);
-        if (bestManifestRid is null) {
+        if (bestManifestRid is null)
+        {
             return null;
         }
         return ridManifestDict[bestManifestRid];
@@ -132,7 +135,8 @@ private static string DeriveRegistryName(Uri baseUri)
     /// <remarks>
     /// Google Artifact Registry locations (one for each availability zone) are of the form "ZONE-docker.pkg.dev".
     /// </remarks>
-    public bool IsGoogleArtifactRegistry {
+    public bool IsGoogleArtifactRegistry
+    {
         get => RegistryName.EndsWith("-docker.pkg.dev", StringComparison.Ordinal);
     }
 
@@ -151,7 +155,7 @@ public async Task<ImageBuilder> GetImageManifestAsync(string repositoryName, str
         {
             SchemaTypes.DockerManifestV2 or SchemaTypes.OciManifestV1 => await ReadSingleImageAsync(
                 repositoryName,
-                await initialManifestResponse.Content.ReadFromJsonAsync<ManifestV2>(cancellationToken: cancellationToken).ConfigureAwait(false),
+                await ReadManifest().ConfigureAwait(false),
                 cancellationToken).ConfigureAwait(false),
             SchemaTypes.DockerManifestListV2 => await PickBestImageFromManifestListAsync(
                 repositoryName,
@@ -167,6 +171,17 @@ await initialManifestResponse.Content.ReadFromJsonAsync<ManifestListV2>(cancella
                 BaseUri,
                 unknownMediaType))
         };
+
+        async Task<ManifestV2> ReadManifest()
+        {
+            initialManifestResponse.Headers.TryGetValues("Docker-Content-Digest", out var knownDigest);
+            var manifest = (await initialManifestResponse.Content.ReadFromJsonAsync<ManifestV2>(cancellationToken: cancellationToken).ConfigureAwait(false))!;
+            if (knownDigest?.FirstOrDefault() is string knownDigestValue)
+            {
+                manifest.KnownDigest = knownDigestValue;
+            }
+            return manifest;
+        }
     }
 
     internal async Task<ManifestListV2?> GetManifestListAsync(string repositoryName, string reference, CancellationToken cancellationToken)
@@ -193,11 +208,12 @@ private async Task<ImageBuilder> ReadSingleImageAsync(string repositoryName, Man
         return new ImageBuilder(manifest, new ImageConfig(configDoc), _logger);
     }
 
-    
+
     private static IReadOnlyDictionary<string, PlatformSpecificManifest> GetManifestsByRid(ManifestListV2 manifestList)
     {
         var ridDict = new Dictionary<string, PlatformSpecificManifest>();
-        foreach (var manifest in manifestList.manifests) {
+        foreach (var manifest in manifestList.manifests)
+        {
             if (CreateRidForPlatform(manifest.platform) is { } rid)
             {
                 ridDict.TryAdd(rid, manifest);
@@ -206,7 +222,7 @@ private static IReadOnlyDictionary<string, PlatformSpecificManifest> GetManifest
 
         return ridDict;
     }
-    
+
     private static string? CreateRidForPlatform(PlatformInformation platform)
     {
         // we only support linux and windows containers explicitly, so anything else we should skip past.
@@ -220,7 +236,7 @@ private static IReadOnlyDictionary<string, PlatformSpecificManifest> GetManifest
         // TODO: we _may_ need OS-specific version parsing. Need to do more research on what the field looks like across more manifest lists.
         var versionPart = platform.version?.Split('.') switch
         {
-            [var major, .. ] => major,
+        [var major, ..] => major,
             _ => null
         };
         var platformPart = platform.architecture switch
@@ -254,12 +270,15 @@ private async Task<ImageBuilder> PickBestImageFromManifestListAsync(
             using HttpResponseMessage manifestResponse = await _registryAPI.Manifest.GetAsync(repositoryName, matchingManifest.digest, cancellationToken).ConfigureAwait(false);
 
             cancellationToken.ThrowIfCancellationRequested();
-
+            var manifest = await manifestResponse.Content.ReadFromJsonAsync<ManifestV2>(cancellationToken: cancellationToken).ConfigureAwait(false);
+            if (manifest is null) throw new BaseImageNotFoundException(runtimeIdentifier, repositoryName, reference, ridManifestDict.Keys);
+            manifest.KnownDigest = matchingManifest.digest;
             return await ReadSingleImageAsync(
                 repositoryName,
-                await manifestResponse.Content.ReadFromJsonAsync<ManifestV2>(cancellationToken: cancellationToken).ConfigureAwait(false),
+                manifest,
                 cancellationToken).ConfigureAwait(false);
-        } else 
+        }
+        else
         {
             throw new BaseImageNotFoundException(runtimeIdentifier, repositoryName, reference, ridManifestDict.Keys);
         }
@@ -332,13 +351,13 @@ internal async Task<FinalizeUploadInformation> UploadBlobChunkedAsync(Stream con
 
             int bytesRead = await contents.ReadAsync(chunkBackingStore, cancellationToken).ConfigureAwait(false);
 
-            ByteArrayContent content = new (chunkBackingStore, offset: 0, count: bytesRead);
+            ByteArrayContent content = new(chunkBackingStore, offset: 0, count: bytesRead);
             content.Headers.ContentLength = bytesRead;
 
             // manual because ACR throws an error with the .NET type {"Range":"bytes 0-84521/*","Reason":"the Content-Range header format is invalid"}
             //    content.Headers.Add("Content-Range", $"0-{contents.Length - 1}");
             Debug.Assert(content.Headers.TryAddWithoutValidation("Content-Range", $"{chunkStart}-{chunkStart + bytesRead - 1}"));
-            
+
             NextChunkUploadInformation nextChunk = await _registryAPI.Blob.Upload.UploadChunkAsync(patchUri, content, cancellationToken).ConfigureAwait(false);
             patchUri = nextChunk.UploadUri;
 
@@ -420,7 +439,7 @@ private async Task PushAsync(BuiltImage builtImage, SourceImageReference source,
             }
 
             // Blob wasn't there; can we tell the server to get it from the base image?
-            if (! await _registryAPI.Blob.Upload.TryMountAsync(destination.Repository, source.Repository, digest, cancellationToken).ConfigureAwait(false))
+            if (!await _registryAPI.Blob.Upload.TryMountAsync(destination.Repository, source.Repository, digest, cancellationToken).ConfigureAwait(false))
             {
                 // The blob wasn't already available in another namespace, so fall back to explicitly uploading it
 
@@ -432,7 +451,8 @@ private async Task PushAsync(BuiltImage builtImage, SourceImageReference source,
                     await destinationRegistry.PushLayerAsync(Layer.FromDescriptor(descriptor), destination.Repository, cancellationToken).ConfigureAwait(false);
                     _logger.LogInformation(Strings.Registry_LayerUploaded, digest, destinationRegistry.RegistryName);
                 }
-                else {
+                else
+                {
                     throw new NotImplementedException(Resource.GetString(nameof(Strings.MissingLinkToRegistry)));
                 }
             }
@@ -444,7 +464,7 @@ private async Task PushAsync(BuiltImage builtImage, SourceImageReference source,
         }
         else
         {
-            foreach(var descriptor in builtImage.LayerDescriptors)
+            foreach (var descriptor in builtImage.LayerDescriptors)
             {
                 await uploadLayerFunc(descriptor).ConfigureAwait(false);
             }