detect and flow through the manifest digest from the registry or manifest list

baronfel · baronfel · commit 1adff27c4baf · 2024-03-06T10:17:33.000-06:00
diff --git a/src/Containers/Microsoft.NET.Build.Containers/ImageBuilder.cs b/src/Containers/Microsoft.NET.Build.Containers/ImageBuilder.cs
@@ -14,6 +14,10 @@ namespace Microsoft.NET.Build.Containers;
 /// </summary>
 internal sealed class ImageBuilder
 {
+    // a snapshot of the manifest that this builder is based on
+    private readonly ManifestV2 _baseImageManifest;
+
+    // the mutable internal manifest that we're building by modifying the base and applying customizations
     private readonly ManifestV2 _manifest;
     private readonly ImageConfig _baseImageConfig;
     private readonly ILogger _logger;
@@ -33,7 +37,8 @@ internal sealed class ImageBuilder
 
     internal ImageBuilder(ManifestV2 manifest, ImageConfig baseImageConfig, ILogger logger)
     {
-        _manifest = manifest;
+        _baseImageManifest = manifest;
+        _manifest = new ManifestV2() { SchemaVersion = manifest.SchemaVersion, Config = manifest.Config, Layers = new(manifest.Layers), MediaType = manifest.MediaType };
         _baseImageConfig = baseImageConfig;
         _logger = logger;
     }
@@ -63,9 +68,12 @@ internal BuiltImage Build()
             size = imageSize
         };
 
-        ManifestV2 newManifest = _manifest with
+        ManifestV2 newManifest = new ManifestV2()
         {
-            Config = newManifestConfig
+            Config = newManifestConfig,
+            SchemaVersion = _manifest.SchemaVersion,
+            MediaType = _manifest.MediaType,
+            Layers = _manifest.Layers
         };
 
         return new BuiltImage()
@@ -89,7 +97,7 @@ internal void AddLayer(Layer l)
 
     internal void AddBaseImageDigestLabel()
     {
-        AddLabel("org.opencontainers.image.base.digest", _manifest.GetDigest());
+        AddLabel("org.opencontainers.image.base.digest", _baseImageManifest.GetDigest());
     }
 
     /// <summary>
diff --git a/src/Containers/Microsoft.NET.Build.Containers/ManifestV2.cs b/src/Containers/Microsoft.NET.Build.Containers/ManifestV2.cs
@@ -12,8 +12,11 @@ namespace Microsoft.NET.Build.Containers;
 /// <remarks>
 /// https://github.com/opencontainers/image-spec/blob/main/manifest.md
 /// </remarks>
-public readonly record struct ManifestV2
+public class ManifestV2
 {
+    [JsonIgnore]
+    public string? KnownDigest { get; set; }
+
     /// <summary>
     /// This REQUIRED property specifies the image manifest schema version.
     /// For this version of the specification, this MUST be 2 to ensure backward compatibility with older versions of Docker.
@@ -47,9 +50,9 @@ public readonly record struct ManifestV2
     /// <summary>
     /// Gets the digest for this manifest.
     /// </summary>
-    public string GetDigest() => DigestUtils.GetDigest(JsonSerializer.SerializeToNode(this)?.ToJsonString() ?? string.Empty);
+    public string GetDigest() => KnownDigest ??= DigestUtils.GetDigest(JsonSerializer.SerializeToNode(this)?.ToJsonString(new JsonSerializerOptions() { WriteIndented = true }).ReplaceLineEndings("\r") ?? string.Empty);
 }
 
 public record struct ManifestConfig(string mediaType, long size, string digest);
 
-public record struct ManifestLayer(string mediaType, long size, string digest, string[]? urls);
+public record struct ManifestLayer(string mediaType, long size, string digest, [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)][field: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string[]? urls);
diff --git a/src/Containers/Microsoft.NET.Build.Containers/PublicAPI/net8.0/PublicAPI.Unshipped.txt b/src/Containers/Microsoft.NET.Build.Containers/PublicAPI/net8.0/PublicAPI.Unshipped.txt
@@ -73,6 +73,8 @@ Microsoft.NET.Build.Containers.ManifestV2
 Microsoft.NET.Build.Containers.ManifestV2.Config.get -> Microsoft.NET.Build.Containers.ManifestConfig
 Microsoft.NET.Build.Containers.ManifestV2.Config.init -> void
 Microsoft.NET.Build.Containers.ManifestV2.GetDigest() -> string!
+Microsoft.NET.Build.Containers.ManifestV2.KnownDigest.get -> string?
+Microsoft.NET.Build.Containers.ManifestV2.KnownDigest.set -> void
 Microsoft.NET.Build.Containers.ManifestV2.Layers.get -> System.Collections.Generic.List<Microsoft.NET.Build.Containers.ManifestLayer>!
 Microsoft.NET.Build.Containers.ManifestV2.Layers.init -> void
 Microsoft.NET.Build.Containers.ManifestV2.ManifestV2() -> void
@@ -256,12 +258,6 @@ Microsoft.NET.Build.Containers.ManifestLayer.Equals(Microsoft.NET.Build.Containe
 ~override Microsoft.NET.Build.Containers.Descriptor.Equals(object obj) -> bool
 Microsoft.NET.Build.Containers.ManifestLayer.Deconstruct(out string! mediaType, out long size, out string! digest, out string![]? urls) -> void
 Microsoft.NET.Build.Containers.Descriptor.Equals(Microsoft.NET.Build.Containers.Descriptor other) -> bool
-~override Microsoft.NET.Build.Containers.ManifestV2.ToString() -> string
-static Microsoft.NET.Build.Containers.ManifestV2.operator !=(Microsoft.NET.Build.Containers.ManifestV2 left, Microsoft.NET.Build.Containers.ManifestV2 right) -> bool
-static Microsoft.NET.Build.Containers.ManifestV2.operator ==(Microsoft.NET.Build.Containers.ManifestV2 left, Microsoft.NET.Build.Containers.ManifestV2 right) -> bool
-override Microsoft.NET.Build.Containers.ManifestV2.GetHashCode() -> int
-~override Microsoft.NET.Build.Containers.ManifestV2.Equals(object obj) -> bool
-Microsoft.NET.Build.Containers.ManifestV2.Equals(Microsoft.NET.Build.Containers.ManifestV2 other) -> bool
 ~override Microsoft.NET.Build.Containers.ManifestListV2.ToString() -> string
 static Microsoft.NET.Build.Containers.ManifestListV2.operator !=(Microsoft.NET.Build.Containers.ManifestListV2 left, Microsoft.NET.Build.Containers.ManifestListV2 right) -> bool
 static Microsoft.NET.Build.Containers.ManifestListV2.operator ==(Microsoft.NET.Build.Containers.ManifestListV2 left, Microsoft.NET.Build.Containers.ManifestListV2 right) -> bool
diff --git a/src/Containers/Microsoft.NET.Build.Containers/Registry/Registry.cs b/src/Containers/Microsoft.NET.Build.Containers/Registry/Registry.cs
@@ -6,12 +6,14 @@
 using NuGet.RuntimeModel;
 using System.Diagnostics;
 using System.Net.Http.Json;
+using System.Text.Json;
 using System.Text.Json.Nodes;
 using System.Text.RegularExpressions;
 
 namespace Microsoft.NET.Build.Containers;
 
-internal interface IManifestPicker {
+internal interface IManifestPicker
+{
     public PlatformSpecificManifest? PickBestManifestForRid(IReadOnlyDictionary<string, PlatformSpecificManifest> manifestList, string runtimeIdentifier);
 }
 
@@ -26,7 +28,8 @@ public RidGraphManifestPicker(string runtimeIdentifierGraphPath)
     public PlatformSpecificManifest? PickBestManifestForRid(IReadOnlyDictionary<string, PlatformSpecificManifest> ridManifestDict, string runtimeIdentifier)
     {
         var bestManifestRid = GetBestMatchingRid(_runtimeGraph, runtimeIdentifier, ridManifestDict.Keys);
-        if (bestManifestRid is null) {
+        if (bestManifestRid is null)
+        {
             return null;
         }
         return ridManifestDict[bestManifestRid];
@@ -132,7 +135,8 @@ private static string DeriveRegistryName(Uri baseUri)
     /// <remarks>
     /// Google Artifact Registry locations (one for each availability zone) are of the form "ZONE-docker.pkg.dev".
     /// </remarks>
-    public bool IsGoogleArtifactRegistry {
+    public bool IsGoogleArtifactRegistry
+    {
         get => RegistryName.EndsWith("-docker.pkg.dev", StringComparison.Ordinal);
     }
 
@@ -151,7 +155,7 @@ public async Task<ImageBuilder> GetImageManifestAsync(string repositoryName, str
         {
             SchemaTypes.DockerManifestV2 or SchemaTypes.OciManifestV1 => await ReadSingleImageAsync(
                 repositoryName,
-                await initialManifestResponse.Content.ReadFromJsonAsync<ManifestV2>(cancellationToken: cancellationToken).ConfigureAwait(false),
+                await ReadManifest().ConfigureAwait(false),
                 cancellationToken).ConfigureAwait(false),
             SchemaTypes.DockerManifestListV2 => await PickBestImageFromManifestListAsync(
                 repositoryName,
@@ -167,6 +171,17 @@ await initialManifestResponse.Content.ReadFromJsonAsync<ManifestListV2>(cancella
                 BaseUri,
                 unknownMediaType))
         };
+
+        async Task<ManifestV2> ReadManifest()
+        {
+            initialManifestResponse.Headers.TryGetValues("Docker-Content-Digest", out var knownDigest);
+            var manifest = (await initialManifestResponse.Content.ReadFromJsonAsync<ManifestV2>(cancellationToken: cancellationToken).ConfigureAwait(false))!;
+            if (knownDigest?.FirstOrDefault() is string knownDigestValue)
+            {
+                manifest.KnownDigest = knownDigestValue;
+            }
+            return manifest;
+        }
     }
 
     internal async Task<ManifestListV2?> GetManifestListAsync(string repositoryName, string reference, CancellationToken cancellationToken)
@@ -193,11 +208,12 @@ private async Task<ImageBuilder> ReadSingleImageAsync(string repositoryName, Man
         return new ImageBuilder(manifest, new ImageConfig(configDoc), _logger);
     }
 
-    
+
     private static IReadOnlyDictionary<string, PlatformSpecificManifest> GetManifestsByRid(ManifestListV2 manifestList)
     {
         var ridDict = new Dictionary<string, PlatformSpecificManifest>();
-        foreach (var manifest in manifestList.manifests) {
+        foreach (var manifest in manifestList.manifests)
+        {
             if (CreateRidForPlatform(manifest.platform) is { } rid)
             {
                 ridDict.TryAdd(rid, manifest);
@@ -206,7 +222,7 @@ private static IReadOnlyDictionary<string, PlatformSpecificManifest> GetManifest
 
         return ridDict;
     }
-    
+
     private static string? CreateRidForPlatform(PlatformInformation platform)
     {
         // we only support linux and windows containers explicitly, so anything else we should skip past.
@@ -220,7 +236,7 @@ private static IReadOnlyDictionary<string, PlatformSpecificManifest> GetManifest
         // TODO: we _may_ need OS-specific version parsing. Need to do more research on what the field looks like across more manifest lists.
         var versionPart = platform.version?.Split('.') switch
         {
-            [var major, .. ] => major,
+        [var major, ..] => major,
             _ => null
         };
         var platformPart = platform.architecture switch
@@ -254,12 +270,14 @@ private async Task<ImageBuilder> PickBestImageFromManifestListAsync(
             using HttpResponseMessage manifestResponse = await _registryAPI.Manifest.GetAsync(repositoryName, matchingManifest.digest, cancellationToken).ConfigureAwait(false);
 
             cancellationToken.ThrowIfCancellationRequested();
-
+            var manifest = await manifestResponse.Content.ReadAsJsonAsync<ManifestV2>(cancellationToken: cancellationToken).ConfigureAwait(false);
+            manifest.KnownDigest = matchingManifest.digest;
             return await ReadSingleImageAsync(
                 repositoryName,
-                await manifestResponse.Content.ReadFromJsonAsync<ManifestV2>(cancellationToken: cancellationToken).ConfigureAwait(false),
+                manifest,
                 cancellationToken).ConfigureAwait(false);
-        } else 
+        }
+        else
         {
             throw new BaseImageNotFoundException(runtimeIdentifier, repositoryName, reference, ridManifestDict.Keys);
         }
@@ -332,13 +350,13 @@ internal async Task<FinalizeUploadInformation> UploadBlobChunkedAsync(Stream con
 
             int bytesRead = await contents.ReadAsync(chunkBackingStore, cancellationToken).ConfigureAwait(false);
 
-            ByteArrayContent content = new (chunkBackingStore, offset: 0, count: bytesRead);
+            ByteArrayContent content = new(chunkBackingStore, offset: 0, count: bytesRead);
             content.Headers.ContentLength = bytesRead;
 
             // manual because ACR throws an error with the .NET type {"Range":"bytes 0-84521/*","Reason":"the Content-Range header format is invalid"}
             //    content.Headers.Add("Content-Range", $"0-{contents.Length - 1}");
             Debug.Assert(content.Headers.TryAddWithoutValidation("Content-Range", $"{chunkStart}-{chunkStart + bytesRead - 1}"));
-            
+
             NextChunkUploadInformation nextChunk = await _registryAPI.Blob.Upload.UploadChunkAsync(patchUri, content, cancellationToken).ConfigureAwait(false);
             patchUri = nextChunk.UploadUri;
 
@@ -420,7 +438,7 @@ private async Task PushAsync(BuiltImage builtImage, SourceImageReference source,
             }
 
             // Blob wasn't there; can we tell the server to get it from the base image?
-            if (! await _registryAPI.Blob.Upload.TryMountAsync(destination.Repository, source.Repository, digest, cancellationToken).ConfigureAwait(false))
+            if (!await _registryAPI.Blob.Upload.TryMountAsync(destination.Repository, source.Repository, digest, cancellationToken).ConfigureAwait(false))
             {
                 // The blob wasn't already available in another namespace, so fall back to explicitly uploading it
 
@@ -432,7 +450,8 @@ private async Task PushAsync(BuiltImage builtImage, SourceImageReference source,
                     await destinationRegistry.PushLayerAsync(Layer.FromDescriptor(descriptor), destination.Repository, cancellationToken).ConfigureAwait(false);
                     _logger.LogInformation(Strings.Registry_LayerUploaded, digest, destinationRegistry.RegistryName);
                 }
-                else {
+                else
+                {
                     throw new NotImplementedException(Resource.GetString(nameof(Strings.MissingLinkToRegistry)));
                 }
             }
@@ -444,7 +463,7 @@ private async Task PushAsync(BuiltImage builtImage, SourceImageReference source,
         }
         else
         {
-            foreach(var descriptor in builtImage.LayerDescriptors)
+            foreach (var descriptor in builtImage.LayerDescriptors)
             {
                 await uploadLayerFunc(descriptor).ConfigureAwait(false);
             }
