Merge pull request #247 from dotnet/support-other-arch-and-os

Author: baronfel

Directory.Packages.props

@@ -13,6 +13,7 @@
         <PackageVersion Include="MSTest.TestAdapter" Version="2.3.0-preview-20220810-02" />
         <PackageVersion Include="MSTest.TestFramework" Version="2.2.10" />
         <PackageVersion Include="Nerdbank.GitVersioning" Version="3.5.109" />
+        <PackageVersion Include="NuGet.Packaging" Version="6.3.1" />
         <PackageVersion Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
         <PackageVersion Include="Valleysoft.DockerCredsProvider" Version="2.1.0" />
     </ItemGroup>

Microsoft.NET.Build.Containers/ContainerBuilder.cs

@@ -7,15 +7,19 @@ namespace Microsoft.NET.Build.Containers;
 
 public static class ContainerBuilder
 {
-    public static async Task Containerize(DirectoryInfo folder, string workingDir, string registryName, string baseName, string baseTag, string[] entrypoint, string[] entrypointArgs, string imageName, string[] imageTags, string? outputRegistry, string[] labels, Port[] exposedPorts, string[] envVars)
+    public static async Task Containerize(DirectoryInfo folder, string workingDir, string registryName, string baseName, string baseTag, string[] entrypoint, string[] entrypointArgs, string imageName, string[] imageTags, string? outputRegistry, string[] labels, Port[] exposedPorts, string[] envVars, string containerRuntimeIdentifier, string ridGraphPath)
     {
         var isDockerPull = String.IsNullOrEmpty(registryName);
         if (isDockerPull) {
             throw new ArgumentException("Don't know how to pull images from local daemons at the moment");
         }
         Registry baseRegistry = new Registry(ContainerHelpers.TryExpandRegistryToUri(registryName));
 
-        Image img = await baseRegistry.GetImageManifest(baseName, baseTag);
+        var img = await baseRegistry.GetImageManifest(baseName, baseTag, containerRuntimeIdentifier, ridGraphPath);
+        if (img is null) {
+            throw new ArgumentException($"Could not find image {baseName}:{baseTag} in registry {registryName} matching RuntimeIdentifier {containerRuntimeIdentifier}");
+        }
+
         img.WorkingDirectory = workingDir;
 
         JsonSerializerOptions options = new()
@@ -82,6 +86,5 @@ public static async Task Containerize(DirectoryInfo folder, string workingDir, s
                 }
             }
         }
-
     }
 }

Microsoft.NET.Build.Containers/ContentStore.cs

@@ -34,6 +34,7 @@ public static string PathForDescriptor(Descriptor descriptor)
         {
             "application/vnd.docker.image.rootfs.diff.tar.gzip"
             or "application/vnd.oci.image.layer.v1.tar+gzip"
+            or "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip"
                 => ".tar.gz",
             "application/vnd.docker.image.rootfs.diff.tar"
             or "application/vnd.oci.image.layer.v1.tar"

Microsoft.NET.Build.Containers/CreateNewImage.Interface.cs

@@ -93,12 +93,25 @@ partial class CreateNewImage
     /// </summary>
     public ITaskItem[] ContainerEnvironmentVariables { get; set; }
 
+    /// <summary>
+    /// The RID to use to determine the host manifest if the parent container is a manifest list
+    /// </summary>
+    [Required]
+    public string ContainerRuntimeIdentifier { get; set; }
+
+    /// <summary>
+    /// The path to the runtime identifier graph file. This is used to compute RID compatibility for Image Manifest List entries.
+    /// </summary>
+    [Required]
+    public string RuntimeIdentifierGraphPath { get; set; }
+
     [Output]
     public string GeneratedContainerManifest { get; set; }
 
     [Output]
     public string GeneratedContainerConfiguration { get; set; }
 
+
     public CreateNewImage()
     {
         ContainerizeDirectory = "";
@@ -117,6 +130,9 @@ public CreateNewImage()
         Labels = Array.Empty<ITaskItem>();
         ExposedPorts = Array.Empty<ITaskItem>();
         ContainerEnvironmentVariables = Array.Empty<ITaskItem>();
+        ContainerRuntimeIdentifier = "";
+        RuntimeIdentifierGraphPath = "";
+
         GeneratedContainerConfiguration = "";
         GeneratedContainerManifest = "";
     }

Microsoft.NET.Build.Containers/CreateNewImage.cs

@@ -1,4 +1,5 @@
-using Microsoft.Build.Framework;
+using System.Text.Json;
+using Microsoft.Build.Framework;
 
 namespace Microsoft.NET.Build.Containers.Tasks;
 
@@ -72,12 +73,12 @@ private void SetEnvironmentVariables(Image img, ITaskItem[] envVars)
         }
     }
 
-    private Image GetBaseImage() {
+    private Image? GetBaseImage() {
         if (IsDockerPull) {
             throw new ArgumentException("Don't know how to pull images from local daemons at the moment");
         } else {
             var reg = new Registry(ContainerHelpers.TryExpandRegistryToUri(BaseRegistry));
-            return reg.GetImageManifest(BaseImageName, BaseImageTag).Result;
+            return reg.GetImageManifest(BaseImageName, BaseImageTag, ContainerRuntimeIdentifier, RuntimeIdentifierGraphPath).Result;
         }
     }
 
@@ -95,6 +96,11 @@ public override bool Execute()
 
         var image = GetBaseImage();
 
+        if (image is null) {
+            Log.LogError($"Couldn't find matching base image for {0}:{1} that matches RuntimeIdentifier {2}", BaseImageName, BaseImageTag, ContainerRuntimeIdentifier);
+            return !Log.HasLoggedErrors;
+        }
+
         SafeLog("Building image '{0}' with tags {1} on top of base image {2}/{3}:{4}", ImageName, String.Join(",", ImageTags), BaseRegistry, BaseImageName, BaseImageTag);
 
         Layer newLayer = Layer.FromDirectory(PublishDirectory, WorkingDirectory);
@@ -118,7 +124,7 @@ public override bool Execute()
         }
 
         // at this point we're done with modifications and are just pushing the data other places
-        GeneratedContainerManifest = image.manifest.ToJsonString();
+        GeneratedContainerManifest =  JsonSerializer.Serialize(image.manifest);
         GeneratedContainerConfiguration = image.config.ToJsonString();
 
         Registry? outputReg = IsDockerPush ? null : new Registry(ContainerHelpers.TryExpandRegistryToUri(OutputRegistry));

Microsoft.NET.Build.Containers/CreateNewImageToolTask.cs

@@ -83,7 +83,9 @@ protected override string GenerateCommandLineCommands()
                (ImageTags.Length > 0 ? " --imagetags " + String.Join(" ", ImageTags.Select((i) => Quote(i))) : "") +
                (EntrypointArgs.Length > 0 ? " --entrypointargs " + String.Join(" ", EntrypointArgs.Select((i) => i.ItemSpec)) : "") +
                (ExposedPorts.Length > 0 ? " --ports " + String.Join(" ", ExposedPorts.Select((i) => i.ItemSpec + "/" + i.GetMetadata("Type"))) : "") +
-               (ContainerEnvironmentVariables.Length > 0 ? " --environmentvariables " + String.Join(" ", ContainerEnvironmentVariables.Select((i) => i.ItemSpec + "=" + Quote(i.GetMetadata("Value")))) : "");
+               (ContainerEnvironmentVariables.Length > 0 ? " --environmentvariables " + String.Join(" ", ContainerEnvironmentVariables.Select((i) => i.ItemSpec + "=" + Quote(i.GetMetadata("Value")))) : "") +
+               $" --rid {Quote(ContainerRuntimeIdentifier)}" +
+               $" --ridgraphpath {Quote(RuntimeIdentifierGraphPath)}";
     }
 
     private string Quote(string path)

Microsoft.NET.Build.Containers/Image.cs

@@ -8,7 +8,7 @@ namespace Microsoft.NET.Build.Containers;
 
 public class Image
 {
-    public JsonNode manifest;
+    public ManifestV2 manifest;
     public JsonNode config;
 
     public readonly string OriginatingName;
@@ -22,7 +22,7 @@ public class Image
 
     internal Dictionary<string, string> environmentVariables;
 
-    public Image(JsonNode manifest, JsonNode config, string name, Registry? registry)
+    public Image(ManifestV2 manifest, JsonNode config, string name, Registry? registry)
     {
         this.manifest = manifest;
         this.config = config;
@@ -38,39 +38,38 @@ public IEnumerable<Descriptor> LayerDescriptors
     {
         get
         {
-            JsonNode? layersNode = manifest["layers"];
+            var layersNode = manifest.layers;
 
             if (layersNode is null)
             {
                 throw new NotImplementedException("Tried to get layer information but there is no layer node?");
             }
 
-            foreach (JsonNode? descriptorJson in layersNode.AsArray())
+            foreach (var layer in layersNode)
             {
-                if (descriptorJson is null)
-                {
-                    throw new NotImplementedException("Null layer descriptor in the list?");
-                }
-
-                yield return descriptorJson.Deserialize<Descriptor>();
+                yield return new(layer.mediaType, layer.digest, layer.size);
             }
         }
     }
 
     public void AddLayer(Layer l)
     {
         newLayers.Add(l);
-        manifest["layers"]!.AsArray().Add(l.Descriptor);
+
+        manifest.layers.Add(new(l.Descriptor.MediaType, l.Descriptor.Size, l.Descriptor.Digest, l.Descriptor.Urls));
         config["rootfs"]!["diff_ids"]!.AsArray().Add(l.Descriptor.UncompressedDigest);
         RecalculateDigest();
     }
 
     private void RecalculateDigest()
     {
         config["created"] = DateTime.UtcNow;
-
-        manifest["config"]!["digest"] = GetDigest(config);
-        manifest["config"]!["size"] = Encoding.UTF8.GetBytes(config.ToJsonString()).Length;
+        var newManifestConfig = manifest.config with
+        {
+            digest = GetDigest(config),
+            size = Encoding.UTF8.GetBytes(config.ToJsonString()).Length
+        };
+        manifest.config = newManifestConfig;
     }
 
     private JsonObject CreatePortMap()
@@ -249,6 +248,13 @@ public string GetDigest(JsonNode json)
         return $"sha256:{hashString}";
     }
 
+    public string GetDigest<T>(T item)
+    {
+        var node = JsonSerializer.SerializeToNode(item);
+        if (node is not null) return GetDigest(node);
+        else return String.Empty;
+    }
+
     public static string GetSha(JsonNode json)
     {
         Span<byte> hash = stackalloc byte[SHA256.HashSizeInBytes];

Microsoft.NET.Build.Containers/LocalDocker.cs

@@ -47,19 +47,20 @@ public static async Task WriteImageToStream(Image x, string name, string tag, St
 
         foreach (var d in x.LayerDescriptors)
         {
-            if (!x.originatingRegistry.HasValue)
+            if (x.originatingRegistry is {} registry)
             {
-                throw new NotImplementedException("Need a good error for 'couldn't download a thing because no link to registry'");
-            }
+                string localPath = await registry.DownloadBlob(x.OriginatingName, d);
 
-            string localPath = await x.originatingRegistry.Value.DownloadBlob(x.OriginatingName, d);
-
-            // Stuff that (uncompressed) tarball into the image tar stream
-            // TODO uncompress!!
-            string layerTarballPath = $"{d.Digest.Substring("sha256:".Length)}/layer.tar";
-            await writer.WriteEntryAsync(localPath, layerTarballPath);
-            layerTarballPaths.Add(layerTarballPath);
-        }
+                // Stuff that (uncompressed) tarball into the image tar stream
+                // TODO uncompress!!
+                string layerTarballPath = $"{d.Digest.Substring("sha256:".Length)}/layer.tar";
+                await writer.WriteEntryAsync(localPath, layerTarballPath);
+                layerTarballPaths.Add(layerTarballPath);
+            }
+            else
+            {
+                throw new NotImplementedException("Need a good error for 'couldn't download a thing because no link to registry'");
+            }        }
 
         // add config
         string configTarballPath = $"{Image.GetSha(x.config)}.json";

Microsoft.NET.Build.Containers/Microsoft.NET.Build.Containers.csproj

@@ -22,6 +22,7 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.Build.Utilities.Core" PrivateAssets="all" ExcludeAssets="runtime" />
+    <PackageReference Include="Nuget.Packaging" /> 
     <PackageReference Include="Valleysoft.DockerCredsProvider" />
   </ItemGroup>