feat(oci): manifest/config updates to support containerd

Signed-off-by: Vaughn Dice <[email protected]>

Author: vdice

Cargo.lock

@@ -14,7 +14,7 @@ dkregistry = { git = "https://github.com/camallo/dkregistry-rs", rev = "37acecb4
 docker_credential = "1.0"
 dirs = "4.0"
 futures-util = "0.3"
-oci-distribution = { git = "https://github.com/fermyon/oci-distribution", rev = "05022618d78feef9b99f20b5da8fd6def6bb80d2" }
+oci-distribution = { git = "https://github.com/fermyon/oci-distribution", rev = "639c907b7c0c4e74716356585410d4abe4aebf4d" }
 reqwest = "0.11"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
@@ -25,7 +25,7 @@ spin-manifest = { path = "../manifest" }
 spin-trigger = { path = "../trigger" }
 tempfile = "3.3"
 tokio = { version = "1", features = ["fs"] }
-tokio-util = "0.7.9"
+tokio-util = { version = "0.7.9", features = ["compat"] }
 tracing = { workspace = true }
 walkdir = "2.3"
 

crates/oci/Cargo.toml

@@ -4,14 +4,13 @@ use anyhow::{bail, Context, Result};
 use docker_credential::DockerCredential;
 use futures_util::future;
 use futures_util::stream::{self, StreamExt, TryStreamExt};
-use oci_distribution::errors::OciDistributionError;
-use oci_distribution::token_cache::RegistryTokenType;
-use oci_distribution::RegistryOperation;
 use oci_distribution::{
     client::{Config, ImageLayer},
-    manifest::OciImageManifest,
+    errors::OciDistributionError,
+    manifest::{OciImageManifest, OCI_IMAGE_MEDIA_TYPE},
     secrets::RegistryAuth,
-    Reference,
+    token_cache::RegistryTokenType,
+    Reference, RegistryOperation,
 };
 use reqwest::Url;
 use spin_app::locked::{ContentPath, ContentRef, LockedApp};
@@ -23,11 +22,16 @@ use walkdir::WalkDir;
 
 use crate::auth::AuthConfig;
 
-// TODO: the media types for application, wasm module, data and archive layer are not final.
-const SPIN_APPLICATION_MEDIA_TYPE: &str = "application/vnd.fermyon.spin.application.v1+config";
-const WASM_LAYER_MEDIA_TYPE: &str = "application/vnd.wasm.content.layer.v1+wasm";
-const DATA_MEDIATYPE: &str = "application/vnd.wasm.content.layer.v1+data";
-const ARCHIVE_MEDIATYPE: &str = "application/vnd.wasm.content.bundle.v1.tar+gzip";
+// TODO: the media types for application, data and archive layer are not final
+pub const SPIN_APPLICATION_MEDIA_TYPE: &str = "application/vnd.fermyon.spin.application.v1+config";
+pub const DATA_MEDIATYPE: &str = "application/vnd.wasm.content.layer.v1+data";
+pub const ARCHIVE_MEDIATYPE: &str = "application/vnd.wasm.content.bundle.v1.tar+gzip";
+// Legacy wasm layer media type used by pre-2.0 versions of Spin
+const WASM_LAYER_MEDIA_TYPE_LEGACY: &str = "application/vnd.wasm.content.layer.v1+wasm";
+
+// TODO: use canonical types defined upstream; see https://github.com/bytecodealliance/registry/pull/146
+const WASM_LAYER_MEDIA_TYPE: &str = "application/vnd.bytecodealliance.wasm.component.layer.v0+wasm";
+const COMPONENT_ARTIFACT_TYPE: &str = "application/vnd.bytecodealliance.component.v1+wasm";
 
 const CONFIG_FILE: &str = "config.json";
 const LATEST_TAG: &str = "latest";
@@ -156,12 +160,24 @@ impl Client {
         locked.components = components;
         locked.metadata.remove("origin");
 
+        // Push layer for locked spin application config
+        let locked_config_layer = ImageLayer::new(
+            serde_json::to_vec(&locked).context("could not serialize locked config")?,
+            SPIN_APPLICATION_MEDIA_TYPE.to_string(),
+            None,
+        );
+        layers.push(locked_config_layer);
+
         let oci_config = Config {
+            // TODO: now that the locked config bytes are pushed as a layer, what should data here be?
+            // Keeping as locked config bytes would make it feasible for older Spin clients to pull/run
+            // apps published by newer Spin clients
             data: serde_json::to_vec(&locked)?,
-            media_type: SPIN_APPLICATION_MEDIA_TYPE.to_string(),
+            media_type: OCI_IMAGE_MEDIA_TYPE.to_string(),
             annotations: None,
         };
-        let manifest = OciImageManifest::build(&layers, &oci_config, None);
+        let mut manifest = OciImageManifest::build(&layers, &oci_config, None);
+        manifest.artifact_type = Some(COMPONENT_ARTIFACT_TYPE.to_string());
         let response = self
             .oci
             .push(&reference, &layers, oci_config, &auth, Some(manifest))
@@ -267,16 +283,17 @@ impl Client {
         let m = self.manifest_path(&reference.to_string()).await?;
         fs::write(&m, &manifest_json).await?;
 
+        // Older published Spin apps feature the locked app config *as* the OCI manifest config layer,
+        // while newer versions publish the locked app config as a generic layer alongside others.
+        // Assume that these bytes may represent the locked app config and write it as such.
+        // TODO: update this assumption if we change the data we write to the OCI manifest config layer.
         let mut cfg_bytes = Vec::new();
         self.oci
             .pull_blob(&reference, &manifest.config.digest, &mut cfg_bytes)
             .await?;
-        let cfg = std::str::from_utf8(&cfg_bytes)?;
-        tracing::debug!("Pulled config: {}", cfg);
-
-        // Write the config object in `<cache_root>/registry/oci/manifests/repository:<tag_or_latest>/config.json`
-        let c = self.lockfile_path(&reference.to_string()).await?;
-        fs::write(&c, &cfg).await?;
+        self.write_locked_app_config(&reference.to_string(), &cfg_bytes)
+            .await
+            .context("unable to write locked app config to cache")?;
 
         // If a layer is a Wasm module, write it in the Wasm directory.
         // Otherwise, write it in the data directory (after unpacking if archive layer)
@@ -300,15 +317,25 @@ impl Client {
                         {
                             Err(e) => return Err(e),
                             _ => match layer.media_type.as_str() {
-                                WASM_LAYER_MEDIA_TYPE => {
+                                // If the locked app config is present as a separate layer, this should take precedence
+                                SPIN_APPLICATION_MEDIA_TYPE => {
+                                    if let Err(e) = this.write_locked_app_config(&reference.to_string(), &bytes)
+                                        .await
+                                        {
+                                            return Err(OciDistributionError::GenericError(
+                                                Some(format!("unable to write locked app config to cache: {}", e))
+                                            ));
+                                        }
+                                }
+                                WASM_LAYER_MEDIA_TYPE | WASM_LAYER_MEDIA_TYPE_LEGACY => {
                                     let _ = this.cache.write_wasm(&bytes, &layer.digest).await;
                                 }
                                 ARCHIVE_MEDIATYPE => {
                                     if let Err(e) =
                                         this.unpack_archive_layer(&bytes, &layer.digest).await
                                     {
                                         return Err(OciDistributionError::GenericError(Some(
-                                            e.to_string(),
+                                            format!("unable to unpack archive layer with digest {}: {}", &layer.digest, e),
                                         )));
                                     }
                                 }
@@ -374,6 +401,19 @@ impl Client {
         Ok(p.join(CONFIG_FILE))
     }
 
+    /// Write the config object in `<cache_root>/registry/oci/manifests/repository:<tag_or_latest>/config.json`
+    async fn write_locked_app_config(
+        &self,
+        reference: impl AsRef<str>,
+        bytes: impl AsRef<[u8]>,
+    ) -> Result<()> {
+        let cfg = std::str::from_utf8(bytes.as_ref())?;
+        tracing::debug!("Pulled config: {}", cfg);
+
+        let c = self.lockfile_path(reference).await?;
+        fs::write(&c, &cfg).await.map_err(anyhow::Error::from)
+    }
+
     /// Create a new wasm layer based on a file.
     async fn wasm_layer(file: &Path) -> Result<ImageLayer> {
         tracing::log::trace!("Reading wasm module from {:?}", file);