Merge pull request #163 from bytecodealliance/common-wasm32

Trim wasm-pkg-common to allow targetting wasm32-wasip1

Author: lann

.github/workflows/ci.yml

@@ -10,24 +10,34 @@ jobs:
       matrix:
         include:
           - os: ubuntu-latest
-            additional_test_flags: ""
+            run_lints: true
+            test_wasm_build: true
           - os: windows-latest
-            additional_test_flags: "--no-default-features"
+            docker_tests_flags: "--no-default-features"
           - os: macos-latest
-            additional_test_flags: "--no-default-features"
+            docker_tests_flags: "--no-default-features"
     steps:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
         with:
           targets: "wasm32-wasip1"
-      # We have to run these separately so we can deactivate a feature for one of the tests
-      - name: Run client tests
-        working-directory: ./crates/wasm-pkg-client
-        run: cargo test ${{ matrix.additional_test_flags }}
-      - name: Run wkg tests
-        working-directory: ./crates/wkg
-        run: cargo test ${{ matrix.additional_test_flags }}
-      - name: Run other tests
+
+      - name: Run lints
+        if: matrix.run_lints
+        run: |
+          cargo clippy --workspace --no-default-features
+          cargo clippy --workspace --all-features
+
+      - name: Run tests
         run: cargo test --workspace --exclude wasm-pkg-client --exclude wkg
-      - name: Run cargo clippy
-        run: cargo clippy --all --workspace 
+
+      # NOTE: Docker tests are only run on linux because other platforms haven't
+      # always worked consistently.
+      - name: Run wasm-pkg-client tests
+        run: cargo test -p wasm-pkg-client ${{ matrix.docker_tests_flags }}
+      - name: Run wkg tests
+        run: cargo test -p wkg ${{ matrix.docker_tests_flags }}
+
+      - name: Test wasm32-wasip1 build for wasm-pkg-common
+        if: matrix.test_wasm_build
+        run: cargo build -p wasm-pkg-common --target wasm32-wasip1

Cargo.lock

@@ -9,9 +9,9 @@ license.workspace = true
 readme = "../../README.md"
 
 [features]
-default = ["_local"]
-# An internal feature for making sure e2e tests can run locally but not in CI for Mac or Windows
-_local = []
+default = ["docker-tests"]
+# This feature enables tests that use Docker
+docker-tests = []
 
 [dependencies]
 anyhow = { workspace = true }
@@ -23,6 +23,13 @@ etcetera = { workspace = true }
 futures-util = { workspace = true, features = ["io"] }
 oci-client = { workspace = true }
 oci-wasm = { workspace = true }
+reqwest = { version = "0.12.0", default-features = false, features = [
+    "charset",
+    "http2",
+    "json",
+    "macos-system-configuration",
+    "rustls-tls",
+]}
 secrecy = { version = "0.8", features = ["serde"] }
 serde = { workspace = true }
 serde_json = { workspace = true }
@@ -38,7 +45,7 @@ warg-client = "0.9.2"
 warg-crypto = "0.9.2"
 wasm-metadata = { workspace = true }
 warg-protocol = "0.9.2"
-wasm-pkg-common = { workspace = true, features = ["metadata-client", "tokio"] }
+wasm-pkg-common = { workspace = true, features = ["registry-config"] }
 wit-component = { workspace = true }
 
 [dev-dependencies]

crates/wasm-pkg-client/Cargo.toml

@@ -29,6 +29,7 @@
 pub mod caching;
 mod loader;
 pub mod local;
+pub mod metadata;
 pub mod oci;
 mod publisher;
 mod release;
@@ -55,6 +56,7 @@ pub use wasm_pkg_common::{
 };
 use wit_component::DecodedWasm;
 
+use crate::metadata::RegistryMetadataExt;
 use crate::{loader::PackageLoader, local::LocalBackend, oci::OciBackend, warg::WargBackend};
 
 pub use release::{Release, VersionInfo};

crates/wasm-pkg-client/src/lib.rs

@@ -2,12 +2,13 @@
 //!
 //! Each package release is a file: `<root>/<namespace>/<name>/<version>.wasm`
 
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
 use anyhow::anyhow;
 use async_trait::async_trait;
 use futures_util::{StreamExt, TryStreamExt};
 use serde::Deserialize;
+use sha2::{Digest, Sha256};
 use tokio_util::io::ReaderStream;
 use wasm_pkg_common::{
     config::RegistryConfig,
@@ -85,7 +86,7 @@ impl PackageLoader for LocalBackend {
     async fn get_release(&self, package: &PackageRef, version: &Version) -> Result<Release, Error> {
         let path = self.version_path(package, version);
         tracing::debug!(path = %path.display(), "Reading content from path");
-        let content_digest = ContentDigest::sha256_from_file(path).await?;
+        let content_digest = sha256_from_file(path).await?;
         Ok(Release {
             version: version.clone(),
             content_digest,
@@ -123,3 +124,18 @@ impl PackagePublisher for LocalBackend {
             .map(|_| ())
     }
 }
+
+async fn sha256_from_file(path: impl AsRef<Path>) -> Result<ContentDigest, std::io::Error> {
+    use tokio::io::AsyncReadExt;
+    let mut file = tokio::fs::File::open(path).await?;
+    let mut hasher = Sha256::new();
+    let mut buf = [0; 4096];
+    loop {
+        let n = file.read(&mut buf).await?;
+        if n == 0 {
+            break;
+        }
+        hasher.update(&buf[..n]);
+    }
+    Ok(hasher.into())
+}

crates/wasm-pkg-client/src/local.rs

@@ -0,0 +1,62 @@
+use anyhow::Context;
+use reqwest::StatusCode;
+use wasm_pkg_common::{
+    metadata::{RegistryMetadata, REGISTRY_METADATA_PATH},
+    registry::Registry,
+    Error,
+};
+
+/// Extension trait for [`RegistryMetadata`] adding client functionality.
+pub trait RegistryMetadataExt: Sized {
+    /// Attempt to fetch [`RegistryMetadata`] from the given [`Registry`]. On
+    /// failure, return defaults.
+    fn fetch_or_default(registry: &Registry) -> impl std::future::Future<Output = Self> + Send;
+
+    /// Fetch [`RegistryMetadata`] from the given [`Registry`].
+    fn fetch(
+        registry: &Registry,
+    ) -> impl std::future::Future<Output = Result<Option<Self>, Error>> + Send;
+}
+
+impl RegistryMetadataExt for RegistryMetadata {
+    async fn fetch_or_default(registry: &Registry) -> Self {
+        match Self::fetch(registry).await {
+            Ok(Some(meta)) => {
+                tracing::debug!(?meta, "Got registry metadata");
+                meta
+            }
+            Ok(None) => {
+                tracing::debug!("Metadata not found");
+                Default::default()
+            }
+            Err(err) => {
+                tracing::warn!(error = ?err, "Error fetching registry metadata");
+                Default::default()
+            }
+        }
+    }
+
+    async fn fetch(registry: &Registry) -> Result<Option<Self>, Error> {
+        let scheme = if registry.host() == "localhost" {
+            "http"
+        } else {
+            "https"
+        };
+        let url = format!("{scheme}://{registry}{REGISTRY_METADATA_PATH}");
+        fetch_url(&url)
+            .await
+            .with_context(|| format!("error fetching registry metadata from {url:?}"))
+            .map_err(Error::RegistryMetadataError)
+    }
+}
+
+async fn fetch_url(url: &str) -> anyhow::Result<Option<RegistryMetadata>> {
+    tracing::debug!(?url, "Fetching registry metadata");
+
+    let resp = reqwest::get(url).await?;
+    if resp.status() == StatusCode::NOT_FOUND {
+        return Ok(None);
+    }
+    let resp = resp.error_for_status()?;
+    Ok(Some(resp.json().await?))
+}

crates/wasm-pkg-client/src/metadata.rs

@@ -1,19 +1,17 @@
 use futures_util::TryStreamExt;
-use testcontainers::{
-    core::{IntoContainerPort, WaitFor},
-    runners::AsyncRunner,
-    GenericImage, ImageExt,
-};
 use wasm_pkg_client::{Client, Config};
 
 const FIXTURE_WASM: &str = "./tests/testdata/binary_wit.wasm";
 
-#[cfg(any(target_os = "linux", feature = "_local"))]
-// NOTE: These are only run on linux for CI purposes, because they rely on the docker client being
-// available, and for various reasons this has proven to be problematic on both the Windows and
-// MacOS runners due to it not being installed (yay licensing).
+#[cfg(feature = "docker-tests")]
 #[tokio::test]
 async fn publish_and_fetch_smoke_test() {
+    use testcontainers::{
+        core::{IntoContainerPort, WaitFor},
+        runners::AsyncRunner,
+        GenericImage, ImageExt,
+    };
+
     let _container = GenericImage::new("registry", "2")
         .with_wait_for(WaitFor::message_on_stderr("listening on [::]:5000"))
         .with_mapped_port(5001, 5000.tcp())

crates/wasm-pkg-client/tests/e2e.rs

@@ -9,32 +9,27 @@ license.workspace = true
 readme = "../../README.md"
 
 [features]
-metadata-client = ["dep:reqwest"]
-tokio = ["tokio/io-util"]
+registry-config = [
+    "dep:etcetera",
+    "dep:tokio",
+    "dep:toml",
+]
 # Extra features to facilitate making working with OCI images easier
 oci_extras = []
 
 [dependencies]
 anyhow = { workspace = true }
 bytes = { workspace = true }
-etcetera = { workspace = true }
+etcetera = { workspace = true, optional = true }
 futures-util = { workspace = true }
 http = "1.1.0"
-reqwest = { version = "0.12.0", default-features = false, features = [
-    "rustls-tls",
-    "charset",
-    "http2",
-    "macos-system-configuration",
-    "json",
-], optional = true }
 semver = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 sha2 = { workspace = true }
-tokio = { workspace = true, features = ["fs"] }
-toml = { workspace = true }
+tokio = { workspace = true, optional = true, features = ["fs"] }
+toml = { workspace = true, optional = true }
 thiserror = { workspace = true }
-tracing = { workspace = true }
 
 [dev-dependencies]
 tokio = { workspace = true, features = ["macros", "rt"] }

crates/wasm-pkg-common/Cargo.toml

@@ -4,7 +4,6 @@ use std::{
     path::{Path, PathBuf},
 };
 
-use etcetera::BaseStrategy;
 use serde::{Deserialize, Serialize};
 
 use crate::{
@@ -103,13 +102,6 @@ impl Config {
         Ok(config)
     }
 
-    /// Returns the default global config file location
-    pub fn global_config_path() -> Option<PathBuf> {
-        etcetera::choose_base_strategy()
-            .ok()
-            .map(|strat| strat.config_dir().join("wasm-pkg").join("config.toml"))
-    }
-
     /// Reads config from the default global config file location
     pub async fn read_global_config() -> Result<Option<Self>, Error> {
         let path = match Config::global_config_path() {
@@ -124,6 +116,14 @@ impl Config {
         Ok(Some(Self::from_toml(&contents)?))
     }
 
+    /// Returns the default global config file location
+    pub fn global_config_path() -> Option<PathBuf> {
+        use etcetera::BaseStrategy;
+        etcetera::choose_base_strategy()
+            .ok()
+            .map(|strat| strat.config_dir().join("wasm-pkg").join("config.toml"))
+    }
+
     /// Reads config from a TOML file at the given path.
     pub async fn from_file(path: impl AsRef<Path>) -> Result<Self, Error> {
         let contents = tokio::fs::read_to_string(path)
@@ -134,14 +134,13 @@ impl Config {
 
     /// Parses config from the given TOML contents.
     pub fn from_toml(contents: &str) -> Result<Self, Error> {
-        let toml_cfg: toml::TomlConfig =
-            ::toml::from_str(contents).map_err(Error::invalid_config)?;
+        let toml_cfg: toml::TomlConfig = ::toml::from_str(contents).map_err(invalid_config)?;
         Ok(toml_cfg.into())
     }
 
     /// Writes the config to a TOML file at the given path.
     pub async fn to_file(&self, path: impl AsRef<Path>) -> Result<(), Error> {
-        let toml_str = ::toml::to_string(&self).map_err(Error::invalid_config)?;
+        let toml_str = ::toml::to_string(&self).map_err(invalid_config)?;
         tokio::fs::write(path, toml_str)
             .await
             .map_err(Error::ConfigFileIoError)
@@ -330,7 +329,7 @@ impl RegistryConfig {
         let Some(table) = self.backend_configs.get(backend_type) else {
             return Ok(None);
         };
-        let config = table.clone().try_into().map_err(Error::invalid_config)?;
+        let config = table.clone().try_into().map_err(invalid_config)?;
         Ok(Some(config))
     }
 
@@ -340,7 +339,7 @@ impl RegistryConfig {
         backend_type: impl Into<String>,
         backend_config: T,
     ) -> Result<(), Error> {
-        let table = ::toml::Table::try_from(backend_config).map_err(Error::invalid_config)?;
+        let table = ::toml::Table::try_from(backend_config).map_err(invalid_config)?;
         self.backend_configs.insert(backend_type.into(), table);
         Ok(())
     }
@@ -368,3 +367,7 @@ impl std::fmt::Debug for DebugBackendConfigs<'_> {
             .finish()
     }
 }
+
+fn invalid_config(err: impl Into<anyhow::Error>) -> Error {
+    Error::InvalidConfig(err.into())
+}