diff --git a/Cargo.lock b/Cargo.lock index 8890e2e066..0719d1e36a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "RustyXML" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b5ace29ee3216de37c0546865ad08edef58b0f9e76838ed8959a84a990e58c5" + [[package]] name = "addr2line" version = "0.24.2" @@ -498,7 +504,7 @@ dependencies = [ "aws-sdk-sts", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.60.7", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -529,13 +535,14 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.4.3" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a10d5c055aa540164d9561a0e2e74ad30f0dcf7393c3a92f6733ddf9c5762468" +checksum = "bee7643696e7fdd74c10f9eb42848a87fe469d35eae9c3323f80aa98f350baac" dependencies = [ "aws-credential-types", "aws-sigv4", "aws-smithy-async", + "aws-smithy-eventstream", "aws-smithy-http", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -562,17 +569,51 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.60.7", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand 2.2.0", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-s3" +version = "1.72.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c7ce6d85596c4bcb3aba8ad5bb134b08e204c8a475c9999c1af9290f80aa8ad" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json 0.61.2", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", + "aws-smithy-xml", "aws-types", "bytes", "fastrand 2.2.0", + "hex", + "hmac", "http 0.2.12", + "http-body 0.4.6", + "lru", "once_cell", + "percent-encoding", "regex-lite", + "sha2", "tracing", + "url", ] [[package]] @@ -585,7 +626,7 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.60.7", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -607,7 +648,7 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.60.7", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -629,7 +670,7 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.60.7", "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -644,11 +685,12 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.2.5" +version = "1.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5619742a0d8f253be760bfbb8e8e8368c69e3587e4637af5754e488a611499b1" +checksum = "9bfe75fad52793ce6dec0dc3d4b1f388f038b5eb866c8d4d7f3a8e21b5ea5051" dependencies = [ "aws-credential-types", + "aws-smithy-eventstream", "aws-smithy-http", "aws-smithy-runtime-api", "aws-smithy-types", @@ -667,21 +709,55 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.1" +version = "1.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62220bc6e97f946ddd51b5f1361f78996e704677afc518a4ff66b7a72ea1378c" +checksum = "1e190749ea56f8c42bf15dd76c65e14f8f765233e6df9b0506d9d934ebef867c" dependencies = [ "futures-util", "pin-project-lite", "tokio", ] +[[package]] +name = "aws-smithy-checksums" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f45a1c384d7a393026bc5f5c177105aa9fa68e4749653b985707ac27d77295" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc32c", + "crc32fast", + "crc64fast-nvme", + "hex", + "http 0.2.12", + "http-body 0.4.6", + "md-5", + "pin-project-lite", + "sha1", + "sha2", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "461e5e02f9864cba17cff30f007c2e37ade94d01e87cdb5204e44a84e6d38c17" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + [[package]] name = "aws-smithy-http" -version = "0.60.11" +version = "0.60.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6" +checksum = "7809c27ad8da6a6a68c454e651d4962479e81472aa19ae99e59f9aba1f9713cc" dependencies = [ + "aws-smithy-eventstream", "aws-smithy-runtime-api", "aws-smithy-types", "bytes", @@ -705,6 +781,15 @@ dependencies = [ "aws-smithy-types", ] +[[package]] +name = "aws-smithy-json" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "623a51127f24c30776c8b374295f2df78d92517386f77ba30773f15a30ce1422" +dependencies = [ + "aws-smithy-types", +] + [[package]] name = "aws-smithy-query" version = "0.60.7" @@ -717,9 +802,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.7.3" +version = "1.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be28bd063fa91fd871d131fc8b68d7cd4c5fa0869bea68daca50dcb1cbd76be2" +checksum = "865f7050bbc7107a6c98a397a9fcd9413690c27fa718446967cf03b2d3ac517e" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -761,9 +846,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.9" +version = "1.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fbd94a32b3a7d55d3806fe27d98d3ad393050439dd05eb53ece36ec5e3d3510" +checksum = "c7b8a53819e42f10d0821f56da995e1470b199686a1809168db6ca485665f042" dependencies = [ "base64-simd", "bytes", @@ -796,9 +881,9 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.3.3" +version = "1.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5221b91b3e441e6675310829fd8984801b772cb1546ef6c0e54dec9f1ac13fef" +checksum = "dfbd0a668309ec1f66c0f6bda4840dd6d4796ae26d699ebc266d7cc95c6d040f" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -855,32 +940,6 @@ dependencies = [ "tower-service", ] -[[package]] -name = "azure_core" -version = "0.20.0" -source = "git+https://github.com/azure/azure-sdk-for-rust?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" -dependencies = [ - "async-trait", - "base64 0.22.1", - "bytes", - "dyn-clone", - "futures", - "getrandom 0.2.15", - "http-types", - "once_cell", - "paste", - "pin-project", - "rand 0.8.5", - "reqwest 0.12.9", - "rustc_version", - "serde", - "serde_json", - "time", - "tracing", - "url", - "uuid", -] - [[package]] name = "azure_core" version = "0.21.0" @@ -898,6 +957,7 @@ dependencies = [ "once_cell", "paste", "pin-project", + "quick-xml 0.31.0", "rand 0.8.5", "reqwest 0.12.9", "rustc_version", @@ -917,7 +977,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0aa5603f2de38c21165a1b5dfed94d64b1ab265526b0686e8557c907a53a0ee2" dependencies = [ "async-trait", - "azure_core 0.21.0", + "azure_core", "bytes", "futures", "serde", @@ -931,13 +991,14 @@ dependencies = [ [[package]] name = "azure_identity" -version = "0.20.0" -source = "git+https://github.com/azure/azure-sdk-for-rust?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ddd80344317c40c04b603807b63a5cefa532f1b43522e72f480a988141f744" dependencies = [ "async-lock", "async-process", "async-trait", - "azure_core 0.20.0", + "azure_core", "futures", "oauth2", "pin-project", @@ -950,34 +1011,70 @@ dependencies = [ ] [[package]] -name = "azure_identity" +name = "azure_security_keyvault" version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ddd80344317c40c04b603807b63a5cefa532f1b43522e72f480a988141f744" +checksum = "bd94f507b75349a0e381c0a23bd77cc654fb509f0e6797ce4f99dd959d9e2d68" +dependencies = [ + "async-trait", + "azure_core", + "futures", + "serde", + "serde_json", + "time", +] + +[[package]] +name = "azure_storage" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59f838159f4d29cb400a14d9d757578ba495ae64feb07a7516bf9e4415127126" dependencies = [ + "RustyXML", "async-lock", - "async-process", "async-trait", - "azure_core 0.21.0", + "azure_core", + "bytes", + "serde", + "serde_derive", + "time", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "azure_storage_blobs" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97e83c3636ae86d9a6a7962b2112e3b19eb3903915c50ce06ff54ff0a2e6a7e4" +dependencies = [ + "RustyXML", + "azure_core", + "azure_storage", + "azure_svc_blobstorage", + "bytes", "futures", - "oauth2", - "pin-project", "serde", + "serde_derive", + "serde_json", "time", "tracing", - "tz-rs", "url", "uuid", ] [[package]] -name = "azure_security_keyvault" -version = "0.20.0" -source = "git+https://github.com/azure/azure-sdk-for-rust?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" +name = "azure_svc_blobstorage" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e6c6f20c5611b885ba94c7bae5e02849a267381aecb8aee577e8c35ff4064c6" dependencies = [ - "async-trait", - "azure_core 0.20.0", + "azure_core", + "bytes", "futures", + "log", + "once_cell", "serde", "serde_json", "time", @@ -2013,6 +2110,30 @@ version = "0.122.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b530783809a55cb68d070e0de60cfbb3db0dc94c8850dd5725411422bedcf6bb" +[[package]] +name = "crc" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + [[package]] name = "crc32fast" version = "1.4.2" @@ -2022,6 +2143,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crc64fast-nvme" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4955638f00a809894c947f85a024020a20815b65a5eea633798ea7924edab2b3" +dependencies = [ + "crc", +] + [[package]] name = "crossbeam" version = "0.8.4" @@ -4136,6 +4266,7 @@ dependencies = [ "hyper 1.5.0", "hyper-util", "rustls 0.23.18", + "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", "tokio-rustls 0.26.0", @@ -5704,6 +5835,36 @@ dependencies = [ "memchr", ] +[[package]] +name = "object_store" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cfccb68961a56facde1163f9319e0d15743352344e7808a11795fb99698dcaf" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "chrono", + "futures", + "humantime", + "hyper 1.5.0", + "itertools 0.13.0", + "md-5", + "parking_lot", + "percent-encoding", + "quick-xml 0.37.5", + "rand 0.8.5", + "reqwest 0.12.9", + "ring", + "serde", + "serde_json", + "snafu", + "tokio", + "tracing", + "url", + "walkdir", +] + [[package]] name = "oci-client" version = "0.14.0" @@ -6710,6 +6871,26 @@ dependencies = [ "version_check", ] +[[package]] +name = "quick-xml" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quick-xml" +version = "0.37.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quinn" version = "0.11.5" @@ -7189,6 +7370,7 @@ dependencies = [ "pin-project-lite", "quinn", "rustls 0.23.18", + "rustls-native-certs 0.8.1", "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", @@ -7543,6 +7725,18 @@ dependencies = [ "security-framework 2.11.1", ] +[[package]] +name = "rustls-native-certs" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework 3.2.0", +] + [[package]] name = "rustls-pemfile" version = "1.0.4" @@ -8119,6 +8313,27 @@ dependencies = [ "version_check", ] +[[package]] +name = "snafu" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320b01e011bf8d5d7a4a4a4be966d9160968935849c83b918827f6a435e7f627" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1961e2ef424c1424204d3a5d6975f934f56b6d50ff5732382d84ebf460e147f7" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "snapbox" version = "0.4.17" @@ -8182,6 +8397,66 @@ dependencies = [ "toml", ] +[[package]] +name = "spin-blobstore-azure" +version = "3.4.0-pre0" +dependencies = [ + "anyhow", + "azure_core", + "azure_storage", + "azure_storage_blobs", + "futures", + "serde", + "spin-core", + "spin-factor-blobstore", + "tokio", + "tokio-stream", + "tokio-util", + "uuid", + "wasmtime-wasi", +] + +[[package]] +name = "spin-blobstore-fs" +version = "3.4.0-pre0" +dependencies = [ + "anyhow", + "futures", + "serde", + "spin-core", + "spin-factor-blobstore", + "tokio", + "tokio-stream", + "tokio-util", + "walkdir", + "wasmtime-wasi", +] + +[[package]] +name = "spin-blobstore-s3" +version = "3.4.0-pre0" +dependencies = [ + "anyhow", + "async-once-cell", + "aws-config", + "aws-credential-types", + "aws-sdk-s3", + "aws-smithy-async", + "bytes", + "futures", + "http-body 1.0.1", + "http-body-util", + "object_store", + "serde", + "spin-core", + "spin-factor-blobstore", + "tokio", + "tokio-stream", + "tokio-util", + "uuid", + "wasmtime-wasi", +] + [[package]] name = "spin-build" version = "3.4.0-pre0" @@ -8377,6 +8652,29 @@ dependencies = [ "toml", ] +[[package]] +name = "spin-factor-blobstore" +version = "3.4.0-pre0" +dependencies = [ + "anyhow", + "bytes", + "futures", + "lru", + "serde", + "spin-core", + "spin-factor-wasi", + "spin-factors", + "spin-factors-test", + "spin-locked-app", + "spin-resource-table", + "spin-world", + "tempfile", + "tokio", + "toml", + "tracing", + "wasmtime-wasi", +] + [[package]] name = "spin-factor-key-value" version = "3.4.0-pre0" @@ -8695,9 +8993,9 @@ version = "3.4.0-pre0" dependencies = [ "anyhow", "async-trait", - "azure_core 0.21.0", + "azure_core", "azure_data_cosmos", - "azure_identity 0.21.0", + "azure_identity", "futures", "reqwest 0.12.9", "serde", @@ -8907,8 +9205,12 @@ version = "3.4.0-pre0" dependencies = [ "anyhow", "serde", + "spin-blobstore-azure", + "spin-blobstore-fs", + "spin-blobstore-s3", "spin-common", "spin-expressions", + "spin-factor-blobstore", "spin-factor-key-value", "spin-factor-llm", "spin-factor-outbound-http", @@ -8945,6 +9247,7 @@ dependencies = [ "anyhow", "clap 3.2.25", "spin-common", + "spin-factor-blobstore", "spin-factor-key-value", "spin-factor-llm", "spin-factor-outbound-http", @@ -9145,9 +9448,10 @@ dependencies = [ name = "spin-variables-azure" version = "3.4.0-pre0" dependencies = [ - "azure_core 0.20.0", - "azure_identity 0.20.0", + "azure_core", + "azure_identity", "azure_security_keyvault", + "dotenvy", "serde", "spin-expressions", "spin-factors", @@ -9198,6 +9502,7 @@ version = "3.4.0-pre0" dependencies = [ "async-trait", "wasmtime", + "wasmtime-wasi", ] [[package]] @@ -10370,6 +10675,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" dependencies = [ "getrandom 0.2.15", + "serde", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index feaa76d388..124b5f0524 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -120,6 +120,12 @@ members = [ [workspace.dependencies] anyhow = "1" async-trait = "0.1" +azure_core = "0.21.0" +azure_data_cosmos = "0.21.0" +azure_identity = "0.21.0" +azure_security_keyvault = "0.21.0" +azure_storage = "0.21.0" +azure_storage_blobs = "0.21.0" base64 = "0.22" bytes = "1" chrono = "0.4" diff --git a/crates/blobstore-azure/Cargo.toml b/crates/blobstore-azure/Cargo.toml new file mode 100644 index 0000000000..53fcf86834 --- /dev/null +++ b/crates/blobstore-azure/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "spin-blobstore-azure" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +anyhow = { workspace = true } +azure_core = { workspace = true } +azure_storage = { workspace = true } +azure_storage_blobs = { workspace = true } +futures = { workspace = true } +serde = { workspace = true } +spin-core = { path = "../core" } +spin-factor-blobstore = { path = "../factor-blobstore" } +tokio = { workspace = true } +tokio-stream = "0.1.16" +tokio-util = { version = "0.7.12", features = ["compat"] } +uuid = { version = "1.0", features = ["v4"] } +wasmtime-wasi = { workspace = true } + +[lints] +workspace = true diff --git a/crates/blobstore-azure/src/lib.rs b/crates/blobstore-azure/src/lib.rs new file mode 100644 index 0000000000..f5169718a0 --- /dev/null +++ b/crates/blobstore-azure/src/lib.rs @@ -0,0 +1,54 @@ +mod store; + +use serde::Deserialize; +use spin_factor_blobstore::runtime_config::spin::MakeBlobStore; +use store::{ + auth::{AzureBlobAuthOptions, AzureKeyAuth}, + AzureContainerManager, +}; + +/// A key-value store that uses Azure Cosmos as the backend. +#[derive(Default)] +pub struct AzureBlobStoreBuilder { + _priv: (), +} + +impl AzureBlobStoreBuilder { + /// Creates a new `AzureBlobStoreBuilder`. + pub fn new() -> Self { + Self::default() + } +} + +/// Runtime configuration for the Azure Cosmos key-value store. +#[derive(Deserialize)] +pub struct AzureBlobStoreRuntimeConfig { + /// The authorization token for the Azure blob storage account. + key: Option, + /// The Azure blob storage account name. + account: String, +} + +impl MakeBlobStore for AzureBlobStoreBuilder { + const RUNTIME_CONFIG_TYPE: &'static str = "azure_blob"; + + type RuntimeConfig = AzureBlobStoreRuntimeConfig; + + type ContainerManager = AzureContainerManager; + + fn make_store( + &self, + runtime_config: Self::RuntimeConfig, + ) -> anyhow::Result { + let auth = match &runtime_config.key { + Some(key) => AzureBlobAuthOptions::AccountKey(AzureKeyAuth::new( + runtime_config.account.clone(), + key.clone(), + )), + None => AzureBlobAuthOptions::Environmental, + }; + + let blob_store = AzureContainerManager::new(auth)?; + Ok(blob_store) + } +} diff --git a/crates/blobstore-azure/src/store.rs b/crates/blobstore-azure/src/store.rs new file mode 100644 index 0000000000..2bb59519b7 --- /dev/null +++ b/crates/blobstore-azure/src/store.rs @@ -0,0 +1,212 @@ +use std::sync::Arc; + +use anyhow::Result; +use azure_storage_blobs::prelude::{BlobServiceClient, ContainerClient}; +use spin_core::async_trait; +use spin_factor_blobstore::{Container, ContainerManager, Error}; + +pub mod auth; +mod incoming_data; +mod object_names; + +use auth::AzureBlobAuthOptions; +use incoming_data::AzureIncomingData; +use object_names::AzureObjectNames; + +pub struct AzureContainerManager { + client: BlobServiceClient, +} + +impl AzureContainerManager { + pub fn new(auth_options: AzureBlobAuthOptions) -> Result { + let (account, credentials) = match auth_options { + AzureBlobAuthOptions::AccountKey(config) => ( + config.account.clone(), + azure_storage::StorageCredentials::access_key(&config.account, config.key.clone()), + ), + AzureBlobAuthOptions::Environmental => { + let account = std::env::var("STORAGE_ACCOUNT").expect("missing STORAGE_ACCOUNT"); + let access_key = + std::env::var("STORAGE_ACCESS_KEY").expect("missing STORAGE_ACCOUNT_KEY"); + ( + account.clone(), + azure_storage::StorageCredentials::access_key(account, access_key), + ) + } + }; + + let client = azure_storage_blobs::prelude::ClientBuilder::new(account, credentials) + .blob_service_client(); + Ok(Self { client }) + } +} + +#[async_trait] +impl ContainerManager for AzureContainerManager { + async fn get(&self, name: &str) -> Result, Error> { + Ok(Arc::new(AzureContainer { + _label: name.to_owned(), + client: self.client.container_client(name), + })) + } + + fn is_defined(&self, _store_name: &str) -> bool { + true + } +} + +struct AzureContainer { + _label: String, + client: ContainerClient, +} + +/// Azure doesn't provide us with a container creation time +const DUMMY_CREATED_AT: u64 = 0; + +#[async_trait] +impl Container for AzureContainer { + async fn exists(&self) -> anyhow::Result { + Ok(self.client.exists().await?) + } + + async fn name(&self) -> String { + self.client.container_name().to_owned() + } + + async fn info(&self) -> anyhow::Result { + let properties = self.client.get_properties().await?; + Ok(spin_factor_blobstore::ContainerMetadata { + name: properties.container.name, + created_at: DUMMY_CREATED_AT, + }) + } + + async fn clear(&self) -> anyhow::Result<()> { + anyhow::bail!("Azure blob storage does not support clearing containers") + } + + async fn delete_object(&self, name: &str) -> anyhow::Result<()> { + self.client.blob_client(name).delete().await?; + Ok(()) + } + + async fn delete_objects(&self, names: &[String]) -> anyhow::Result<()> { + // TODO: are atomic semantics required? or efficiency guarantees? + let futures = names.iter().map(|name| self.delete_object(name)); + futures::future::try_join_all(futures).await?; + Ok(()) + } + + async fn has_object(&self, name: &str) -> anyhow::Result { + Ok(self.client.blob_client(name).exists().await?) + } + + async fn object_info( + &self, + name: &str, + ) -> anyhow::Result { + let response = self.client.blob_client(name).get_properties().await?; + Ok(spin_factor_blobstore::ObjectMetadata { + name: name.to_string(), + container: self.client.container_name().to_string(), + created_at: response + .blob + .properties + .creation_time + .unix_timestamp() + .try_into() + .unwrap(), + size: response.blob.properties.content_length, + }) + } + + async fn get_data( + &self, + name: &str, + start: u64, + end: u64, + ) -> anyhow::Result> { + // We can't use a Rust range because the Azure type does not accept inclusive ranges, + // and we don't want to add 1 to `end` if it's already at MAX! + let range = if end == u64::MAX { + azure_core::request_options::Range::RangeFrom(start..) + } else { + azure_core::request_options::Range::Range(start..(end + 1)) + }; + let client = self.client.blob_client(name); + Ok(Box::new(AzureIncomingData::new(client, range))) + } + + async fn write_data( + &self, + name: &str, + data: tokio::io::ReadHalf, + finished_tx: tokio::sync::mpsc::Sender>, + ) -> anyhow::Result<()> { + let client = self.client.blob_client(name); + + tokio::spawn(async move { + let write_result = Self::write_data_core(data, client).await; + finished_tx + .send(write_result) + .await + .expect("should sent finish tx"); + }); + + Ok(()) + } + + async fn list_objects(&self) -> anyhow::Result> { + let stm = self.client.list_blobs().into_stream(); + Ok(Box::new(AzureObjectNames::new(stm))) + } +} + +impl AzureContainer { + async fn write_data_core( + mut data: tokio::io::ReadHalf, + client: azure_storage_blobs::prelude::BlobClient, + ) -> anyhow::Result<()> { + use tokio::io::AsyncReadExt; + + // Azure limits us to 50k blocks per blob. At 2MB/block that allows 100GB, which will be + // enough for most use cases. If users need flexibility for larger blobs, we could make + // the block size configurable via the runtime config ("size hint" or something). + const BLOCK_SIZE: usize = 2 * 1024 * 1024; + + let mut blocks = vec![]; + + 'put_blocks: loop { + let mut bytes = Vec::with_capacity(BLOCK_SIZE); + loop { + let read = data.read_buf(&mut bytes).await?; + let len = bytes.len(); + + if read == 0 { + // end of stream - send the last block and go + let id_bytes = uuid::Uuid::new_v4().as_bytes().to_vec(); + let block_id = azure_storage_blobs::prelude::BlockId::new(id_bytes); + client.put_block(block_id.clone(), bytes).await?; + blocks.push(azure_storage_blobs::blob::BlobBlockType::Uncommitted( + block_id, + )); + break 'put_blocks; + } + if len >= BLOCK_SIZE { + let id_bytes = uuid::Uuid::new_v4().as_bytes().to_vec(); + let block_id = azure_storage_blobs::prelude::BlockId::new(id_bytes); + client.put_block(block_id.clone(), bytes).await?; + blocks.push(azure_storage_blobs::blob::BlobBlockType::Uncommitted( + block_id, + )); + break; + } + } + } + + let block_list = azure_storage_blobs::blob::BlockList { blocks }; + client.put_block_list(block_list).await?; + + Ok(()) + } +} diff --git a/crates/blobstore-azure/src/store/auth.rs b/crates/blobstore-azure/src/store/auth.rs new file mode 100644 index 0000000000..4d818c0627 --- /dev/null +++ b/crates/blobstore-azure/src/store/auth.rs @@ -0,0 +1,27 @@ +/// Azure blob storage runtime config literal options for authentication +#[derive(Clone, Debug)] +pub struct AzureKeyAuth { + pub account: String, + pub key: String, +} + +impl AzureKeyAuth { + pub fn new(account: String, key: String) -> Self { + Self { account, key } + } +} + +/// Azure blob storage enumeration for the possible authentication options +#[derive(Clone, Debug)] +pub enum AzureBlobAuthOptions { + /// The account and key have been specified directly + AccountKey(AzureKeyAuth), + /// Spin should use the environment variables of the process to + /// create the StorageCredentials for the storage client. For now this uses old school credentials: + /// + /// STORAGE_ACCOUNT + /// STORAGE_ACCESS_KEY + /// + /// TODO: Thorsten pls make this proper with *hand waving* managed identity and stuff! + Environmental, +} diff --git a/crates/blobstore-azure/src/store/incoming_data.rs b/crates/blobstore-azure/src/store/incoming_data.rs new file mode 100644 index 0000000000..424e40ff2e --- /dev/null +++ b/crates/blobstore-azure/src/store/incoming_data.rs @@ -0,0 +1,86 @@ +use anyhow::Result; +use azure_core::Pageable; +use azure_storage_blobs::blob::operations::GetBlobResponse; +use azure_storage_blobs::prelude::BlobClient; +use futures::StreamExt; +use spin_core::async_trait; +use tokio::sync::Mutex; + +pub struct AzureIncomingData { + // The Mutex is used to make it Send + stm: Mutex>>, + client: BlobClient, +} + +impl AzureIncomingData { + pub fn new(client: BlobClient, range: azure_core::request_options::Range) -> Self { + let stm = client.get().range(range).into_stream(); + Self { + stm: Mutex::new(Some(stm)), + client, + } + } + + fn consume_async_impl(&mut self) -> wasmtime_wasi::p2::pipe::AsyncReadStream { + use futures::TryStreamExt; + use tokio_util::compat::FuturesAsyncReadCompatExt; + let stm = self.consume_as_stream(); + let ar = stm.into_async_read(); + let arr = ar.compat(); + wasmtime_wasi::p2::pipe::AsyncReadStream::new(arr) + } + + fn consume_as_stream( + &mut self, + ) -> impl futures::stream::Stream, std::io::Error>> { + let opt_stm = self.stm.get_mut(); + let stm = opt_stm.take().unwrap(); + stm.flat_map(|chunk| streamify_chunk(chunk.unwrap().data)) + } +} + +fn streamify_chunk( + chunk: azure_core::ResponseBody, +) -> impl futures::stream::Stream, std::io::Error>> { + chunk.map(|c| Ok(c.unwrap().to_vec())) +} + +#[async_trait] +impl spin_factor_blobstore::IncomingData for AzureIncomingData { + async fn consume_sync(&mut self) -> anyhow::Result> { + let mut data = vec![]; + let Some(pageable) = self.stm.get_mut() else { + anyhow::bail!("oh no"); + }; + + loop { + let Some(chunk) = pageable.next().await else { + break; + }; + let chunk = chunk.unwrap(); + let by = chunk.data.collect().await.unwrap(); + data.extend(by.to_vec()); + } + + Ok(data) + } + + fn consume_async(&mut self) -> wasmtime_wasi::p2::pipe::AsyncReadStream { + self.consume_async_impl() + } + + async fn size(&mut self) -> anyhow::Result { + // TODO: in theory this should be infallible once we have the IncomingData + // object. But in practice if we use the Pageable for that we don't get it until + // we do the first read. So that would force us to either pre-fetch the + // first chunk or to issue a properties request *just in case* size() was + // called. So I'm making it fallible for now. + Ok(self + .client + .get_properties() + .await? + .blob + .properties + .content_length) + } +} diff --git a/crates/blobstore-azure/src/store/object_names.rs b/crates/blobstore-azure/src/store/object_names.rs new file mode 100644 index 0000000000..64681f10bf --- /dev/null +++ b/crates/blobstore-azure/src/store/object_names.rs @@ -0,0 +1,83 @@ +use azure_core::Pageable; +use azure_storage_blobs::container::operations::ListBlobsResponse; +use tokio::sync::Mutex; + +use spin_core::async_trait; + +pub struct AzureObjectNames { + // The Mutex is used to make it Send + stm: Mutex>, + read_but_not_yet_returned: Vec, + end_stm_after_read_but_not_yet_returned: bool, +} + +impl AzureObjectNames { + pub fn new(stm: Pageable) -> Self { + Self { + stm: Mutex::new(stm), + read_but_not_yet_returned: Default::default(), + end_stm_after_read_but_not_yet_returned: false, + } + } + + async fn read_impl(&mut self, len: u64) -> anyhow::Result<(Vec, bool)> { + use futures::StreamExt; + + let len: usize = len.try_into().unwrap(); + + // If we have names outstanding, send that first. (We are allowed to send less than len, + // and so sending all pending stuff before paging, rather than trying to manage a mix of + // pending stuff with newly retrieved chunks, simplifies the code.) + if !self.read_but_not_yet_returned.is_empty() { + if self.read_but_not_yet_returned.len() <= len { + // We are allowed to send all pending names + let to_return = self.read_but_not_yet_returned.drain(..).collect(); + return Ok((to_return, self.end_stm_after_read_but_not_yet_returned)); + } else { + // Send as much as we can. The rest remains in the pending buffer to send, + // so this does not represent end of stream. + let to_return = self.read_but_not_yet_returned.drain(0..len).collect(); + return Ok((to_return, false)); + } + } + + // Get one chunk and send as much as we can of it. Aagin, we don't need to try to + // pack the full length here - we can send chunk by chunk. + + let Some(chunk) = self.stm.get_mut().next().await else { + return Ok((vec![], false)); + }; + let chunk = chunk.unwrap(); + + // TODO: do we need to prefix these with a prefix from somewhere or do they include it? + let mut names: Vec<_> = chunk.blobs.blobs().map(|blob| blob.name.clone()).collect(); + let at_end = chunk.next_marker.is_none(); + + if names.len() <= len { + // We can send them all! + Ok((names, at_end)) + } else { + // We have more names than we can send in this response. Send what we can and + // stash the rest. + let to_return: Vec<_> = names.drain(0..len).collect(); + self.read_but_not_yet_returned = names; + self.end_stm_after_read_but_not_yet_returned = at_end; + Ok((to_return, false)) + } + } +} + +#[async_trait] +impl spin_factor_blobstore::ObjectNames for AzureObjectNames { + async fn read(&mut self, len: u64) -> anyhow::Result<(Vec, bool)> { + self.read_impl(len).await // Separate function because rust-analyser gives better intellisense when async_trait isn't in the picture! + } + + async fn skip(&mut self, num: u64) -> anyhow::Result<(u64, bool)> { + // TODO: there is a question (raised as an issue on the repo) about the required behaviour + // here. For now I assume that skipping fewer than `num` is allowed as long as we are + // honest about it. Because it is easier that is why. + let (skipped, at_end) = self.read_impl(num).await?; + Ok((skipped.len().try_into().unwrap(), at_end)) + } +} diff --git a/crates/blobstore-fs/Cargo.toml b/crates/blobstore-fs/Cargo.toml new file mode 100644 index 0000000000..d393829b83 --- /dev/null +++ b/crates/blobstore-fs/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "spin-blobstore-fs" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +anyhow = { workspace = true } +futures = { workspace = true } +serde = { workspace = true } +spin-core = { path = "../core" } +spin-factor-blobstore = { path = "../factor-blobstore" } +tokio = { workspace = true } +tokio-stream = "0.1.16" +tokio-util = { version = "0.7.12", features = ["codec", "compat"] } +walkdir = "2.5" +wasmtime-wasi = { workspace = true } + +[lints] +workspace = true diff --git a/crates/blobstore-fs/src/lib.rs b/crates/blobstore-fs/src/lib.rs new file mode 100644 index 0000000000..9402e4a2c5 --- /dev/null +++ b/crates/blobstore-fs/src/lib.rs @@ -0,0 +1,372 @@ +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; + +use anyhow::Context; +use serde::{Deserialize, Serialize}; + +use spin_core::async_trait; +use spin_factor_blobstore::runtime_config::spin::MakeBlobStore; + +/// A blob store that uses a persistent file system volume +/// as a back end. +#[derive(Default)] +pub struct FileSystemBlobStore { + _priv: (), +} + +impl FileSystemBlobStore { + /// Creates a new `FileSystemBlobStore`. + pub fn new() -> Self { + Self::default() + } +} + +impl MakeBlobStore for FileSystemBlobStore { + const RUNTIME_CONFIG_TYPE: &'static str = "file_system"; + + type RuntimeConfig = FileSystemBlobStoreRuntimeConfig; + + type ContainerManager = BlobStoreFileSystem; + + fn make_store( + &self, + runtime_config: Self::RuntimeConfig, + ) -> anyhow::Result { + Ok(BlobStoreFileSystem::new(runtime_config.path)) + } +} + +pub struct BlobStoreFileSystem { + path: PathBuf, +} + +impl BlobStoreFileSystem { + fn new(path: PathBuf) -> Self { + Self { path } + } +} + +/// The serialized runtime configuration for the in memory blob store. +#[derive(Deserialize, Serialize)] +pub struct FileSystemBlobStoreRuntimeConfig { + path: PathBuf, +} + +#[async_trait] +impl spin_factor_blobstore::ContainerManager for BlobStoreFileSystem { + async fn get(&self, name: &str) -> Result, String> { + let container = FileSystemContainer::new(name, &self.path); + Ok(Arc::new(container)) + } + + fn is_defined(&self, _container_name: &str) -> bool { + true + } +} + +struct FileSystemContainer { + name: String, + path: PathBuf, +} + +impl FileSystemContainer { + fn new(name: &str, path: &Path) -> Self { + Self { + name: name.to_string(), + path: path.to_owned(), + } + } + + fn object_path(&self, name: &str) -> anyhow::Result { + validate_no_escape(name)?; + Ok(self.path.join(name)) + } +} + +fn validate_no_escape(name: &str) -> anyhow::Result<()> { + // TODO: this is hopelessly naive but will do for testing + if name.contains("..") { + anyhow::bail!("path tries to escape from base directory"); + } + Ok(()) +} + +#[async_trait] +impl spin_factor_blobstore::Container for FileSystemContainer { + async fn exists(&self) -> anyhow::Result { + Ok(true) + } + async fn name(&self) -> String { + self.name.clone() + } + async fn info(&self) -> anyhow::Result { + let meta = self.path.metadata()?; + let created_at = created_at_nanos(&meta)?; + + Ok(spin_factor_blobstore::ContainerMetadata { + name: self.name.to_owned(), + created_at, + }) + } + async fn clear(&self) -> anyhow::Result<()> { + let entries = std::fs::read_dir(&self.path)?.collect::>(); + + for entry in entries { + let entry = entry?; + if entry.metadata()?.is_dir() { + std::fs::remove_dir_all(entry.path())?; + } else { + std::fs::remove_file(entry.path())?; + } + } + + Ok(()) + } + async fn delete_object(&self, name: &str) -> anyhow::Result<()> { + tokio::fs::remove_file(self.object_path(name)?).await?; + Ok(()) + } + async fn delete_objects(&self, names: &[String]) -> anyhow::Result<()> { + let futs = names.iter().map(|name| self.delete_object(name)); + let results = futures::future::join_all(futs).await; + + if let Some(err_result) = results.into_iter().find(|r| r.is_err()) { + err_result + } else { + Ok(()) + } + } + async fn has_object(&self, name: &str) -> anyhow::Result { + Ok(self.object_path(name)?.exists()) + } + async fn object_info( + &self, + name: &str, + ) -> anyhow::Result { + let meta = tokio::fs::metadata(self.object_path(name)?).await?; + let created_at = created_at_nanos(&meta)?; + Ok(spin_factor_blobstore::ObjectMetadata { + name: name.to_string(), + container: self.name.to_string(), + created_at, + size: meta.len(), + }) + } + async fn get_data( + &self, + name: &str, + start: u64, + end: u64, + ) -> anyhow::Result> { + let path = self.object_path(name)?; + let file = tokio::fs::File::open(&path).await?; + + Ok(Box::new(BlobContent { + file: Some(file), + start, + end, + })) + } + + async fn write_data( + &self, + name: &str, + data: tokio::io::ReadHalf, + finished_tx: tokio::sync::mpsc::Sender>, + ) -> anyhow::Result<()> { + let path = self.object_path(name)?; + if let Some(dir) = path.parent() { + tokio::fs::create_dir_all(dir).await?; + } + let file = tokio::fs::File::create(&path).await?; + + tokio::spawn(async move { + let write_result = Self::write_data_core(data, file).await; + finished_tx + .send(write_result) + .await + .expect("shoulda sent finished_tx"); + }); + + Ok(()) + } + + async fn list_objects(&self) -> anyhow::Result> { + if !self.path.is_dir() { + anyhow::bail!( + "Backing store for {} does not exist or is not a directory", + self.name + ); + } + Ok(Box::new(BlobNames::new(&self.path))) + } +} + +impl FileSystemContainer { + async fn write_data_core( + data: tokio::io::ReadHalf, + file: tokio::fs::File, + ) -> anyhow::Result<()> { + use futures::SinkExt; + use tokio_util::codec::{BytesCodec, FramedWrite}; + + // Ceremonies to turn `file` and `data` into Sink and Stream + let mut file_sink = FramedWrite::new(file, BytesCodec::new()); + let mut data_stm = tokio_util::io::ReaderStream::new(data); + + file_sink.send_all(&mut data_stm).await?; + + Ok(()) + } +} + +struct BlobContent { + file: Option, + start: u64, + end: u64, +} + +#[async_trait] +impl spin_factor_blobstore::IncomingData for BlobContent { + async fn consume_sync(&mut self) -> anyhow::Result> { + use tokio::io::{AsyncReadExt, AsyncSeekExt}; + + let mut file = self.file.take().context("already consumed")?; + + let mut buf = Vec::with_capacity(1000); + + file.seek(std::io::SeekFrom::Start(self.start)).await?; + file.take(self.end - self.start) + .read_to_end(&mut buf) + .await?; + + Ok(buf) + } + + fn consume_async(&mut self) -> wasmtime_wasi::p2::pipe::AsyncReadStream { + use futures::StreamExt; + use futures::TryStreamExt; + use tokio_util::compat::FuturesAsyncReadCompatExt; + + let file = self.file.take().unwrap(); + let stm = tokio_util::io::ReaderStream::new(file) + .skip(self.start.try_into().unwrap()) + .take((self.end - self.start).try_into().unwrap()); + + let ar = stm.into_async_read().compat(); + wasmtime_wasi::p2::pipe::AsyncReadStream::new(ar) + } + + async fn size(&mut self) -> anyhow::Result { + let file = self.file.as_ref().context("already consumed")?; + let meta = file.metadata().await?; + Ok(meta.len()) + } +} + +struct BlobNames { + // This isn't async like tokio ReadDir, but it saves us having + // to manage state ourselves as we traverse into subdirectories. + walk_dir: Box> + Send + Sync>, + + base_path: PathBuf, +} + +impl BlobNames { + fn new(path: &Path) -> Self { + let walk_dir = walkdir::WalkDir::new(path) + .into_iter() + .filter_map(as_file_path); + Self { + walk_dir: Box::new(walk_dir), + base_path: path.to_owned(), + } + } + + fn object_name(&self, path: &Path) -> anyhow::Result { + Ok(path + .strip_prefix(&self.base_path) + .map(|p| format!("{}", p.display()))?) + } +} + +fn as_file_path( + entry: Result, +) -> Option> { + match entry { + Err(err) => Some(Err(err)), + Ok(entry) => { + if entry.file_type().is_file() { + Some(Ok(entry.into_path())) + } else { + None + } + } + } +} + +#[async_trait] +impl spin_factor_blobstore::ObjectNames for BlobNames { + async fn read(&mut self, len: u64) -> anyhow::Result<(Vec, bool)> { + let mut names = Vec::with_capacity(len.try_into().unwrap_or_default()); + let mut at_end = false; + + for _ in 0..len { + match self.walk_dir.next() { + None => { + at_end = true; + break; + } + Some(Err(e)) => { + anyhow::bail!(e); + } + Some(Ok(path)) => { + names.push(self.object_name(&path)?); + } + } + } + + // We could report "at end" when we actually just returned the last file. + // It's not worth messing around with peeking ahead because the cost to the + // guest of making a call that returns nothing is (hopefully) small. + Ok((names, at_end)) + } + + async fn skip(&mut self, num: u64) -> anyhow::Result<(u64, bool)> { + // TODO: we could save semi-duplicate code by delegating to `read`? + // The cost would be a bunch of allocation but that seems minor when + // you're dealing with the filesystem. + + let mut count = 0; + let mut at_end = false; + + for _ in 0..num { + match self.walk_dir.next() { + None => { + at_end = true; + break; + } + Some(Err(e)) => { + anyhow::bail!(e); + } + Some(Ok(_)) => { + count += 1; + } + } + } + + Ok((count, at_end)) + } +} + +fn created_at_nanos(meta: &std::fs::Metadata) -> anyhow::Result { + let time_nanos = meta + .created()? + .duration_since(std::time::SystemTime::UNIX_EPOCH)? + .as_nanos() + .try_into() + .unwrap_or_default(); + Ok(time_nanos) +} diff --git a/crates/blobstore-s3/Cargo.toml b/crates/blobstore-s3/Cargo.toml new file mode 100644 index 0000000000..b570b64296 --- /dev/null +++ b/crates/blobstore-s3/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "spin-blobstore-s3" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +anyhow = { workspace = true } +async-once-cell = "0.5.4" +# Turn off default features to avoid pulling in "aws-smithy-runtime/default-https-client" which messes up tls provider selection +aws-config = { version = "1.1.7", default-features = false, features = ["rt-tokio", "credentials-process", "sso"] } +aws-credential-types = "1.1.7" +aws-sdk-s3 = { version = "1.68", default-features = false, features = ["rustls", "rt-tokio"] } +aws-smithy-async = "1.2.5" +bytes = { workspace = true } +futures = { workspace = true } +http-body = "1.0" +http-body-util = "0.1" +object_store = { version = "0.11", features = ["aws"] } +serde = { workspace = true } +spin-core = { path = "../core" } +spin-factor-blobstore = { path = "../factor-blobstore" } +tokio = { workspace = true } +tokio-stream = "0.1.16" +tokio-util = { version = "0.7.12", features = ["compat"] } +uuid = { version = "1.0", features = ["v4"] } +wasmtime-wasi = { workspace = true } + +[lints] +workspace = true diff --git a/crates/blobstore-s3/src/lib.rs b/crates/blobstore-s3/src/lib.rs new file mode 100644 index 0000000000..db53706b59 --- /dev/null +++ b/crates/blobstore-s3/src/lib.rs @@ -0,0 +1,67 @@ +mod store; + +use serde::Deserialize; +use spin_factor_blobstore::runtime_config::spin::MakeBlobStore; +use store::S3ContainerManager; + +/// A blob store that uses a S3-compatible service as the backend. +/// This currently supports only AWS S3 +#[derive(Default)] +pub struct S3BlobStore { + _priv: (), +} + +impl S3BlobStore { + /// Creates a new `S3BlobStore`. + pub fn new() -> Self { + Self::default() + } +} + +// TODO: allow URL configuration for compatible non-AWS services + +/// Runtime configuration for the S3 blob store. +#[derive(Deserialize)] +pub struct S3BlobStoreRuntimeConfig { + /// The access key for the AWS S3 account role. + access_key: Option, + /// The secret key for authorization on the AWS S3 account. + secret_key: Option, + /// The token for authorization on the AWS S3 account. + token: Option, + /// The AWS region where the S3 account is located + region: String, + /// The name of the bucket backing the store. The default is the store label. + bucket: Option, +} + +impl MakeBlobStore for S3BlobStore { + const RUNTIME_CONFIG_TYPE: &'static str = "s3"; + + type RuntimeConfig = S3BlobStoreRuntimeConfig; + + type ContainerManager = S3ContainerManager; + + fn make_store( + &self, + runtime_config: Self::RuntimeConfig, + ) -> anyhow::Result { + let auth = match (&runtime_config.access_key, &runtime_config.secret_key) { + (Some(access_key), Some(secret_key)) => { + store::S3AuthOptions::AccessKey(store::S3KeyAuth::new( + access_key.clone(), + secret_key.clone(), + runtime_config.token.clone(), + )) + } + (None, None) => store::S3AuthOptions::Environmental, + _ => anyhow::bail!( + "either both of access_key and secret_key must be provided, or neither" + ), + }; + + let blob_store = + S3ContainerManager::new(runtime_config.region, auth, runtime_config.bucket)?; + Ok(blob_store) + } +} diff --git a/crates/blobstore-s3/src/store.rs b/crates/blobstore-s3/src/store.rs new file mode 100644 index 0000000000..50d60cb29f --- /dev/null +++ b/crates/blobstore-s3/src/store.rs @@ -0,0 +1,262 @@ +use std::sync::Arc; + +use anyhow::Result; +use spin_core::async_trait; +use spin_factor_blobstore::{Container, ContainerManager, Error}; + +mod auth; +mod incoming_data; +mod object_names; + +pub use auth::{S3AuthOptions, S3KeyAuth}; +use incoming_data::S3IncomingData; +use object_names::S3ObjectNames; + +pub struct S3ContainerManager { + builder: object_store::aws::AmazonS3Builder, + client: async_once_cell::Lazy< + aws_sdk_s3::Client, + std::pin::Pin + Send>>, + >, + bucket: Option, +} + +impl S3ContainerManager { + pub fn new( + region: String, + auth_options: S3AuthOptions, + bucket: Option, + ) -> Result { + let builder = match &auth_options { + S3AuthOptions::AccessKey(config) => object_store::aws::AmazonS3Builder::new() + .with_region(®ion) + .with_access_key_id(&config.access_key) + .with_secret_access_key(&config.secret_key) + .with_token(config.token.clone().unwrap_or_default()), + S3AuthOptions::Environmental => object_store::aws::AmazonS3Builder::from_env(), + }; + + let region_clone = region.clone(); + let client_fut = Box::pin(async move { + let sdk_config = match auth_options { + S3AuthOptions::AccessKey(config) => aws_config::SdkConfig::builder() + .credentials_provider(aws_sdk_s3::config::SharedCredentialsProvider::new( + config, + )) + .region(aws_config::Region::new(region_clone)) + .behavior_version(aws_config::BehaviorVersion::latest()) + .build(), + S3AuthOptions::Environmental => { + aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await + } + }; + aws_sdk_s3::Client::new(&sdk_config) + }); + + Ok(Self { + builder, + client: async_once_cell::Lazy::from_future(client_fut), + bucket, + }) + } +} + +#[async_trait] +impl ContainerManager for S3ContainerManager { + async fn get(&self, name: &str) -> Result, Error> { + let name = self.bucket.clone().unwrap_or_else(|| name.to_owned()); + + let store = self + .builder + .clone() + .with_bucket_name(&name) + .build() + .map_err(|e| e.to_string())?; + + Ok(Arc::new(S3Container { + name, + store, + client: self.client.get_unpin().await.clone(), + })) + } + + fn is_defined(&self, _store_name: &str) -> bool { + true + } +} + +struct S3Container { + name: String, + store: object_store::aws::AmazonS3, + client: aws_sdk_s3::Client, +} + +/// S3 doesn't provide us with a container creation time +const DUMMY_CREATED_AT: u64 = 0; + +#[async_trait] +impl Container for S3Container { + async fn exists(&self) -> anyhow::Result { + match self.client.head_bucket().bucket(&self.name).send().await { + Ok(_) => Ok(true), + Err(e) => match e.as_service_error() { + Some(se) => Ok(!se.is_not_found()), + None => anyhow::bail!(e), + }, + } + } + + async fn name(&self) -> String { + self.name.clone() + } + + async fn info(&self) -> anyhow::Result { + Ok(spin_factor_blobstore::ContainerMetadata { + name: self.name.clone(), + created_at: DUMMY_CREATED_AT, + }) + } + + async fn clear(&self) -> anyhow::Result<()> { + anyhow::bail!("AWS S3 blob storage does not support clearing containers") + } + + async fn delete_object(&self, name: &str) -> anyhow::Result<()> { + self.client + .delete_object() + .bucket(&self.name) + .key(name) + .send() + .await?; + Ok(()) + } + + async fn delete_objects(&self, names: &[String]) -> anyhow::Result<()> { + // TODO: are atomic semantics required? or efficiency guarantees? + let futures = names.iter().map(|name| self.delete_object(name)); + futures::future::try_join_all(futures).await?; + Ok(()) + } + + async fn has_object(&self, name: &str) -> anyhow::Result { + match self + .client + .head_object() + .bucket(&self.name) + .key(name) + .send() + .await + { + Ok(_) => Ok(true), + Err(e) => match e.as_service_error() { + Some(se) => Ok(!se.is_not_found()), + None => anyhow::bail!(e), + }, + } + } + + async fn object_info( + &self, + name: &str, + ) -> anyhow::Result { + let response = self + .client + .head_object() + .bucket(&self.name) + .key(name) + .send() + .await?; + Ok(spin_factor_blobstore::ObjectMetadata { + name: name.to_string(), + container: self.name.clone(), + created_at: response + .last_modified() + .and_then(|t| t.secs().try_into().ok()) + .unwrap_or(DUMMY_CREATED_AT), + size: response + .content_length + .and_then(|l| l.try_into().ok()) + .unwrap_or_default(), + }) + } + + async fn get_data( + &self, + name: &str, + start: u64, + end: u64, + ) -> anyhow::Result> { + let range = if end == u64::MAX { + format!("bytes={start}-") + } else { + format!("bytes={start}-{end}") + }; + let resp = self + .client + .get_object() + .bucket(&self.name) + .key(name) + .range(range) + .send() + .await?; + Ok(Box::new(S3IncomingData::new(resp))) + } + + async fn write_data( + &self, + name: &str, + data: tokio::io::ReadHalf, + finished_tx: tokio::sync::mpsc::Sender>, + ) -> anyhow::Result<()> { + let store = self.store.clone(); + let path = object_store::path::Path::from(name); + + tokio::spawn(async move { + let write_result = Self::write_data_core(data, store, path).await; + finished_tx + .send(write_result) + .await + .expect("should sent finish tx"); + }); + + Ok(()) + } + + async fn list_objects(&self) -> anyhow::Result> { + let stm = self + .client + .list_objects_v2() + .bucket(&self.name) + .into_paginator() + .send(); + Ok(Box::new(S3ObjectNames::new(stm))) + } +} + +impl S3Container { + async fn write_data_core( + mut data: tokio::io::ReadHalf, + store: object_store::aws::AmazonS3, + path: object_store::path::Path, + ) -> anyhow::Result<()> { + use object_store::ObjectStore; + + const BUF_SIZE: usize = 5 * 1024 * 1024; + + let mupload = store.put_multipart(&path).await?; + let mut writer = object_store::WriteMultipart::new(mupload); + loop { + use tokio::io::AsyncReadExt; + let mut buf = vec![0; BUF_SIZE]; + let read_amount = data.read(&mut buf).await?; + if read_amount == 0 { + break; + } + buf.truncate(read_amount); + writer.put(buf.into()); + } + writer.finish().await?; + + Ok(()) + } +} diff --git a/crates/blobstore-s3/src/store/auth.rs b/crates/blobstore-s3/src/store/auth.rs new file mode 100644 index 0000000000..855732bd03 --- /dev/null +++ b/crates/blobstore-s3/src/store/auth.rs @@ -0,0 +1,48 @@ +/// AWS S3 runtime config literal options for authentication +#[derive(Clone, Debug)] +pub struct S3KeyAuth { + /// The access key for the AWS S3 account role. + pub access_key: String, + /// The secret key for authorization on the AWS S3 account. + pub secret_key: String, + /// The token for authorization on the AWS S3 account. + pub token: Option, +} + +impl S3KeyAuth { + pub fn new(access_key: String, secret_key: String, token: Option) -> Self { + Self { + access_key, + secret_key, + token, + } + } +} + +impl aws_credential_types::provider::ProvideCredentials for S3KeyAuth { + fn provide_credentials<'a>( + &'a self, + ) -> aws_credential_types::provider::future::ProvideCredentials<'a> + where + Self: 'a, + { + aws_credential_types::provider::future::ProvideCredentials::ready(Ok( + aws_credential_types::Credentials::new( + self.access_key.clone(), + self.secret_key.clone(), + self.token.clone(), + None, // Optional expiration time + "spin_custom_s3_provider", + ), + )) + } +} + +/// AWS S3 authentication options +#[derive(Clone, Debug)] +pub enum S3AuthOptions { + /// The account and key have been specified directly + AccessKey(S3KeyAuth), + /// Use environment variables + Environmental, +} diff --git a/crates/blobstore-s3/src/store/incoming_data.rs b/crates/blobstore-s3/src/store/incoming_data.rs new file mode 100644 index 0000000000..dc61149405 --- /dev/null +++ b/crates/blobstore-s3/src/store/incoming_data.rs @@ -0,0 +1,67 @@ +use aws_sdk_s3::operation::get_object; + +use anyhow::Result; +use spin_core::async_trait; + +pub struct S3IncomingData { + get_obj_output: Option, +} + +impl S3IncomingData { + pub fn new(get_obj_output: get_object::GetObjectOutput) -> Self { + Self { + get_obj_output: Some(get_obj_output), + } + } + + /// Destructively takes the GetObjectOutput from self. + /// After this self will be unusable; but this cannot + /// consume self for resource lifetime reasons. + fn take_output(&mut self) -> get_object::GetObjectOutput { + self.get_obj_output + .take() + .expect("GetObject response was already consumed") + } + + fn consume_async_impl(&mut self) -> wasmtime_wasi::p2::pipe::AsyncReadStream { + use futures::TryStreamExt; + use tokio_util::compat::FuturesAsyncReadCompatExt; + let stream = self.consume_as_stream(); + let reader = stream.into_async_read().compat(); + wasmtime_wasi::p2::pipe::AsyncReadStream::new(reader) + } + + fn consume_as_stream( + &mut self, + ) -> impl futures::stream::Stream, std::io::Error>> { + use futures::StreamExt; + let get_obj_output = self.take_output(); + let reader = get_obj_output.body.into_async_read(); + let stream = tokio_util::io::ReaderStream::new(reader); + stream.map(|chunk| chunk.map(|b| b.to_vec())) + } +} + +#[async_trait] +impl spin_factor_blobstore::IncomingData for S3IncomingData { + async fn consume_sync(&mut self) -> anyhow::Result> { + let get_obj_output = self.take_output(); + Ok(get_obj_output.body.collect().await?.to_vec()) + } + + fn consume_async(&mut self) -> wasmtime_wasi::p2::pipe::AsyncReadStream { + self.consume_async_impl() + } + + async fn size(&mut self) -> anyhow::Result { + use anyhow::Context; + let goo = self + .get_obj_output + .as_ref() + .context("object was already consumed")?; + Ok(goo + .content_length() + .context("content-length not returned")? + .try_into()?) + } +} diff --git a/crates/blobstore-s3/src/store/object_names.rs b/crates/blobstore-s3/src/store/object_names.rs new file mode 100644 index 0000000000..7539c4219f --- /dev/null +++ b/crates/blobstore-s3/src/store/object_names.rs @@ -0,0 +1,101 @@ +use aws_sdk_s3::config::http::HttpResponse as AwsHttpResponse; +use aws_sdk_s3::error::SdkError; +use aws_sdk_s3::operation::list_objects_v2; +use aws_smithy_async::future::pagination_stream::PaginationStream; +use tokio::sync::Mutex; + +use anyhow::Result; +use spin_core::async_trait; + +pub struct S3ObjectNames { + stm: Mutex< + PaginationStream< + Result< + list_objects_v2::ListObjectsV2Output, + SdkError, + >, + >, + >, + read_but_not_yet_returned: Vec, + end_stm_after_read_but_not_yet_returned: bool, +} + +impl S3ObjectNames { + pub fn new( + stm: PaginationStream< + Result< + list_objects_v2::ListObjectsV2Output, + SdkError, + >, + >, + ) -> Self { + Self { + stm: Mutex::new(stm), + read_but_not_yet_returned: Default::default(), + end_stm_after_read_but_not_yet_returned: false, + } + } + + async fn read_impl(&mut self, len: u64) -> anyhow::Result<(Vec, bool)> { + let len: usize = len.try_into().unwrap(); + + // If we have names outstanding, send that first. (We are allowed to send less than len, + // and so sending all pending stuff before paging, rather than trying to manage a mix of + // pending stuff with newly retrieved chunks, simplifies the code.) + if !self.read_but_not_yet_returned.is_empty() { + if self.read_but_not_yet_returned.len() <= len { + // We are allowed to send all pending names + let to_return = self.read_but_not_yet_returned.drain(..).collect(); + return Ok((to_return, self.end_stm_after_read_but_not_yet_returned)); + } else { + // Send as much as we can. The rest remains in the pending buffer to send, + // so this does not represent end of stream. + let to_return = self.read_but_not_yet_returned.drain(0..len).collect(); + return Ok((to_return, false)); + } + } + + // Get one chunk and send as much as we can of it. Aagin, we don't need to try to + // pack the full length here - we can send chunk by chunk. + + let Some(chunk) = self.stm.get_mut().next().await else { + return Ok((vec![], false)); + }; + let chunk = chunk.unwrap(); + + let at_end = chunk.continuation_token().is_none(); + let mut names: Vec<_> = chunk + .contents + .unwrap_or_default() + .into_iter() + .flat_map(|blob| blob.key) + .collect(); + + if names.len() <= len { + // We can send them all! + Ok((names, at_end)) + } else { + // We have more names than we can send in this response. Send what we can and + // stash the rest. + let to_return: Vec<_> = names.drain(0..len).collect(); + self.read_but_not_yet_returned = names; + self.end_stm_after_read_but_not_yet_returned = at_end; + Ok((to_return, false)) + } + } +} + +#[async_trait] +impl spin_factor_blobstore::ObjectNames for S3ObjectNames { + async fn read(&mut self, len: u64) -> anyhow::Result<(Vec, bool)> { + self.read_impl(len).await // Separate function because rust-analyser gives better intellisense when async_trait isn't in the picture! + } + + async fn skip(&mut self, num: u64) -> anyhow::Result<(u64, bool)> { + // TODO: there is a question (raised as an issue on the repo) about the required behaviour + // here. For now I assume that skipping fewer than `num` is allowed as long as we are + // honest about it. Because it is easier that is why. + let (skipped, at_end) = self.read_impl(num).await?; + Ok((skipped.len().try_into().unwrap(), at_end)) + } +} diff --git a/crates/factor-blobstore/Cargo.toml b/crates/factor-blobstore/Cargo.toml new file mode 100644 index 0000000000..ae329bb539 --- /dev/null +++ b/crates/factor-blobstore/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "spin-factor-blobstore" +version = { workspace = true } +authors = { workspace = true } +edition = { workspace = true } + +[dependencies] +anyhow = { workspace = true } +bytes = { workspace = true } +futures = { workspace = true } +lru = "0.12" +serde = { workspace = true } +spin-core = { path = "../core" } +spin-factor-wasi = { path = "../factor-wasi" } +spin-factors = { path = "../factors" } +spin-locked-app = { path = "../locked-app" } +spin-resource-table = { path = "../table" } +spin-world = { path = "../world" } +tokio = { workspace = true, features = ["macros", "sync", "rt", "io-util"] } +toml = { workspace = true } +tracing = { workspace = true } +wasmtime-wasi = { workspace = true } + +[dev-dependencies] +spin-factors-test = { path = "../factors-test" } +tempfile = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt"] } + +[lints] +workspace = true diff --git a/crates/factor-blobstore/src/host.rs b/crates/factor-blobstore/src/host.rs new file mode 100644 index 0000000000..a6249388e9 --- /dev/null +++ b/crates/factor-blobstore/src/host.rs @@ -0,0 +1,195 @@ +use anyhow::Result; +use spin_core::wasmtime::component::ResourceTable; +use spin_core::{async_trait, wasmtime::component::Resource}; +use spin_resource_table::Table; +use spin_world::wasi::blobstore::{self as bs}; +use std::{collections::HashSet, sync::Arc}; +use tokio::io::{ReadHalf, SimplexStream}; +use tokio::sync::mpsc; +use tokio::sync::RwLock; + +pub use bs::types::Error; + +mod container; +mod incoming_value; +mod object_names; +mod outgoing_value; + +pub(crate) use outgoing_value::OutgoingValue; + +use crate::DelegatingContainerManager; + +// TODO: I feel like the notions of "container" and "container manager" are muddled. +// This was kinda modelled on the KV StoreManager but I am not sure it has worked. +// A "container manager" actually manages only one container, making the `get` and +// `is_defined` functions seemingly redundant. More clarity and better definition +// is needed here, although the existing code does work! +// +// Part of the trouble is, I think, that the WIT has operations for "create container" +// etc. which implies a level above "container" but whose semantics are very poorly +// defined (the implication in the WIT is that a `blobstore` implementation backs +// onto exactly one provider, and if you need to deal with multiple providers then +// you need to do some double-import trickery, which does not seem right). Clarification +// sought via https://github.com/WebAssembly/wasi-blobstore/issues/27, so we may need +// to do some rework once the authors define it more fully. + +/// Allows obtaining a container. The only interesting implementation is +/// [DelegatingContainerManager] (which is what [BlobStoreDispatch] uses); +/// other implementations currently manage only one container. (See comments.) +#[async_trait] +pub trait ContainerManager: Sync + Send { + async fn get(&self, name: &str) -> Result, Error>; + fn is_defined(&self, container_name: &str) -> bool; +} + +/// A container. This represents the system or network resource defined by +/// a label mapping in the runtime config, e.g. a file system directory, +/// Azure blob storage account, or S3 bucket. This trait is implemented +/// by providers; it is the interface through which the [BlobStoreDispatch] +/// WASI host talks to the different implementations. +#[async_trait] +pub trait Container: Sync + Send { + async fn exists(&self) -> anyhow::Result; + async fn name(&self) -> String; + async fn info(&self) -> anyhow::Result; + async fn clear(&self) -> anyhow::Result<()>; + async fn delete_object(&self, name: &str) -> anyhow::Result<()>; + async fn delete_objects(&self, names: &[String]) -> anyhow::Result<()>; + async fn has_object(&self, name: &str) -> anyhow::Result; + async fn object_info(&self, name: &str) -> anyhow::Result; + async fn get_data( + &self, + name: &str, + start: u64, + end: u64, + ) -> anyhow::Result>; + async fn write_data( + &self, + name: &str, + data: ReadHalf, + finished_tx: mpsc::Sender>, + ) -> anyhow::Result<()>; + async fn list_objects(&self) -> anyhow::Result>; +} + +/// An interface implemented by providers when listing objects. +#[async_trait] +pub trait ObjectNames: Send + Sync { + async fn read(&mut self, len: u64) -> anyhow::Result<(Vec, bool)>; + async fn skip(&mut self, num: u64) -> anyhow::Result<(u64, bool)>; +} + +/// The content of a blob being read from a container. Called by the host to +/// handle WIT incoming-value methods, and implemented by providers. +/// providers +#[async_trait] +pub trait IncomingData: Send + Sync { + async fn consume_sync(&mut self) -> anyhow::Result>; + fn consume_async(&mut self) -> wasmtime_wasi::p2::pipe::AsyncReadStream; + async fn size(&mut self) -> anyhow::Result; +} + +/// Implements all the WIT host interfaces for wasi-blobstore. +pub struct BlobStoreDispatch<'a> { + allowed_containers: &'a HashSet, + manager: &'a DelegatingContainerManager, + wasi_resources: &'a mut ResourceTable, + containers: &'a RwLock>>, + incoming_values: &'a RwLock>>, + outgoing_values: &'a RwLock>, + object_names: &'a RwLock>>, +} + +impl<'a> BlobStoreDispatch<'a> { + pub(crate) fn new( + allowed_containers: &'a HashSet, + manager: &'a DelegatingContainerManager, + wasi_resources: &'a mut ResourceTable, + containers: &'a RwLock>>, + incoming_values: &'a RwLock>>, + outgoing_values: &'a RwLock>, + object_names: &'a RwLock>>, + ) -> Self { + Self { + allowed_containers, + manager, + wasi_resources, + containers, + incoming_values, + outgoing_values, + object_names, + } + } + + pub fn allowed_containers(&self) -> &HashSet { + self.allowed_containers + } + + async fn take_incoming_value( + &mut self, + resource: Resource, + ) -> Result, String> { + self.incoming_values + .write() + .await + .remove(resource.rep()) + .ok_or_else(|| "invalid incoming-value resource".to_string()) + } +} + +impl bs::blobstore::Host for BlobStoreDispatch<'_> { + async fn create_container( + &mut self, + _name: String, + ) -> Result, String> { + Err("This version of Spin does not support creating containers".to_owned()) + } + + async fn get_container( + &mut self, + name: String, + ) -> Result, String> { + if self.allowed_containers.contains(&name) { + let container = self.manager.get(&name).await?; + let rep = self.containers.write().await.push(container).unwrap(); + Ok(Resource::new_own(rep)) + } else { + Err(format!("Container {name:?} not defined or access denied")) + } + } + + async fn delete_container(&mut self, _name: String) -> Result<(), String> { + Err("This version of Spin does not support deleting containers".to_owned()) + } + + async fn container_exists(&mut self, name: String) -> Result { + if self.allowed_containers.contains(&name) { + let container = self.manager.get(&name).await?; + container.exists().await.map_err(|e| e.to_string()) + } else { + Ok(false) + } + } + + async fn copy_object( + &mut self, + _src: bs::blobstore::ObjectId, + _dest: bs::blobstore::ObjectId, + ) -> Result<(), String> { + Err("This version of Spin does not support copying objects".to_owned()) + } + + async fn move_object( + &mut self, + _src: bs::blobstore::ObjectId, + _dest: bs::blobstore::ObjectId, + ) -> Result<(), String> { + Err("This version of Spin does not support moving objects".to_owned()) + } +} + +impl bs::types::Host for BlobStoreDispatch<'_> { + fn convert_error(&mut self, error: String) -> anyhow::Result { + Ok(error) + } +} diff --git a/crates/factor-blobstore/src/host/container.rs b/crates/factor-blobstore/src/host/container.rs new file mode 100644 index 0000000000..ba39a430b8 --- /dev/null +++ b/crates/factor-blobstore/src/host/container.rs @@ -0,0 +1,154 @@ +use anyhow::Result; +use spin_core::wasmtime::component::Resource; +use spin_world::wasi::blobstore::{self as bs}; + +use super::BlobStoreDispatch; + +impl bs::container::Host for BlobStoreDispatch<'_> {} + +impl bs::container::HostContainer for BlobStoreDispatch<'_> { + async fn name(&mut self, self_: Resource) -> Result { + let lock = self.containers.read().await; + let container = lock + .get(self_.rep()) + .ok_or_else(|| "invalid container resource".to_string())?; + Ok(container.name().await) + } + + async fn info( + &mut self, + self_: Resource, + ) -> Result { + let lock = self.containers.read().await; + let container = lock + .get(self_.rep()) + .ok_or_else(|| "invalid container resource".to_string())?; + container.info().await.map_err(|e| e.to_string()) + } + + async fn get_data( + &mut self, + self_: Resource, + name: bs::container::ObjectName, + start: u64, + end: u64, + ) -> Result, String> { + let lock = self.containers.read().await; + let container = lock + .get(self_.rep()) + .ok_or_else(|| "invalid container resource".to_string())?; + let incoming = container + .get_data(&name, start, end) + .await + .map_err(|e| e.to_string())?; + let rep = self.incoming_values.write().await.push(incoming).unwrap(); + Ok(Resource::new_own(rep)) + } + + async fn write_data( + &mut self, + self_: Resource, + name: bs::container::ObjectName, + data: Resource, + ) -> Result<(), String> { + let lock_c = self.containers.read().await; + let container = lock_c + .get(self_.rep()) + .ok_or_else(|| "invalid container resource".to_string())?; + let mut lock_ov = self.outgoing_values.write().await; + let outgoing = lock_ov + .get_mut(data.rep()) + .ok_or_else(|| "invalid outgoing-value resource".to_string())?; + + let (stm, finished_tx) = outgoing.take_read_stream().map_err(|e| e.to_string())?; + container + .write_data(&name, stm, finished_tx) + .await + .map_err(|e| e.to_string())?; + + Ok(()) + } + + async fn list_objects( + &mut self, + self_: Resource, + ) -> Result, String> { + let lock = self.containers.read().await; + let container = lock + .get(self_.rep()) + .ok_or_else(|| "invalid container resource".to_string())?; + let names = container.list_objects().await.map_err(|e| e.to_string())?; + let rep = self.object_names.write().await.push(names).unwrap(); + Ok(Resource::new_own(rep)) + } + + async fn delete_object( + &mut self, + self_: Resource, + name: String, + ) -> Result<(), String> { + let lock = self.containers.read().await; + let container = lock + .get(self_.rep()) + .ok_or_else(|| "invalid container resource".to_string())?; + container + .delete_object(&name) + .await + .map_err(|e| e.to_string()) + } + + async fn delete_objects( + &mut self, + self_: Resource, + names: Vec, + ) -> Result<(), String> { + let lock = self.containers.read().await; + let container = lock + .get(self_.rep()) + .ok_or_else(|| "invalid container resource".to_string())?; + container + .delete_objects(&names) + .await + .map_err(|e| e.to_string()) + } + + async fn has_object( + &mut self, + self_: Resource, + name: String, + ) -> Result { + let lock = self.containers.read().await; + let container = lock + .get(self_.rep()) + .ok_or_else(|| "invalid container resource".to_string())?; + container.has_object(&name).await.map_err(|e| e.to_string()) + } + + async fn object_info( + &mut self, + self_: Resource, + name: String, + ) -> Result { + let lock = self.containers.read().await; + let container = lock + .get(self_.rep()) + .ok_or_else(|| "invalid container resource".to_string())?; + container + .object_info(&name) + .await + .map_err(|e| e.to_string()) + } + + async fn clear(&mut self, self_: Resource) -> Result<(), String> { + let lock = self.containers.read().await; + let container = lock + .get(self_.rep()) + .ok_or_else(|| "invalid container resource".to_string())?; + container.clear().await.map_err(|e| e.to_string()) + } + + async fn drop(&mut self, rep: Resource) -> anyhow::Result<()> { + self.containers.write().await.remove(rep.rep()); + Ok(()) + } +} diff --git a/crates/factor-blobstore/src/host/incoming_value.rs b/crates/factor-blobstore/src/host/incoming_value.rs new file mode 100644 index 0000000000..0fc7cd3511 --- /dev/null +++ b/crates/factor-blobstore/src/host/incoming_value.rs @@ -0,0 +1,44 @@ +use spin_core::wasmtime::component::Resource; +use spin_world::wasi::blobstore::{self as bs}; +use wasmtime_wasi::p2::bindings::io::streams::InputStream; +use wasmtime_wasi::p2::InputStream as HostInputStream; + +use super::BlobStoreDispatch; + +impl bs::types::HostIncomingValue for BlobStoreDispatch<'_> { + async fn incoming_value_consume_sync( + &mut self, + self_: Resource, + ) -> Result, String> { + let mut incoming = self.take_incoming_value(self_).await?; + incoming + .as_mut() + .consume_sync() + .await + .map_err(|e| e.to_string()) + } + + async fn incoming_value_consume_async( + &mut self, + self_: Resource, + ) -> Result, String> { + let mut incoming = self.take_incoming_value(self_).await?; + let async_body = incoming.as_mut().consume_async(); + let input_stream: Box = Box::new(async_body); + let resource = self.wasi_resources.push(input_stream).unwrap(); + Ok(resource) + } + + async fn size(&mut self, self_: Resource) -> anyhow::Result { + let mut lock = self.incoming_values.write().await; + let incoming = lock + .get_mut(self_.rep()) + .ok_or_else(|| anyhow::anyhow!("invalid incoming-value resource"))?; + incoming.size().await + } + + async fn drop(&mut self, rep: Resource) -> anyhow::Result<()> { + self.incoming_values.write().await.remove(rep.rep()); + Ok(()) + } +} diff --git a/crates/factor-blobstore/src/host/object_names.rs b/crates/factor-blobstore/src/host/object_names.rs new file mode 100644 index 0000000000..c9a49a33bd --- /dev/null +++ b/crates/factor-blobstore/src/host/object_names.rs @@ -0,0 +1,35 @@ +use spin_core::wasmtime::component::Resource; +use spin_world::wasi::blobstore::container::{HostStreamObjectNames, StreamObjectNames}; + +use super::BlobStoreDispatch; + +impl HostStreamObjectNames for BlobStoreDispatch<'_> { + async fn read_stream_object_names( + &mut self, + self_: Resource, + len: u64, + ) -> Result<(Vec, bool), String> { + let mut lock = self.object_names.write().await; + let object_names = lock + .get_mut(self_.rep()) + .ok_or_else(|| "invalid stream-object-names resource".to_string())?; + object_names.read(len).await.map_err(|e| e.to_string()) + } + + async fn skip_stream_object_names( + &mut self, + self_: Resource, + num: u64, + ) -> Result<(u64, bool), String> { + let mut lock = self.object_names.write().await; + let object_names = lock + .get_mut(self_.rep()) + .ok_or_else(|| "invalid stream-object-names resource".to_string())?; + object_names.skip(num).await.map_err(|e| e.to_string()) + } + + async fn drop(&mut self, rep: Resource) -> anyhow::Result<()> { + self.object_names.write().await.remove(rep.rep()); + Ok(()) + } +} diff --git a/crates/factor-blobstore/src/host/outgoing_value.rs b/crates/factor-blobstore/src/host/outgoing_value.rs new file mode 100644 index 0000000000..62e49e4989 --- /dev/null +++ b/crates/factor-blobstore/src/host/outgoing_value.rs @@ -0,0 +1,117 @@ +use spin_core::wasmtime::component::Resource; +use spin_world::wasi::blobstore::types::HostOutgoingValue; +use spin_world::wasi::blobstore::{self as bs}; +use tokio::io::{ReadHalf, SimplexStream, WriteHalf}; +use tokio::sync::mpsc; + +use super::BlobStoreDispatch; + +pub struct OutgoingValue { + read: Option>, + write: Option>, + stop_tx: Option>, + finished_rx: Option>>, +} + +const OUTGOING_VALUE_BUF_SIZE: usize = 16 * 1024; + +impl OutgoingValue { + fn new() -> Self { + let (read, write) = tokio::io::simplex(OUTGOING_VALUE_BUF_SIZE); + Self { + read: Some(read), + write: Some(write), + stop_tx: None, + finished_rx: None, + } + } + + fn write_stream(&mut self) -> anyhow::Result { + let Some(write) = self.write.take() else { + anyhow::bail!("OutgoingValue has already returned its write stream"); + }; + + let (stop_tx, stop_rx) = mpsc::channel(1); + + self.stop_tx = Some(stop_tx); + + let stm = crate::AsyncWriteStream::new_closeable(OUTGOING_VALUE_BUF_SIZE, write, stop_rx); + Ok(stm) + } + + fn syncers( + &mut self, + ) -> ( + Option<&mpsc::Sender<()>>, + Option<&mut mpsc::Receiver>>, + ) { + (self.stop_tx.as_ref(), self.finished_rx.as_mut()) + } + + pub(crate) fn take_read_stream( + &mut self, + ) -> anyhow::Result<(ReadHalf, mpsc::Sender>)> { + let Some(read) = self.read.take() else { + anyhow::bail!("OutgoingValue has already been connected to a blob"); + }; + + let (finished_tx, finished_rx) = mpsc::channel(1); + self.finished_rx = Some(finished_rx); + + Ok((read, finished_tx)) + } +} + +impl HostOutgoingValue for BlobStoreDispatch<'_> { + async fn new_outgoing_value(&mut self) -> anyhow::Result> { + let outgoing_value = OutgoingValue::new(); + let rep = self + .outgoing_values + .write() + .await + .push(outgoing_value) + .unwrap(); + Ok(Resource::new_own(rep)) + } + + async fn outgoing_value_write_body( + &mut self, + self_: Resource, + ) -> anyhow::Result, ()>> + { + let mut lock = self.outgoing_values.write().await; + let outgoing = lock + .get_mut(self_.rep()) + .ok_or_else(|| anyhow::anyhow!("invalid outgoing-value resource"))?; + let stm = outgoing.write_stream()?; + + let host_stm: Box = Box::new(stm); + let resource = self.wasi_resources.push(host_stm).unwrap(); + + Ok(Ok(resource)) + } + + async fn finish(&mut self, self_: Resource) -> Result<(), String> { + let mut lock = self.outgoing_values.write().await; + let outgoing = lock + .get_mut(self_.rep()) + .ok_or_else(|| "invalid outgoing-value resource".to_string())?; + // Separate methods cause "mutable borrow while immutably borrowed" so get it all in one go + let (stop_tx, finished_rx) = outgoing.syncers(); + let stop_tx = stop_tx.expect("shoulda had a stop_tx"); + let finished_rx = finished_rx.expect("shoulda had a finished_rx"); + + stop_tx.send(()).await.expect("shoulda sent a stop"); + let result = finished_rx.recv().await; + + match result { + None | Some(Ok(())) => Ok(()), + Some(Err(e)) => Err(format!("{e}")), + } + } + + async fn drop(&mut self, rep: Resource) -> anyhow::Result<()> { + self.outgoing_values.write().await.remove(rep.rep()); + Ok(()) + } +} diff --git a/crates/factor-blobstore/src/lib.rs b/crates/factor-blobstore/src/lib.rs new file mode 100644 index 0000000000..5c4a66e41f --- /dev/null +++ b/crates/factor-blobstore/src/lib.rs @@ -0,0 +1,215 @@ +//! Example usage: +//! +//! -------------------- +//! +//! spin.toml: +//! +//! [component.foo] +//! blob_containers = ["default"] +//! +//! -------------------- +//! +//! runtime-config.toml +//! +//! [blob_store.default] +//! type = "file_system" | "s3" | "azure_blob" +//! # further config settings per type +//! +//! -------------------- +//! +//! TODO: the naming here is not very consistent and we should make a more conscious +//! decision about whether these things are "blob stores" or "containers" or what + +mod host; +pub mod runtime_config; +mod stream; +mod util; + +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; + +use anyhow::ensure; +use spin_factors::{ConfigureAppContext, Factor, InitContext, PrepareContext, RuntimeFactors}; +use spin_locked_app::MetadataKey; +use spin_resource_table::Table; + +pub use host::{BlobStoreDispatch, Container, ContainerManager, Error, IncomingData, ObjectNames}; +pub use runtime_config::RuntimeConfig; +pub use spin_world::wasi::blobstore::types::{ContainerMetadata, ObjectMetadata}; +pub use stream::AsyncWriteStream; +use tokio::sync::RwLock; +pub use util::DelegatingContainerManager; + +/// Lockfile metadata key for blob stores. +pub const BLOB_CONTAINERS_KEY: MetadataKey> = MetadataKey::new("blob_containers"); + +/// A factor that provides blob storage. +#[derive(Default)] +pub struct BlobStoreFactor { + _priv: (), +} + +impl BlobStoreFactor { + /// Create a new BlobStoreFactor. + pub fn new() -> Self { + Self { _priv: () } + } +} + +struct HasBlobStore; + +impl spin_core::wasmtime::component::HasData for HasBlobStore { + type Data<'a> = BlobStoreDispatch<'a>; +} + +fn get_blob_store(t: &mut T::StoreData) -> BlobStoreDispatch<'_> +where + T: InitContext + ?Sized, +{ + let (state, table) = T::get_data_with_table(t); + + BlobStoreDispatch::new( + &state.allowed_containers, + &state.container_manager, + table, + &state.containers, + &state.incoming_values, + &state.outgoing_values, + &state.object_names, + ) +} + +trait InitContextExt: InitContext { + fn link_blob_store_interfaces(&mut self) -> anyhow::Result<()> { + spin_world::wasi::blobstore::blobstore::add_to_linker::( + self.linker(), + get_blob_store::, + )?; + spin_world::wasi::blobstore::container::add_to_linker::( + self.linker(), + get_blob_store::, + )?; + spin_world::wasi::blobstore::types::add_to_linker::( + self.linker(), + get_blob_store::, + )?; + Ok(()) + } +} + +impl> InitContextExt for T {} + +impl Factor for BlobStoreFactor { + type RuntimeConfig = RuntimeConfig; + type AppState = AppState; + type InstanceBuilder = InstanceBuilder; + + fn init(&mut self, ctx: &mut impl InitContext) -> anyhow::Result<()> { + ctx.link_blob_store_interfaces()?; + + Ok(()) + } + + fn configure_app( + &self, + mut ctx: ConfigureAppContext, + ) -> anyhow::Result { + let runtime_config = ctx.take_runtime_config().unwrap_or_default(); + + let delegating_manager = DelegatingContainerManager::new(runtime_config); + let container_manager = Arc::new(delegating_manager); + + // Build component -> allowed containers map + let mut component_allowed_containers = HashMap::new(); + for component in ctx.app().components() { + let component_id = component.id().to_string(); + let containers = component + .get_metadata(BLOB_CONTAINERS_KEY)? + .unwrap_or_default() + .into_iter() + .collect::>(); + for label in &containers { + ensure!( + container_manager.is_defined(label), + "unknown {} label {label:?} for component {component_id:?}", + BLOB_CONTAINERS_KEY.as_ref(), + ); + } + component_allowed_containers.insert(component_id, Arc::new(containers)); + } + + Ok(AppState { + container_manager, + component_allowed_containers, + }) + } + + fn prepare( + &self, + ctx: PrepareContext, + ) -> anyhow::Result { + let app_state = ctx.app_state(); + let allowed_containers = app_state + .component_allowed_containers + .get(ctx.app_component().id()) + .expect("component should be in component_allowed_containers") + .clone(); + let capacity = u32::MAX; + Ok(InstanceBuilder { + container_manager: app_state.container_manager.clone(), + allowed_containers, + containers: Arc::new(RwLock::new(Table::new(capacity))), + incoming_values: Arc::new(RwLock::new(Table::new(capacity))), + object_names: Arc::new(RwLock::new(Table::new(capacity))), + outgoing_values: Arc::new(RwLock::new(Table::new(capacity))), + }) + } +} + +pub struct AppState { + /// The container manager for the app. + container_manager: Arc, + /// The allowed containers for each component. + /// + /// This is a map from component ID to the set of container labels that the + /// component is allowed to use. + component_allowed_containers: HashMap>>, +} + +pub struct InstanceBuilder { + /// The container manager for the app. This contains *all* container mappings. + container_manager: Arc, + /// The allowed containers for this component instance. + allowed_containers: Arc>, + /// There are multiple WASI interfaces in play here. The factor adds each of them + /// to the linker, passing a closure that derives the interface implementation + /// from the InstanceBuilder. + /// + /// For the different interfaces to agree on their resource tables, each closure + /// needs to derive the same resource table from the InstanceBuilder. + /// So the InstanceBuilder (which is also the instance state) sets up all the resource + /// tables and RwLocks them, then the dispatch object borrows them. + containers: Arc>>>, + incoming_values: Arc>>>, + outgoing_values: Arc>>, + object_names: Arc>>>, +} + +impl spin_factors::SelfInstanceBuilder for InstanceBuilder { + // type InstanceState = BlobStoreDispatch; + + // fn build(self) -> anyhow::Result { + // let blobstore = BlobStoreDispatch::new( + // self.allowed_containers, + // self.container_manager, + // todo!(), + // self.containers, + // self.incoming_values, + // self.outgoing_values, + // self.object_names, + // ); + // Ok(blobstore) + // } +} diff --git a/crates/factor-blobstore/src/runtime_config.rs b/crates/factor-blobstore/src/runtime_config.rs new file mode 100644 index 0000000000..9abbc938bf --- /dev/null +++ b/crates/factor-blobstore/src/runtime_config.rs @@ -0,0 +1,44 @@ +pub mod spin; + +use std::{collections::HashMap, sync::Arc}; + +use crate::ContainerManager; + +/// Runtime configuration for all blob containers. +#[derive(Default, Clone)] +pub struct RuntimeConfig { + /// Map of container names to container managers. + container_managers: HashMap>, +} + +impl RuntimeConfig { + /// Adds a container manager for the container with the given label to the runtime configuration. + /// + /// If a container manager already exists for the given label, it will be replaced. + pub fn add_container_manager( + &mut self, + label: String, + container_manager: Arc, + ) { + self.container_managers.insert(label, container_manager); + } + + /// Returns whether a container manager exists for the given label. + pub fn has_container_manager(&self, label: &str) -> bool { + self.container_managers.contains_key(label) + } + + /// Returns the container manager for the container with the given label. + pub fn get_container_manager(&self, label: &str) -> Option> { + self.container_managers.get(label).cloned() + } +} + +impl IntoIterator for RuntimeConfig { + type Item = (String, Arc); + type IntoIter = std::collections::hash_map::IntoIter>; + + fn into_iter(self) -> Self::IntoIter { + self.container_managers.into_iter() + } +} diff --git a/crates/factor-blobstore/src/runtime_config/spin.rs b/crates/factor-blobstore/src/runtime_config/spin.rs new file mode 100644 index 0000000000..1f709e0e40 --- /dev/null +++ b/crates/factor-blobstore/src/runtime_config/spin.rs @@ -0,0 +1,135 @@ +//! Runtime configuration implementation used by Spin CLI. + +use crate::{ContainerManager, RuntimeConfig}; +use anyhow::Context as _; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use spin_factors::runtime_config::toml::GetTomlValue; +use std::{collections::HashMap, sync::Arc}; + +/// Defines the construction of a blob store from a serialized runtime config. +pub trait MakeBlobStore: 'static + Send + Sync { + /// Unique type identifier for the store. + const RUNTIME_CONFIG_TYPE: &'static str; + /// Runtime configuration for the store. + type RuntimeConfig: DeserializeOwned; + /// The store manager for the store. + type ContainerManager: ContainerManager; + + /// Creates a new store manager from the runtime configuration. + fn make_store( + &self, + runtime_config: Self::RuntimeConfig, + ) -> anyhow::Result; +} + +/// A function that creates a container manager from a TOML table. +type StoreFromToml = + Arc anyhow::Result> + Send + Sync>; + +/// Creates a `StoreFromToml` function from a `MakeBlobStore` implementation. +fn store_from_toml_fn(provider_type: T) -> StoreFromToml { + Arc::new(move |table| { + let runtime_config: T::RuntimeConfig = table + .try_into() + .context("could not parse blobstore runtime config")?; + let provider = provider_type + .make_store(runtime_config) + .context("could not make blobstore from runtime config")?; + Ok(Arc::new(provider)) + }) +} + +/// Converts from toml based runtime configuration into a [`RuntimeConfig`]. +/// +/// The various container types (i.e., the "type" field in the toml field) are registered with the +/// resolver using `add_store_type`. The default store for a label is registered using `add_default_store`. +#[derive(Default, Clone)] +pub struct RuntimeConfigResolver { + /// A map of store types to a function that returns the appropriate store + /// manager from runtime config TOML. + store_types: HashMap<&'static str, StoreFromToml>, +} + +impl RuntimeConfigResolver { + /// Create a new RuntimeConfigResolver. + pub fn new() -> Self { + ::default() + } + + /// Registers a store type to the resolver. + pub fn register_store_type(&mut self, store_type: T) -> anyhow::Result<()> { + if self + .store_types + .insert(T::RUNTIME_CONFIG_TYPE, store_from_toml_fn(store_type)) + .is_some() + { + anyhow::bail!( + "duplicate key value store type {:?}", + T::RUNTIME_CONFIG_TYPE + ); + } + Ok(()) + } + + /// Resolves a toml table into a runtime config. + pub fn resolve(&self, table: Option<&impl GetTomlValue>) -> anyhow::Result { + let runtime_config = self.resolve_from_toml(table)?.unwrap_or_default(); + Ok(runtime_config) + } + + fn resolve_from_toml( + &self, + table: Option<&impl GetTomlValue>, + ) -> anyhow::Result> { + let Some(table) = table.and_then(|t| t.get("blob_store")) else { + return Ok(None); + }; + let table: HashMap = table.clone().try_into()?; + + let mut runtime_config = RuntimeConfig::default(); + for (label, config) in table { + let store_manager = self + .container_manager_from_config(config) + .with_context(|| format!("could not configure blob store with label '{label}'"))?; + runtime_config.add_container_manager(label.clone(), store_manager); + } + + Ok(Some(runtime_config)) + } + + /// Given a [`ContainerConfig`], returns a container manager. + /// + /// Errors if there is no [`MakeBlobStore`] registered for the container config's type + /// or if the container manager cannot be created from the config. + fn container_manager_from_config( + &self, + config: ContainerConfig, + ) -> anyhow::Result> { + let config_type = config.type_.as_str(); + let maker = self.store_types.get(config_type).with_context(|| { + format!("the store type '{config_type}' was not registered with the config resolver") + })?; + maker(config.config) + } +} + +#[derive(Deserialize, Clone)] +pub struct ContainerConfig { + #[serde(rename = "type")] + pub type_: String, + #[serde(flatten)] + pub config: toml::Table, +} + +impl ContainerConfig { + pub fn new(type_: String, config: T) -> anyhow::Result + where + T: Serialize, + { + Ok(Self { + type_, + config: toml::value::Table::try_from(config)?, + }) + } +} diff --git a/crates/factor-blobstore/src/stream/mod.rs b/crates/factor-blobstore/src/stream/mod.rs new file mode 100644 index 0000000000..33075b5f0b --- /dev/null +++ b/crates/factor-blobstore/src/stream/mod.rs @@ -0,0 +1,15 @@ +//! Adapts the WASI streams to allow closing without resource mapping. +//! +//! The solution that the Wasmtime/WASI folks advice is to map your child +//! resources to a custom type, and have the parent "close child" function +//! get the child resource and call a suitable function to termimate it. +//! Unfortunately, that requires (as far as I know) the binding expression +//! to know about the custom type. And since we do all our binding in +//! `spin-world`, which cannot depend on factor crates because it would make +//! things circular, we need to work around it by implementing a close +//! side channel on our own OutputStream implementation. +//! And that is what this module does. + +mod write_stream; + +pub use write_stream::AsyncWriteStream; diff --git a/crates/factor-blobstore/src/stream/write_stream.rs b/crates/factor-blobstore/src/stream/write_stream.rs new file mode 100644 index 0000000000..68b3907899 --- /dev/null +++ b/crates/factor-blobstore/src/stream/write_stream.rs @@ -0,0 +1,279 @@ +use anyhow::anyhow; +use bytes::Bytes; +use std::sync::{Arc, Mutex}; +use wasmtime_wasi::p2::{OutputStream, Pollable, StreamError}; + +#[derive(Debug)] +struct WorkerState { + alive: bool, + items: std::collections::VecDeque, + write_budget: usize, + flush_pending: bool, + shutdown_pending: bool, + error: Option, +} + +impl WorkerState { + fn check_error(&mut self) -> Result<(), StreamError> { + if let Some(e) = self.error.take() { + return Err(StreamError::LastOperationFailed(e)); + } + if !self.alive { + return Err(StreamError::Closed); + } + Ok(()) + } +} + +struct Worker { + state: Mutex, + new_work: tokio::sync::Notify, + write_ready_changed: tokio::sync::Notify, +} + +enum Job { + Shutdown, + Flush, + Write(Bytes), +} + +impl Worker { + fn new(write_budget: usize) -> Self { + Self { + state: Mutex::new(WorkerState { + alive: true, + items: std::collections::VecDeque::new(), + write_budget, + flush_pending: false, + shutdown_pending: false, + error: None, + }), + new_work: tokio::sync::Notify::new(), + write_ready_changed: tokio::sync::Notify::new(), + } + } + async fn ready(&self) { + loop { + { + let state = self.state(); + if state.error.is_some() + || !state.alive + || (!state.flush_pending && !state.shutdown_pending && state.write_budget > 0) + { + return; + } + } + self.write_ready_changed.notified().await; + } + } + fn check_write(&self) -> Result { + let mut state = self.state(); + state.check_error()?; + + if state.flush_pending || state.shutdown_pending || state.write_budget == 0 { + return Ok(0); + } + + Ok(state.write_budget) + } + fn state(&self) -> std::sync::MutexGuard { + self.state.lock().unwrap() + } + fn pop(&self) -> Option { + let mut state = self.state(); + if state.items.is_empty() { + if state.flush_pending { + return Some(Job::Flush); + } + if state.shutdown_pending { + return Some(Job::Shutdown); + } + } else if let Some(bytes) = state.items.pop_front() { + return Some(Job::Write(bytes)); + } + + None + } + fn report_error(&self, e: std::io::Error) { + { + let mut state = self.state(); + state.alive = false; + state.error = Some(e.into()); + state.flush_pending = false; + state.shutdown_pending = false; + } + self.write_ready_changed.notify_one(); + } + async fn work(&self, mut writer: T) { + use tokio::io::AsyncWriteExt; + loop { + while let Some(job) = self.pop() { + match job { + Job::Flush => { + if let Err(e) = writer.flush().await { + self.report_error(e); + return; + } + + tracing::debug!("worker marking flush complete"); + self.state().flush_pending = false; + } + + Job::Shutdown => { + if let Err(e) = writer.shutdown().await { + self.report_error(e); + return; + } + self.state().shutdown_pending = false; + } + + Job::Write(mut bytes) => { + tracing::debug!("worker writing: {bytes:?}"); + let len = bytes.len(); + match writer.write_all_buf(&mut bytes).await { + Err(e) => { + self.report_error(e); + return; + } + Ok(_) => { + self.state().write_budget += len; + } + } + } + } + + self.write_ready_changed.notify_one(); + } + self.new_work.notified().await; + } + } +} + +/// Provides a [`OutputStream`] impl from a [`tokio::io::AsyncWrite`] impl +pub struct AsyncWriteStream { + worker: Arc, + join_handle: Option>, + shutdown_join_handle: Option, +} + +impl AsyncWriteStream { + /// Create a [`AsyncWriteStream`]. In order to use the [`OutputStream`] impl + /// provided by this struct, the argument must impl [`tokio::io::AsyncWrite`]. + pub fn new( + write_budget: usize, + writer: T, + ) -> Self { + let worker = Arc::new(Worker::new(write_budget)); + + let w = Arc::clone(&worker); + let join_handle = wasmtime_wasi::runtime::spawn(async move { w.work(writer).await }); + + AsyncWriteStream { + worker, + join_handle: Some(join_handle), + shutdown_join_handle: None, + } + } + + /// Create a [`AsyncWriteStream`]. In order to use the [`OutputStream`] impl + /// provided by this struct, the argument must impl [`tokio::io::AsyncWrite`]. + /// + /// The [`AsyncWriteStream`] created by this constructor can be shut down (that is, + /// graceful EOF) by sending a message through the sender side of the `shutdown_rx` + /// sync channel. + pub fn new_closeable( + write_budget: usize, + writer: T, + mut shutdown_rx: tokio::sync::mpsc::Receiver<()>, + ) -> Self { + let worker = Arc::new(Worker::new(write_budget)); + + let w = Arc::clone(&worker); + let join_handle = wasmtime_wasi::runtime::spawn(async move { w.work(writer).await }); + + let w_clone = worker.clone(); + let shutdown_join_handle = tokio::spawn(async move { + let shutdown_msg = shutdown_rx.recv().await; + if shutdown_msg.is_some() { + let mut state = w_clone.state(); + if state.check_error().is_err() { + // The stream is already failing - no point shutting it down. + return; + } + + state.shutdown_pending = true; + w_clone.new_work.notify_one(); + } + }) + .abort_handle(); + + AsyncWriteStream { + worker, + join_handle: Some(join_handle), + shutdown_join_handle: Some(shutdown_join_handle), + } + } +} + +#[spin_core::async_trait] +impl OutputStream for AsyncWriteStream { + fn write(&mut self, bytes: Bytes) -> Result<(), StreamError> { + let mut state = self.worker.state(); + state.check_error()?; + if state.flush_pending { + return Err(StreamError::Trap(anyhow!( + "write not permitted while flush pending" + ))); + } + match state.write_budget.checked_sub(bytes.len()) { + Some(remaining_budget) => { + state.write_budget = remaining_budget; + state.items.push_back(bytes); + } + None => return Err(StreamError::Trap(anyhow!("write exceeded budget"))), + } + drop(state); + self.worker.new_work.notify_one(); + Ok(()) + } + fn flush(&mut self) -> Result<(), StreamError> { + let mut state = self.worker.state(); + state.check_error()?; + + state.flush_pending = true; + self.worker.new_work.notify_one(); + + Ok(()) + } + + fn check_write(&mut self) -> Result { + self.worker.check_write() + } + + async fn cancel(&mut self) { + if let Some(handle) = self.shutdown_join_handle.take() { + handle.abort(); + }; + if let Some(task) = self.join_handle.take() { + _ = cancel(task).await; + }; + } +} + +#[spin_core::async_trait] +impl Pollable for AsyncWriteStream { + async fn ready(&mut self) { + self.worker.ready().await; + } +} + +async fn cancel(mut handle: wasmtime_wasi::runtime::AbortOnDropJoinHandle<()>) -> Option<()> { + use std::ops::DerefMut; + + handle.deref_mut().abort(); + match handle.deref_mut().await { + Ok(value) => Some(value), + Err(err) if err.is_cancelled() => None, + Err(err) => std::panic::resume_unwind(err.into_panic()), + } +} diff --git a/crates/factor-blobstore/src/util.rs b/crates/factor-blobstore/src/util.rs new file mode 100644 index 0000000000..df8d4266d0 --- /dev/null +++ b/crates/factor-blobstore/src/util.rs @@ -0,0 +1,29 @@ +use crate::{Container, ContainerManager, Error}; +use spin_core::async_trait; +use std::{collections::HashMap, sync::Arc}; + +/// A [`ContainerManager`] which delegates to other `ContainerManager`s based on the label. +pub struct DelegatingContainerManager { + delegates: HashMap>, +} + +impl DelegatingContainerManager { + pub fn new(delegates: impl IntoIterator)>) -> Self { + let delegates = delegates.into_iter().collect(); + Self { delegates } + } +} + +#[async_trait] +impl ContainerManager for DelegatingContainerManager { + async fn get(&self, name: &str) -> Result, Error> { + match self.delegates.get(name) { + Some(cm) => cm.get(name).await, + None => Err("no such container".to_string()), + } + } + + fn is_defined(&self, label: &str) -> bool { + self.delegates.contains_key(label) + } +} diff --git a/crates/factor-blobstore/tests/factor_test.rs b/crates/factor-blobstore/tests/factor_test.rs new file mode 100644 index 0000000000..d116c82028 --- /dev/null +++ b/crates/factor-blobstore/tests/factor_test.rs @@ -0,0 +1,146 @@ +// use anyhow::bail; +// use spin_core::async_trait; +// use spin_factor_blobstore::{BlobStoreFactor, RuntimeConfig, Store, StoreManager}; +// use spin_factors::RuntimeFactors; +// use spin_factors_test::{toml, TestEnvironment}; +// use spin_world::wasi::blobstore::types::Error; +// use std::{collections::HashSet, sync::Arc}; + +// #[derive(RuntimeFactors)] +// struct TestFactors { +// blobstore: BlobStoreFactor, +// } + +// impl From for TestFactorsRuntimeConfig { +// fn from(value: RuntimeConfig) -> Self { +// Self { +// blobstore: Some(value), +// } +// } +// } + +// #[tokio::test] +// async fn works_when_allowed_store_is_defined() -> anyhow::Result<()> { +// todo!("this test") +// // let mut runtime_config = RuntimeConfig::default(); +// // runtime_config.add_store_manager("default".into(), mock_store_manager()); +// // let factors = TestFactors { +// // key_value: KeyValueFactor::new(), +// // }; +// // let env = TestEnvironment::new(factors).extend_manifest(toml! { +// // [component.test-component] +// // source = "does-not-exist.wasm" +// // key_value_stores = ["default"] +// // }); +// // let mut state = env +// // .runtime_config(runtime_config)? +// // .build_instance_state() +// // .await?; + +// // assert_eq!( +// // state.key_value.allowed_stores(), +// // &["default".into()].into_iter().collect::>() +// // ); + +// // assert!(state.key_value.open("default".to_owned()).await?.is_ok()); +// // Ok(()) +// } + +// #[tokio::test] +// async fn errors_when_store_is_not_defined() -> anyhow::Result<()> { +// todo!("this test") +// // let runtime_config = RuntimeConfig::default(); +// // let factors = TestFactors { +// // key_value: KeyValueFactor::new(), +// // }; +// // let env = TestEnvironment::new(factors).extend_manifest(toml! { +// // [component.test-component] +// // source = "does-not-exist.wasm" +// // key_value_stores = ["default"] +// // }); +// // let Err(err) = env +// // .runtime_config(runtime_config)? +// // .build_instance_state() +// // .await +// // else { +// // bail!("expected instance build to fail but it didn't"); +// // }; + +// // assert!(err +// // .to_string() +// // .contains(r#"unknown key_value_stores label "default""#)); + +// // Ok(()) +// } + +// #[tokio::test] +// async fn errors_when_store_is_not_allowed() -> anyhow::Result<()> { +// todo!("this test") +// // let mut runtime_config = RuntimeConfig::default(); +// // runtime_config.add_store_manager("default".into(), mock_store_manager()); +// // let factors = TestFactors { +// // key_value: KeyValueFactor::new(), +// // }; +// // let env = TestEnvironment::new(factors).extend_manifest(toml! { +// // [component.test-component] +// // source = "does-not-exist.wasm" +// // key_value_stores = [] +// // }); +// // let mut state = env +// // .runtime_config(runtime_config)? +// // .build_instance_state() +// // .await?; + +// // assert_eq!(state.key_value.allowed_stores(), &HashSet::new()); + +// // assert!(matches!( +// // state.key_value.open("default".to_owned()).await?, +// // Err(Error::AccessDenied) +// // )); + +// // Ok(()) +// } + +// fn mock_store_manager() -> Arc { +// Arc::new(MockStoreManager) +// } + +// struct MockStoreManager; + +// #[async_trait] +// impl StoreManager for MockStoreManager { +// async fn get(&self, name: &str) -> Result, Error> { +// let _ = name; +// Ok(Arc::new(MockStore)) +// } + +// fn is_defined(&self, store_name: &str) -> bool { +// let _ = store_name; +// todo!() +// } +// } + +// struct MockStore; + +// #[async_trait] +// impl Store for MockStore { +// async fn get(&self, key: &str) -> Result>, Error> { +// let _ = key; +// todo!() +// } +// async fn set(&self, key: &str, value: &[u8]) -> Result<(), Error> { +// let _ = (key, value); +// todo!() +// } +// async fn delete(&self, key: &str) -> Result<(), Error> { +// let _ = key; +// todo!() +// } +// async fn exists(&self, key: &str) -> Result { +// let _ = key; +// todo!() +// } +// async fn get_keys(&self) -> Result, Error> { +// todo!() +// } +// } diff --git a/crates/key-value-azure/Cargo.toml b/crates/key-value-azure/Cargo.toml index ff69e84429..8e61be3a14 100644 --- a/crates/key-value-azure/Cargo.toml +++ b/crates/key-value-azure/Cargo.toml @@ -11,9 +11,9 @@ rust-version.workspace = true [dependencies] anyhow = { workspace = true } async-trait = { workspace = true } -azure_core = "0.21.0" -azure_data_cosmos = "0.21.0" -azure_identity = "0.21.0" +azure_data_cosmos = { workspace = true } +azure_identity = { workspace = true } +azure_core = { workspace = true } futures = { workspace = true } reqwest = { version = "0.12", default-features = false } serde = { workspace = true } diff --git a/crates/loader/src/local.rs b/crates/loader/src/local.rs index cd6af7d389..3167bf4f2f 100644 --- a/crates/loader/src/local.rs +++ b/crates/loader/src/local.rs @@ -152,6 +152,7 @@ impl LocalLoader { .string_array("allowed_outbound_hosts", allowed_outbound_hosts) .string_array("key_value_stores", component.key_value_stores) .string_array("databases", component.sqlite_databases) + .string_array("blob_containers", component.blob_containers) .string_array("ai_models", component.ai_models) .serializable("build", component.build)? .take(); diff --git a/crates/manifest/src/compat.rs b/crates/manifest/src/compat.rs index b16dd0bf63..69a4c9c704 100644 --- a/crates/manifest/src/compat.rs +++ b/crates/manifest/src/compat.rs @@ -70,6 +70,7 @@ pub fn v1_to_v2_app(manifest: v1::AppManifestV1) -> Result, + /// `blob_containers = ["default", "my-container"]` + #[serde( + default, + with = "kebab_or_snake_case", + skip_serializing_if = "Vec::is_empty" + )] + #[schemars(with = "Vec")] + pub blob_containers: Vec, /// The AI models which the component is allowed to access. For local execution, you must /// download all models; for hosted execution, you should check which models are available /// in your target environment. @@ -731,7 +739,8 @@ mod tests { allowed_http_hosts: vec![], allowed_outbound_hosts: vec![], key_value_stores: labels.clone(), - sqlite_databases: labels, + sqlite_databases: labels.clone(), + blob_containers: labels, ai_models: vec![], build: None, tool: Map::new(), diff --git a/crates/runtime-config/Cargo.toml b/crates/runtime-config/Cargo.toml index 5a8aa05ad0..6d53490636 100644 --- a/crates/runtime-config/Cargo.toml +++ b/crates/runtime-config/Cargo.toml @@ -11,8 +11,12 @@ rust-version.workspace = true [dependencies] anyhow = { workspace = true } serde = { workspace = true, features = ["derive"] } +spin-blobstore-s3 = { path = "../blobstore-s3" } +spin-blobstore-azure = { path = "../blobstore-azure" } +spin-blobstore-fs = { path = "../blobstore-fs" } spin-common = { path = "../common" } spin-expressions = { path = "../expressions" } +spin-factor-blobstore = { path = "../factor-blobstore" } spin-factor-key-value = { path = "../factor-key-value" } spin-factor-llm = { path = "../factor-llm" } spin-factor-outbound-http = { path = "../factor-outbound-http" } diff --git a/crates/runtime-config/src/lib.rs b/crates/runtime-config/src/lib.rs index ee81d930cb..6b5894990d 100644 --- a/crates/runtime-config/src/lib.rs +++ b/crates/runtime-config/src/lib.rs @@ -2,6 +2,7 @@ use std::path::{Path, PathBuf}; use anyhow::Context as _; use spin_common::ui::quoted_path; +use spin_factor_blobstore::BlobStoreFactor; use spin_factor_key_value::runtime_config::spin::{self as key_value}; use spin_factor_key_value::KeyValueFactor; use spin_factor_llm::{spin as llm, LlmFactor}; @@ -140,6 +141,7 @@ where let key_value_resolver = key_value_config_resolver(runtime_config_dir, state_dir.clone()); let sqlite_resolver = sqlite_config_resolver(state_dir.clone()) .context("failed to resolve sqlite runtime config")?; + let blobstore_config_resolver = blobstore_config_resolver(toml_resolver.state_dir()?); let toml = toml_resolver.toml(); let log_dir = toml_resolver.log_dir()?; @@ -150,6 +152,7 @@ where &key_value_resolver, outbound_networking.as_ref(), &sqlite_resolver, + &blobstore_config_resolver, ); // Note: all valid fields in the runtime config must have been referenced at @@ -302,6 +305,7 @@ pub struct TomlRuntimeConfigSource<'a, 'b> { key_value: &'a key_value::RuntimeConfigResolver, outbound_networking: Option<&'a OutboundNetworkingSpinRuntimeConfig>, sqlite: &'a sqlite::RuntimeConfigResolver, + blob_store: &'a spin_factor_blobstore::runtime_config::spin::RuntimeConfigResolver, } impl<'a, 'b> TomlRuntimeConfigSource<'a, 'b> { @@ -310,12 +314,14 @@ impl<'a, 'b> TomlRuntimeConfigSource<'a, 'b> { key_value: &'a key_value::RuntimeConfigResolver, outbound_networking: Option<&'a OutboundNetworkingSpinRuntimeConfig>, sqlite: &'a sqlite::RuntimeConfigResolver, + blob_store: &'a spin_factor_blobstore::runtime_config::spin::RuntimeConfigResolver, ) -> Self { Self { toml: toml_resolver, key_value, outbound_networking, sqlite, + blob_store, } } } @@ -396,6 +402,15 @@ impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_, '_> } } +impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_, '_> { + fn get_runtime_config( + &mut self, + ) -> anyhow::Result> { + // TODO: actually + Ok(Some(self.blob_store.resolve(Some(&self.toml.table))?)) + } +} + impl RuntimeConfigSourceFinalizer for TomlRuntimeConfigSource<'_, '_> { fn finalize(&mut self) -> anyhow::Result<()> { Ok(self.toml.validate_all_keys_used()?) @@ -463,6 +478,29 @@ fn sqlite_config_resolver( )) } +/// The blob store runtime configuration resolver. +pub fn blobstore_config_resolver( + // local_store_base_path: Option, + _default_store_base_path: Option, // TODO: used? +) -> spin_factor_blobstore::runtime_config::spin::RuntimeConfigResolver { + let mut blobstore_resolver = + spin_factor_blobstore::runtime_config::spin::RuntimeConfigResolver::new(); + + // Register the supported store types. + // Unwraps are safe because the store types are known to not overlap. + blobstore_resolver + .register_store_type(spin_blobstore_fs::FileSystemBlobStore::new()) + .unwrap(); + blobstore_resolver + .register_store_type(spin_blobstore_azure::AzureBlobStoreBuilder::new()) + .unwrap(); + blobstore_resolver + .register_store_type(spin_blobstore_s3::S3BlobStore::new()) + .unwrap(); + + blobstore_resolver +} + #[cfg(test)] mod tests { use std::{collections::HashMap, sync::Arc}; diff --git a/crates/runtime-factors/Cargo.toml b/crates/runtime-factors/Cargo.toml index 257c3f36fa..26231621ab 100644 --- a/crates/runtime-factors/Cargo.toml +++ b/crates/runtime-factors/Cargo.toml @@ -17,6 +17,7 @@ llm-cublas = ["spin-factor-llm/llm-cublas"] anyhow = { workspace = true } clap = { workspace = true, features = ["derive", "env"] } spin-common = { path = "../common" } +spin-factor-blobstore = { path = "../factor-blobstore" } spin-factor-key-value = { path = "../factor-key-value" } spin-factor-llm = { path = "../factor-llm" } spin-factor-outbound-http = { path = "../factor-outbound-http" } diff --git a/crates/runtime-factors/src/lib.rs b/crates/runtime-factors/src/lib.rs index 45110cbc18..e850f26d24 100644 --- a/crates/runtime-factors/src/lib.rs +++ b/crates/runtime-factors/src/lib.rs @@ -8,6 +8,8 @@ use std::path::PathBuf; use anyhow::Context as _; use spin_common::arg_parser::parse_kv; + +use spin_factor_blobstore::BlobStoreFactor; use spin_factor_key_value::KeyValueFactor; use spin_factor_llm::LlmFactor; use spin_factor_outbound_http::OutboundHttpFactor; @@ -36,6 +38,7 @@ pub struct TriggerFactors { pub pg: OutboundPgFactor, pub mysql: OutboundMysqlFactor, pub llm: LlmFactor, + pub blobstore: BlobStoreFactor, } impl TriggerFactors { @@ -59,6 +62,7 @@ impl TriggerFactors { spin_factor_llm::spin::default_engine_creator(state_dir) .context("failed to configure LLM factor")?, ), + blobstore: BlobStoreFactor::new(), }) } } diff --git a/crates/variables-azure/Cargo.toml b/crates/variables-azure/Cargo.toml index 78d9584728..67348d528e 100644 --- a/crates/variables-azure/Cargo.toml +++ b/crates/variables-azure/Cargo.toml @@ -9,9 +9,10 @@ repository.workspace = true rust-version.workspace = true [dependencies] -azure_core = { git = "https://github.com/azure/azure-sdk-for-rust", rev = "8c4caa251c3903d5eae848b41bb1d02a4d65231c" } -azure_identity = { git = "https://github.com/azure/azure-sdk-for-rust", rev = "8c4caa251c3903d5eae848b41bb1d02a4d65231c" } -azure_security_keyvault = { git = "https://github.com/azure/azure-sdk-for-rust", rev = "8c4caa251c3903d5eae848b41bb1d02a4d65231c" } +azure_core = { workspace = true } +azure_identity = { workspace = true } +azure_security_keyvault = { workspace = true } +dotenvy = "0.15" serde = { workspace = true } spin-expressions = { path = "../expressions" } spin-factors = { path = "../factors" } diff --git a/crates/world/Cargo.toml b/crates/world/Cargo.toml index fd4763100d..3fbffddf52 100644 --- a/crates/world/Cargo.toml +++ b/crates/world/Cargo.toml @@ -7,3 +7,4 @@ edition = { workspace = true } [dependencies] async-trait = { workspace = true } wasmtime = { workspace = true } +wasmtime-wasi = { workspace = true } diff --git a/crates/world/src/lib.rs b/crates/world/src/lib.rs index cbc2283aba..17365919e4 100644 --- a/crates/world/src/lib.rs +++ b/crates/world/src/lib.rs @@ -38,8 +38,12 @@ wasmtime::component::bindgen!({ "wasi:config/store@0.2.0-draft-2024-09-27/error" => wasi::config::store::Error, "wasi:keyvalue/store/error" => wasi::keyvalue::store::Error, "wasi:keyvalue/atomics/cas-error" => wasi::keyvalue::atomics::CasError, + "wasi:blobstore/types@0.2.0-draft-2024-09-01/error" => wasi::blobstore::types::Error, }, trappable_imports: true, + with: { + "wasi:io": wasmtime_wasi::p2::bindings::io, + }, }); pub use fermyon::spin as v1; diff --git a/examples/spin-timer/Cargo.lock b/examples/spin-timer/Cargo.lock index ec073d9b6c..cac86e4346 100644 --- a/examples/spin-timer/Cargo.lock +++ b/examples/spin-timer/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "RustyXML" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b5ace29ee3216de37c0546865ad08edef58b0f9e76838ed8959a84a990e58c5" + [[package]] name = "addr2line" version = "0.24.2" @@ -17,6 +23,17 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +[[package]] +name = "ahash" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" +dependencies = [ + "getrandom 0.2.15", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" version = "1.1.3" @@ -71,6 +88,12 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "async-channel" version = "1.9.0" @@ -305,6 +328,7 @@ dependencies = [ "aws-credential-types", "aws-sigv4", "aws-smithy-async", + "aws-smithy-eventstream", "aws-smithy-http", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -344,6 +368,40 @@ dependencies = [ "tracing", ] +[[package]] +name = "aws-sdk-s3" +version = "1.68.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc5ddf1dc70287dc9a2f953766a1fe15e3e74aef02fd1335f2afa475c9b4f4fc" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "fastrand 2.3.0", + "hex", + "hmac", + "http 0.2.12", + "http-body 0.4.6", + "lru", + "once_cell", + "percent-encoding", + "regex-lite", + "sha2", + "tracing", + "url", +] + [[package]] name = "aws-sdk-sso" version = "1.53.0" @@ -418,6 +476,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d3820e0c08d0737872ff3c7c1f21ebbb6693d832312d6152bf18ef50a5471c2" dependencies = [ "aws-credential-types", + "aws-smithy-eventstream", "aws-smithy-http", "aws-smithy-runtime-api", "aws-smithy-types", @@ -436,21 +495,54 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.3" +version = "1.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "427cb637d15d63d6f9aae26358e1c9a9c09d5aa490d64b09354c8217cfef0f28" +checksum = "1e190749ea56f8c42bf15dd76c65e14f8f765233e6df9b0506d9d934ebef867c" dependencies = [ "futures-util", "pin-project-lite", "tokio", ] +[[package]] +name = "aws-smithy-checksums" +version = "0.60.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba1a71073fca26775c8b5189175ea8863afb1c9ea2cceb02a5de5ad9dfbaa795" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc32c", + "crc32fast", + "hex", + "http 0.2.12", + "http-body 0.4.6", + "md-5", + "pin-project-lite", + "sha1", + "sha2", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef7d0a272725f87e51ba2bf89f8c21e4df61b9e49ae1ac367a6d69916ef7c90" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + [[package]] name = "aws-smithy-http" version = "0.60.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6" dependencies = [ + "aws-smithy-eventstream", "aws-smithy-runtime-api", "aws-smithy-types", "bytes", @@ -624,32 +716,6 @@ dependencies = [ "tower-service", ] -[[package]] -name = "azure_core" -version = "0.20.0" -source = "git+https://github.com/azure/azure-sdk-for-rust.git?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" -dependencies = [ - "async-trait", - "base64 0.22.1", - "bytes", - "dyn-clone", - "futures", - "getrandom 0.2.15", - "http-types", - "once_cell", - "paste", - "pin-project", - "rand 0.8.5", - "reqwest", - "rustc_version", - "serde", - "serde_json", - "time", - "tracing", - "url", - "uuid", -] - [[package]] name = "azure_core" version = "0.21.0" @@ -667,6 +733,7 @@ dependencies = [ "once_cell", "paste", "pin-project", + "quick-xml 0.31.0", "rand 0.8.5", "reqwest", "rustc_version", @@ -686,7 +753,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0aa5603f2de38c21165a1b5dfed94d64b1ab265526b0686e8557c907a53a0ee2" dependencies = [ "async-trait", - "azure_core 0.21.0", + "azure_core", "bytes", "futures", "serde", @@ -700,13 +767,14 @@ dependencies = [ [[package]] name = "azure_identity" -version = "0.20.0" -source = "git+https://github.com/azure/azure-sdk-for-rust.git?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ddd80344317c40c04b603807b63a5cefa532f1b43522e72f480a988141f744" dependencies = [ "async-lock", "async-process", "async-trait", - "azure_core 0.20.0", + "azure_core", "futures", "oauth2", "pin-project", @@ -719,34 +787,70 @@ dependencies = [ ] [[package]] -name = "azure_identity" +name = "azure_security_keyvault" version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ddd80344317c40c04b603807b63a5cefa532f1b43522e72f480a988141f744" +checksum = "bd94f507b75349a0e381c0a23bd77cc654fb509f0e6797ce4f99dd959d9e2d68" +dependencies = [ + "async-trait", + "azure_core", + "futures", + "serde", + "serde_json", + "time", +] + +[[package]] +name = "azure_storage" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59f838159f4d29cb400a14d9d757578ba495ae64feb07a7516bf9e4415127126" dependencies = [ + "RustyXML", "async-lock", - "async-process", "async-trait", - "azure_core 0.21.0", + "azure_core", + "bytes", + "serde", + "serde_derive", + "time", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "azure_storage_blobs" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97e83c3636ae86d9a6a7962b2112e3b19eb3903915c50ce06ff54ff0a2e6a7e4" +dependencies = [ + "RustyXML", + "azure_core", + "azure_storage", + "azure_svc_blobstorage", + "bytes", "futures", - "oauth2", - "pin-project", "serde", + "serde_derive", + "serde_json", "time", "tracing", - "tz-rs", "url", "uuid", ] [[package]] -name = "azure_security_keyvault" -version = "0.20.0" -source = "git+https://github.com/azure/azure-sdk-for-rust.git?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" +name = "azure_svc_blobstorage" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e6c6f20c5611b885ba94c7bae5e02849a267381aecb8aee577e8c35ff4064c6" dependencies = [ - "async-trait", - "azure_core 0.20.0", + "azure_core", + "bytes", "futures", + "log", + "once_cell", "serde", "serde_json", "time", @@ -816,6 +920,18 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -838,6 +954,29 @@ dependencies = [ "piper", ] +[[package]] +name = "borsh" +version = "1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8646f98db542e39fc66e68a20b2144f6a732636df7c2354e74645faaa433ce" +dependencies = [ + "borsh-derive", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd1d3c0c2f5833f22386f252fe8ed005c7f59fdcddeef025c01b4c3b9fd9ac3" +dependencies = [ + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "btoi" version = "0.4.3" @@ -856,6 +995,28 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "bytecheck" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" +dependencies = [ + "bytecheck_derive", + "ptr_meta", + "simdutf8", +] + +[[package]] +name = "bytecheck_derive" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "byteorder" version = "1.5.0" @@ -1090,6 +1251,16 @@ dependencies = [ "libc", ] +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -1251,6 +1422,15 @@ version = "0.122.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b530783809a55cb68d070e0de60cfbb3db0dc94c8850dd5725411422bedcf6bb" +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + [[package]] name = "crc32fast" version = "1.4.2" @@ -1719,6 +1899,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures" version = "0.3.31" @@ -1980,6 +2166,9 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash", +] [[package]] name = "hashbrown" @@ -2138,6 +2327,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "humantime" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b112acc8b3adf4b107a8ec20977da0273a8c386765a3ec0229bd500a1443f9f" + [[package]] name = "hyper" version = "0.14.32" @@ -2228,6 +2423,7 @@ dependencies = [ "hyper 1.6.0", "hyper-util", "rustls 0.23.25", + "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", "tokio-rustls 0.26.2", @@ -2548,6 +2744,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.14.0" @@ -2950,7 +3155,7 @@ dependencies = [ "openssl-probe", "openssl-sys", "schannel", - "security-framework", + "security-framework 2.11.1", "security-framework-sys", "tempfile", ] @@ -3061,6 +3266,36 @@ dependencies = [ "memchr", ] +[[package]] +name = "object_store" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cfccb68961a56facde1163f9319e0d15743352344e7808a11795fb99698dcaf" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "chrono", + "futures", + "humantime", + "hyper 1.6.0", + "itertools 0.13.0", + "md-5", + "parking_lot", + "percent-encoding", + "quick-xml 0.37.5", + "rand 0.8.5", + "reqwest", + "ring", + "serde", + "serde_json", + "snafu", + "tokio", + "tracing", + "url", + "walkdir", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -3457,6 +3692,16 @@ dependencies = [ "uuid", ] +[[package]] +name = "postgres_range" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6dce28dc5ba143d8eb157b62aac01ae5a1c585c40792158b720e86a87642101" +dependencies = [ + "postgres-protocol", + "postgres-types", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -3472,6 +3717,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "proc-macro-crate" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -3551,6 +3805,26 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "ptr_meta" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "pulley-interpreter" version = "35.0.0" @@ -3574,6 +3848,26 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "quick-xml" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quick-xml" +version = "0.37.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quinn" version = "0.11.7" @@ -3643,6 +3937,12 @@ version = "5.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rand" version = "0.7.3" @@ -3917,6 +4217,15 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "rend" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" +dependencies = [ + "bytecheck", +] + [[package]] name = "reqwest" version = "0.12.15" @@ -3948,6 +4257,7 @@ dependencies = [ "pin-project-lite", "quinn", "rustls 0.23.25", + "rustls-native-certs 0.8.1", "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", @@ -3984,6 +4294,35 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rkyv" +version = "0.7.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9008cd6385b9e161d8229e1f6549dd23c3d022f132a2ea37ac3a10ac4935779b" +dependencies = [ + "bitvec", + "bytecheck", + "bytes", + "hashbrown 0.12.3", + "ptr_meta", + "rend", + "rkyv_derive", + "seahash", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv_derive" +version = "0.7.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "503d1d27590a2b0a3a4ca4c94755aa2875657196ecbf401a42eff41d7de532c0" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "rumqttc" version = "0.24.0" @@ -4017,6 +4356,23 @@ dependencies = [ "smallvec", ] +[[package]] +name = "rust_decimal" +version = "1.37.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b203a6425500a03e0919c42d3c47caca51e79f1132046626d2c8871c5092035d" +dependencies = [ + "arrayvec", + "borsh", + "bytes", + "num-traits", + "postgres-types", + "rand 0.8.5", + "rkyv", + "serde", + "serde_json", +] + [[package]] name = "rustc-demangle" version = "0.1.24" @@ -4158,7 +4514,7 @@ dependencies = [ "openssl-probe", "rustls-pemfile 1.0.4", "schannel", - "security-framework", + "security-framework 2.11.1", ] [[package]] @@ -4171,7 +4527,19 @@ dependencies = [ "rustls-pemfile 2.2.0", "rustls-pki-types", "schannel", - "security-framework", + "security-framework 2.11.1", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework 3.3.0", ] [[package]] @@ -4246,6 +4614,15 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "sanitize-filename" version = "0.5.0" @@ -4319,6 +4696,12 @@ dependencies = [ "untrusted", ] +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" + [[package]] name = "security-framework" version = "2.11.1" @@ -4326,7 +4709,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ "bitflags 2.9.0", - "core-foundation", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80fb1d92c5028aa318b4b8bd7302a5bfcf48be96a37fc6fc790f806b0004ee0c" +dependencies = [ + "bitflags 2.9.0", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -4488,6 +4884,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "siphasher" version = "1.0.1" @@ -4512,6 +4914,27 @@ dependencies = [ "serde", ] +[[package]] +name = "snafu" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320b01e011bf8d5d7a4a4a4be966d9160968935849c83b918827f6a435e7f627" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1961e2ef424c1424204d3a5d6975f934f56b6d50ff5732382d84ebf460e147f7" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "socket2" version = "0.5.9" @@ -4550,6 +4973,66 @@ dependencies = [ "spin-locked-app", ] +[[package]] +name = "spin-blobstore-azure" +version = "3.4.0-pre0" +dependencies = [ + "anyhow", + "azure_core", + "azure_storage", + "azure_storage_blobs", + "futures", + "serde", + "spin-core", + "spin-factor-blobstore", + "tokio", + "tokio-stream", + "tokio-util", + "uuid", + "wasmtime-wasi", +] + +[[package]] +name = "spin-blobstore-fs" +version = "3.4.0-pre0" +dependencies = [ + "anyhow", + "futures", + "serde", + "spin-core", + "spin-factor-blobstore", + "tokio", + "tokio-stream", + "tokio-util", + "walkdir", + "wasmtime-wasi", +] + +[[package]] +name = "spin-blobstore-s3" +version = "3.4.0-pre0" +dependencies = [ + "anyhow", + "async-once-cell", + "aws-config", + "aws-credential-types", + "aws-sdk-s3", + "aws-smithy-async", + "bytes", + "futures", + "http-body 1.0.1", + "http-body-util", + "object_store", + "serde", + "spin-core", + "spin-factor-blobstore", + "tokio", + "tokio-stream", + "tokio-util", + "uuid", + "wasmtime-wasi", +] + [[package]] name = "spin-common" version = "3.4.0-pre0" @@ -4613,6 +5096,27 @@ dependencies = [ "thiserror 2.0.12", ] +[[package]] +name = "spin-factor-blobstore" +version = "3.4.0-pre0" +dependencies = [ + "anyhow", + "bytes", + "futures", + "lru", + "serde", + "spin-core", + "spin-factor-wasi", + "spin-factors", + "spin-locked-app", + "spin-resource-table", + "spin-world", + "tokio", + "toml", + "tracing", + "wasmtime-wasi", +] + [[package]] name = "spin-factor-key-value" version = "3.4.0-pre0" @@ -4732,11 +5236,14 @@ name = "spin-factor-outbound-pg" version = "3.4.0-pre0" dependencies = [ "anyhow", + "bytes", "chrono", "deadpool-postgres", "moka", "native-tls", "postgres-native-tls", + "postgres_range", + "rust_decimal", "serde_json", "spin-core", "spin-factor-outbound-networking", @@ -4852,9 +5359,9 @@ version = "3.4.0-pre0" dependencies = [ "anyhow", "async-trait", - "azure_core 0.21.0", + "azure_core", "azure_data_cosmos", - "azure_identity 0.21.0", + "azure_identity", "futures", "reqwest", "serde", @@ -4955,8 +5462,12 @@ version = "3.4.0-pre0" dependencies = [ "anyhow", "serde", + "spin-blobstore-azure", + "spin-blobstore-fs", + "spin-blobstore-s3", "spin-common", "spin-expressions", + "spin-factor-blobstore", "spin-factor-key-value", "spin-factor-llm", "spin-factor-outbound-http", @@ -4989,6 +5500,7 @@ dependencies = [ "anyhow", "clap", "spin-common", + "spin-factor-blobstore", "spin-factor-key-value", "spin-factor-llm", "spin-factor-outbound-http", @@ -5004,6 +5516,7 @@ dependencies = [ "spin-factors-executor", "spin-runtime-config", "spin-trigger", + "spin-variables-static", "terminal", "tracing", ] @@ -5102,9 +5615,10 @@ dependencies = [ name = "spin-variables-azure" version = "3.4.0-pre0" dependencies = [ - "azure_core 0.20.0", - "azure_identity 0.20.0", + "azure_core", + "azure_identity", "azure_security_keyvault", + "dotenvy", "serde", "spin-expressions", "spin-factors", @@ -5130,8 +5644,11 @@ name = "spin-variables-static" version = "3.4.0-pre0" dependencies = [ "serde", + "serde_json", + "spin-common", "spin-expressions", "spin-factors", + "toml", ] [[package]] @@ -5151,6 +5668,7 @@ version = "3.4.0-pre0" dependencies = [ "async-trait", "wasmtime", + "wasmtime-wasi", ] [[package]] @@ -5253,7 +5771,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ "bitflags 2.9.0", - "core-foundation", + "core-foundation 0.9.4", "system-configuration-sys", ] @@ -5289,6 +5807,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "target-lexicon" version = "0.13.2" @@ -5564,6 +6088,7 @@ checksum = "6b9590b93e6fcc1739458317cccd391ad3955e2bde8913edf6f95f9e65a8f034" dependencies = [ "bytes", "futures-core", + "futures-io", "futures-sink", "pin-project-lite", "tokio", @@ -5923,6 +6448,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" dependencies = [ "getrandom 0.3.2", + "serde", ] [[package]] @@ -6008,6 +6534,16 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -7089,6 +7625,15 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "xmlparser" version = "0.13.6" diff --git a/wit/deps/wasi-blobstore-2024-09-01/blobstore.wit b/wit/deps/wasi-blobstore-2024-09-01/blobstore.wit new file mode 100644 index 0000000000..5a50214f6b --- /dev/null +++ b/wit/deps/wasi-blobstore-2024-09-01/blobstore.wit @@ -0,0 +1,27 @@ +// wasi-cloud Blobstore service definition +interface blobstore { + use container.{container}; + use types.{error, container-name, object-id}; + + // creates a new empty container + create-container: func(name: container-name) -> result; + + // retrieves a container by name + get-container: func(name: container-name) -> result; + + // deletes a container and all objects within it + delete-container: func(name: container-name) -> result<_, error>; + + // returns true if the container exists + container-exists: func(name: container-name) -> result; + + // copies (duplicates) an object, to the same or a different container. + // returns an error if the target container does not exist. + // overwrites destination object if it already existed. + copy-object: func(src: object-id, dest: object-id) -> result<_, error>; + + // moves or renames an object, to the same or a different container + // returns an error if the destination container does not exist. + // overwrites destination object if it already existed. + move-object: func(src:object-id, dest: object-id) -> result<_, error>; +} \ No newline at end of file diff --git a/wit/deps/wasi-blobstore-2024-09-01/container.wit b/wit/deps/wasi-blobstore-2024-09-01/container.wit new file mode 100644 index 0000000000..f4c577e5d7 --- /dev/null +++ b/wit/deps/wasi-blobstore-2024-09-01/container.wit @@ -0,0 +1,66 @@ +// a Container is a collection of objects +interface container { + use wasi:io/streams@0.2.0.{ + input-stream, + output-stream, + }; + + use types.{ + container-metadata, + error, + incoming-value, + object-metadata, + object-name, + outgoing-value, + }; + + // this defines the `container` resource + resource container { + // returns container name + name: func() -> result; + + // returns container metadata + info: func() -> result; + + // retrieves an object or portion of an object, as a resource. + // Start and end offsets are inclusive. + // Once a data-blob resource has been created, the underlying bytes are held by the blobstore service for the lifetime + // of the data-blob resource, even if the object they came from is later deleted. + get-data: func(name: object-name, start: u64, end: u64) -> result; + + // creates or replaces an object with the data blob. + write-data: func(name: object-name, data: borrow) -> result<_, error>; + + // returns list of objects in the container. Order is undefined. + list-objects: func() -> result; + + // deletes object. + // does not return error if object did not exist. + delete-object: func(name: object-name) -> result<_, error>; + + // deletes multiple objects in the container + delete-objects: func(names: list) -> result<_, error>; + + // returns true if the object exists in this container + has-object: func(name: object-name) -> result; + + // returns metadata for the object + object-info: func(name: object-name) -> result; + + // removes all objects within the container, leaving the container empty. + clear: func() -> result<_, error>; + } + + // this defines the `stream-object-names` resource which is a representation of stream + resource stream-object-names { + // reads the next number of objects from the stream + // + // This function returns the list of objects read, and a boolean indicating if the end of the stream was reached. + read-stream-object-names: func(len: u64) -> result, bool>, error>; + + // skip the next number of objects in the stream + // + // This function returns the number of objects skipped, and a boolean indicating if the end of the stream was reached. + skip-stream-object-names: func(num: u64) -> result, error>; + } +} \ No newline at end of file diff --git a/wit/deps/wasi-blobstore-2024-09-01/types.wit b/wit/deps/wasi-blobstore-2024-09-01/types.wit new file mode 100644 index 0000000000..ca5972494b --- /dev/null +++ b/wit/deps/wasi-blobstore-2024-09-01/types.wit @@ -0,0 +1,91 @@ +// Types used by blobstore +interface types { + use wasi:io/streams@0.2.0.{input-stream, output-stream}; + + // name of a container, a collection of objects. + // The container name may be any valid UTF-8 string. + type container-name = string; + + // name of an object within a container + // The object name may be any valid UTF-8 string. + type object-name = string; + + // TODO: define timestamp to include seconds since + // Unix epoch and nanoseconds + // https://github.com/WebAssembly/wasi-blob-store/issues/7 + type timestamp = u64; + + // size of an object, in bytes + type object-size = u64; + + type error = string; + + // information about a container + record container-metadata { + // the container's name + name: container-name, + // date and time container was created + created-at: timestamp, + } + + // information about an object + record object-metadata { + // the object's name + name: object-name, + // the object's parent container + container: container-name, + // date and time the object was created + created-at: timestamp, + // size of the object, in bytes + size: object-size, + } + + // identifier for an object that includes its container name + record object-id { + container: container-name, + object: object-name + } + + /// A data is the data stored in a data blob. The value can be of any type + /// that can be represented in a byte array. It provides a way to write the value + /// to the output-stream defined in the `wasi-io` interface. + // Soon: switch to `resource value { ... }` + resource outgoing-value { + new-outgoing-value: static func() -> outgoing-value; + + /// Returns a stream for writing the value contents. + /// + /// The returned `output-stream` is a child resource: it must be dropped + /// before the parent `outgoing-value` resource is dropped (or finished), + /// otherwise the `outgoing-value` drop or `finish` will trap. + /// + /// Returns success on the first call: the `output-stream` resource for + /// this `outgoing-value` may be retrieved at most once. Subsequent calls + /// will return error. + outgoing-value-write-body: func() -> result; + + /// Finalize an outgoing value. This must be + /// called to signal that the outgoing value is complete. If the `outgoing-value` + /// is dropped without calling `outgoing-value.finalize`, the implementation + /// should treat the value as corrupted. + finish: static func(this: outgoing-value) -> result<_, error>; + } + + /// A incoming-value is a wrapper around a value. It provides a way to read the value + /// from the input-stream defined in the `wasi-io` interface. + /// + /// The incoming-value provides two ways to consume the value: + /// 1. `incoming-value-consume-sync` consumes the value synchronously and returns the + /// value as a list of bytes. + /// 2. `incoming-value-consume-async` consumes the value asynchronously and returns the + /// value as an input-stream. + // Soon: switch to `resource incoming-value { ... }` + resource incoming-value { + incoming-value-consume-sync: static func(this: incoming-value) -> result; + incoming-value-consume-async: static func(this: incoming-value) -> result; + size: func() -> u64; + } + + type incoming-value-async-body = input-stream; + type incoming-value-sync-body = list; +} diff --git a/wit/deps/wasi-blobstore-2024-09-01/world.wit b/wit/deps/wasi-blobstore-2024-09-01/world.wit new file mode 100644 index 0000000000..61f4225282 --- /dev/null +++ b/wit/deps/wasi-blobstore-2024-09-01/world.wit @@ -0,0 +1,5 @@ +package wasi:blobstore@0.2.0-draft-2024-09-01; + +world imports { + import blobstore; +} \ No newline at end of file diff --git a/wit/world.wit b/wit/world.wit index b5d66b3b2f..abbff6a341 100644 --- a/wit/world.wit +++ b/wit/world.wit @@ -13,5 +13,6 @@ world platform { import spin:postgres/postgres@3.0.0; import spin:postgres/postgres@4.0.0; import spin:sqlite/sqlite@3.0.0; + import wasi:blobstore/blobstore@0.2.0-draft-2024-09-01; import wasi:config/store@0.2.0-draft-2024-09-27; }