From 788f92f451060c84f39a889f256c1034d2a13214 Mon Sep 17 00:00:00 2001 From: Daniel Vigovszky Date: Tue, 14 Jan 2025 16:24:47 +0100 Subject: [PATCH 1/6] Final durability related refactorings Test dsl improvements Adjust to latest golem-wit changes (host side) Adjust to latest golem-wit changes Changed how wit dependencies are fetched --- .github/workflows/ci.yaml | 2 + .gitignore | 3 + Cargo.lock | 8 +- Makefile.toml | 42 +- golem-client/Cargo.toml | 2 +- golem-common/Cargo.toml | 11 +- golem-common/src/base_model.rs | 358 ++++++++++ golem-common/src/lib.rs | 9 + golem-common/src/model/mod.rs | 288 +------- golem-common/src/model/oplog.rs | 65 +- golem-common/src/newtype.rs | 46 +- golem-common/src/tracing.rs | 2 + golem-test-framework/Cargo.toml | 4 + .../component_service/filesystem.rs | 96 +-- .../src/components/component_service/mod.rs | 108 +-- golem-test-framework/src/dsl/mod.rs | 367 +++++----- golem-worker-executor-base/Cargo.toml | 4 - golem-worker-executor-base/build.rs | 106 --- .../src/durable_host/durability.rs | 106 ++- .../src/durable_host/dynamic_linking/mod.rs | 2 +- .../durable_host/dynamic_linking/wasm_rpc.rs | 2 +- .../src/durable_host/golem/mod.rs | 2 +- .../src/durable_host/http/mod.rs | 1 + .../src/durable_host/http/outgoing_http.rs | 1 + .../src/durable_host/wasm_rpc/mod.rs | 60 +- .../src/preview2/mod.rs | 24 +- .../src/services/golem_config.rs | 2 + golem-worker-executor-base/tests/api.rs | 98 +-- golem-worker-executor-base/tests/blobstore.rs | 4 +- .../tests/common/mod.rs | 23 +- .../tests/guest_languages1.rs | 20 +- .../tests/guest_languages2.rs | 8 +- .../tests/guest_languages3.rs | 6 +- .../tests/hot_update.rs | 24 +- golem-worker-executor-base/tests/keyvalue.rs | 24 +- .../tests/measure_test_component_mem.rs | 11 +- .../tests/observability.rs | 12 +- golem-worker-executor-base/tests/rust_rpc.rs | 64 +- .../tests/rust_rpc_stubless.rs | 651 +++++++++--------- .../tests/scalability.rs | 8 +- .../tests/transactions.rs | 36 +- golem-worker-executor-base/tests/ts_rpc1.rs | 12 +- .../tests/ts_rpc1_stubless.rs | 93 ++- golem-worker-executor-base/tests/ts_rpc2.rs | 8 +- .../tests/ts_rpc2_stubless.rs | 62 +- golem-worker-executor-base/tests/wasi.rs | 86 ++- golem-worker-executor/src/context.rs | 2 +- golem-worker-executor/src/lib.rs | 9 +- .../src/benchmarks/durability_overhead.rs | 4 +- integration-tests/src/benchmarks/mod.rs | 2 +- integration-tests/src/benchmarks/rpc.rs | 8 +- .../src/benchmarks/rpc_cpu_intensive.rs | 8 +- .../src/benchmarks/rpc_large_input.rs | 8 +- integration-tests/tests/fork.rs | 16 +- integration-tests/tests/plugins.rs | 6 +- integration-tests/tests/sharding.rs | 2 +- integration-tests/tests/worker.rs | 80 +-- wasm-ast/Cargo.toml | 2 +- wasm-rpc-stubgen/Cargo.toml | 5 +- wasm-rpc-stubgen/src/wit_generate.rs | 4 +- wasm-rpc-stubgen/tests/add_dep.rs | 2 +- wasm-rpc/Cargo.toml | 4 +- wasm-rpc/src/bindings.rs | 14 +- wasm-rpc/src/lib.rs | 12 +- wasm-rpc/src/value_and_type.rs | 6 +- wasm-rpc/wit/deps/io/poll.wit | 41 -- wasm-rpc/wit/wasm-rpc.wit | 110 +-- wit/deps.lock | 61 ++ wit/deps.toml | 1 + wit/host.wit | 19 + 70 files changed, 1688 insertions(+), 1709 deletions(-) create mode 100644 golem-common/src/base_model.rs delete mode 100644 golem-worker-executor-base/build.rs delete mode 100644 wasm-rpc/wit/deps/io/poll.wit create mode 100644 wit/deps.lock create mode 100644 wit/deps.toml create mode 100644 wit/host.wit diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ee61f12afd..d1605d0c71 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -277,6 +277,8 @@ jobs: uses: arduino/setup-protoc@v3 with: repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Prepare WIT dependencies + run: cargo make wit - name: Build wasm-rpc in stub mode run: cargo component build -p golem-wasm-rpc --no-default-features --features stub diff --git a/.gitignore b/.gitignore index ad22ff803b..3984ee9cfd 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,6 @@ golem-client/* logs data .DS_Store +wit/deps +wasm-rpc/wit/deps +durable-wasi/wit/deps diff --git a/Cargo.lock b/Cargo.lock index bac742953a..601940d502 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4250,6 +4250,7 @@ dependencies = [ "cli-table", "colored", "console-subscriber", + "futures-util", "golem-api-grpc", "golem-common", "golem-service-base", @@ -4346,6 +4347,7 @@ dependencies = [ "glob", "golem-wasm-ast", "golem-wasm-rpc", + "golem-wit", "heck 0.5.0", "id-arena", "indexmap 2.7.0", @@ -4376,9 +4378,9 @@ dependencies = [ [[package]] name = "golem-wit" -version = "1.1.2" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf53fc25db491f96c480d7f195175f6da5f18dcc6ef548d04b2ae648b7796bbf" +checksum = "04cb9d855d8604587ccf16a402f759ee6e85b555201b6d6b1c9f0764a0fbc6d2" [[package]] name = "golem-worker-executor" @@ -4433,7 +4435,6 @@ dependencies = [ "cap-fs-ext", "cap-std", "cap-time-ext", - "cargo_metadata 0.19.1", "chrono", "console-subscriber", "dashmap", @@ -4454,7 +4455,6 @@ dependencies = [ "golem-test-framework", "golem-wasm-ast", "golem-wasm-rpc", - "golem-wit", "hex", "http 1.2.0", "http-body 1.0.1", diff --git a/Makefile.toml b/Makefile.toml index 49d0289d5a..bcc1393ecd 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -1,6 +1,7 @@ # List of top-level tasks intended for use: # # - `cargo make dev-flow` or just `cargo make`: runs a full development flow, including fixing format and clippy, building and running tests and generating OpenAPI specs +# - `cargo make wit`: fetches the WIT dependencies based on wit/deps.toml # - `cargo make build`: builds everything in debug mode # - `cargo make build-release`: builds everything in release mode. customizable with PLATFORM_OVERRIDE env variable for docker builds # - `cargo make check`: runs rustfmt and clippy checks without applying any fix @@ -46,13 +47,35 @@ alias = "dev-flow" [tasks.dev-flow] description = "Runs a full development flow, including fixing format and clippy, building and running tests" dependencies = [ + "wit", "fix", "check", "build", # "test" ] +# WIT DEPENDENCIES +[tasks.wit] +description = "Fetches the WIT dependencies based on wit/deps.toml" +dependencies = ["wit-host", "wit-wasm-rpc"] + +[tasks.wit-wasm-rpc] +dependencies = ["wit-host"] +script_runner = "@duckscript" +script = """ +rm -r wasm-rpc/wit/deps +mkdir wasm-rpc/wit/deps +cp wit/deps/io wasm-rpc/wit/deps +cp wit/deps/wasm-rpc wasm-rpc/wit/deps +""" + +[tasks.wit-host] +install_crate = { crate_name = "wit-deps-cli", binary = "wit-deps", test_arg = "--help" } +command = "wit-deps" +args = ["update"] + [tasks.build] +dependencies = ["wit"] description = "Builds everything in debug mode" command = "cargo" args = ["build", "--workspace", "--all-targets"] @@ -64,16 +87,19 @@ condition = { env_not_set = [ run_task = "build-bins" [tasks.build-bins] +dependencies = ["wit"] description = "Builds all executables in debug mode" command = "cargo" args = ["build", "--workspace", "--bins"] [tasks.build-worker-service] +dependencies = ["wit"] description = "Builds the worker-service" command = "cargo" args = ["build", "-p", "golem-worker-service"] [tasks.build-component-service] +dependencies = ["wit"] description = "Builds the component-service" command = "cargo" args = ["build", "-p", "golem-component-service"] @@ -83,6 +109,7 @@ description = """This is the top-level task that builds everything in release mo to build for other target than the current one, can be linux/amd64 or linux/arm64. This is used for cross-compiling for docker images.""" dependencies = [ + "wit", "set-version", "build-release-default", "build-release-override-linux-amd64", @@ -115,7 +142,7 @@ args = ["build", "--release", "--target", "aarch64-unknown-linux-gnu"] [tasks.check] description = "Runs rustfmt and clippy checks without applying any fix" -dependencies = ["check-clippy", "check-rustfmt"] +dependencies = ["wit", "check-clippy", "check-rustfmt"] [tasks.check-rustfmt] description = "Runs rustfmt checks without applying any fix" @@ -133,7 +160,7 @@ args = ["clippy", "--all-targets", "--", "--no-deps", "-Dwarnings"] [tasks.fix] description = "Runs rustfmt and clippy checks and applies fixes" -dependencies = ["fix-clippy", "fix-rustfmt"] +dependencies = ["wit", "fix-clippy", "fix-rustfmt"] [tasks.fix-rustfmt] description = "Runs rustfmt checks and applies fixes" @@ -154,7 +181,6 @@ args = [ "--no-deps", "-Dwarnings", ] - ## ** TEST ** [tasks.test] @@ -168,6 +194,7 @@ dependencies = [ ] [tasks.unit-tests] +dependencies = ["wit"] description = "Runs unit tests only" script = ''' cargo test --workspace --lib --all-features --exclude wasm-rpc-stubgen-tests-integration -- --nocapture --report-time $JUNIT_OPTS @@ -175,6 +202,7 @@ cargo test -p golem-wasm-ast --tests --all-features -- --nocapture --report-time ''' [tasks.worker-executor-tests] +dependencies = ["wit"] description = "Runs worker executor tests only" env = { "WASMTIME_BACKTRACE_DETAILS" = "1", "RUST_BACKTRACE" = "1", "RUST_LOG" = "info" } command = "cargo" @@ -189,6 +217,7 @@ args = [ ] [tasks.worker-executor-tests-group1] +dependencies = ["wit"] description = "Runs worker executor tests only (group 1/8)" env = { "RUST_BACKTRACE" = "1", "WASMTIME_BACKTRACE_DETAILS" = "1", "RUST_LOG" = "info", "RUST_TEST_TIME_INTEGRATION" = "5000,30000" } script = ''' @@ -197,6 +226,7 @@ cargo test --package golem-worker-executor-base --test integration :tag:group1 - ''' [tasks.worker-executor-tests-group2] +dependencies = ["wit"] description = "Runs worker executor tests only (group 2/8)" env = { "RUST_BACKTRACE" = "1", "WASMTIME_BACKTRACE_DETAILS" = "1", "RUST_LOG" = "info", "RUST_TEST_TIME_INTEGRATION" = "5000,30000" } script = ''' @@ -204,6 +234,7 @@ cargo test --package golem-worker-executor-base --test integration :tag:group2 - ''' [tasks.worker-executor-tests-group3] +dependencies = ["wit"] description = "Runs worker executor tests only (group 3/8)" env = { "RUST_BACKTRACE" = "1", "WASMTIME_BACKTRACE_DETAILS" = "1", "RUST_LOG" = "info", "RUST_TEST_TIME_INTEGRATION" = "5000,30000" } script = ''' @@ -211,6 +242,7 @@ cargo test --package golem-worker-executor-base --test integration :tag:group3 - ''' [tasks.worker-executor-tests-group4] +dependencies = ["wit"] description = "Runs worker executor tests only (group 4/8)" env = { "RUST_BACKTRACE" = "1", "WASMTIME_BACKTRACE_DETAILS" = "1", "RUST_LOG" = "info", "RUST_TEST_TIME_INTEGRATION" = "5000,30000" } script = ''' @@ -219,6 +251,7 @@ cargo test --package golem-worker-executor-base --test integration :tag:group4 - # NOTE: temporarily set test-threads=1 to debug flakyness [tasks.worker-executor-tests-group5] +dependencies = ["wit"] description = "Runs worker executor tests only (group 5/8)" env = { "RUST_BACKTRACE" = "1", "WASMTIME_BACKTRACE_DETAILS" = "1", "RUST_LOG" = "info", "RUST_TEST_TIME_INTEGRATION" = "5000,30000" } script = ''' @@ -227,6 +260,7 @@ cargo test --package golem-worker-executor-base --test integration :tag:group5 - [tasks.worker-executor-tests-group6] +dependencies = ["wit"] description = "Runs worker executor tests only (group 6/8)" env = { "RUST_BACKTRACE" = "1", "WASMTIME_BACKTRACE_DETAILS" = "1", "RUST_LOG" = "info", "RUST_TEST_TIME_INTEGRATION" = "5000,30000" } script = ''' @@ -234,6 +268,7 @@ cargo test --package golem-worker-executor-base --test integration :tag:group6 - ''' [tasks.worker-executor-tests-group7] +dependencies = ["wit"] description = "Runs worker executor tests only (group 7/8)" env = { "RUST_BACKTRACE" = "1", "WASMTIME_BACKTRACE_DETAILS" = "1", "RUST_LOG" = "info", "RUST_TEST_TIME_INTEGRATION" = "5000,30000" } script = ''' @@ -241,6 +276,7 @@ cargo test --package golem-worker-executor-base --test integration :tag:group7 - ''' [tasks.worker-executor-tests-group8] +dependencies = ["wit"] description = "Runs worker executor tests only (group 8/8)" env = { "RUST_BACKTRACE" = "1", "WASMTIME_BACKTRACE_DETAILS" = "1", "RUST_LOG" = "info", "RUST_TEST_TIME_INTEGRATION" = "5000,30000" } script = ''' diff --git a/golem-client/Cargo.toml b/golem-client/Cargo.toml index e56ddd0b44..3e4fa89e8f 100644 --- a/golem-client/Cargo.toml +++ b/golem-client/Cargo.toml @@ -13,7 +13,7 @@ include = ["src/**/*", "Cargo.toml", "build.rs", "openapi/**/*"] harness = false [dependencies] -golem-common = { path = "../golem-common", version = "0.0.0", default-features = false } +golem-common = { path = "../golem-common", version = "0.0.0", default-features = false, features = ["model"] } golem-wasm-ast = { path = "../wasm-ast", version = "0.0.0", default-features = false, features = ["analysis"] } golem-wasm-rpc = { path = "../wasm-rpc", version = "0.0.0", default-features = false } diff --git a/golem-common/Cargo.toml b/golem-common/Cargo.toml index 0c0be48346..7655cfe5f1 100644 --- a/golem-common/Cargo.toml +++ b/golem-common/Cargo.toml @@ -8,13 +8,16 @@ repository = "https://github.com/golemcloud/golem" description = "Shared code between Golem services" [features] -default = ["config", "observability", "poem", "protobuf", "redis", "sql", "tokio"] +default = ["base-model", "config", "model", "observability", "poem", "protobuf", "redis", "serialization", "sql", "tokio"] +base-model = [] config = ["dep:figment"] +model = ["base-model", "dep:golem-wasm-rpc", "dep:golem-wasm-ast", "dep:golem-rib"] observability = ["dep:console-subscriber", "dep:prometheus", "dep:tracing", "dep:tracing-subscriber", "dep:tracing-serde"] poem = ["dep:poem", "dep:poem-openapi", "golem-wasm-ast/poem_openapi", "golem-wasm-ast/poem_openapi", "golem-rib/poem"] protobuf = ["dep:prost", "dep:prost-types", "dep:tonic", "dep:golem-api-grpc", "golem-rib/protobuf", "golem-wasm-ast/protobuf", "golem-wasm-rpc/protobuf"] redis = ["dep:fred"] +serialization = [] sql = ["dep:sqlx"] tokio = ["dep:tokio"] @@ -23,9 +26,9 @@ harness = false [dependencies] golem-api-grpc = { path = "../golem-api-grpc", version = "0.0.0", optional = true } -golem-rib = { path = "../golem-rib", version = "0.0.0", default-features = false, features = ["protobuf", "poem", "json_in_errors"] } # TODO: make these optionals -golem-wasm-ast = { path = "../wasm-ast", version = "0.0.0", default-features = false, features = ["metadata"] } -golem-wasm-rpc = { path = "../wasm-rpc", version = "0.0.0", default-features = false, features = ["host"] } +golem-rib = { path = "../golem-rib", version = "0.0.0", default-features = false, features = ["protobuf", "poem", "json_in_errors"], optional = true } # TODO: make these optionals +golem-wasm-ast = { path = "../wasm-ast", version = "0.0.0", default-features = false, features = ["metadata"], optional = true } +golem-wasm-rpc = { path = "../wasm-rpc", version = "0.0.0", default-features = false, features = ["host"], optional = true } anyhow = { workspace = true } async-trait = { workspace = true } diff --git a/golem-common/src/base_model.rs b/golem-common/src/base_model.rs new file mode 100644 index 0000000000..9b2993f326 --- /dev/null +++ b/golem-common/src/base_model.rs @@ -0,0 +1,358 @@ +// Copyright 2024-2025 Golem Cloud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::newtype_uuid; +use bincode::{Decode, Encode}; +use std::collections::HashSet; +use std::fmt::{Display, Formatter}; +use std::str::FromStr; +use uuid::Uuid; + +newtype_uuid!( + ComponentId, + golem_api_grpc::proto::golem::component::ComponentId +); + +newtype_uuid!(ProjectId, golem_api_grpc::proto::golem::common::ProjectId); + +newtype_uuid!( + PluginInstallationId, + golem_api_grpc::proto::golem::common::PluginInstallationId +); + +#[derive(Clone, Copy, Debug, Eq, PartialEq, PartialOrd, Ord, Hash, Encode, Decode)] +#[cfg_attr(feature = "model", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "poem", derive(poem_openapi::Object))] +#[cfg_attr(feature = "poem", oai(rename_all = "camelCase"))] +#[cfg_attr(feature = "model", serde(rename_all = "camelCase"))] +pub struct ShardId { + pub(crate) value: i64, +} + +impl ShardId { + pub fn new(value: i64) -> Self { + Self { value } + } + + pub fn from_worker_id(worker_id: &WorkerId, number_of_shards: usize) -> Self { + let hash = Self::hash_worker_id(worker_id); + let value = hash.abs() % number_of_shards as i64; + Self { value } + } + + pub fn hash_worker_id(worker_id: &WorkerId) -> i64 { + let (high_bits, low_bits) = ( + (worker_id.component_id.0.as_u128() >> 64) as i64, + worker_id.component_id.0.as_u128() as i64, + ); + let high = Self::hash_string(&high_bits.to_string()); + let worker_name = &worker_id.worker_name; + let component_worker_name = format!("{}{}", low_bits, worker_name); + let low = Self::hash_string(&component_worker_name); + ((high as i64) << 32) | ((low as i64) & 0xFFFFFFFF) + } + + fn hash_string(string: &str) -> i32 { + let mut hash = 0; + if hash == 0 && !string.is_empty() { + for val in &mut string.bytes() { + hash = 31_i32.wrapping_mul(hash).wrapping_add(val as i32); + } + } + hash + } + + pub fn is_left_neighbor(&self, other: &ShardId) -> bool { + other.value == self.value + 1 + } +} + +impl Display for ShardId { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "<{}>", self.value) + } +} + +#[cfg(feature = "model")] +impl golem_wasm_rpc::IntoValue for ShardId { + fn into_value(self) -> golem_wasm_rpc::Value { + golem_wasm_rpc::Value::S64(self.value) + } + + fn get_type() -> golem_wasm_ast::analysis::AnalysedType { + golem_wasm_ast::analysis::analysed_type::s64() + } +} + +pub type ComponentVersion = u64; + +#[derive(Clone, Debug, Eq, PartialEq, Hash, Encode, Decode)] +#[cfg_attr(feature = "model", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "poem", derive(poem_openapi::Object))] +#[cfg_attr(feature = "poem", oai(rename_all = "camelCase"))] +#[cfg_attr(feature = "model", serde(rename_all = "camelCase"))] +pub struct WorkerId { + pub component_id: ComponentId, + pub worker_name: String, +} + +impl WorkerId { + pub fn to_redis_key(&self) -> String { + format!("{}:{}", self.component_id.0, self.worker_name) + } + + #[cfg(feature = "model")] + pub fn uri(&self) -> String { + crate::uri::oss::urn::WorkerUrn { + id: self.clone().into_target_worker_id(), + } + .to_string() + } + + /// The dual of `TargetWorkerId::into_worker_id` + pub fn into_target_worker_id(self) -> TargetWorkerId { + TargetWorkerId { + component_id: self.component_id, + worker_name: Some(self.worker_name), + } + } +} + +impl FromStr for WorkerId { + type Err = String; + + fn from_str(s: &str) -> Result { + let parts: Vec<&str> = s.split(':').collect(); + if parts.len() == 2 { + let component_id_uuid = Uuid::from_str(parts[0]) + .map_err(|_| format!("invalid component id: {s} - expected uuid"))?; + let component_id = ComponentId(component_id_uuid); + let worker_name = parts[1].to_string(); + Ok(Self { + component_id, + worker_name, + }) + } else { + Err(format!( + "invalid worker id: {s} - expected format: :" + )) + } + } +} + +impl Display for WorkerId { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(&format!("{}/{}", self.component_id, self.worker_name)) + } +} + +#[cfg(feature = "model")] +impl golem_wasm_rpc::IntoValue for WorkerId { + fn into_value(self) -> golem_wasm_rpc::Value { + golem_wasm_rpc::Value::Record(vec![ + self.component_id.into_value(), + self.worker_name.into_value(), + ]) + } + + fn get_type() -> golem_wasm_ast::analysis::AnalysedType { + use golem_wasm_ast::analysis::analysed_type::{field, record}; + record(vec![ + field("component_id", ComponentId::get_type()), + field("worker_name", String::get_type()), + ]) + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Hash, Encode, Decode)] +#[cfg_attr(feature = "model", derive(serde::Serialize, serde::Deserialize))] +pub struct TargetWorkerId { + pub component_id: ComponentId, + pub worker_name: Option, +} + +impl TargetWorkerId { + #[cfg(feature = "model")] + pub fn uri(&self) -> String { + crate::uri::oss::urn::WorkerUrn { id: self.clone() }.to_string() + } + + /// Converts a `TargetWorkerId` to a `WorkerId` if the worker name is specified + pub fn try_into_worker_id(self) -> Option { + self.worker_name.map(|worker_name| WorkerId { + component_id: self.component_id, + worker_name, + }) + } + + /// Converts a `TargetWorkerId` to a `WorkerId`. If the worker name was not specified, + /// it generates a new unique one, and if the `force_in_shard` set is not empty, it guarantees + /// that the generated worker ID will belong to one of the provided shards. + /// + /// If the worker name was specified, `force_in_shard` is ignored. + pub fn into_worker_id( + self, + force_in_shard: &HashSet, + number_of_shards: usize, + ) -> WorkerId { + let TargetWorkerId { + component_id, + worker_name, + } = self; + match worker_name { + Some(worker_name) => WorkerId { + component_id, + worker_name, + }, + None => { + if force_in_shard.is_empty() || number_of_shards == 0 { + let worker_name = Uuid::new_v4().to_string(); + WorkerId { + component_id, + worker_name, + } + } else { + let mut current = Uuid::new_v4().to_u128_le(); + loop { + let uuid = Uuid::from_u128_le(current); + let worker_name = uuid.to_string(); + let worker_id = WorkerId { + component_id: component_id.clone(), + worker_name, + }; + let shard_id = ShardId::from_worker_id(&worker_id, number_of_shards); + if force_in_shard.contains(&shard_id) { + return worker_id; + } + current += 1; + } + } + } + } + } +} + +impl Display for TargetWorkerId { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match &self.worker_name { + Some(worker_name) => write!(f, "{}/{}", self.component_id, worker_name), + None => write!(f, "{}/*", self.component_id), + } + } +} + +impl From for TargetWorkerId { + fn from(value: WorkerId) -> Self { + value.into_target_worker_id() + } +} + +impl From<&WorkerId> for TargetWorkerId { + fn from(value: &WorkerId) -> Self { + value.clone().into_target_worker_id() + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Hash, Encode, Decode)] +#[cfg_attr(feature = "model", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "poem", derive(poem_openapi::Object))] +#[cfg_attr(feature = "poem", oai(rename_all = "camelCase"))] +#[cfg_attr(feature = "model", serde(rename_all = "camelCase"))] +pub struct PromiseId { + pub worker_id: WorkerId, + pub oplog_idx: OplogIndex, +} + +impl PromiseId { + pub fn to_redis_key(&self) -> String { + format!("{}:{}", self.worker_id.to_redis_key(), self.oplog_idx) + } +} + +impl Display for PromiseId { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}/{}", self.worker_id, self.oplog_idx) + } +} + +#[cfg(feature = "model")] +impl golem_wasm_rpc::IntoValue for PromiseId { + fn into_value(self) -> golem_wasm_rpc::Value { + golem_wasm_rpc::Value::Record(vec![ + self.worker_id.into_value(), + self.oplog_idx.into_value(), + ]) + } + + fn get_type() -> golem_wasm_ast::analysis::AnalysedType { + use golem_wasm_ast::analysis::analysed_type::{field, record}; + record(vec![ + field("worker_id", WorkerId::get_type()), + field("oplog_idx", OplogIndex::get_type()), + ]) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Encode, Decode, Default)] +#[cfg_attr(feature = "poem", derive(poem_openapi::NewType))] +#[cfg_attr(feature = "model", derive(serde::Serialize, serde::Deserialize))] +pub struct OplogIndex(pub(crate) u64); + +impl OplogIndex { + pub const NONE: OplogIndex = OplogIndex(0); + pub const INITIAL: OplogIndex = OplogIndex(1); + + pub const fn from_u64(value: u64) -> OplogIndex { + OplogIndex(value) + } + + /// Gets the previous oplog index + pub fn previous(&self) -> OplogIndex { + OplogIndex(self.0 - 1) + } + + /// Gets the next oplog index + pub fn next(&self) -> OplogIndex { + OplogIndex(self.0 + 1) + } + + /// Gets the last oplog index belonging to an inclusive range starting at this oplog index, + /// having `count` elements. + pub fn range_end(&self, count: u64) -> OplogIndex { + OplogIndex(self.0 + count - 1) + } +} + +impl Display for OplogIndex { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +impl From for u64 { + fn from(value: OplogIndex) -> Self { + value.0 + } +} + +#[cfg(feature = "model")] +impl golem_wasm_rpc::IntoValue for OplogIndex { + fn into_value(self) -> golem_wasm_rpc::Value { + golem_wasm_rpc::Value::U64(self.0) + } + + fn get_type() -> golem_wasm_ast::analysis::AnalysedType { + golem_wasm_ast::analysis::analysed_type::u64() + } +} diff --git a/golem-common/src/lib.rs b/golem-common/src/lib.rs index 5d084defe4..47f71b3145 100644 --- a/golem-common/src/lib.rs +++ b/golem-common/src/lib.rs @@ -15,6 +15,9 @@ use std::fmt; use std::fmt::{Display, Formatter}; +#[cfg(feature = "base-model")] +pub mod base_model; + #[cfg(feature = "tokio")] pub mod cache; @@ -35,7 +38,10 @@ pub mod json_yaml; #[cfg(feature = "observability")] pub mod metrics; +#[cfg(feature = "model")] pub mod model; + +#[cfg(any(feature = "model", feature = "base-model"))] pub mod newtype; #[cfg(feature = "redis")] @@ -44,16 +50,19 @@ pub mod redis; #[cfg(feature = "sql")] pub mod repo; +#[cfg(feature = "model")] pub mod retriable_error; #[cfg(feature = "tokio")] pub mod retries; +#[cfg(feature = "serialization")] pub mod serialization; #[cfg(feature = "observability")] pub mod tracing; +#[cfg(feature = "model")] pub mod uri; #[cfg(test)] diff --git a/golem-common/src/model/mod.rs b/golem-common/src/model/mod.rs index 33ade7dfd5..45742172e0 100644 --- a/golem-common/src/model/mod.rs +++ b/golem-common/src/model/mod.rs @@ -13,21 +13,15 @@ // limitations under the License. use crate::model::oplog::{ - IndexedResourceKey, OplogEntry, OplogIndex, TimestampedUpdateDescription, WorkerResourceId, + IndexedResourceKey, OplogEntry, TimestampedUpdateDescription, WorkerResourceId, }; use crate::model::regions::DeletedRegions; -use crate::newtype_uuid; -use crate::uri::oss::urn::WorkerUrn; -use bincode::de::read::Reader; use bincode::de::{BorrowDecoder, Decoder}; -use bincode::enc::write::Writer; use bincode::enc::Encoder; use bincode::error::{DecodeError, EncodeError}; use bincode::{BorrowDecode, Decode, Encode}; -use golem_wasm_ast::analysis::analysed_type::{ - field, list, r#enum, record, s64, str, tuple, u32, u64, -}; +use golem_wasm_ast::analysis::analysed_type::{field, list, r#enum, record, str, tuple, u32, u64}; use golem_wasm_ast::analysis::{analysed_type, AnalysedType}; use golem_wasm_rpc::IntoValue; use http::Uri; @@ -43,6 +37,8 @@ use std::time::{Duration, SystemTime}; use typed_path::Utf8UnixPathBuf; use uuid::{uuid, Uuid}; +pub use crate::base_model::*; + pub mod component; pub mod component_constraint; pub mod component_metadata; @@ -81,18 +77,6 @@ impl< #[cfg(not(feature = "poem"))] impl PoemTypeRequirements for T {} -newtype_uuid!( - ComponentId, - golem_api_grpc::proto::golem::component::ComponentId -); - -newtype_uuid!(ProjectId, golem_api_grpc::proto::golem::common::ProjectId); - -newtype_uuid!( - PluginInstallationId, - golem_api_grpc::proto::golem::common::PluginInstallationId -); - #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] #[repr(transparent)] pub struct Timestamp(iso8601_timestamp::Timestamp); @@ -203,82 +187,6 @@ impl IntoValue for Timestamp { } } -pub type ComponentVersion = u64; - -#[derive(Clone, Debug, Eq, PartialEq, Hash, Encode, Decode, Serialize, Deserialize)] -#[cfg_attr(feature = "poem", derive(poem_openapi::Object))] -#[cfg_attr(feature = "poem", oai(rename_all = "camelCase"))] -#[serde(rename_all = "camelCase")] -pub struct WorkerId { - pub component_id: ComponentId, - pub worker_name: String, -} - -impl WorkerId { - pub fn to_redis_key(&self) -> String { - format!("{}:{}", self.component_id.0, self.worker_name) - } - - pub fn uri(&self) -> String { - WorkerUrn { - id: self.clone().into_target_worker_id(), - } - .to_string() - } - - /// The dual of `TargetWorkerId::into_worker_id` - pub fn into_target_worker_id(self) -> TargetWorkerId { - TargetWorkerId { - component_id: self.component_id, - worker_name: Some(self.worker_name), - } - } -} - -impl FromStr for WorkerId { - type Err = String; - - fn from_str(s: &str) -> Result { - let parts: Vec<&str> = s.split(':').collect(); - if parts.len() == 2 { - let component_id_uuid = Uuid::from_str(parts[0]) - .map_err(|_| format!("invalid component id: {s} - expected uuid"))?; - let component_id = ComponentId(component_id_uuid); - let worker_name = parts[1].to_string(); - Ok(Self { - component_id, - worker_name, - }) - } else { - Err(format!( - "invalid worker id: {s} - expected format: :" - )) - } - } -} - -impl Display for WorkerId { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.write_str(&format!("{}/{}", self.component_id, self.worker_name)) - } -} - -impl IntoValue for WorkerId { - fn into_value(self) -> golem_wasm_rpc::Value { - golem_wasm_rpc::Value::Record(vec![ - self.component_id.into_value(), - self.worker_name.into_value(), - ]) - } - - fn get_type() -> AnalysedType { - record(vec![ - field("component_id", ComponentId::get_type()), - field("worker_name", std::string::String::get_type()), - ]) - } -} - /// Associates a worker-id with its owner account #[derive(Clone, Debug, Eq, PartialEq, Hash, Encode, Decode)] pub struct OwnedWorkerId { @@ -317,130 +225,6 @@ impl Display for OwnedWorkerId { } } -#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize, Encode, Decode)] -pub struct TargetWorkerId { - pub component_id: ComponentId, - pub worker_name: Option, -} - -impl TargetWorkerId { - pub fn uri(&self) -> String { - WorkerUrn { id: self.clone() }.to_string() - } - - /// Converts a `TargetWorkerId` to a `WorkerId` if the worker name is specified - pub fn try_into_worker_id(self) -> Option { - self.worker_name.map(|worker_name| WorkerId { - component_id: self.component_id, - worker_name, - }) - } - - /// Converts a `TargetWorkerId` to a `WorkerId`. If the worker name was not specified, - /// it generates a new unique one, and if the `force_in_shard` set is not empty, it guarantees - /// that the generated worker ID will belong to one of the provided shards. - /// - /// If the worker name was specified, `force_in_shard` is ignored. - pub fn into_worker_id( - self, - force_in_shard: &HashSet, - number_of_shards: usize, - ) -> WorkerId { - let TargetWorkerId { - component_id, - worker_name, - } = self; - match worker_name { - Some(worker_name) => WorkerId { - component_id, - worker_name, - }, - None => { - if force_in_shard.is_empty() || number_of_shards == 0 { - let worker_name = Uuid::new_v4().to_string(); - WorkerId { - component_id, - worker_name, - } - } else { - let mut current = Uuid::new_v4().to_u128_le(); - loop { - let uuid = Uuid::from_u128_le(current); - let worker_name = uuid.to_string(); - let worker_id = WorkerId { - component_id: component_id.clone(), - worker_name, - }; - let shard_id = ShardId::from_worker_id(&worker_id, number_of_shards); - if force_in_shard.contains(&shard_id) { - return worker_id; - } - current += 1; - } - } - } - } - } -} - -impl Display for TargetWorkerId { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match &self.worker_name { - Some(worker_name) => write!(f, "{}/{}", self.component_id, worker_name), - None => write!(f, "{}/*", self.component_id), - } - } -} - -impl From for TargetWorkerId { - fn from(value: WorkerId) -> Self { - value.into_target_worker_id() - } -} - -impl From<&WorkerId> for TargetWorkerId { - fn from(value: &WorkerId) -> Self { - value.clone().into_target_worker_id() - } -} - -#[derive(Clone, Debug, Eq, PartialEq, Hash, Encode, Decode, Serialize, Deserialize)] -#[cfg_attr(feature = "poem", derive(poem_openapi::Object))] -#[cfg_attr(feature = "poem", oai(rename_all = "camelCase"))] -#[serde(rename_all = "camelCase")] -pub struct PromiseId { - pub worker_id: WorkerId, - pub oplog_idx: OplogIndex, -} - -impl PromiseId { - pub fn to_redis_key(&self) -> String { - format!("{}:{}", self.worker_id.to_redis_key(), self.oplog_idx) - } -} - -impl Display for PromiseId { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{}/{}", self.worker_id, self.oplog_idx) - } -} - -impl IntoValue for PromiseId { - fn into_value(self) -> golem_wasm_rpc::Value { - golem_wasm_rpc::Value::Record(vec![ - self.worker_id.into_value(), - self.oplog_idx.into_value(), - ]) - } - - fn get_type() -> AnalysedType { - record(vec![ - field("worker_id", WorkerId::get_type()), - field("oplog_idx", OplogIndex::get_type()), - ]) - } -} - /// Actions that can be scheduled to be executed at a given point in time #[derive(Debug, Clone, Hash, Eq, PartialEq, Encode, Decode)] pub enum ScheduledAction { @@ -500,70 +284,6 @@ impl Display for ScheduleId { } } -#[derive( - Clone, Copy, Debug, Eq, PartialEq, PartialOrd, Ord, Hash, Serialize, Deserialize, Encode, Decode, -)] -#[cfg_attr(feature = "poem", derive(poem_openapi::Object))] -#[cfg_attr(feature = "poem", oai(rename_all = "camelCase"))] -#[serde(rename_all = "camelCase")] -pub struct ShardId { - value: i64, -} - -impl ShardId { - pub fn new(value: i64) -> Self { - Self { value } - } - - pub fn from_worker_id(worker_id: &WorkerId, number_of_shards: usize) -> Self { - let hash = Self::hash_worker_id(worker_id); - let value = hash.abs() % number_of_shards as i64; - Self { value } - } - - pub fn hash_worker_id(worker_id: &WorkerId) -> i64 { - let (high_bits, low_bits) = ( - (worker_id.component_id.0.as_u128() >> 64) as i64, - worker_id.component_id.0.as_u128() as i64, - ); - let high = Self::hash_string(&high_bits.to_string()); - let worker_name = &worker_id.worker_name; - let component_worker_name = format!("{}{}", low_bits, worker_name); - let low = Self::hash_string(&component_worker_name); - ((high as i64) << 32) | ((low as i64) & 0xFFFFFFFF) - } - - fn hash_string(string: &str) -> i32 { - let mut hash = 0; - if hash == 0 && !string.is_empty() { - for val in &mut string.bytes() { - hash = 31_i32.wrapping_mul(hash).wrapping_add(val as i32); - } - } - hash - } - - pub fn is_left_neighbor(&self, other: &ShardId) -> bool { - other.value == self.value + 1 - } -} - -impl Display for ShardId { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "<{}>", self.value) - } -} - -impl IntoValue for ShardId { - fn into_value(self) -> golem_wasm_rpc::Value { - golem_wasm_rpc::Value::S64(self.value) - } - - fn get_type() -> AnalysedType { - s64() - } -} - #[derive(Clone)] pub struct NumberOfShards { pub value: usize, diff --git a/golem-common/src/model/oplog.rs b/golem-common/src/model/oplog.rs index e95d446f55..e290d5a6c2 100644 --- a/golem-common/src/model/oplog.rs +++ b/golem-common/src/model/oplog.rs @@ -34,48 +34,7 @@ use std::sync::atomic::AtomicU64; use std::sync::Arc; use uuid::Uuid; -#[derive( - Debug, - Copy, - Clone, - PartialEq, - Eq, - PartialOrd, - Ord, - Hash, - Serialize, - Deserialize, - Encode, - Decode, - Default, -)] -#[cfg_attr(feature = "poem", derive(poem_openapi::NewType))] -pub struct OplogIndex(u64); - -impl OplogIndex { - pub const NONE: OplogIndex = OplogIndex(0); - pub const INITIAL: OplogIndex = OplogIndex(1); - - pub const fn from_u64(value: u64) -> OplogIndex { - OplogIndex(value) - } - - /// Gets the previous oplog index - pub fn previous(&self) -> OplogIndex { - OplogIndex(self.0 - 1) - } - - /// Gets the next oplog index - pub fn next(&self) -> OplogIndex { - OplogIndex(self.0 + 1) - } - - /// Gets the last oplog index belonging to an inclusive range starting at this oplog index, - /// having `count` elements. - pub fn range_end(&self, count: u64) -> OplogIndex { - OplogIndex(self.0 + count - 1) - } -} +pub use crate::base_model::OplogIndex; pub struct OplogIndexRange { current: u64, @@ -105,28 +64,6 @@ impl OplogIndexRange { } } -impl Display for OplogIndex { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.0) - } -} - -impl From for u64 { - fn from(value: OplogIndex) -> Self { - value.0 - } -} - -impl IntoValue for OplogIndex { - fn into_value(self) -> Value { - Value::U64(self.0) - } - - fn get_type() -> AnalysedType { - u64() - } -} - #[derive(Clone)] pub struct AtomicOplogIndex(Arc); diff --git a/golem-common/src/newtype.rs b/golem-common/src/newtype.rs index 0e07527559..2fed81447d 100644 --- a/golem-common/src/newtype.rs +++ b/golem-common/src/newtype.rs @@ -15,19 +15,9 @@ #[macro_export] macro_rules! newtype_uuid { ($name:ident, $proto_type:path) => { - #[derive( - Clone, - Debug, - PartialOrd, - Ord, - derive_more::FromStr, - Eq, - Hash, - PartialEq, - Serialize, - Deserialize, - )] - #[serde(transparent)] + #[derive(Clone, Debug, PartialOrd, Ord, derive_more::FromStr, Eq, Hash, PartialEq)] + #[cfg_attr(feature = "model", derive(serde::Serialize, serde::Deserialize))] + #[cfg_attr(feature = "model", serde(transparent))] pub struct $name(pub Uuid); impl $name { @@ -36,22 +26,35 @@ macro_rules! newtype_uuid { } } - impl Encode for $name { - fn encode(&self, encoder: &mut E) -> Result<(), EncodeError> { + impl bincode::Encode for $name { + fn encode( + &self, + encoder: &mut E, + ) -> Result<(), bincode::error::EncodeError> { + use bincode::enc::write::Writer; + encoder.writer().write(self.0.as_bytes()) } } - impl Decode for $name { - fn decode(decoder: &mut D) -> Result { + impl bincode::Decode for $name { + fn decode( + decoder: &mut D, + ) -> Result { + use bincode::de::read::Reader; + let mut bytes = [0u8; 16]; decoder.reader().read(&mut bytes)?; Ok(Self(Uuid::from_bytes(bytes))) } } - impl<'de> BorrowDecode<'de> for $name { - fn borrow_decode>(decoder: &mut D) -> Result { + impl<'de> bincode::BorrowDecode<'de> for $name { + fn borrow_decode>( + decoder: &mut D, + ) -> Result { + use bincode::de::read::Reader; + let mut bytes = [0u8; 16]; decoder.reader().read(&mut bytes)?; Ok(Self(Uuid::from_bytes(bytes))) @@ -147,12 +150,13 @@ macro_rules! newtype_uuid { } } - impl Display for $name { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + impl std::fmt::Display for $name { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", &self.0) } } + #[cfg(feature = "model")] impl golem_wasm_rpc::IntoValue for $name { fn into_value(self) -> golem_wasm_rpc::Value { let (hi, lo) = self.0.as_u64_pair(); diff --git a/golem-common/src/tracing.rs b/golem-common/src/tracing.rs index b5fd0b7f44..9165c50cdf 100644 --- a/golem-common/src/tracing.rs +++ b/golem-common/src/tracing.rs @@ -266,6 +266,8 @@ pub mod directive { warn("hyper"), warn("tower"), warn("fred"), + warn("wac_graph"), + warn("wasmtime_environ"), ] } } diff --git a/golem-test-framework/Cargo.toml b/golem-test-framework/Cargo.toml index 9c63b8bd84..1562ca8105 100644 --- a/golem-test-framework/Cargo.toml +++ b/golem-test-framework/Cargo.toml @@ -28,6 +28,7 @@ clap = { workspace = true } cli-table = { workspace = true } colored = "2.1.0" console-subscriber = { workspace = true } +futures-util = { workspace = true } itertools = { workspace = true } k8s-openapi = { workspace = true } kill_tree = { version = "0.2.4", features = ["tokio"] } @@ -53,3 +54,6 @@ uuid = { workspace = true } [dev-dependencies] test-r = { workspace = true } + +[features] +default = [] diff --git a/golem-test-framework/src/components/component_service/filesystem.rs b/golem-test-framework/src/components/component_service/filesystem.rs index adde5663d2..6449b13c97 100644 --- a/golem-test-framework/src/components/component_service/filesystem.rs +++ b/golem-test-framework/src/components/component_service/filesystem.rs @@ -148,103 +148,62 @@ impl ComponentService for FileSystemComponentService { async fn get_or_add_component( &self, local_path: &Path, + name: &str, component_type: ComponentType, + files: &[InitialComponentFile], + dynamic_linking: &HashMap, + unverified: bool, ) -> ComponentId { - self.add_component(local_path, component_type) - .await - .expect("Failed to add component") - } - - async fn get_or_add_component_unverified( - &self, - local_path: &Path, - component_type: ComponentType, - ) -> ComponentId { - self.write_component_to_filesystem( + self.add_component( local_path, - &ComponentId(Uuid::new_v4()), - 0, + name, component_type, - &[], - true, - &HashMap::new(), + files, + dynamic_linking, + unverified, ) .await .expect("Failed to add component") } - async fn add_component_with_id( - &self, - local_path: &Path, - component_id: &ComponentId, - component_type: ComponentType, - ) -> Result<(), AddComponentError> { - self.write_component_to_filesystem( - local_path, - component_id, - 0, - component_type, - &[], - false, - &HashMap::new(), - ) - .await?; - Ok(()) - } - async fn add_component( &self, local_path: &Path, + _name: &str, component_type: ComponentType, + files: &[InitialComponentFile], + dynamic_linking: &HashMap, + unverified: bool, ) -> Result { self.write_component_to_filesystem( local_path, &ComponentId(Uuid::new_v4()), 0, component_type, - &[], - false, - &HashMap::new(), + files, + unverified, + dynamic_linking, ) .await } - async fn add_component_with_name( + async fn add_component_with_id( &self, local_path: &Path, - _name: &str, + component_id: &ComponentId, component_type: ComponentType, - ) -> Result { + ) -> Result<(), AddComponentError> { self.write_component_to_filesystem( local_path, - &ComponentId(Uuid::new_v4()), + component_id, 0, component_type, &[], false, &HashMap::new(), ) - .await - } - - async fn add_component_with_files( - &self, - local_path: &Path, - _name: &str, - component_type: ComponentType, - files: &[InitialComponentFile], - dynamic_linking: &HashMap, - ) -> Result { - self.write_component_to_filesystem( - local_path, - &ComponentId(Uuid::new_v4()), - 0, - component_type, - files, - false, - dynamic_linking, - ) - .await + .await?; + Ok(()) } async fn update_component( @@ -306,6 +265,17 @@ impl ComponentService for FileSystemComponentService { *versions.last().unwrap_or(&0) } + async fn get_component_size( + &self, + component_id: &ComponentId, + component_version: ComponentVersion, + ) -> crate::Result> { + let target_dir = &self.root; + let path = target_dir.join(format!("{component_id}-{component_version}.wasm")); + let metadata = tokio::fs::metadata(&path).await?; + Ok(Some(metadata.len())) + } + fn private_host(&self) -> String { panic!("No real component service running") } diff --git a/golem-test-framework/src/components/component_service/mod.rs b/golem-test-framework/src/components/component_service/mod.rs index f75493d1c4..fada9405ce 100644 --- a/golem-test-framework/src/components/component_service/mod.rs +++ b/golem-test-framework/src/components/component_service/mod.rs @@ -15,12 +15,13 @@ use anyhow::anyhow; use async_trait::async_trait; use create_component_request::Data; +use futures_util::TryStreamExt; use golem_api_grpc::proto::golem::component::v1::{ component_error, create_component_request, create_component_response, create_plugin_response, - get_component_metadata_response, get_components_response, install_plugin_response, - update_component_request, update_component_response, CreateComponentRequest, - CreateComponentRequestChunk, CreateComponentRequestHeader, CreatePluginRequest, - GetComponentsRequest, GetLatestComponentRequest, UpdateComponentRequest, + download_component_response, get_component_metadata_response, get_components_response, + install_plugin_response, update_component_request, update_component_response, + CreateComponentRequest, CreateComponentRequestChunk, CreateComponentRequestHeader, + CreatePluginRequest, GetComponentsRequest, GetLatestComponentRequest, UpdateComponentRequest, UpdateComponentRequestChunk, UpdateComponentRequestHeader, }; use std::collections::HashMap; @@ -42,7 +43,9 @@ use golem_api_grpc::proto::golem::component::v1::component_service_client::Compo use golem_api_grpc::proto::golem::component::v1::plugin_service_client::PluginServiceClient; use golem_common::model::component_metadata::DynamicLinkedInstance; use golem_common::model::plugin::{DefaultPluginOwner, DefaultPluginScope, PluginDefinition}; -use golem_common::model::{ComponentId, ComponentType, InitialComponentFile, PluginInstallationId}; +use golem_common::model::{ + ComponentId, ComponentType, ComponentVersion, InitialComponentFile, PluginInstallationId, +}; pub mod docker; pub mod filesystem; @@ -58,24 +61,19 @@ pub trait ComponentService { async fn get_or_add_component( &self, local_path: &Path, + name: &str, component_type: ComponentType, + files: &[InitialComponentFile], + dynamic_linking: &HashMap, + unverified: bool, ) -> ComponentId { let mut retries = 5; loop { - let mut file_name: String = local_path - .file_name() - .unwrap() - .to_string_lossy() - .to_string(); - if component_type == ComponentType::Ephemeral { - file_name = format!("{}-ephemeral", file_name); - } - let mut client = self.client().await; let response = client .get_components(GetComponentsRequest { project_id: None, - component_name: Some(file_name.to_string()), + component_name: Some(name.to_string()), }) .await .expect("Failed to call get-components") @@ -106,24 +104,29 @@ pub trait ComponentService { } _ => { match self - .add_component_with_name(local_path, &file_name, component_type) + .add_component( + local_path, + name, + component_type, + files, + dynamic_linking, + unverified, + ) .await { Ok(component_id) => break component_id, Err(AddComponentError::AlreadyExists) => { if retries > 0 { - info!("Component with name {file_name} got created in parallel, retrying get_or_add_component"); + info!("Component with name {name} got created in parallel, retrying get_or_add_component"); retries -= 1; sleep(Duration::from_secs(1)).await; continue; } else { - panic!("Component with name {file_name} already exists in golem-component-service"); + panic!("Component with name {name} already exists in golem-component-service"); } } Err(AddComponentError::Other(message)) => { - panic!( - "Failed to add component with name {file_name}: {message}" - ); + panic!("Failed to add component with name {name}: {message}"); } } } @@ -136,16 +139,6 @@ pub trait ComponentService { } } - // Forward to get_or_add_component. This method is only used in tests for adding a 'broken' component using the - // filesystem component service, which will skip verification here. - async fn get_or_add_component_unverified( - &self, - local_path: &Path, - component_type: ComponentType, - ) -> ComponentId { - self.get_or_add_component(local_path, component_type).await - } - async fn add_component_with_id( &self, _local_path: &Path, @@ -158,32 +151,13 @@ pub trait ComponentService { } async fn add_component( - &self, - local_path: &Path, - component_type: ComponentType, - ) -> Result { - let file_name = local_path.file_name().unwrap().to_string_lossy(); - self.add_component_with_name(local_path, &file_name, component_type) - .await - } - - async fn add_component_with_name( - &self, - local_path: &Path, - name: &str, - component_type: ComponentType, - ) -> Result { - self.add_component_with_files(local_path, name, component_type, &[], &HashMap::new()) - .await - } - - async fn add_component_with_files( &self, local_path: &Path, name: &str, component_type: ComponentType, files: &[InitialComponentFile], dynamic_linking: &HashMap, + _unverified: bool, ) -> Result { let mut client = self.client().await; let mut file = File::open(local_path).await.map_err(|_| { @@ -452,6 +426,38 @@ pub trait ComponentService { } } + async fn get_component_size( + &self, + component_id: &ComponentId, + component_version: ComponentVersion, + ) -> crate::Result> { + let mut client = self.client().await; + let response = client + .download_component( + golem_api_grpc::proto::golem::component::v1::DownloadComponentRequest { + component_id: Some(component_id.clone().into()), + version: Some(component_version), + }, + ) + .await? + .into_inner(); + + let chunks = response.into_stream().try_collect::>().await?; + let bytes = chunks + .into_iter() + .map(|chunk| match chunk.result { + None => Err(anyhow!("Empty response")), + Some(download_component_response::Result::SuccessChunk(chunk)) => Ok(chunk), + Some(download_component_response::Result::Error(error)) => { + Err(anyhow!("Failed to download component: {error:?}")) + } + }) + .collect::>>>()?; + + let bytes: Vec = bytes.into_iter().flatten().collect(); + Ok(Some(bytes.len() as u64)) + } + fn private_host(&self) -> String; fn private_http_port(&self) -> u16; fn private_grpc_port(&self) -> u16; diff --git a/golem-test-framework/src/dsl/mod.rs b/golem-test-framework/src/dsl/mod.rs index 37048370c0..3fa76d43db 100644 --- a/golem-test-framework/src/dsl/mod.rs +++ b/golem-test-framework/src/dsl/mod.rs @@ -62,36 +62,127 @@ use tokio::sync::oneshot::Sender; use tracing::{debug, info}; use uuid::Uuid; +pub struct StoreComponentBuilder<'a, DSL: TestDsl + ?Sized> { + dsl: &'a DSL, + name: String, + component_type: ComponentType, + unique: bool, + unverified: bool, + files: Vec, + dynamic_linking: Vec<(&'static str, DynamicLinkedInstance)>, +} + +impl<'a, DSL: TestDsl> StoreComponentBuilder<'a, DSL> { + pub fn new(dsl: &'a DSL, name: impl AsRef) -> Self { + Self { + dsl, + name: name.as_ref().to_string(), + component_type: ComponentType::Durable, + unique: false, + unverified: false, + files: vec![], + dynamic_linking: vec![], + } + } + + /// Set the component type to ephemeral. + pub fn ephemeral(mut self) -> Self { + self.component_type = ComponentType::Ephemeral; + self + } + + /// Set the component type to durable. + pub fn durable(mut self) -> Self { + self.component_type = ComponentType::Durable; + self + } + + /// Always create as a new component - otherwise, if the same component was already uploaded, it will be reused + pub fn unique(mut self) -> Self { + self.unique = true; + self + } + + /// Reuse an existing component of the same WASM if it exists + pub fn reused(mut self) -> Self { + self.unique = false; + self + } + + /// Local filesystem mode only - do not try to parse the component + pub fn unverified(mut self) -> Self { + self.unverified = true; + self + } + + /// Local filesystem mode only - parse the component before storing + pub fn verified(mut self) -> Self { + self.unverified = false; + self + } + + /// Set the initial files for the component + pub fn with_files(mut self, files: &[InitialComponentFile]) -> Self { + self.files = files.to_vec(); + self + } + + /// Adds an initial file to the component + pub fn add_file(mut self, file: InitialComponentFile) -> Self { + self.files.push(file); + self + } + + /// Set the dynamic linking for the component + pub fn with_dynamic_linking( + mut self, + dynamic_linking: &[(&'static str, DynamicLinkedInstance)], + ) -> Self { + self.dynamic_linking = dynamic_linking.to_vec(); + self + } + + /// Adds a dynamic linked instance to the component + pub fn add_dynamic_linking( + mut self, + name: &'static str, + instance: DynamicLinkedInstance, + ) -> Self { + self.dynamic_linking.push((name, instance)); + self + } + + /// Stores the component + pub async fn store(self) -> ComponentId { + self.dsl + .store_component_with( + &self.name, + self.component_type, + self.unique, + self.unverified, + &self.files, + &self.dynamic_linking, + ) + .await + } +} + #[async_trait] pub trait TestDsl { - async fn store_component(&self, name: &str) -> ComponentId; - async fn store_ephemeral_component(&self, name: &str) -> ComponentId; - async fn store_unique_component(&self, name: &str) -> ComponentId; - async fn store_component_unverified(&self, name: &str) -> ComponentId; - async fn store_component_with_id(&self, name: &str, component_id: &ComponentId); - async fn store_unique_component_with_files( - &self, - name: &str, - component_type: ComponentType, - files: &[InitialComponentFile], - ) -> ComponentId; - async fn store_component_with_files( + fn component(&self, name: &str) -> StoreComponentBuilder<'_, Self>; + + async fn store_component_with( &self, name: &str, component_type: ComponentType, + unique: bool, + unverified: bool, files: &[InitialComponentFile], - ) -> ComponentId; - async fn store_component_with_dynamic_linking( - &self, - name: &str, - dynamic_linking: &[(&'static str, DynamicLinkedInstance)], - ) -> ComponentId; - async fn store_unique_component_with_dynamic_linking( - &self, - name: &str, dynamic_linking: &[(&'static str, DynamicLinkedInstance)], ) -> ComponentId; + async fn store_component_with_id(&self, name: &str, component_id: &ComponentId); + async fn update_component(&self, component_id: &ComponentId, name: &str) -> ComponentVersion; async fn update_component_with_files( @@ -275,127 +366,67 @@ pub trait TestDsl { #[async_trait] impl TestDsl for T { - async fn store_component(&self, name: &str) -> ComponentId { - let source_path = self.component_directory().join(format!("{name}.wasm")); - - self.component_service() - .get_or_add_component(&source_path, ComponentType::Durable) - .await - } - - async fn store_ephemeral_component(&self, name: &str) -> ComponentId { - let source_path = self.component_directory().join(format!("{name}.wasm")); - - self.component_service() - .get_or_add_component(&source_path, ComponentType::Ephemeral) - .await - } - - async fn store_unique_component(&self, name: &str) -> ComponentId { - let source_path = self.component_directory().join(format!("{name}.wasm")); - let uuid = Uuid::new_v4(); - let unique_name = format!("{name}-{uuid}"); - self.component_service() - .add_component_with_name(&source_path, &unique_name, ComponentType::Durable) - .await - .expect("Failed to store unique component") - } - - async fn store_component_unverified(&self, name: &str) -> ComponentId { - let source_path = self.component_directory().join(format!("{name}.wasm")); - self.component_service() - .get_or_add_component_unverified(&source_path, ComponentType::Durable) - .await - } - - async fn store_component_with_id(&self, name: &str, component_id: &ComponentId) { - let source_path = self.component_directory().join(format!("{name}.wasm")); - self.component_service() - .add_component_with_id(&source_path, component_id, ComponentType::Durable) - .await - .expect("Failed to store component"); - } - - async fn store_unique_component_with_files( - &self, - name: &str, - component_type: ComponentType, - files: &[InitialComponentFile], - ) -> ComponentId { - let source_path = self.component_directory().join(format!("{name}.wasm")); - let uuid = Uuid::new_v4(); - let unique_name = format!("{name}-{uuid}"); - self.component_service() - .add_component_with_files( - &source_path, - &unique_name, - component_type, - files, - &HashMap::new(), - ) - .await - .expect("Failed to store component") + fn component(&self, name: &str) -> StoreComponentBuilder<'_, Self> { + StoreComponentBuilder::new(self, name) } - async fn store_component_with_files( + async fn store_component_with( &self, name: &str, component_type: ComponentType, + unique: bool, + unverified: bool, files: &[InitialComponentFile], - ) -> ComponentId { - let source_path = self.component_directory().join(format!("{name}.wasm")); - self.component_service() - .add_component_with_files(&source_path, name, component_type, files, &HashMap::new()) - .await - .expect("Failed to store component with id {component_id}") - } - - async fn store_component_with_dynamic_linking( - &self, - name: &str, dynamic_linking: &[(&'static str, DynamicLinkedInstance)], ) -> ComponentId { let source_path = self.component_directory().join(format!("{name}.wasm")); + let component_name = if unique { + let uuid = Uuid::new_v4(); + format!("{name}-{uuid}") + } else { + match component_type { + ComponentType::Durable => name.to_string(), + ComponentType::Ephemeral => format!("{name}-ephemeral"), + } + }; let dynamic_linking = HashMap::from_iter( dynamic_linking .iter() .map(|(k, v)| (k.to_string(), v.clone())), ); - self.component_service() - .add_component_with_files( - &source_path, - name, - ComponentType::Durable, - &[], - &dynamic_linking, - ) - .await - .expect("Failed to store component with id {component_id}") + + if unique { + self.component_service() + .add_component( + &source_path, + &component_name, + component_type, + files, + &dynamic_linking, + unverified, + ) + .await + .expect("Failed to add component") + } else { + self.component_service() + .get_or_add_component( + &source_path, + &component_name, + component_type, + files, + &dynamic_linking, + unverified, + ) + .await + } } - async fn store_unique_component_with_dynamic_linking( - &self, - name: &str, - dynamic_linking: &[(&'static str, DynamicLinkedInstance)], - ) -> ComponentId { + async fn store_component_with_id(&self, name: &str, component_id: &ComponentId) { let source_path = self.component_directory().join(format!("{name}.wasm")); - let uuid = Uuid::new_v4(); - let unique_name = format!("{name}-{uuid}"); - let dynamic_linking = HashMap::from_iter( - dynamic_linking - .iter() - .map(|(k, v)| (k.to_string(), v.clone())), - ); self.component_service() - .add_component_with_files( - &source_path, - &unique_name, - ComponentType::Durable, - &[], - &dynamic_linking, - ) + .add_component_with_id(&source_path, component_id, ComponentType::Durable) .await - .expect("Failed to store component") + .expect("Failed to store component"); } async fn add_initial_component_file( @@ -1526,34 +1557,22 @@ pub fn to_worker_metadata( #[async_trait] pub trait TestDslUnsafe { - async fn store_component(&self, name: &str) -> ComponentId; - async fn store_ephemeral_component(&self, name: &str) -> ComponentId; - async fn store_unique_component(&self, name: &str) -> ComponentId; - async fn store_component_unverified(&self, name: &str) -> ComponentId; - async fn store_component_with_id(&self, name: &str, component_id: &ComponentId); - async fn store_unique_component_with_files( - &self, - name: &str, - component_type: ComponentType, - files: &[InitialComponentFile], - ) -> ComponentId; - async fn store_component_with_files( + type Safe: TestDsl; + + fn component(&self, name: &str) -> StoreComponentBuilder<'_, Self::Safe>; + + async fn store_component_with( &self, name: &str, component_type: ComponentType, + unique: bool, + unverified: bool, files: &[InitialComponentFile], - ) -> ComponentId; - async fn store_component_with_dynamic_linking( - &self, - name: &str, - dynamic_linking: &[(&'static str, DynamicLinkedInstance)], - ) -> ComponentId; - async fn store_unique_component_with_dynamic_linking( - &self, - name: &str, dynamic_linking: &[(&'static str, DynamicLinkedInstance)], ) -> ComponentId; + async fn store_component_with_id(&self, name: &str, component_id: &ComponentId); + async fn update_component(&self, component_id: &ComponentId, name: &str) -> ComponentVersion; async fn update_component_with_files( &self, @@ -1707,59 +1726,35 @@ pub trait TestDslUnsafe { #[async_trait] impl TestDslUnsafe for T { - async fn store_component(&self, name: &str) -> ComponentId { - ::store_component(self, name).await - } + type Safe = T; - async fn store_ephemeral_component(&self, name: &str) -> ComponentId { - ::store_ephemeral_component(self, name).await + fn component(&self, name: &str) -> StoreComponentBuilder<'_, T> { + StoreComponentBuilder::new(self, name) } - async fn store_unique_component(&self, name: &str) -> ComponentId { - ::store_unique_component(self, name).await - } - - async fn store_component_unverified(&self, name: &str) -> ComponentId { - ::store_component_unverified(self, name).await - } - - async fn store_component_with_id(&self, name: &str, component_id: &ComponentId) { - ::store_component_with_id(self, name, component_id).await - } - - async fn store_unique_component_with_files( + async fn store_component_with( &self, name: &str, component_type: ComponentType, + unique: bool, + unverified: bool, files: &[InitialComponentFile], - ) -> ComponentId { - ::store_unique_component_with_files(self, name, component_type, files).await - } - - async fn store_component_with_files( - &self, - name: &str, - component_type: ComponentType, - files: &[InitialComponentFile], - ) -> ComponentId { - ::store_component_with_files(self, name, component_type, files).await - } - - async fn store_component_with_dynamic_linking( - &self, - name: &str, dynamic_linking: &[(&'static str, DynamicLinkedInstance)], ) -> ComponentId { - ::store_component_with_dynamic_linking(self, name, dynamic_linking).await + ::store_component_with( + self, + name, + component_type, + unique, + unverified, + files, + dynamic_linking, + ) + .await } - async fn store_unique_component_with_dynamic_linking( - &self, - name: &str, - dynamic_linking: &[(&'static str, DynamicLinkedInstance)], - ) -> ComponentId { - ::store_unique_component_with_dynamic_linking(self, name, dynamic_linking) - .await + async fn store_component_with_id(&self, name: &str, component_id: &ComponentId) { + ::store_component_with_id(self, name, component_id).await } async fn update_component(&self, component_id: &ComponentId, name: &str) -> ComponentVersion { diff --git a/golem-worker-executor-base/Cargo.toml b/golem-worker-executor-base/Cargo.toml index 56ed5b0352..bb6709f2dc 100644 --- a/golem-worker-executor-base/Cargo.toml +++ b/golem-worker-executor-base/Cargo.toml @@ -107,10 +107,6 @@ testcontainers-modules = { workspace = true } test-r = { workspace = true } tracing-subscriber = { workspace = true } -[build-dependencies] -cargo_metadata = "0.19.1" -golem-wit = { workspace = true } - [[test]] name = "integration" path = "tests/lib.rs" diff --git a/golem-worker-executor-base/build.rs b/golem-worker-executor-base/build.rs deleted file mode 100644 index f48566249e..0000000000 --- a/golem-worker-executor-base/build.rs +++ /dev/null @@ -1,106 +0,0 @@ -use cargo_metadata::MetadataCommand; -use std::env::var_os; -use std::path::Path; - -fn main() -> Result<(), Box> { - let golem_wit_root = find_package_root("golem-wit"); - let out_dir = var_os("OUT_DIR").unwrap(); - let target_file = Path::new(&out_dir).join("preview2_mod.rs"); - - std::fs::write(target_file.clone(), preview2_mod_gen(&golem_wit_root)).unwrap(); - - println!("cargo:rerun-if-changed=build.rs"); - println!("cargo:rerun-if-changed=Cargo.toml"); - - Ok(()) -} - -fn find_package_root(name: &str) -> String { - let metadata = MetadataCommand::new() - .manifest_path("./Cargo.toml") - .exec() - .unwrap(); - - let package = metadata - .packages - .into_iter() - .fold(None, |acc, p| { - if p.name == name { - match acc { - None => Some(p), - Some(cp) - if (cp.version < p.version - && (cp.version.major != 0 - || cp.version.minor != 0 - || cp.version.patch != 0)) - || (p.version.major == 0 - && p.version.minor == 0 - && p.version.patch == 0) => - { - Some(p) - } - _ => acc, - } - } else { - acc - } - }) - .unwrap(); - - package.manifest_path.parent().unwrap().to_string() -} - -fn preview2_mod_gen(golem_wit_path: &str) -> String { - let golem_wit_path = Path::new(golem_wit_path) - .join("wit") - .to_str() - .map(|s| s.to_string()) - .unwrap(); - format!( - r#"wasmtime::component::bindgen!({{ - path: r"{golem_wit_path}", - inline: " - package golem:api; - - world golem {{ - import golem:api/host@0.2.0; - import golem:api/host@1.1.0; - import golem:api/oplog@1.1.0; - import golem:api/durability@1.2.0; - - import wasi:blobstore/blobstore; - import wasi:blobstore/container; - import wasi:blobstore/types; - import wasi:keyvalue/atomic@0.1.0; - import wasi:keyvalue/eventual-batch@0.1.0; - import wasi:keyvalue/cache@0.1.0; - import wasi:keyvalue/eventual@0.1.0; - import wasi:keyvalue/types@0.1.0; - import wasi:keyvalue/wasi-keyvalue-error@0.1.0; - import wasi:logging/logging; - }} - ", - world: "golem:api/golem", - tracing: false, - async: true, - trappable_imports: true, - with: {{ - "wasi:io/streams/input-stream": InputStream, - "wasi:io/streams/output-stream": OutputStream, - "wasi:io/poll/pollable": Pollable, - "wasi:blobstore/container/container": super::durable_host::blobstore::types::ContainerEntry, - "wasi:blobstore/container/stream-object-names": super::durable_host::blobstore::types::StreamObjectNamesEntry, - "wasi:blobstore/types/incoming-value": super::durable_host::blobstore::types::IncomingValueEntry, - "wasi:blobstore/types/outgoing-value": super::durable_host::blobstore::types::OutgoingValueEntry, - "wasi:keyvalue/wasi-keyvalue-error/error": super::durable_host::keyvalue::error::ErrorEntry, - "wasi:keyvalue/types/bucket": super::durable_host::keyvalue::types::BucketEntry, - "wasi:keyvalue/types/incoming-value": super::durable_host::keyvalue::types::IncomingValueEntry, - "wasi:keyvalue/types/outgoing-value": super::durable_host::keyvalue::types::OutgoingValueEntry, - "golem:api/host/get-workers": super::durable_host::golem::GetWorkersEntry, - "golem:api/oplog/get-oplog": super::durable_host::golem::v11::GetOplogEntry, - "golem:api/oplog/search-oplog": super::durable_host::golem::v11::SearchOplogEntry, - }}, - }}); - "# - ) -} diff --git a/golem-worker-executor-base/src/durable_host/durability.rs b/golem-worker-executor-base/src/durable_host/durability.rs index ab0807cedb..4a85c0aced 100644 --- a/golem-worker-executor-base/src/durable_host/durability.rs +++ b/golem-worker-executor-base/src/durable_host/durability.rs @@ -16,8 +16,7 @@ use crate::durable_host::DurableWorkerCtx; use crate::error::GolemError; use crate::metrics::wasm::record_host_function_call; use crate::model::PersistenceLevel; -use crate::preview2::golem; -use crate::preview2::golem::api1_2_0; +use crate::preview2::golem::durability::durability; use crate::services::oplog::{CommitLevel, OplogOps}; use crate::workerctx::WorkerCtx; use async_trait::async_trait; @@ -72,6 +71,7 @@ pub trait DurabilityHost { &mut self, function_type: &DurableFunctionType, begin_index: OplogIndex, + forced_commit: bool, ) -> Result<(), GolemError>; /// Gets the current durable execution state @@ -105,62 +105,48 @@ pub trait DurabilityHost { ) -> Result; } -impl From for DurableFunctionType { - fn from(value: api1_2_0::durability::DurableFunctionType) -> Self { +impl From for DurableFunctionType { + fn from(value: durability::DurableFunctionType) -> Self { match value { - api1_2_0::durability::DurableFunctionType::WriteRemote => { - DurableFunctionType::WriteRemote - } - api1_2_0::durability::DurableFunctionType::WriteLocal => { - DurableFunctionType::WriteLocal - } - api1_2_0::durability::DurableFunctionType::WriteRemoteBatched(oplog_index) => { + durability::DurableFunctionType::WriteRemote => DurableFunctionType::WriteRemote, + durability::DurableFunctionType::WriteLocal => DurableFunctionType::WriteLocal, + durability::DurableFunctionType::WriteRemoteBatched(oplog_index) => { DurableFunctionType::WriteRemoteBatched(oplog_index.map(OplogIndex::from_u64)) } - api1_2_0::durability::DurableFunctionType::ReadRemote => { - DurableFunctionType::ReadRemote - } - api1_2_0::durability::DurableFunctionType::ReadLocal => DurableFunctionType::ReadLocal, + durability::DurableFunctionType::ReadRemote => DurableFunctionType::ReadRemote, + durability::DurableFunctionType::ReadLocal => DurableFunctionType::ReadLocal, } } } -impl From for api1_2_0::durability::DurableFunctionType { +impl From for durability::DurableFunctionType { fn from(value: DurableFunctionType) -> Self { match value { - DurableFunctionType::WriteRemote => { - api1_2_0::durability::DurableFunctionType::WriteRemote - } - DurableFunctionType::WriteLocal => { - api1_2_0::durability::DurableFunctionType::WriteLocal - } + DurableFunctionType::WriteRemote => durability::DurableFunctionType::WriteRemote, + DurableFunctionType::WriteLocal => durability::DurableFunctionType::WriteLocal, DurableFunctionType::WriteRemoteBatched(oplog_index) => { - api1_2_0::durability::DurableFunctionType::WriteRemoteBatched( + durability::DurableFunctionType::WriteRemoteBatched( oplog_index.map(|idx| idx.into()), ) } - DurableFunctionType::ReadRemote => { - api1_2_0::durability::DurableFunctionType::ReadRemote - } - DurableFunctionType::ReadLocal => api1_2_0::durability::DurableFunctionType::ReadLocal, + DurableFunctionType::ReadRemote => durability::DurableFunctionType::ReadRemote, + DurableFunctionType::ReadLocal => durability::DurableFunctionType::ReadLocal, } } } -impl From for api1_2_0::durability::OplogEntryVersion { +impl From for durability::OplogEntryVersion { fn from(value: OplogEntryVersion) -> Self { match value { - OplogEntryVersion::V1 => api1_2_0::durability::OplogEntryVersion::V1, - OplogEntryVersion::V2 => api1_2_0::durability::OplogEntryVersion::V2, + OplogEntryVersion::V1 => durability::OplogEntryVersion::V1, + OplogEntryVersion::V2 => durability::OplogEntryVersion::V2, } } } -impl From - for api1_2_0::durability::PersistedDurableFunctionInvocation -{ +impl From for durability::PersistedDurableFunctionInvocation { fn from(value: PersistedDurableFunctionInvocation) -> Self { - api1_2_0::durability::PersistedDurableFunctionInvocation { + durability::PersistedDurableFunctionInvocation { timestamp: value.timestamp.into(), function_name: value.function_name, response: value.response, @@ -171,7 +157,7 @@ impl From } #[async_trait] -impl api1_2_0::durability::Host for DurableWorkerCtx { +impl durability::Host for DurableWorkerCtx { async fn observe_function_call( &mut self, iface: String, @@ -183,21 +169,23 @@ impl api1_2_0::durability::Host for DurableWorkerCtx { async fn begin_durable_function( &mut self, - function_type: api1_2_0::durability::DurableFunctionType, - ) -> anyhow::Result { + function_type: durability::DurableFunctionType, + ) -> anyhow::Result { let oplog_idx = DurabilityHost::begin_durable_function(self, &function_type.into()).await?; Ok(oplog_idx.into()) } async fn end_durable_function( &mut self, - function_type: api1_2_0::durability::DurableFunctionType, - begin_index: api1_2_0::durability::OplogIndex, + function_type: durability::DurableFunctionType, + begin_index: durability::OplogIndex, + forced_commit: bool, ) -> anyhow::Result<()> { DurabilityHost::end_durable_function( self, &function_type.into(), OplogIndex::from_u64(begin_index), + forced_commit, ) .await?; Ok(()) @@ -205,13 +193,17 @@ impl api1_2_0::durability::Host for DurableWorkerCtx { async fn current_durable_execution_state( &mut self, - ) -> anyhow::Result { + ) -> anyhow::Result { let state = DurabilityHost::durable_execution_state(self); - let persistence_level: golem::api0_2_0::host::PersistenceLevel = - state.persistence_level.into(); - Ok(api1_2_0::durability::DurableExecutionState { + Ok(durability::DurableExecutionState { is_live: state.is_live, - persistence_level: persistence_level.into(), + persistence_level: match state.persistence_level { + PersistenceLevel::PersistNothing => durability::PersistenceLevel::PersistNothing, + PersistenceLevel::PersistRemoteSideEffects => { + durability::PersistenceLevel::PersistRemoteSideEffects + } + PersistenceLevel::Smart => durability::PersistenceLevel::Smart, + }, }) } @@ -220,7 +212,7 @@ impl api1_2_0::durability::Host for DurableWorkerCtx { function_name: String, request: Vec, response: Vec, - function_type: api1_2_0::durability::DurableFunctionType, + function_type: durability::DurableFunctionType, ) -> anyhow::Result<()> { DurabilityHost::persist_durable_function_invocation( self, @@ -236,20 +228,20 @@ impl api1_2_0::durability::Host for DurableWorkerCtx { async fn persist_typed_durable_function_invocation( &mut self, function_name: String, - request: api1_2_0::durability::ValueAndType, - response: api1_2_0::durability::ValueAndType, - function_type: api1_2_0::durability::DurableFunctionType, + request: durability::ValueAndType, + response: durability::ValueAndType, + function_type: durability::DurableFunctionType, ) -> anyhow::Result<()> { let request = unsafe { transmute::< - api1_2_0::durability::ValueAndType, - golem_wasm_rpc::golem::rpc::types::ValueAndType, + durability::ValueAndType, + golem_wasm_rpc::golem::rpc0_1_1::types::ValueAndType, >(request) }; let response = unsafe { transmute::< - api1_2_0::durability::ValueAndType, - golem_wasm_rpc::golem::rpc::types::ValueAndType, + durability::ValueAndType, + golem_wasm_rpc::golem::rpc0_1_1::types::ValueAndType, >(response) }; DurabilityHost::persist_typed_durable_function_invocation( @@ -265,7 +257,7 @@ impl api1_2_0::durability::Host for DurableWorkerCtx { async fn read_persisted_durable_function_invocation( &mut self, - ) -> anyhow::Result { + ) -> anyhow::Result { let invocation = DurabilityHost::read_persisted_durable_function_invocation(self).await?; Ok(invocation.into()) } @@ -288,10 +280,12 @@ impl DurabilityHost for DurableWorkerCtx { &mut self, function_type: &DurableFunctionType, begin_index: OplogIndex, + forced_commit: bool, ) -> Result<(), GolemError> { self.state.end_function(function_type, begin_index).await?; if function_type == &DurableFunctionType::WriteRemote || matches!(function_type, DurableFunctionType::WriteRemoteBatched(_)) + || forced_commit { self.state.oplog.commit(CommitLevel::DurableOnly).await; } @@ -495,7 +489,7 @@ impl Durability { self.function_type.clone(), ) .await; - ctx.end_durable_function(&self.function_type, self.begin_index) + ctx.end_durable_function(&self.function_type, self.begin_index, false) .await?; } Ok(()) @@ -524,7 +518,7 @@ impl Durability { self.function_type.clone(), ) .await; - ctx.end_durable_function(&self.function_type, self.begin_index) + ctx.end_durable_function(&self.function_type, self.begin_index, false) .await?; } Ok(()) @@ -539,7 +533,7 @@ impl Durability { let function_name = self.function_name(); Self::validate_oplog_entry(&oplog_entry, &function_name)?; - ctx.end_durable_function(&self.function_type, self.begin_index) + ctx.end_durable_function(&self.function_type, self.begin_index, false) .await?; Ok((oplog_entry.response.into(), oplog_entry.oplog_entry_version)) diff --git a/golem-worker-executor-base/src/durable_host/dynamic_linking/mod.rs b/golem-worker-executor-base/src/durable_host/dynamic_linking/mod.rs index 0aa03aaf39..10524400e5 100644 --- a/golem-worker-executor-base/src/durable_host/dynamic_linking/mod.rs +++ b/golem-worker-executor-base/src/durable_host/dynamic_linking/mod.rs @@ -18,7 +18,7 @@ use crate::services::component::ComponentMetadata; use crate::workerctx::{DynamicLinking, WorkerCtx}; use async_trait::async_trait; use golem_common::model::component_metadata::DynamicLinkedInstance; -use golem_wasm_rpc::golem::rpc::types::HostFutureInvokeResult; +use golem_wasm_rpc::golem::rpc0_1_1::types::HostFutureInvokeResult; use golem_wasm_rpc::HostWasmRpc; use wasmtime::component::types::ComponentItem; use wasmtime::component::{Component, Linker}; diff --git a/golem-worker-executor-base/src/durable_host/dynamic_linking/wasm_rpc.rs b/golem-worker-executor-base/src/durable_host/dynamic_linking/wasm_rpc.rs index 298112d654..62154af5e3 100644 --- a/golem-worker-executor-base/src/durable_host/dynamic_linking/wasm_rpc.rs +++ b/golem-worker-executor-base/src/durable_host/dynamic_linking/wasm_rpc.rs @@ -18,7 +18,7 @@ use crate::workerctx::WorkerCtx; use anyhow::{anyhow, Context}; use golem_common::model::component_metadata::DynamicLinkedWasmRpc; use golem_common::model::OwnedWorkerId; -use golem_wasm_rpc::golem::rpc::types::{FutureInvokeResult, HostFutureInvokeResult}; +use golem_wasm_rpc::golem::rpc0_1_1::types::{FutureInvokeResult, HostFutureInvokeResult}; use golem_wasm_rpc::wasmtime::{decode_param, encode_output, ResourceStore}; use golem_wasm_rpc::{HostWasmRpc, Uri, Value, WasmRpcEntry, WitValue}; use itertools::Itertools; diff --git a/golem-worker-executor-base/src/durable_host/golem/mod.rs b/golem-worker-executor-base/src/durable_host/golem/mod.rs index 45f9a890d3..de18b34d9e 100644 --- a/golem-worker-executor-base/src/durable_host/golem/mod.rs +++ b/golem-worker-executor-base/src/durable_host/golem/mod.rs @@ -224,7 +224,7 @@ impl golem::api0_2_0::host::Host for DurableWorkerCtx { function_name: String, ) -> Result { self.observe_function_call("golem::api", "get_self_uri"); - let uri = golem_wasm_rpc::golem::rpc::types::Uri::golem_urn( + let uri = golem_wasm_rpc::golem::rpc0_1_1::types::Uri::golem_urn( &self.owned_worker_id.worker_id, Some(&function_name), ); diff --git a/golem-worker-executor-base/src/durable_host/http/mod.rs b/golem-worker-executor-base/src/durable_host/http/mod.rs index 6cc0a7e4c0..2d770acc47 100644 --- a/golem-worker-executor-base/src/durable_host/http/mod.rs +++ b/golem-worker-executor-base/src/durable_host/http/mod.rs @@ -35,6 +35,7 @@ pub(crate) async fn end_http_request( ctx.end_durable_function( &DurableFunctionType::WriteRemoteBatched(None), *begin_index, + false, ) .await?; ctx.state.open_function_table.remove(&state.root_handle); diff --git a/golem-worker-executor-base/src/durable_host/http/outgoing_http.rs b/golem-worker-executor-base/src/durable_host/http/outgoing_http.rs index 175d1d0471..a9413897f4 100644 --- a/golem-worker-executor-base/src/durable_host/http/outgoing_http.rs +++ b/golem-worker-executor-base/src/durable_host/http/outgoing_http.rs @@ -92,6 +92,7 @@ impl Host for DurableWorkerCtx { self.end_durable_function( &DurableFunctionType::WriteRemoteBatched(None), begin_index, + false, ) .await .map_err(|err| HttpError::trap(anyhow!(err)))?; diff --git a/golem-worker-executor-base/src/durable_host/wasm_rpc/mod.rs b/golem-worker-executor-base/src/durable_host/wasm_rpc/mod.rs index fab7feab01..dd8a7c2e0a 100644 --- a/golem-worker-executor-base/src/durable_host/wasm_rpc/mod.rs +++ b/golem-worker-executor-base/src/durable_host/wasm_rpc/mod.rs @@ -35,7 +35,7 @@ use golem_common::model::{ }; use golem_common::serialization::try_deserialize; use golem_common::uri::oss::urn::{WorkerFunctionUrn, WorkerOrFunctionUrn}; -use golem_wasm_rpc::golem::rpc::types::{ +use golem_wasm_rpc::golem::rpc0_1_1::types::{ FutureInvokeResult, HostFutureInvokeResult, Pollable, Uri, }; use golem_wasm_rpc::protobuf::type_annotated_value::TypeAnnotatedValue; @@ -89,8 +89,8 @@ impl HostWasmRpc for DurableWorkerCtx { function_name: String, mut function_params: Vec, ) -> anyhow::Result> { - let args = self.get_arguments().await?; - let env = self.get_environment().await?; + let args = persisted_get_arguments(self).await?; + let env = persisted_get_environment(self).await?; let entry = self.table().get(&self_)?; let payload = entry.payload.downcast_ref::().unwrap(); @@ -221,8 +221,8 @@ impl HostWasmRpc for DurableWorkerCtx { function_name: String, mut function_params: Vec, ) -> anyhow::Result> { - let args = self.get_arguments().await?; - let env = self.get_environment().await?; + let args = persisted_get_arguments(self).await?; + let env = persisted_get_environment(self).await?; let entry = self.table().get(&self_)?; let payload = entry.payload.downcast_ref::().unwrap(); @@ -312,8 +312,8 @@ impl HostWasmRpc for DurableWorkerCtx { function_name: String, mut function_params: Vec, ) -> anyhow::Result> { - let args = self.get_arguments().await?; - let env = self.get_environment().await?; + let args = persisted_get_arguments(self).await?; + let env = persisted_get_environment(self).await?; let begin_index = self .state @@ -780,14 +780,14 @@ impl golem_wasm_rpc::Host for DurableWorkerCtx { // generator does not include types that are not used in any exported _functions_ async fn extract_value( &mut self, - vnt: golem_wasm_rpc::golem::rpc::types::ValueAndType, + vnt: golem_wasm_rpc::golem::rpc0_1_1::types::ValueAndType, ) -> anyhow::Result { Ok(vnt.value) } async fn extract_type( &mut self, - vnt: golem_wasm_rpc::golem::rpc::types::ValueAndType, + vnt: golem_wasm_rpc::golem::rpc0_1_1::types::ValueAndType, ) -> anyhow::Result { Ok(vnt.typ) } @@ -824,6 +824,48 @@ async fn try_get_typed_parameters( Vec::new() } +// For backward compatibility with Golem 1.1 - before compositional durability the RPC calls +// were calling the durable get_environment() +async fn persisted_get_environment( + ctx: &mut DurableWorkerCtx, +) -> anyhow::Result> { + let durability = Durability::, SerializableError>::new( + ctx, + "golem_environment", + "get_environment", + DurableFunctionType::ReadLocal, + ) + .await?; + + if durability.is_live() { + let result = ctx.as_wasi_view().get_environment().await; + durability.persist(ctx, (), result).await + } else { + durability.replay(ctx).await + } +} + +// For backward compatibility with Golem 1.1 - before compositional durability the RPC calls +// were calling the durable get_Arguments() +async fn persisted_get_arguments( + ctx: &mut DurableWorkerCtx, +) -> anyhow::Result> { + let durability = Durability::, SerializableError>::new( + ctx, + "golem_environment", + "get_arguments", + DurableFunctionType::ReadLocal, + ) + .await?; + + if durability.is_live() { + let result = ctx.as_wasi_view().get_arguments().await; + durability.persist(ctx, (), result).await + } else { + durability.replay(ctx).await + } +} + pub enum WasmRpcEntryPayload { Interface { #[allow(dead_code)] diff --git a/golem-worker-executor-base/src/preview2/mod.rs b/golem-worker-executor-base/src/preview2/mod.rs index 5789447233..47e42b8aa6 100644 --- a/golem-worker-executor-base/src/preview2/mod.rs +++ b/golem-worker-executor-base/src/preview2/mod.rs @@ -15,7 +15,29 @@ use golem_wasm_rpc::ValueAndType; use std::mem; -include!(concat!(env!("OUT_DIR"), "/preview2_mod.rs")); +wasmtime::component::bindgen!({ + path: r"../wit", + world: "golem:api/golem", + tracing: false, + async: true, + trappable_imports: true, + with: { + "wasi:io/streams/input-stream": InputStream, + "wasi:io/streams/output-stream": OutputStream, + "wasi:io/poll/pollable": Pollable, + "wasi:blobstore/container/container": super::durable_host::blobstore::types::ContainerEntry, + "wasi:blobstore/container/stream-object-names": super::durable_host::blobstore::types::StreamObjectNamesEntry, + "wasi:blobstore/types/incoming-value": super::durable_host::blobstore::types::IncomingValueEntry, + "wasi:blobstore/types/outgoing-value": super::durable_host::blobstore::types::OutgoingValueEntry, + "wasi:keyvalue/wasi-keyvalue-error/error": super::durable_host::keyvalue::error::ErrorEntry, + "wasi:keyvalue/types/bucket": super::durable_host::keyvalue::types::BucketEntry, + "wasi:keyvalue/types/incoming-value": super::durable_host::keyvalue::types::IncomingValueEntry, + "wasi:keyvalue/types/outgoing-value": super::durable_host::keyvalue::types::OutgoingValueEntry, + "golem:api/host/get-workers": super::durable_host::golem::GetWorkersEntry, + "golem:api/oplog/get-oplog": super::durable_host::golem::v11::GetOplogEntry, + "golem:api/oplog/search-oplog": super::durable_host::golem::v11::SearchOplogEntry, + }, +}); pub type InputStream = wasmtime_wasi::InputStream; pub type OutputStream = wasmtime_wasi::OutputStream; diff --git a/golem-worker-executor-base/src/services/golem_config.rs b/golem-worker-executor-base/src/services/golem_config.rs index 39c665682b..f9e58a45fd 100644 --- a/golem-worker-executor-base/src/services/golem_config.rs +++ b/golem-worker-executor-base/src/services/golem_config.rs @@ -55,6 +55,7 @@ pub struct GolemConfig { pub port: u16, pub http_address: String, pub http_port: u16, + pub debug_worker_output: bool, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -362,6 +363,7 @@ impl Default for GolemConfig { port: 9000, http_address: "0.0.0.0".to_string(), http_port: 8082, + debug_worker_output: true, } } } diff --git a/golem-worker-executor-base/tests/api.rs b/golem-worker-executor-base/tests/api.rs index 99fd5fa56a..a0721a5d6f 100644 --- a/golem-worker-executor-base/tests/api.rs +++ b/golem-worker-executor-base/tests/api.rs @@ -61,7 +61,7 @@ async fn interruption( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("interruption").await; + let component_id = executor.component("interruption").store().await; let worker_id = executor.start_worker(&component_id, "interruption-1").await; let executor_clone = executor.clone(); @@ -95,7 +95,7 @@ async fn simulated_crash( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("interruption").await; + let component_id = executor.component("interruption").store().await; let worker_id = executor .start_worker(&component_id, "simulated-crash-1") .await; @@ -141,7 +141,7 @@ async fn shopping_cart_example( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("shopping-cart").await; + let component_id = executor.component("shopping-cart").store().await; let worker_id = executor .start_worker(&component_id, "shopping-cart-1") .await; @@ -246,7 +246,7 @@ async fn dynamic_worker_creation( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("environment-service").await; + let component_id = executor.component("environment-service").store().await; let worker_id = WorkerId { component_id: component_id.clone(), worker_name: "dynamic-worker-creation-1".to_string(), @@ -320,7 +320,7 @@ async fn dynamic_worker_creation_without_name( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("environment-service").await; + let component_id = executor.component("environment-service").store().await; let worker_id = TargetWorkerId { component_id: component_id.clone(), worker_name: None, @@ -359,7 +359,9 @@ async fn ephemeral_worker_creation_without_name( let executor = start(deps, &context).await.unwrap(); let component_id = executor - .store_ephemeral_component("environment-service") + .component("environment-service") + .ephemeral() + .store() .await; let worker_id = TargetWorkerId { component_id: component_id.clone(), @@ -398,7 +400,7 @@ async fn ephemeral_worker_creation_with_name_is_not_persistent( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_ephemeral_component("counters").await; + let component_id = executor.component("counters").ephemeral().store().await; let worker_id = TargetWorkerId { component_id: component_id.clone(), worker_name: Some("test".to_string()), @@ -437,7 +439,7 @@ async fn promise( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("promise").await; + let component_id = executor.component("promise").store().await; let worker_id = executor.start_worker(&component_id, "promise-1").await; let executor_clone = executor.clone(); @@ -493,7 +495,7 @@ async fn get_self_uri( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("runtime-service").await; + let component_id = executor.component("runtime-service").store().await; let worker_id = executor .start_worker(&component_id, "runtime-service-1") .await; @@ -526,7 +528,7 @@ async fn get_workers_from_worker( let context = TestContext::new(last_unique_id); let mut executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("runtime-service").await; + let component_id = executor.component("runtime-service").store().await; let worker_id1 = executor .start_worker(&component_id, "runtime-service-1") @@ -605,7 +607,7 @@ async fn get_metadata_from_worker( let context = TestContext::new(last_unique_id); let mut executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("runtime-service").await; + let component_id = executor.component("runtime-service").store().await; let worker_id1 = executor .start_worker(&component_id, "runtime-service-1") @@ -698,7 +700,7 @@ async fn invoking_with_same_idempotency_key_is_idempotent( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("shopping-cart").await; + let component_id = executor.component("shopping-cart").store().await; let worker_id = executor .start_worker(&component_id, "shopping-cart-2") .await; @@ -761,7 +763,7 @@ async fn invoking_with_same_idempotency_key_is_idempotent_after_restart( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("shopping-cart").await; + let component_id = executor.component("shopping-cart").store().await; let worker_id = executor .start_worker(&component_id, "shopping-cart-4") .await; @@ -828,7 +830,7 @@ async fn optional_parameters( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("option-service").await; + let component_id = executor.component("option-service").store().await; let worker_id = executor .start_worker(&component_id, "optional-service-1") .await; @@ -896,7 +898,7 @@ async fn flags_parameters( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("flags-service").await; + let component_id = executor.component("flags-service").store().await; let worker_id = executor .start_worker(&component_id, "flags-service-1") .await; @@ -951,7 +953,7 @@ async fn variants_with_no_payloads( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("variant-service").await; + let component_id = executor.component("variant-service").store().await; let worker_id = executor .start_worker(&component_id, "variant-service-1") .await; @@ -975,7 +977,7 @@ async fn delete_worker( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("option-service").await; + let component_id = executor.component("option-service").store().await; let worker_id = executor .start_worker(&component_id, "delete-worker-1") .await; @@ -1042,7 +1044,7 @@ async fn get_workers( let context = TestContext::new(last_unique_id); let mut executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("option-service").await; + let component_id = executor.component("option-service").store().await; let workers_count = 10; let mut worker_ids = vec![]; @@ -1164,7 +1166,7 @@ async fn error_handling_when_worker_is_invoked_with_fewer_than_expected_paramete let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("option-service").await; + let component_id = executor.component("option-service").store().await; let worker_id = executor .start_worker(&component_id, "fewer-than-expected-parameters-1") .await; @@ -1185,7 +1187,7 @@ async fn error_handling_when_worker_is_invoked_with_more_than_expected_parameter let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("option-service").await; + let component_id = executor.component("option-service").store().await; let worker_id = executor .start_worker(&component_id, "more-than-expected-parameters-1") .await; @@ -1215,9 +1217,14 @@ async fn get_worker_metadata( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("clock-service").await; - let component_path = executor.component_directory().join("clock-service.wasm"); - let expected_component_size = tokio::fs::metadata(&component_path).await.unwrap().len(); + let component_id = executor.component("clock-service").store().await; + + let expected_component_size = deps + .component_service + .get_component_size(&component_id, 0) + .await + .unwrap() + .unwrap(); let worker_id = executor .start_worker(&component_id, "get-worker-metadata-1") @@ -1266,7 +1273,7 @@ async fn get_worker_metadata( ); check!(metadata2.last_known_status.component_size == expected_component_size); - check!(metadata2.last_known_status.total_linear_memory_size == 1245184); + check!(metadata2.last_known_status.total_linear_memory_size == 2293760); } #[test] @@ -1278,7 +1285,7 @@ async fn create_invoke_delete_create_invoke( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("shopping-cart").await; + let component_id = executor.component("shopping-cart").store().await; let worker_id = executor .start_worker(&component_id, "create-invoke-delete-create-invoke-1") .await; @@ -1330,7 +1337,7 @@ async fn recovering_an_old_worker_after_updating_a_component( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_unique_component("shopping-cart").await; + let component_id = executor.component("shopping-cart").unique().store().await; let worker_id = executor .start_worker( &component_id, @@ -1413,7 +1420,7 @@ async fn recreating_a_worker_after_it_got_deleted_with_a_different_version( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_unique_component("shopping-cart").await; + let component_id = executor.component("shopping-cart").unique().store().await; let worker_id = executor .start_worker( &component_id, @@ -1483,7 +1490,11 @@ async fn trying_to_use_an_old_wasm_provides_good_error_message( let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component_unverified("old-component").await; + let component_id = executor + .component("old-component") + .unverified() + .store() + .await; let result = executor .try_start_worker(&component_id, "old-component-1") .await; @@ -1508,7 +1519,7 @@ async fn trying_to_use_a_wasm_that_wasmtime_cannot_load_provides_good_error_mess let context = TestContext::new(last_unique_id); // case: WASM can be parsed but wasmtime does not support it let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("write-stdout").await; + let component_id = executor.component("write-stdout").store().await; let cwd = env::current_dir().expect("Failed to get current directory"); debug!("Current directory: {cwd:?}"); @@ -1547,7 +1558,7 @@ async fn trying_to_use_a_wasm_that_wasmtime_cannot_load_provides_good_error_mess ) { let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("write-stdout").await; + let component_id = executor.component("write-stdout").store().await; let worker_id = executor .try_start_worker(&component_id, "bad-wasm-2") @@ -1629,9 +1640,10 @@ async fn long_running_poll_loop_works_as_expected( axum::serve(listener, route).await.unwrap(); }); - let component_id = executor.store_component("http-client-2").await; + let component_id = executor.component("http-client-2").store().await; let mut env = HashMap::new(); env.insert("PORT".to_string(), host_http_port.to_string()); + env.insert("RUST_BACKTRACE".to_string(), "1".to_string()); let worker_id = executor .start_worker_with(&component_id, "poll-loop-component-0", vec![], env) @@ -1697,7 +1709,7 @@ async fn long_running_poll_loop_interrupting_and_resuming_by_second_invocation( axum::serve(listener, route).await.unwrap(); }); - let component_id = executor.store_component("http-client-2").await; + let component_id = executor.component("http-client-2").store().await; let mut env = HashMap::new(); env.insert("PORT".to_string(), host_http_port.to_string()); let worker_id = executor @@ -1824,7 +1836,7 @@ async fn long_running_poll_loop_connection_breaks_on_interrupt( axum::serve(listener, route).await.unwrap(); }); - let component_id = executor.store_component("http-client-2").await; + let component_id = executor.component("http-client-2").store().await; let mut env = HashMap::new(); env.insert("PORT".to_string(), host_http_port.to_string()); let worker_id = executor @@ -1905,7 +1917,7 @@ async fn long_running_poll_loop_connection_retry_does_not_resume_interrupted_wor axum::serve(listener, route).await.unwrap(); }); - let component_id = executor.store_component("http-client-2").await; + let component_id = executor.component("http-client-2").store().await; let mut env = HashMap::new(); env.insert("PORT".to_string(), host_http_port.to_string()); @@ -1976,7 +1988,7 @@ async fn long_running_poll_loop_connection_can_be_restored_after_resume( axum::serve(listener, route).await.unwrap(); }); - let component_id = executor.store_component("http-client-2").await; + let component_id = executor.component("http-client-2").store().await; let mut env = HashMap::new(); env.insert("PORT".to_string(), host_http_port.to_string()); @@ -2078,7 +2090,7 @@ async fn long_running_poll_loop_worker_can_be_deleted_after_interrupt( axum::serve(listener, route).await.unwrap(); }); - let component_id = executor.store_component("http-client-2").await; + let component_id = executor.component("http-client-2").store().await; let mut env = HashMap::new(); env.insert("PORT".to_string(), host_http_port.to_string()); @@ -2123,7 +2135,7 @@ async fn shopping_cart_resource_example( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("shopping-cart-resource").await; + let component_id = executor.component("shopping-cart-resource").store().await; let worker_id = executor .start_worker(&component_id, "shopping-cart-resource-1") .await; @@ -2250,7 +2262,7 @@ async fn counter_resource_test_1( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("counters").await; + let component_id = executor.component("counters").store().await; let worker_id = executor.start_worker(&component_id, "counters-1").await; executor.log_output(&worker_id).await; @@ -2364,7 +2376,7 @@ async fn counter_resource_test_2( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("counters").await; + let component_id = executor.component("counters").store().await; let worker_id = executor.start_worker(&component_id, "counters-2").await; executor.log_output(&worker_id).await; @@ -2516,7 +2528,7 @@ async fn reconstruct_interrupted_state( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("interruption").await; + let component_id = executor.component("interruption").store().await; let worker_id = executor.start_worker(&component_id, "interruption-1").await; let executor_clone = executor.clone(); @@ -2593,7 +2605,7 @@ async fn invocation_queue_is_persistent( axum::serve(listener, route).await.unwrap(); }); - let component_id = executor.store_component("http-client-2").await; + let component_id = executor.component("http-client-2").store().await; let mut env = HashMap::new(); env.insert("PORT".to_string(), host_http_port.to_string()); @@ -2677,7 +2689,7 @@ async fn invoke_with_non_existing_function( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("option-service").await; + let component_id = executor.component("option-service").store().await; let worker_id = executor .start_worker(&component_id, "invoke_with_non_existing_function") .await; @@ -2716,7 +2728,7 @@ async fn stderr_returned_for_failed_component( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("failing-component").await; + let component_id = executor.component("failing-component").store().await; let worker_id = executor .start_worker(&component_id, "failing-worker-1") .await; diff --git a/golem-worker-executor-base/tests/blobstore.rs b/golem-worker-executor-base/tests/blobstore.rs index 6339c27ae6..371b397dc1 100644 --- a/golem-worker-executor-base/tests/blobstore.rs +++ b/golem-worker-executor-base/tests/blobstore.rs @@ -34,7 +34,7 @@ async fn blobstore_exists_return_true_if_the_container_was_created( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("blob-store-service").await; + let component_id = executor.component("blob-store-service").store().await; let worker_name = "blob-store-service-1"; let worker_id = executor.start_worker(&component_id, worker_name).await; @@ -75,7 +75,7 @@ async fn blobstore_exists_return_false_if_the_container_was_not_created( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("blob-store-service").await; + let component_id = executor.component("blob-store-service").store().await; let worker_name = "blob-store-service-1"; let worker_id = executor.start_worker(&component_id, worker_name).await; diff --git a/golem-worker-executor-base/tests/common/mod.rs b/golem-worker-executor-base/tests/common/mod.rs index acbe36a5f1..13164ebd24 100644 --- a/golem-worker-executor-base/tests/common/mod.rs +++ b/golem-worker-executor-base/tests/common/mod.rs @@ -79,10 +79,10 @@ use golem_test_framework::components::shard_manager::ShardManager; use golem_test_framework::components::worker_executor_cluster::WorkerExecutorCluster; use golem_test_framework::config::TestDependencies; use golem_test_framework::dsl::to_worker_metadata; -use golem_wasm_rpc::golem::rpc::types::{FutureInvokeResult, WasmRpc}; -use golem_wasm_rpc::golem::rpc::types::{HostFutureInvokeResult, Pollable}; +use golem_wasm_rpc::golem::rpc0_1_1::types::{FutureInvokeResult, WasmRpc}; +use golem_wasm_rpc::golem::rpc0_1_1::types::{HostFutureInvokeResult, Pollable}; use golem_worker_executor_base::preview2::golem; -use golem_worker_executor_base::preview2::golem::{api1_1_0, api1_2_0}; +use golem_worker_executor_base::preview2::golem::{api1_1_0, durability}; use golem_worker_executor_base::services::events::Events; use golem_worker_executor_base::services::oplog::plugin::OplogProcessorPlugin; use golem_worker_executor_base::services::plugins::{Plugins, PluginsObservations}; @@ -97,6 +97,7 @@ use golem_worker_executor_base::services::worker_proxy::WorkerProxy; use golem_worker_executor_base::worker::{RetryDecision, Worker}; use tonic::transport::Channel; use tracing::{debug, info}; +use uuid::Uuid; use wasmtime::component::{Component, Instance, Linker, Resource, ResourceAny}; use wasmtime::{AsContextMut, Engine, ResourceLimiterAsync}; use wasmtime_wasi::WasiView; @@ -259,17 +260,22 @@ impl TestDependencies for TestWorkerExecutor { } pub struct TestContext { + base_prefix: String, unique_id: u16, } impl TestContext { pub fn new(last_unique_id: &LastUniqueId) -> Self { + let base_prefix = Uuid::new_v4().to_string(); let unique_id = last_unique_id.id.fetch_add(1, Ordering::Relaxed); - Self { unique_id } + Self { + base_prefix, + unique_id, + } } pub fn redis_prefix(&self) -> String { - format!("test-{}:", self.unique_id) + format!("test-{}-{}:", self.base_prefix, self.unique_id) } pub fn grpc_port(&self) -> u16 { @@ -1014,8 +1020,11 @@ impl Bootstrap for ServerBootstrap { api0_2_0::host::add_to_linker_get_host(&mut linker, get_durable_ctx)?; api1_1_0::host::add_to_linker_get_host(&mut linker, get_durable_ctx)?; api1_1_0::oplog::add_to_linker_get_host(&mut linker, get_durable_ctx)?; - api1_2_0::durability::add_to_linker_get_host(&mut linker, get_durable_ctx)?; - golem_wasm_rpc::golem::rpc::types::add_to_linker_get_host(&mut linker, get_durable_ctx)?; + durability::durability::add_to_linker_get_host(&mut linker, get_durable_ctx)?; + golem_wasm_rpc::golem::rpc0_1_1::types::add_to_linker_get_host( + &mut linker, + get_durable_ctx, + )?; Ok(linker) } } diff --git a/golem-worker-executor-base/tests/guest_languages1.rs b/golem-worker-executor-base/tests/guest_languages1.rs index 1536151d67..b3369a4370 100644 --- a/golem-worker-executor-base/tests/guest_languages1.rs +++ b/golem-worker-executor-base/tests/guest_languages1.rs @@ -46,7 +46,7 @@ async fn zig_example_3( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("zig-3").await; + let component_id = executor.component("zig-3").store().await; let worker_id = executor.start_worker(&component_id, "zig-3").await; let _ = executor @@ -77,7 +77,7 @@ async fn tinygo_example( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("tinygo-wasi").await; + let component_id = executor.component("tinygo-wasi").store().await; let worker_id = executor .start_worker_with( &component_id, @@ -181,7 +181,7 @@ async fn tinygo_http_client( axum::serve(listener, route).await.unwrap(); }); - let component_id = executor.store_component("tinygo-wasi-http").await; + let component_id = executor.component("tinygo-wasi-http").store().await; let mut env = HashMap::new(); env.insert("PORT".to_string(), context.host_http_port().to_string()); @@ -227,7 +227,7 @@ async fn grain_example_1( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("grain-1").await; + let component_id = executor.component("grain-1").store().await; let worker_id = executor.start_worker(&component_id, "grain-1").await; let mut rx = executor.capture_output(&worker_id).await; @@ -271,7 +271,7 @@ async fn java_example_1( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("java-1").await; + let component_id = executor.component("java-1").store().await; let worker_id = executor.start_worker(&component_id, "java-1").await; let mut rx = executor.capture_output(&worker_id).await; @@ -317,7 +317,7 @@ async fn java_shopping_cart( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("java-2").await; + let component_id = executor.component("java-2").store().await; let worker_id = executor.start_worker(&component_id, "java-2").await; let _ = executor @@ -420,7 +420,7 @@ async fn c_example_1( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("c-1").await; + let component_id = executor.component("c-1").store().await; let worker_id = executor.start_worker(&component_id, "c-1").await; let mut rx = executor.capture_output(&worker_id).await; @@ -458,7 +458,7 @@ async fn c_example_2( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("c-1").await; + let component_id = executor.component("c-1").store().await; let worker_id = executor.start_worker(&component_id, "c-2").await; let mut rx = executor.capture_output(&worker_id).await; @@ -502,7 +502,7 @@ async fn c_example_3( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("large-initial-memory").await; + let component_id = executor.component("large-initial-memory").store().await; let worker_id = executor .start_worker(&component_id, "large-initial-memory") .await; @@ -530,7 +530,7 @@ async fn c_example_4( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("large-dynamic-memory").await; + let component_id = executor.component("large-dynamic-memory").store().await; let worker_id = executor .start_worker(&component_id, "large-dynamic-memory") .await; diff --git a/golem-worker-executor-base/tests/guest_languages2.rs b/golem-worker-executor-base/tests/guest_languages2.rs index 5a46b37a90..557bd1381c 100644 --- a/golem-worker-executor-base/tests/guest_languages2.rs +++ b/golem-worker-executor-base/tests/guest_languages2.rs @@ -36,7 +36,7 @@ async fn javascript_example_3( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("js-3").await; + let component_id = executor.component("js-3").store().await; let worker_id = executor.start_worker(&component_id, "js-3").await; let result_fetch_get = executor @@ -65,7 +65,7 @@ async fn javascript_example_4( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("js-4").await; + let component_id = executor.component("js-4").store().await; let worker_id = executor.start_worker(&component_id, "js-4").await; let result = executor @@ -88,7 +88,7 @@ async fn python_example_1( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("python-1").await; + let component_id = executor.component("python-1").store().await; let worker_id = executor.start_worker(&component_id, "python-1").await; let _ = executor @@ -121,7 +121,7 @@ async fn swift_example_1( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("swift-1").await; + let component_id = executor.component("swift-1").store().await; let worker_id = executor.start_worker(&component_id, "swift-1").await; let mut rx = executor.capture_output(&worker_id).await; diff --git a/golem-worker-executor-base/tests/guest_languages3.rs b/golem-worker-executor-base/tests/guest_languages3.rs index 3551fe90f7..8427bc6629 100644 --- a/golem-worker-executor-base/tests/guest_languages3.rs +++ b/golem-worker-executor-base/tests/guest_languages3.rs @@ -37,7 +37,7 @@ async fn javascript_example_1( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("js-1").await; + let component_id = executor.component("js-1").store().await; let worker_id = executor.start_worker(&component_id, "js-1").await; let mut rx = executor.capture_output(&worker_id).await; @@ -106,7 +106,7 @@ async fn javascript_example_2( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("js-2").await; + let component_id = executor.component("js-2").store().await; let worker_id = executor.start_worker(&component_id, "js-2").await; let _ = executor @@ -139,7 +139,7 @@ async fn csharp_example_1( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("csharp-1").await; + let component_id = executor.component("csharp-1").store().await; let mut env = HashMap::new(); env.insert("TEST_ENV".to_string(), "test-value".to_string()); let worker_id = executor diff --git a/golem-worker-executor-base/tests/hot_update.rs b/golem-worker-executor-base/tests/hot_update.rs index 57394711b6..36718e6d77 100644 --- a/golem-worker-executor-base/tests/hot_update.rs +++ b/golem-worker-executor-base/tests/hot_update.rs @@ -144,7 +144,7 @@ async fn auto_update_on_running( let mut env = HashMap::new(); env.insert("PORT".to_string(), context.host_http_port().to_string()); - let component_id = executor.store_unique_component("update-test-v1").await; + let component_id = executor.component("update-test-v1").unique().store().await; let worker_id = executor .start_worker_with(&component_id, "auto_update_on_running", vec![], env) .await; @@ -213,7 +213,7 @@ async fn auto_update_on_idle( let context = common::TestContext::new(last_unique_id); let executor = common::start(deps, &context).await.unwrap(); - let component_id = executor.store_unique_component("update-test-v1").await; + let component_id = executor.component("update-test-v1").unique().store().await; let worker_id = executor .start_worker(&component_id, "auto_update_on_idle") .await; @@ -260,7 +260,7 @@ async fn failing_auto_update_on_idle( let mut env = HashMap::new(); env.insert("PORT".to_string(), context.host_http_port().to_string()); - let component_id = executor.store_unique_component("update-test-v1").await; + let component_id = executor.component("update-test-v1").unique().store().await; let worker_id = executor .start_worker_with(&component_id, "failing_auto_update_on_idle", vec![], env) .await; @@ -312,7 +312,7 @@ async fn auto_update_on_idle_with_non_diverging_history( let context = common::TestContext::new(last_unique_id); let executor = common::start(deps, &context).await.unwrap(); - let component_id = executor.store_unique_component("update-test-v1").await; + let component_id = executor.component("update-test-v1").unique().store().await; let worker_id = executor .start_worker( &component_id, @@ -372,7 +372,7 @@ async fn failing_auto_update_on_running( let mut env = HashMap::new(); env.insert("PORT".to_string(), context.host_http_port().to_string()); - let component_id = executor.store_unique_component("update-test-v1").await; + let component_id = executor.component("update-test-v1").unique().store().await; let worker_id = executor .start_worker_with(&component_id, "failing_auto_update_on_running", vec![], env) .await; @@ -453,7 +453,7 @@ async fn manual_update_on_idle( let mut env = HashMap::new(); env.insert("PORT".to_string(), context.host_http_port().to_string()); - let component_id = executor.store_unique_component("update-test-v2").await; + let component_id = executor.component("update-test-v2").unique().store().await; let worker_id = executor .start_worker_with(&component_id, "manual_update_on_idle", vec![], env) .await; @@ -514,7 +514,7 @@ async fn manual_update_on_idle_without_save_snapshot( let mut env = HashMap::new(); env.insert("PORT".to_string(), context.host_http_port().to_string()); - let component_id = executor.store_unique_component("update-test-v1").await; + let component_id = executor.component("update-test-v1").unique().store().await; let worker_id = executor .start_worker_with( &component_id, @@ -574,7 +574,7 @@ async fn auto_update_on_running_followed_by_manual( let mut env = HashMap::new(); env.insert("PORT".to_string(), context.host_http_port().to_string()); - let component_id = executor.store_unique_component("update-test-v1").await; + let component_id = executor.component("update-test-v1").unique().store().await; let worker_id = executor .start_worker_with( &component_id, @@ -666,7 +666,7 @@ async fn manual_update_on_idle_with_failing_load( let mut env = HashMap::new(); env.insert("PORT".to_string(), context.host_http_port().to_string()); - let component_id = executor.store_unique_component("update-test-v2").await; + let component_id = executor.component("update-test-v2").unique().store().await; let worker_id = executor .start_worker_with( &component_id, @@ -725,7 +725,11 @@ async fn manual_update_on_idle_using_v11( let mut env = HashMap::new(); env.insert("PORT".to_string(), context.host_http_port().to_string()); - let component_id = executor.store_unique_component("update-test-v2-11").await; + let component_id = executor + .component("update-test-v2-11") + .unique() + .store() + .await; let worker_id = executor .start_worker_with( &component_id, diff --git a/golem-worker-executor-base/tests/keyvalue.rs b/golem-worker-executor-base/tests/keyvalue.rs index 963f991290..83aafcf7a5 100644 --- a/golem-worker-executor-base/tests/keyvalue.rs +++ b/golem-worker-executor-base/tests/keyvalue.rs @@ -34,7 +34,7 @@ async fn readwrite_get_returns_the_value_that_was_set( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("key-value-service").await; + let component_id = executor.component("key-value-service").store().await; let worker_name = "key-value-service-1"; let worker_id = executor.start_worker(&component_id, worker_name).await; @@ -85,7 +85,7 @@ async fn readwrite_get_fails_if_the_value_was_not_set( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("key-value-service").await; + let component_id = executor.component("key-value-service").store().await; let worker_name = "key-value-service-2"; let worker_id = executor.start_worker(&component_id, worker_name).await; @@ -116,7 +116,7 @@ async fn readwrite_set_replaces_the_value_if_it_was_already_set( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("key-value-service").await; + let component_id = executor.component("key-value-service").store().await; let worker_name = "key-value-service-3"; let worker_id = executor.start_worker(&component_id, worker_name).await; @@ -180,7 +180,7 @@ async fn readwrite_delete_removes_the_value_if_it_was_already_set( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("key-value-service").await; + let component_id = executor.component("key-value-service").store().await; let worker_name = "key-value-service-4"; let worker_id = executor.start_worker(&component_id, worker_name).await; @@ -236,7 +236,7 @@ async fn readwrite_exists_returns_true_if_the_value_was_set( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("key-value-service").await; + let component_id = executor.component("key-value-service").store().await; let worker_name = "key-value-service-5"; let worker_id = executor.start_worker(&component_id, worker_name).await; @@ -280,7 +280,7 @@ async fn readwrite_exists_returns_false_if_the_value_was_not_set( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("key-value-service").await; + let component_id = executor.component("key-value-service").store().await; let worker_name = "key-value-service-6"; let worker_id = executor.start_worker(&component_id, worker_name).await; @@ -311,7 +311,7 @@ async fn readwrite_buckets_can_be_shared_between_workers( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("key-value-service").await; + let component_id = executor.component("key-value-service").store().await; let worker_id_1 = executor .start_worker(&component_id, "key-value-service-7") .await; @@ -366,7 +366,7 @@ async fn batch_get_many_gets_multiple_values( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("key-value-service").await; + let component_id = executor.component("key-value-service").store().await; let worker_name = "key-value-service-9"; let worker_id = executor.start_worker(&component_id, worker_name).await; @@ -447,7 +447,7 @@ async fn batch_get_many_fails_if_any_value_was_not_set( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("key-value-service").await; + let component_id = executor.component("key-value-service").store().await; let worker_name = "key-value-service-10"; let worker_id = executor.start_worker(&component_id, worker_name).await; @@ -508,7 +508,7 @@ async fn batch_set_many_sets_multiple_values( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("key-value-service").await; + let component_id = executor.component("key-value-service").store().await; let worker_name = "key-value-service-11"; let worker_id = executor.start_worker(&component_id, worker_name).await; @@ -611,7 +611,7 @@ async fn batch_delete_many_deletes_multiple_values( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("key-value-service").await; + let component_id = executor.component("key-value-service").store().await; let worker_name = "key-value-service-12"; let worker_id = executor.start_worker(&component_id, worker_name).await; @@ -723,7 +723,7 @@ async fn batch_get_keys_returns_multiple_keys( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("key-value-service").await; + let component_id = executor.component("key-value-service").store().await; let worker_name = "key-value-service-13"; let worker_id = executor.start_worker(&component_id, worker_name).await; diff --git a/golem-worker-executor-base/tests/measure_test_component_mem.rs b/golem-worker-executor-base/tests/measure_test_component_mem.rs index dd24d76745..b74c819f3c 100644 --- a/golem-worker-executor-base/tests/measure_test_component_mem.rs +++ b/golem-worker-executor-base/tests/measure_test_component_mem.rs @@ -12,7 +12,7 @@ use golem_wasm_ast::IgnoreAllButMetadata; use humansize::{ISizeFormatter, BINARY}; use rand::prelude::SliceRandom; use rand::thread_rng; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; use std::fmt::Write; use std::path::Path; use sysinfo::{Pid, ProcessesToUpdate, System}; @@ -102,7 +102,14 @@ async fn measure_component( let component_id = executor .component_service() - .get_or_add_component(path, ComponentType::Durable) + .get_or_add_component( + path, + path.file_name().unwrap().to_string_lossy().as_ref(), + ComponentType::Durable, + &[], + &HashMap::new(), + false, + ) .await; let data = std::fs::read(path)?; diff --git a/golem-worker-executor-base/tests/observability.rs b/golem-worker-executor-base/tests/observability.rs index f3b88ffe86..079af9f6fd 100644 --- a/golem-worker-executor-base/tests/observability.rs +++ b/golem-worker-executor-base/tests/observability.rs @@ -38,7 +38,7 @@ async fn get_oplog_1( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("runtime-service").await; + let component_id = executor.component("runtime-service").store().await; let worker_id = WorkerId { component_id, @@ -106,7 +106,7 @@ async fn search_oplog_1( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("shopping-cart").await; + let component_id = executor.component("shopping-cart").store().await; let worker_id = WorkerId { component_id, @@ -208,7 +208,7 @@ async fn get_oplog_with_api_changing_updates( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_unique_component("update-test-v1").await; + let component_id = executor.component("update-test-v1").unique().store().await; let worker_id = executor .start_worker(&component_id, "get_oplog_with_api_changing_updates") .await; @@ -244,8 +244,10 @@ async fn get_oplog_with_api_changing_updates( .filter(|entry| !matches!(entry, PublicOplogEntry::PendingWorkerInvocation(_))) .collect::>(); + println!("oplog\n{:#?}", oplog); + check!(result[0] == Value::U64(11)); - assert_eq!(oplog.len(), 17); + assert_eq!(oplog.len(), 15); } #[test] @@ -258,7 +260,7 @@ async fn get_oplog_starting_with_updated_component( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_unique_component("update-test-v1").await; + let component_id = executor.component("update-test-v1").unique().store().await; let target_version = executor .update_component(&component_id, "update-test-v2") .await; diff --git a/golem-worker-executor-base/tests/rust_rpc.rs b/golem-worker-executor-base/tests/rust_rpc.rs index 43f2cc7257..d32640cf03 100644 --- a/golem-worker-executor-base/tests/rust_rpc.rs +++ b/golem-worker-executor-base/tests/rust_rpc.rs @@ -37,8 +37,11 @@ async fn auction_example_1( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let registry_component_id = executor.store_component("auction_registry_composed").await; - let auction_component_id = executor.store_component("auction").await; + let registry_component_id = executor + .component("auction_registry_composed") + .store() + .await; + let auction_component_id = executor.component("auction").store().await; let mut env = HashMap::new(); env.insert( @@ -106,8 +109,11 @@ async fn auction_example_2( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let registry_component_id = executor.store_component("auction_registry_composed").await; - let auction_component_id = executor.store_component("auction").await; + let registry_component_id = executor + .component("auction_registry_composed") + .store() + .await; + let auction_component_id = executor.component("auction").store().await; let mut env = HashMap::new(); env.insert( @@ -175,8 +181,8 @@ async fn counter_resource_test_1( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let counters_component_id = executor.store_component("counters").await; - let caller_component_id = executor.store_component("caller_composed").await; + let counters_component_id = executor.component("counters").store().await; + let caller_component_id = executor.component("caller_composed").store().await; let mut env = HashMap::new(); env.insert( @@ -217,8 +223,8 @@ async fn counter_resource_test_2( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let counters_component_id = executor.store_component("counters").await; - let caller_component_id = executor.store_component("caller_composed").await; + let counters_component_id = executor.component("counters").store().await; + let caller_component_id = executor.component("caller_composed").store().await; let mut env = HashMap::new(); env.insert( @@ -260,8 +266,8 @@ async fn counter_resource_test_2_with_restart( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let counters_component_id = executor.store_component("counters").await; - let caller_component_id = executor.store_component("caller_composed").await; + let counters_component_id = executor.component("counters").store().await; + let caller_component_id = executor.component("caller_composed").store().await; let mut env = HashMap::new(); env.insert( @@ -307,8 +313,8 @@ async fn counter_resource_test_3( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let counters_component_id = executor.store_component("counters").await; - let caller_component_id = executor.store_component("caller_composed").await; + let counters_component_id = executor.component("counters").store().await; + let caller_component_id = executor.component("caller_composed").store().await; let mut env = HashMap::new(); env.insert( @@ -350,8 +356,8 @@ async fn counter_resource_test_3_with_restart( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let counters_component_id = executor.store_component("counters").await; - let caller_component_id = executor.store_component("caller_composed").await; + let counters_component_id = executor.component("counters").store().await; + let caller_component_id = executor.component("caller_composed").store().await; let mut env = HashMap::new(); env.insert( @@ -397,8 +403,8 @@ async fn context_inheritance( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let counters_component_id = executor.store_component("counters").await; - let caller_component_id = executor.store_component("caller_composed").await; + let counters_component_id = executor.component("counters").store().await; + let caller_component_id = executor.component("caller_composed").store().await; let mut env = HashMap::new(); env.insert( @@ -487,8 +493,8 @@ async fn counter_resource_test_5( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let counters_component_id = executor.store_component("counters").await; - let caller_component_id = executor.store_component("caller_composed").await; + let counters_component_id = executor.component("counters").store().await; + let caller_component_id = executor.component("caller_composed").store().await; let mut env = HashMap::new(); env.insert( @@ -532,8 +538,8 @@ async fn counter_resource_test_5_with_restart( let executor = start(deps, &context).await.unwrap(); // using store_unique_component to avoid collision with counter_resource_test_5 - let counters_component_id = executor.store_unique_component("counters").await; - let caller_component_id = executor.store_unique_component("caller_composed").await; + let counters_component_id = executor.component("counters").unique().store().await; + let caller_component_id = executor.component("caller_composed").unique().store().await; let mut env = HashMap::new(); env.insert( @@ -597,8 +603,8 @@ async fn wasm_rpc_bug_32_test( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let counters_component_id = executor.store_component("counters").await; - let caller_component_id = executor.store_component("caller_composed").await; + let counters_component_id = executor.component("counters").store().await; + let caller_component_id = executor.component("caller_composed").store().await; let mut env = HashMap::new(); env.insert( @@ -641,7 +647,10 @@ async fn error_message_invalid_uri( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let registry_component_id = executor.store_component("auction_registry_composed").await; + let registry_component_id = executor + .component("auction_registry_composed") + .store() + .await; let mut env = HashMap::new(); env.insert( @@ -696,7 +705,10 @@ async fn error_message_non_existing_target_component( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let registry_component_id = executor.store_component("auction_registry_composed").await; + let registry_component_id = executor + .component("auction_registry_composed") + .store() + .await; let mut env = HashMap::new(); env.insert( @@ -747,8 +759,8 @@ async fn ephemeral_worker_invocation_via_rpc1( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let ephemeral_component_id = executor.store_ephemeral_component("ephemeral").await; - let caller_component_id = executor.store_component("caller_composed").await; + let ephemeral_component_id = executor.component("ephemeral").ephemeral().store().await; + let caller_component_id = executor.component("caller_composed").store().await; let mut env = HashMap::new(); env.insert( diff --git a/golem-worker-executor-base/tests/rust_rpc_stubless.rs b/golem-worker-executor-base/tests/rust_rpc_stubless.rs index ccf420658a..00d482770a 100644 --- a/golem-worker-executor-base/tests/rust_rpc_stubless.rs +++ b/golem-worker-executor-base/tests/rust_rpc_stubless.rs @@ -39,23 +39,22 @@ async fn auction_example_1( let executor = start(deps, &context).await.unwrap(); let registry_component_id = executor - .store_component_with_dynamic_linking( - "auction_registry", - &[( - "auction:auction-client/auction-client", - DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { - target_interface_name: HashMap::from_iter(vec![ - ("api".to_string(), "auction:auction-exports/api".to_string()), - ( - "running-auction".to_string(), - "auction:auction-exports/api".to_string(), - ), - ]), - }), - )], - ) + .component("auction_registry") + .with_dynamic_linking(&[( + "auction:auction-client/auction-client", + DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { + target_interface_name: HashMap::from_iter(vec![ + ("api".to_string(), "auction:auction-exports/api".to_string()), + ( + "running-auction".to_string(), + "auction:auction-exports/api".to_string(), + ), + ]), + }), + )]) + .store() .await; - let auction_component_id = executor.store_component("auction").await; + let auction_component_id = executor.component("auction").store().await; let mut env = HashMap::new(); env.insert( @@ -124,23 +123,22 @@ async fn auction_example_2( let executor = start(deps, &context).await.unwrap(); let registry_component_id = executor - .store_component_with_dynamic_linking( - "auction_registry", - &[( - "auction:auction-client/auction-client", - DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { - target_interface_name: HashMap::from_iter(vec![ - ("api".to_string(), "auction:auction-exports/api".to_string()), - ( - "running-auction".to_string(), - "auction:auction-exports/api".to_string(), - ), - ]), - }), - )], - ) + .component("auction_registry") + .with_dynamic_linking(&[( + "auction:auction-client/auction-client", + DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { + target_interface_name: HashMap::from_iter(vec![ + ("api".to_string(), "auction:auction-exports/api".to_string()), + ( + "running-auction".to_string(), + "auction:auction-exports/api".to_string(), + ), + ]), + }), + )]) + .store() .await; - let auction_component_id = executor.store_component("auction").await; + let auction_component_id = executor.component("auction").store().await; let mut env = HashMap::new(); env.insert( @@ -208,34 +206,33 @@ async fn counter_resource_test_1( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let counters_component_id = executor.store_component("counters").await; + let counters_component_id = executor.component("counters").store().await; let caller_component_id = executor - .store_component_with_dynamic_linking( - "caller", - &[ - ( - "rpc:counters-client/counters-client", - DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { - target_interface_name: HashMap::from_iter(vec![ - ("api".to_string(), "rpc:counters-exports/api".to_string()), - ( - "counter".to_string(), - "rpc:counters-exports/api".to_string(), - ), - ]), - }), - ), - ( - "rpc:ephemeral-client/ephemeral-client", - DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { - target_interface_name: HashMap::from_iter(vec![( - "api".to_string(), - "rpc:ephemeral-exports/api".to_string(), - )]), - }), - ), - ], - ) + .component("caller") + .with_dynamic_linking(&[ + ( + "rpc:counters-client/counters-client", + DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { + target_interface_name: HashMap::from_iter(vec![ + ("api".to_string(), "rpc:counters-exports/api".to_string()), + ( + "counter".to_string(), + "rpc:counters-exports/api".to_string(), + ), + ]), + }), + ), + ( + "rpc:ephemeral-client/ephemeral-client", + DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { + target_interface_name: HashMap::from_iter(vec![( + "api".to_string(), + "rpc:ephemeral-exports/api".to_string(), + )]), + }), + ), + ]) + .store() .await; let mut env = HashMap::new(); @@ -277,34 +274,33 @@ async fn counter_resource_test_2( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let counters_component_id = executor.store_component("counters").await; + let counters_component_id = executor.component("counters").store().await; let caller_component_id = executor - .store_component_with_dynamic_linking( - "caller", - &[ - ( - "rpc:counters-client/counters-client", - DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { - target_interface_name: HashMap::from_iter(vec![ - ("api".to_string(), "rpc:counters-exports/api".to_string()), - ( - "counter".to_string(), - "rpc:counters-exports/api".to_string(), - ), - ]), - }), - ), - ( - "rpc:ephemeral-client/ephemeral-client", - DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { - target_interface_name: HashMap::from_iter(vec![( - "api".to_string(), - "rpc:ephemeral-exports/api".to_string(), - )]), - }), - ), - ], - ) + .component("caller") + .with_dynamic_linking(&[ + ( + "rpc:counters-client/counters-client", + DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { + target_interface_name: HashMap::from_iter(vec![ + ("api".to_string(), "rpc:counters-exports/api".to_string()), + ( + "counter".to_string(), + "rpc:counters-exports/api".to_string(), + ), + ]), + }), + ), + ( + "rpc:ephemeral-client/ephemeral-client", + DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { + target_interface_name: HashMap::from_iter(vec![( + "api".to_string(), + "rpc:ephemeral-exports/api".to_string(), + )]), + }), + ), + ]) + .store() .await; let mut env = HashMap::new(); @@ -347,34 +343,33 @@ async fn counter_resource_test_2_with_restart( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let counters_component_id = executor.store_component("counters").await; + let counters_component_id = executor.component("counters").store().await; let caller_component_id = executor - .store_component_with_dynamic_linking( - "caller", - &[ - ( - "rpc:counters-client/counters-client", - DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { - target_interface_name: HashMap::from_iter(vec![ - ("api".to_string(), "rpc:counters-exports/api".to_string()), - ( - "counter".to_string(), - "rpc:counters-exports/api".to_string(), - ), - ]), - }), - ), - ( - "rpc:ephemeral-client/ephemeral-client", - DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { - target_interface_name: HashMap::from_iter(vec![( - "api".to_string(), - "rpc:ephemeral-exports/api".to_string(), - )]), - }), - ), - ], - ) + .component("caller") + .with_dynamic_linking(&[ + ( + "rpc:counters-client/counters-client", + DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { + target_interface_name: HashMap::from_iter(vec![ + ("api".to_string(), "rpc:counters-exports/api".to_string()), + ( + "counter".to_string(), + "rpc:counters-exports/api".to_string(), + ), + ]), + }), + ), + ( + "rpc:ephemeral-client/ephemeral-client", + DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { + target_interface_name: HashMap::from_iter(vec![( + "api".to_string(), + "rpc:ephemeral-exports/api".to_string(), + )]), + }), + ), + ]) + .store() .await; let mut env = HashMap::new(); @@ -421,34 +416,33 @@ async fn counter_resource_test_3( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let counters_component_id = executor.store_component("counters").await; + let counters_component_id = executor.component("counters").store().await; let caller_component_id = executor - .store_component_with_dynamic_linking( - "caller", - &[ - ( - "rpc:counters-client/counters-client", - DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { - target_interface_name: HashMap::from_iter(vec![ - ("api".to_string(), "rpc:counters-exports/api".to_string()), - ( - "counter".to_string(), - "rpc:counters-exports/api".to_string(), - ), - ]), - }), - ), - ( - "rpc:ephemeral-client/ephemeral-client", - DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { - target_interface_name: HashMap::from_iter(vec![( - "api".to_string(), - "rpc:ephemeral-exports/api".to_string(), - )]), - }), - ), - ], - ) + .component("caller") + .with_dynamic_linking(&[ + ( + "rpc:counters-client/counters-client", + DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { + target_interface_name: HashMap::from_iter(vec![ + ("api".to_string(), "rpc:counters-exports/api".to_string()), + ( + "counter".to_string(), + "rpc:counters-exports/api".to_string(), + ), + ]), + }), + ), + ( + "rpc:ephemeral-client/ephemeral-client", + DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { + target_interface_name: HashMap::from_iter(vec![( + "api".to_string(), + "rpc:ephemeral-exports/api".to_string(), + )]), + }), + ), + ]) + .store() .await; let mut env = HashMap::new(); @@ -491,34 +485,33 @@ async fn counter_resource_test_3_with_restart( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let counters_component_id = executor.store_component("counters").await; + let counters_component_id = executor.component("counters").store().await; let caller_component_id = executor - .store_component_with_dynamic_linking( - "caller", - &[ - ( - "rpc:counters-client/counters-client", - DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { - target_interface_name: HashMap::from_iter(vec![ - ("api".to_string(), "rpc:counters-exports/api".to_string()), - ( - "counter".to_string(), - "rpc:counters-exports/api".to_string(), - ), - ]), - }), - ), - ( - "rpc:ephemeral-client/ephemeral-client", - DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { - target_interface_name: HashMap::from_iter(vec![( - "api".to_string(), - "rpc:ephemeral-exports/api".to_string(), - )]), - }), - ), - ], - ) + .component("caller") + .with_dynamic_linking(&[ + ( + "rpc:counters-client/counters-client", + DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { + target_interface_name: HashMap::from_iter(vec![ + ("api".to_string(), "rpc:counters-exports/api".to_string()), + ( + "counter".to_string(), + "rpc:counters-exports/api".to_string(), + ), + ]), + }), + ), + ( + "rpc:ephemeral-client/ephemeral-client", + DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { + target_interface_name: HashMap::from_iter(vec![( + "api".to_string(), + "rpc:ephemeral-exports/api".to_string(), + )]), + }), + ), + ]) + .store() .await; let mut env = HashMap::new(); @@ -565,34 +558,33 @@ async fn context_inheritance( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let counters_component_id = executor.store_component("counters").await; + let counters_component_id = executor.component("counters").store().await; let caller_component_id = executor - .store_component_with_dynamic_linking( - "caller", - &[ - ( - "rpc:counters-client/counters-client", - DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { - target_interface_name: HashMap::from_iter(vec![ - ("api".to_string(), "rpc:counters-exports/api".to_string()), - ( - "counter".to_string(), - "rpc:counters-exports/api".to_string(), - ), - ]), - }), - ), - ( - "rpc:ephemeral-client/ephemeral-client", - DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { - target_interface_name: HashMap::from_iter(vec![( - "api".to_string(), - "rpc:ephemeral-exports/api".to_string(), - )]), - }), - ), - ], - ) + .component("caller") + .with_dynamic_linking(&[ + ( + "rpc:counters-client/counters-client", + DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { + target_interface_name: HashMap::from_iter(vec![ + ("api".to_string(), "rpc:counters-exports/api".to_string()), + ( + "counter".to_string(), + "rpc:counters-exports/api".to_string(), + ), + ]), + }), + ), + ( + "rpc:ephemeral-client/ephemeral-client", + DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { + target_interface_name: HashMap::from_iter(vec![( + "api".to_string(), + "rpc:ephemeral-exports/api".to_string(), + )]), + }), + ), + ]) + .store() .await; let mut env = HashMap::new(); @@ -682,34 +674,33 @@ async fn counter_resource_test_5( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let counters_component_id = executor.store_component("counters").await; + let counters_component_id = executor.component("counters").store().await; let caller_component_id = executor - .store_component_with_dynamic_linking( - "caller", - &[ - ( - "rpc:counters-client/counters-client", - DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { - target_interface_name: HashMap::from_iter(vec![ - ("api".to_string(), "rpc:counters-exports/api".to_string()), - ( - "counter".to_string(), - "rpc:counters-exports/api".to_string(), - ), - ]), - }), - ), - ( - "rpc:ephemeral-client/ephemeral-client", - DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { - target_interface_name: HashMap::from_iter(vec![( - "api".to_string(), - "rpc:ephemeral-exports/api".to_string(), - )]), - }), - ), - ], - ) + .component("caller") + .with_dynamic_linking(&[ + ( + "rpc:counters-client/counters-client", + DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { + target_interface_name: HashMap::from_iter(vec![ + ("api".to_string(), "rpc:counters-exports/api".to_string()), + ( + "counter".to_string(), + "rpc:counters-exports/api".to_string(), + ), + ]), + }), + ), + ( + "rpc:ephemeral-client/ephemeral-client", + DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { + target_interface_name: HashMap::from_iter(vec![( + "api".to_string(), + "rpc:ephemeral-exports/api".to_string(), + )]), + }), + ), + ]) + .store() .await; let mut env = HashMap::new(); @@ -754,34 +745,34 @@ async fn counter_resource_test_5_with_restart( let executor = start(deps, &context).await.unwrap(); // using store_unique_component to avoid collision with counter_resource_test_5 - let counters_component_id = executor.store_unique_component("counters").await; + let counters_component_id = executor.component("counters").unique().store().await; let caller_component_id = executor - .store_unique_component_with_dynamic_linking( - "caller", - &[ - ( - "rpc:counters-client/counters-client", - DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { - target_interface_name: HashMap::from_iter(vec![ - ("api".to_string(), "rpc:counters-exports/api".to_string()), - ( - "counter".to_string(), - "rpc:counters-exports/api".to_string(), - ), - ]), - }), - ), - ( - "rpc:ephemeral-client/ephemeral-client", - DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { - target_interface_name: HashMap::from_iter(vec![( - "api".to_string(), - "rpc:ephemeral-exports/api".to_string(), - )]), - }), - ), - ], - ) + .component("caller") + .unique() + .with_dynamic_linking(&[ + ( + "rpc:counters-client/counters-client", + DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { + target_interface_name: HashMap::from_iter(vec![ + ("api".to_string(), "rpc:counters-exports/api".to_string()), + ( + "counter".to_string(), + "rpc:counters-exports/api".to_string(), + ), + ]), + }), + ), + ( + "rpc:ephemeral-client/ephemeral-client", + DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { + target_interface_name: HashMap::from_iter(vec![( + "api".to_string(), + "rpc:ephemeral-exports/api".to_string(), + )]), + }), + ), + ]) + .store() .await; let mut env = HashMap::new(); @@ -846,34 +837,33 @@ async fn wasm_rpc_bug_32_test( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let counters_component_id = executor.store_component("counters").await; + let counters_component_id = executor.component("counters").store().await; let caller_component_id = executor - .store_component_with_dynamic_linking( - "caller", - &[ - ( - "rpc:counters-client/counters-client", - DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { - target_interface_name: HashMap::from_iter(vec![ - ("api".to_string(), "rpc:counters-exports/api".to_string()), - ( - "counter".to_string(), - "rpc:counters-exports/api".to_string(), - ), - ]), - }), - ), - ( - "rpc:ephemeral-client/ephemeral-client", - DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { - target_interface_name: HashMap::from_iter(vec![( - "api".to_string(), - "rpc:ephemeral-exports/api".to_string(), - )]), - }), - ), - ], - ) + .component("caller") + .with_dynamic_linking(&[ + ( + "rpc:counters-client/counters-client", + DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { + target_interface_name: HashMap::from_iter(vec![ + ("api".to_string(), "rpc:counters-exports/api".to_string()), + ( + "counter".to_string(), + "rpc:counters-exports/api".to_string(), + ), + ]), + }), + ), + ( + "rpc:ephemeral-client/ephemeral-client", + DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { + target_interface_name: HashMap::from_iter(vec![( + "api".to_string(), + "rpc:ephemeral-exports/api".to_string(), + )]), + }), + ), + ]) + .store() .await; let mut env = HashMap::new(); @@ -918,21 +908,20 @@ async fn error_message_invalid_uri( let executor = start(deps, &context).await.unwrap(); let registry_component_id = executor - .store_component_with_dynamic_linking( - "auction_registry", - &[( - "auction:auction-client/auction-client", - DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { - target_interface_name: HashMap::from_iter(vec![ - ("api".to_string(), "auction:auction-exports/api".to_string()), - ( - "running-auction".to_string(), - "auction:auction-exports/api".to_string(), - ), - ]), - }), - )], - ) + .component("auction_registry") + .with_dynamic_linking(&[( + "auction:auction-client/auction-client", + DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { + target_interface_name: HashMap::from_iter(vec![ + ("api".to_string(), "auction:auction-exports/api".to_string()), + ( + "running-auction".to_string(), + "auction:auction-exports/api".to_string(), + ), + ]), + }), + )]) + .store() .await; let mut env = HashMap::new(); @@ -989,21 +978,20 @@ async fn error_message_non_existing_target_component( let executor = start(deps, &context).await.unwrap(); let registry_component_id = executor - .store_component_with_dynamic_linking( - "auction_registry", - &[( - "auction:auction-client/auction-client", - DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { - target_interface_name: HashMap::from_iter(vec![ - ("api".to_string(), "auction:auction-exports/api".to_string()), - ( - "running-auction".to_string(), - "auction:auction-exports/api".to_string(), - ), - ]), - }), - )], - ) + .component("auction_registry") + .with_dynamic_linking(&[( + "auction:auction-client/auction-client", + DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { + target_interface_name: HashMap::from_iter(vec![ + ("api".to_string(), "auction:auction-exports/api".to_string()), + ( + "running-auction".to_string(), + "auction:auction-exports/api".to_string(), + ), + ]), + }), + )]) + .store() .await; let mut env = HashMap::new(); @@ -1055,34 +1043,33 @@ async fn ephemeral_worker_invocation_via_rpc1( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let ephemeral_component_id = executor.store_ephemeral_component("ephemeral").await; + let ephemeral_component_id = executor.component("ephemeral").ephemeral().store().await; let caller_component_id = executor - .store_component_with_dynamic_linking( - "caller", - &[ - ( - "rpc:counters-client/counters-client", - DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { - target_interface_name: HashMap::from_iter(vec![ - ("api".to_string(), "rpc:counters-exports/api".to_string()), - ( - "counter".to_string(), - "rpc:counters-exports/api".to_string(), - ), - ]), - }), - ), - ( - "rpc:ephemeral-client/ephemeral-client", - DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { - target_interface_name: HashMap::from_iter(vec![( - "api".to_string(), - "rpc:ephemeral-exports/api".to_string(), - )]), - }), - ), - ], - ) + .component("caller") + .with_dynamic_linking(&[ + ( + "rpc:counters-client/counters-client", + DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { + target_interface_name: HashMap::from_iter(vec![ + ("api".to_string(), "rpc:counters-exports/api".to_string()), + ( + "counter".to_string(), + "rpc:counters-exports/api".to_string(), + ), + ]), + }), + ), + ( + "rpc:ephemeral-client/ephemeral-client", + DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { + target_interface_name: HashMap::from_iter(vec![( + "api".to_string(), + "rpc:ephemeral-exports/api".to_string(), + )]), + }), + ), + ]) + .store() .await; let mut env = HashMap::new(); diff --git a/golem-worker-executor-base/tests/scalability.rs b/golem-worker-executor-base/tests/scalability.rs index c3de68846f..0f26016026 100644 --- a/golem-worker-executor-base/tests/scalability.rs +++ b/golem-worker-executor-base/tests/scalability.rs @@ -55,7 +55,7 @@ async fn spawning_many_workers_that_sleep( } let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("clocks").await; + let component_id = executor.component("clocks").store().await; let warmup_worker = executor.start_worker(&component_id, &worker_name(0)).await; @@ -144,7 +144,7 @@ async fn spawning_many_workers_that_sleep_long_enough_to_get_suspended( } let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("clocks").await; + let component_id = executor.component("clocks").store().await; let warmup_worker = executor.start_worker(&component_id, &worker_name(0)).await; @@ -238,7 +238,7 @@ async fn initial_large_memory_allocation( let executor = start_limited(deps, &context, Some(768 * 1024 * 1024)) .await .unwrap(); - let component_id = executor.store_component("large-initial-memory").await; + let component_id = executor.component("large-initial-memory").store().await; let mut handles = JoinSet::new(); let mut results: Vec> = Vec::new(); @@ -280,7 +280,7 @@ async fn dynamic_large_memory_allocation( let executor = start_limited(deps, &context, Some(768 * 1024 * 1024)) .await .unwrap(); - let component_id = executor.store_component("large-dynamic-memory").await; + let component_id = executor.component("large-dynamic-memory").store().await; let mut handles = JoinSet::new(); let mut results: Vec> = Vec::new(); diff --git a/golem-worker-executor-base/tests/transactions.rs b/golem-worker-executor-base/tests/transactions.rs index 8cd0716d3d..431a2751b6 100644 --- a/golem-worker-executor-base/tests/transactions.rs +++ b/golem-worker-executor-base/tests/transactions.rs @@ -133,7 +133,7 @@ async fn jump( let http_server = TestHttpServer::start(host_http_port, 1); - let component_id = executor.store_component("runtime-service").await; + let component_id = executor.component("runtime-service").store().await; let mut env = HashMap::new(); env.insert("PORT".to_string(), context.host_http_port().to_string()); @@ -192,7 +192,7 @@ async fn explicit_oplog_commit( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("runtime-service").await; + let component_id = executor.component("runtime-service").store().await; let worker_id = executor .start_worker(&component_id, "runtime-service-explicit-oplog-commit") @@ -223,7 +223,7 @@ async fn set_retry_policy( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("runtime-service").await; + let component_id = executor.component("runtime-service").store().await; let worker_id = executor .start_worker(&component_id, "set-retry-policy-1") .await; @@ -271,7 +271,7 @@ async fn atomic_region( let host_http_port = context.host_http_port(); let http_server = TestHttpServer::start(host_http_port, 2); - let component_id = executor.store_component("runtime-service").await; + let component_id = executor.component("runtime-service").store().await; let mut env = HashMap::new(); env.insert("PORT".to_string(), context.host_http_port().to_string()); @@ -307,7 +307,7 @@ async fn idempotence_on( let host_http_port = context.host_http_port(); let http_server = TestHttpServer::start(host_http_port, 1); - let component_id = executor.store_component("runtime-service").await; + let component_id = executor.component("runtime-service").store().await; let mut env = HashMap::new(); env.insert("PORT".to_string(), context.host_http_port().to_string()); @@ -347,7 +347,7 @@ async fn idempotence_off( let host_http_port = context.host_http_port(); let http_server = TestHttpServer::start(host_http_port, 1); - let component_id = executor.store_component("runtime-service").await; + let component_id = executor.component("runtime-service").store().await; let mut env = HashMap::new(); env.insert("PORT".to_string(), context.host_http_port().to_string()); @@ -388,7 +388,7 @@ async fn persist_nothing( let host_http_port = context.host_http_port(); let http_server = TestHttpServer::start(host_http_port, 2); - let component_id = executor.store_component("runtime-service").await; + let component_id = executor.component("runtime-service").store().await; let mut env = HashMap::new(); env.insert("PORT".to_string(), context.host_http_port().to_string()); @@ -424,7 +424,7 @@ async fn golem_rust_explicit_oplog_commit( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("golem-rust-tests").await; + let component_id = executor.component("golem-rust-tests").store().await; let worker_id = executor .start_worker(&component_id, "golem-rust-tests-explicit-oplog-commit") @@ -455,7 +455,7 @@ async fn golem_rust_set_retry_policy( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("golem-rust-tests").await; + let component_id = executor.component("golem-rust-tests").store().await; let worker_id = executor .start_worker(&component_id, "golem-rust-tests-set-retry-policy-1") .await; @@ -503,7 +503,7 @@ async fn golem_rust_atomic_region( let host_http_port = context.host_http_port(); let http_server = TestHttpServer::start(host_http_port, 2); - let component_id = executor.store_component("golem-rust-tests").await; + let component_id = executor.component("golem-rust-tests").store().await; let mut env = HashMap::new(); env.insert("PORT".to_string(), context.host_http_port().to_string()); @@ -539,7 +539,7 @@ async fn golem_rust_idempotence_on( let host_http_port = context.host_http_port(); let http_server = TestHttpServer::start(host_http_port, 1); - let component_id = executor.store_component("golem-rust-tests").await; + let component_id = executor.component("golem-rust-tests").store().await; let mut env = HashMap::new(); env.insert("PORT".to_string(), context.host_http_port().to_string()); @@ -584,7 +584,7 @@ async fn golem_rust_idempotence_off( let host_http_port = context.host_http_port(); let http_server = TestHttpServer::start(host_http_port, 1); - let component_id = executor.store_component("golem-rust-tests").await; + let component_id = executor.component("golem-rust-tests").store().await; let mut env = HashMap::new(); env.insert("PORT".to_string(), context.host_http_port().to_string()); @@ -630,7 +630,7 @@ async fn golem_rust_persist_nothing( let host_http_port = context.host_http_port(); let http_server = TestHttpServer::start(host_http_port, 2); - let component_id = executor.store_component("golem-rust-tests").await; + let component_id = executor.component("golem-rust-tests").store().await; let mut env = HashMap::new(); env.insert("PORT".to_string(), context.host_http_port().to_string()); @@ -679,7 +679,7 @@ async fn golem_rust_fallible_transaction( true, ); - let component_id = executor.store_component("golem-rust-tests").await; + let component_id = executor.component("golem-rust-tests").store().await; let mut env = HashMap::new(); env.insert("PORT".to_string(), context.host_http_port().to_string()); @@ -741,7 +741,7 @@ async fn golem_rust_infallible_transaction( true, ); - let component_id = executor.store_component("golem-rust-tests").await; + let component_id = executor.component("golem-rust-tests").store().await; let mut env = HashMap::new(); env.insert("PORT".to_string(), context.host_http_port().to_string()); @@ -794,7 +794,11 @@ async fn idempotency_keys_in_ephemeral_workers( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_ephemeral_component("runtime-service").await; + let component_id = executor + .component("runtime-service") + .ephemeral() + .store() + .await; let target_worker_id = TargetWorkerId { component_id, diff --git a/golem-worker-executor-base/tests/ts_rpc1.rs b/golem-worker-executor-base/tests/ts_rpc1.rs index 904e9db3d6..832ec1fbcc 100644 --- a/golem-worker-executor-base/tests/ts_rpc1.rs +++ b/golem-worker-executor-base/tests/ts_rpc1.rs @@ -37,8 +37,8 @@ async fn counter_resource_test_1( let context = common::TestContext::new(last_unique_id); let executor = common::start(deps, &context).await.unwrap(); - let counters_component_id = executor.store_component(COUNTER_COMPONENT_NAME).await; - let caller_component_id = executor.store_component(CALLER_COMPONENT_NAME).await; + let counters_component_id = executor.component(COUNTER_COMPONENT_NAME).store().await; + let caller_component_id = executor.component(CALLER_COMPONENT_NAME).store().await; let mut env = HashMap::new(); env.insert( @@ -80,8 +80,8 @@ async fn counter_resource_test_1_with_restart( let context = common::TestContext::new(last_unique_id); let executor = common::start(deps, &context).await.unwrap(); - let counters_component_id = executor.store_component(COUNTER_COMPONENT_NAME).await; - let caller_component_id = executor.store_component(CALLER_COMPONENT_NAME).await; + let counters_component_id = executor.component(COUNTER_COMPONENT_NAME).store().await; + let caller_component_id = executor.component(CALLER_COMPONENT_NAME).store().await; let mut env = HashMap::new(); env.insert( @@ -127,8 +127,8 @@ async fn context_inheritance( let context = common::TestContext::new(last_unique_id); let executor = common::start(deps, &context).await.unwrap(); - let counters_component_id = executor.store_component(COUNTER_COMPONENT_NAME).await; - let caller_component_id = executor.store_component(CALLER_COMPONENT_NAME).await; + let counters_component_id = executor.component(COUNTER_COMPONENT_NAME).store().await; + let caller_component_id = executor.component(CALLER_COMPONENT_NAME).store().await; let mut env = HashMap::new(); env.insert( diff --git a/golem-worker-executor-base/tests/ts_rpc1_stubless.rs b/golem-worker-executor-base/tests/ts_rpc1_stubless.rs index 9877fed4ef..6279b88e37 100644 --- a/golem-worker-executor-base/tests/ts_rpc1_stubless.rs +++ b/golem-worker-executor-base/tests/ts_rpc1_stubless.rs @@ -38,23 +38,22 @@ async fn counter_resource_test_1( let context = common::TestContext::new(last_unique_id); let executor = common::start(deps, &context).await.unwrap(); - let counters_component_id = executor.store_component(COUNTER_COMPONENT_NAME).await; + let counters_component_id = executor.component(COUNTER_COMPONENT_NAME).store().await; let caller_component_id = executor - .store_component_with_dynamic_linking( - CALLER_COMPONENT_NAME, - &[( - "rpc:counters-client/counters-client", - DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { - target_interface_name: HashMap::from_iter(vec![ - ("api".to_string(), "rpc:counters-exports/api".to_string()), - ( - "counter".to_string(), - "rpc:counters-exports/api".to_string(), - ), - ]), - }), - )], - ) + .component(CALLER_COMPONENT_NAME) + .with_dynamic_linking(&[( + "rpc:counters-client/counters-client", + DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { + target_interface_name: HashMap::from_iter(vec![ + ("api".to_string(), "rpc:counters-exports/api".to_string()), + ( + "counter".to_string(), + "rpc:counters-exports/api".to_string(), + ), + ]), + }), + )]) + .store() .await; let mut env = HashMap::new(); @@ -97,23 +96,22 @@ async fn counter_resource_test_1_with_restart( let context = common::TestContext::new(last_unique_id); let executor = common::start(deps, &context).await.unwrap(); - let counters_component_id = executor.store_component(COUNTER_COMPONENT_NAME).await; + let counters_component_id = executor.component(COUNTER_COMPONENT_NAME).store().await; let caller_component_id = executor - .store_component_with_dynamic_linking( - CALLER_COMPONENT_NAME, - &[( - "rpc:counters-client/counters-client", - DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { - target_interface_name: HashMap::from_iter(vec![ - ("api".to_string(), "rpc:counters-exports/api".to_string()), - ( - "counter".to_string(), - "rpc:counters-exports/api".to_string(), - ), - ]), - }), - )], - ) + .component(CALLER_COMPONENT_NAME) + .with_dynamic_linking(&[( + "rpc:counters-client/counters-client", + DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { + target_interface_name: HashMap::from_iter(vec![ + ("api".to_string(), "rpc:counters-exports/api".to_string()), + ( + "counter".to_string(), + "rpc:counters-exports/api".to_string(), + ), + ]), + }), + )]) + .store() .await; let mut env = HashMap::new(); @@ -160,23 +158,22 @@ async fn context_inheritance( let context = common::TestContext::new(last_unique_id); let executor = common::start(deps, &context).await.unwrap(); - let counters_component_id = executor.store_component(COUNTER_COMPONENT_NAME).await; + let counters_component_id = executor.component(COUNTER_COMPONENT_NAME).store().await; let caller_component_id = executor - .store_component_with_dynamic_linking( - CALLER_COMPONENT_NAME, - &[( - "rpc:counters-client/counters-client", - DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { - target_interface_name: HashMap::from_iter(vec![ - ("api".to_string(), "rpc:counters-exports/api".to_string()), - ( - "counter".to_string(), - "rpc:counters-exports/api".to_string(), - ), - ]), - }), - )], - ) + .component(CALLER_COMPONENT_NAME) + .with_dynamic_linking(&[( + "rpc:counters-client/counters-client", + DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { + target_interface_name: HashMap::from_iter(vec![ + ("api".to_string(), "rpc:counters-exports/api".to_string()), + ( + "counter".to_string(), + "rpc:counters-exports/api".to_string(), + ), + ]), + }), + )]) + .store() .await; let mut env = HashMap::new(); diff --git a/golem-worker-executor-base/tests/ts_rpc2.rs b/golem-worker-executor-base/tests/ts_rpc2.rs index 418c10cacf..0631a2a026 100644 --- a/golem-worker-executor-base/tests/ts_rpc2.rs +++ b/golem-worker-executor-base/tests/ts_rpc2.rs @@ -37,8 +37,8 @@ async fn counter_resource_test_2( let context = common::TestContext::new(last_unique_id); let executor = common::start(deps, &context).await.unwrap(); - let counters_component_id = executor.store_component(COUNTER_COMPONENT_NAME).await; - let caller_component_id = executor.store_component(CALLER_COMPONENT_NAME).await; + let counters_component_id = executor.component(COUNTER_COMPONENT_NAME).store().await; + let caller_component_id = executor.component(CALLER_COMPONENT_NAME).store().await; let mut env = HashMap::new(); env.insert( @@ -80,8 +80,8 @@ async fn counter_resource_test_2_with_restart( let context = common::TestContext::new(last_unique_id); let executor = common::start(deps, &context).await.unwrap(); - let counters_component_id = executor.store_component(COUNTER_COMPONENT_NAME).await; - let caller_component_id = executor.store_component(CALLER_COMPONENT_NAME).await; + let counters_component_id = executor.component(COUNTER_COMPONENT_NAME).store().await; + let caller_component_id = executor.component(CALLER_COMPONENT_NAME).store().await; let mut env = HashMap::new(); env.insert( diff --git a/golem-worker-executor-base/tests/ts_rpc2_stubless.rs b/golem-worker-executor-base/tests/ts_rpc2_stubless.rs index 4383ac3bd5..f8be503861 100644 --- a/golem-worker-executor-base/tests/ts_rpc2_stubless.rs +++ b/golem-worker-executor-base/tests/ts_rpc2_stubless.rs @@ -38,23 +38,22 @@ async fn counter_resource_test_2( let context = common::TestContext::new(last_unique_id); let executor = common::start(deps, &context).await.unwrap(); - let counters_component_id = executor.store_component(COUNTER_COMPONENT_NAME).await; + let counters_component_id = executor.component(COUNTER_COMPONENT_NAME).store().await; let caller_component_id = executor - .store_component_with_dynamic_linking( - CALLER_COMPONENT_NAME, - &[( - "rpc:counters-client/counters-client", - DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { - target_interface_name: HashMap::from_iter(vec![ - ("api".to_string(), "rpc:counters-exports/api".to_string()), - ( - "counter".to_string(), - "rpc:counters-exports/api".to_string(), - ), - ]), - }), - )], - ) + .component(CALLER_COMPONENT_NAME) + .with_dynamic_linking(&[( + "rpc:counters-client/counters-client", + DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { + target_interface_name: HashMap::from_iter(vec![ + ("api".to_string(), "rpc:counters-exports/api".to_string()), + ( + "counter".to_string(), + "rpc:counters-exports/api".to_string(), + ), + ]), + }), + )]) + .store() .await; let mut env = HashMap::new(); @@ -97,23 +96,22 @@ async fn counter_resource_test_2_with_restart( let context = common::TestContext::new(last_unique_id); let executor = common::start(deps, &context).await.unwrap(); - let counters_component_id = executor.store_component(COUNTER_COMPONENT_NAME).await; + let counters_component_id = executor.component(COUNTER_COMPONENT_NAME).store().await; let caller_component_id = executor - .store_component_with_dynamic_linking( - CALLER_COMPONENT_NAME, - &[( - "rpc:counters-client/counters-client", - DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { - target_interface_name: HashMap::from_iter(vec![ - ("api".to_string(), "rpc:counters-exports/api".to_string()), - ( - "counter".to_string(), - "rpc:counters-exports/api".to_string(), - ), - ]), - }), - )], - ) + .component(CALLER_COMPONENT_NAME) + .with_dynamic_linking(&[( + "rpc:counters-client/counters-client", + DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { + target_interface_name: HashMap::from_iter(vec![ + ("api".to_string(), "rpc:counters-exports/api".to_string()), + ( + "counter".to_string(), + "rpc:counters-exports/api".to_string(), + ), + ]), + }), + )]) + .store() .await; let mut env = HashMap::new(); diff --git a/golem-worker-executor-base/tests/wasi.rs b/golem-worker-executor-base/tests/wasi.rs index c7de8dc350..c55c71eced 100644 --- a/golem-worker-executor-base/tests/wasi.rs +++ b/golem-worker-executor-base/tests/wasi.rs @@ -31,8 +31,7 @@ use bytes::Bytes; use futures_util::stream; use golem_common::model::{ AccountId, ComponentFilePath, ComponentFilePermissions, ComponentFileSystemNode, - ComponentFileSystemNodeDetails, ComponentType, IdempotencyKey, InitialComponentFile, - WorkerStatus, + ComponentFileSystemNodeDetails, IdempotencyKey, InitialComponentFile, WorkerStatus, }; use golem_test_framework::dsl::{ drain_connection, stderr_events, stdout_events, worker_error_message, TestDslUnsafe, @@ -58,7 +57,7 @@ async fn write_stdout( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("write-stdout").await; + let component_id = executor.component("write-stdout").store().await; let worker_id = executor.start_worker(&component_id, "write-stdout-1").await; let mut rx = executor.capture_output(&worker_id).await; @@ -90,7 +89,7 @@ async fn write_stderr( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("write-stderr").await; + let component_id = executor.component("write-stderr").store().await; let worker_id = executor.start_worker(&component_id, "write-stderr-1").await; let mut rx = executor.capture_output(&worker_id).await; @@ -122,7 +121,7 @@ async fn read_stdin( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("read-stdin").await; + let component_id = executor.component("read-stdin").store().await; let worker_id = executor.start_worker(&component_id, "read-stdin-1").await; let result = executor.invoke_and_await(&worker_id, "run", vec![]).await; @@ -142,7 +141,7 @@ async fn clocks( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("clocks").await; + let component_id = executor.component("clocks").store().await; let worker_id = executor.start_worker(&component_id, "clocks-1").await; let result = executor @@ -192,7 +191,7 @@ async fn file_write_read_delete( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("file-write-read-delete").await; + let component_id = executor.component("file-write-read-delete").store().await; let mut env = HashMap::new(); env.insert("RUST_BACKTRACE".to_string(), "full".to_string()); let worker_id = executor @@ -257,11 +256,10 @@ async fn initial_file_read_write( ]; let component_id = executor - .store_unique_component_with_files( - "initial-file-read-write", - ComponentType::Durable, - &component_files, - ) + .component("initial-file-read-write") + .unique() + .with_files(&component_files) + .store() .await; let mut env = HashMap::new(); env.insert("RUST_BACKTRACE".to_string(), "full".to_string()); @@ -334,11 +332,10 @@ async fn initial_file_listing_through_api( ]; let component_id = executor - .store_unique_component_with_files( - "initial-file-read-write", - ComponentType::Durable, - &component_files, - ) + .component("initial-file-read-write") + .unique() + .with_files(&component_files) + .store() .await; let mut env = HashMap::new(); env.insert("RUST_BACKTRACE".to_string(), "full".to_string()); @@ -429,11 +426,10 @@ async fn initial_file_reading_through_api( ]; let component_id = executor - .store_unique_component_with_files( - "initial-file-read-write", - ComponentType::Durable, - &component_files, - ) + .component("initial-file-read-write") + .unique() + .with_files(&component_files) + .store() .await; let mut env = HashMap::new(); env.insert("RUST_BACKTRACE".to_string(), "full".to_string()); @@ -469,7 +465,7 @@ async fn directories( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("directories").await; + let component_id = executor.component("directories").store().await; let worker_id = executor.start_worker(&component_id, "directories-1").await; let result = executor @@ -527,7 +523,7 @@ async fn directories_replay( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("directories").await; + let component_id = executor.component("directories").store().await; let worker_id = executor.start_worker(&component_id, "directories-1").await; let result = executor @@ -594,7 +590,7 @@ async fn file_write_read( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("file-service").await; + let component_id = executor.component("file-service").store().await; let worker_id = executor.start_worker(&component_id, "file-service-1").await; let _ = executor @@ -660,7 +656,7 @@ async fn http_client( axum::serve(listener, route).await.unwrap(); }); - let component_id = executor.store_component("http-client").await; + let component_id = executor.component("http-client").store().await; let mut env = HashMap::new(); env.insert("PORT".to_string(), host_http_port.to_string()); env.insert("RUST_BACKTRACE".to_string(), "full".to_string()); @@ -728,7 +724,7 @@ async fn http_client_using_reqwest( axum::serve(listener, route).await.unwrap(); }); - let component_id = executor.store_component("http-client-2").await; + let component_id = executor.component("http-client-2").store().await; let mut env = HashMap::new(); env.insert("PORT".to_string(), host_http_port.to_string()); @@ -763,7 +759,7 @@ async fn environment_service( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("environment-service").await; + let component_id = executor.component("environment-service").store().await; let args = vec!["test-arg".to_string()]; let mut env = HashMap::new(); env.insert("TEST_ENV".to_string(), "test-value".to_string()); @@ -849,7 +845,7 @@ async fn http_client_response_persisted_between_invocations( axum::serve(listener, route).await.unwrap(); }); - let component_id = executor.store_component("http-client").await; + let component_id = executor.component("http-client").store().await; let mut env = HashMap::new(); env.insert("PORT".to_string(), host_http_port.to_string()); @@ -927,7 +923,7 @@ async fn http_client_interrupting_response_stream( axum::serve(listener, route).await.unwrap(); }); - let component_id = executor.store_component("http-client-2").await; + let component_id = executor.component("http-client-2").store().await; let mut env = HashMap::new(); env.insert("PORT".to_string(), host_http_port.to_string()); @@ -986,7 +982,7 @@ async fn sleep( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("clock-service").await; + let component_id = executor.component("clock-service").store().await; let worker_id = executor .start_worker(&component_id, "clock-service-1") .await; @@ -1019,7 +1015,7 @@ async fn resuming_sleep( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("clock-service").await; + let component_id = executor.component("clock-service").store().await; let worker_id = executor .start_worker(&component_id, "clock-service-2") .await; @@ -1069,7 +1065,7 @@ async fn failing_worker( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("failing-component").await; + let component_id = executor.component("failing-component").store().await; let worker_id = executor .start_worker(&component_id, "failing-worker-1") .await; @@ -1112,7 +1108,7 @@ async fn file_service_write_direct( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("file-service").await; + let component_id = executor.component("file-service").store().await; let worker_id = executor.start_worker(&component_id, "file-service-2").await; let _ = executor @@ -1157,7 +1153,7 @@ async fn filesystem_write_replay_restores_file_times( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("file-service").await; + let component_id = executor.component("file-service").store().await; let worker_id = executor.start_worker(&component_id, "file-service-3").await; let _ = executor @@ -1205,7 +1201,7 @@ async fn filesystem_create_dir_replay_restores_file_times( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("file-service").await; + let component_id = executor.component("file-service").store().await; let worker_id = executor.start_worker(&component_id, "file-service-4").await; let _ = executor @@ -1250,7 +1246,7 @@ async fn file_hard_link( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("file-service").await; + let component_id = executor.component("file-service").store().await; let worker_id = executor.start_worker(&component_id, "file-service-5").await; let _ = executor @@ -1304,7 +1300,7 @@ async fn filesystem_link_replay_restores_file_times( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("file-service").await; + let component_id = executor.component("file-service").store().await; let worker_id = executor.start_worker(&component_id, "file-service-6").await; let _ = executor @@ -1397,7 +1393,7 @@ async fn filesystem_remove_dir_replay_restores_file_times( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("file-service").await; + let component_id = executor.component("file-service").store().await; let worker_id = executor.start_worker(&component_id, "file-service-7").await; let _ = executor @@ -1458,7 +1454,7 @@ async fn filesystem_symlink_replay_restores_file_times( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("file-service").await; + let component_id = executor.component("file-service").store().await; let worker_id = executor.start_worker(&component_id, "file-service-8").await; let _ = executor @@ -1552,7 +1548,7 @@ async fn filesystem_rename_replay_restores_file_times( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("file-service").await; + let component_id = executor.component("file-service").store().await; let worker_id = executor.start_worker(&component_id, "file-service-9").await; let _ = executor @@ -1662,7 +1658,7 @@ async fn filesystem_remove_file_replay_restores_file_times( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("file-service").await; + let component_id = executor.component("file-service").store().await; let worker_id = executor .start_worker(&component_id, "file-service-10") .await; @@ -1728,7 +1724,7 @@ async fn filesystem_write_via_stream_replay_restores_file_times( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("file-service").await; + let component_id = executor.component("file-service").store().await; let worker_id = executor.start_worker(&component_id, "file-service-3").await; let _ = executor @@ -1776,7 +1772,7 @@ async fn filesystem_metadata_hash( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("file-service").await; + let component_id = executor.component("file-service").store().await; let worker_id = executor.start_worker(&component_id, "file-service-3").await; let _ = executor @@ -1824,7 +1820,7 @@ async fn ip_address_resolve( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let component_id = executor.store_component("networking").await; + let component_id = executor.component("networking").store().await; let worker_id = executor .start_worker(&component_id, "ip-address-resolve-1") .await; diff --git a/golem-worker-executor/src/context.rs b/golem-worker-executor/src/context.rs index 28585f6bb3..fc6f26d336 100644 --- a/golem-worker-executor/src/context.rs +++ b/golem-worker-executor/src/context.rs @@ -23,7 +23,7 @@ use golem_common::model::{ WorkerMetadata, WorkerStatus, WorkerStatusRecord, }; use golem_common::model::{ComponentFilePath, PluginInstallationId}; -use golem_wasm_rpc::golem::rpc::types::{ +use golem_wasm_rpc::golem::rpc0_1_1::types::{ FutureInvokeResult, HostFutureInvokeResult, Pollable, WasmRpc, }; use golem_wasm_rpc::protobuf::type_annotated_value::TypeAnnotatedValue; diff --git a/golem-worker-executor/src/lib.rs b/golem-worker-executor/src/lib.rs index 6db2782df0..b29b01ab9b 100644 --- a/golem-worker-executor/src/lib.rs +++ b/golem-worker-executor/src/lib.rs @@ -23,7 +23,7 @@ use async_trait::async_trait; use golem_common::model::component::ComponentOwner; use golem_common::model::plugin::{DefaultPluginOwner, DefaultPluginScope}; use golem_worker_executor_base::durable_host::DurableWorkerCtx; -use golem_worker_executor_base::preview2::golem::{api0_2_0, api1_1_0, api1_2_0}; +use golem_worker_executor_base::preview2::golem::{api0_2_0, api1_1_0, durability}; use golem_worker_executor_base::services::active_workers::ActiveWorkers; use golem_worker_executor_base::services::blob_store::BlobStoreService; use golem_worker_executor_base::services::component::ComponentService; @@ -205,8 +205,11 @@ impl Bootstrap for ServerBootstrap { api0_2_0::host::add_to_linker_get_host(&mut linker, get_durable_ctx)?; api1_1_0::host::add_to_linker_get_host(&mut linker, get_durable_ctx)?; api1_1_0::oplog::add_to_linker_get_host(&mut linker, get_durable_ctx)?; - api1_2_0::durability::add_to_linker_get_host(&mut linker, get_durable_ctx)?; - golem_wasm_rpc::golem::rpc::types::add_to_linker_get_host(&mut linker, get_durable_ctx)?; + durability::durability::add_to_linker_get_host(&mut linker, get_durable_ctx)?; + golem_wasm_rpc::golem::rpc0_1_1::types::add_to_linker_get_host( + &mut linker, + get_durable_ctx, + )?; Ok(linker) } } diff --git a/integration-tests/src/benchmarks/durability_overhead.rs b/integration-tests/src/benchmarks/durability_overhead.rs index fa9bbc1de7..c8010bffc2 100644 --- a/integration-tests/src/benchmarks/durability_overhead.rs +++ b/integration-tests/src/benchmarks/durability_overhead.rs @@ -67,7 +67,9 @@ impl Benchmark for DurabilityOverhead { ) -> Self::IterationContext { let component_id = benchmark_context .deps - .store_unique_component("durability-overhead") + .component("durability-overhead") + .unique() + .store() .await; let durable_worker_ids = diff --git a/integration-tests/src/benchmarks/mod.rs b/integration-tests/src/benchmarks/mod.rs index 2dd9c4f223..b93d997ed8 100644 --- a/integration-tests/src/benchmarks/mod.rs +++ b/integration-tests/src/benchmarks/mod.rs @@ -62,7 +62,7 @@ pub async fn setup_iteration( // Initialize infrastructure // Upload test component - let component_id = deps.store_unique_component(component_name).await; + let component_id = deps.component(component_name).unique().store().await; // Create 'size' workers let worker_ids = generate_worker_ids(size, &component_id, worker_name_prefix); diff --git a/integration-tests/src/benchmarks/rpc.rs b/integration-tests/src/benchmarks/rpc.rs index 65af145fae..d4080006ba 100644 --- a/integration-tests/src/benchmarks/rpc.rs +++ b/integration-tests/src/benchmarks/rpc.rs @@ -88,11 +88,15 @@ impl Benchmark for Rpc { ) -> Self::IterationContext { let child_component_id = benchmark_context .deps - .store_unique_component("child_component") + .component("child_component") + .unique() + .store() .await; let component_id = benchmark_context .deps - .store_unique_component("parent_component_composed") + .component("parent_component_composed") + .unique() + .store() .await; let mut worker_ids = Vec::new(); diff --git a/integration-tests/src/benchmarks/rpc_cpu_intensive.rs b/integration-tests/src/benchmarks/rpc_cpu_intensive.rs index bfaec2dda5..2ae7f90bc1 100644 --- a/integration-tests/src/benchmarks/rpc_cpu_intensive.rs +++ b/integration-tests/src/benchmarks/rpc_cpu_intensive.rs @@ -88,11 +88,15 @@ impl Benchmark for RpcCpuIntensive { ) -> Self::IterationContext { let child_component_id = benchmark_context .deps - .store_unique_component("child_component") + .component("child_component") + .unique() + .store() .await; let component_id = benchmark_context .deps - .store_unique_component("parent_component_composed") + .component("parent_component_composed") + .unique() + .store() .await; let mut worker_ids = Vec::new(); diff --git a/integration-tests/src/benchmarks/rpc_large_input.rs b/integration-tests/src/benchmarks/rpc_large_input.rs index 8a9bb0ac17..1b742c3e0f 100644 --- a/integration-tests/src/benchmarks/rpc_large_input.rs +++ b/integration-tests/src/benchmarks/rpc_large_input.rs @@ -89,11 +89,15 @@ impl Benchmark for RpcLargeInput { ) -> Self::IterationContext { let child_component_id = benchmark_context .deps - .store_unique_component("child_component") + .component("child_component") + .unique() + .store() .await; let component_id = benchmark_context .deps - .store_unique_component("parent_component_composed") + .component("parent_component_composed") + .unique() + .store() .await; let mut worker_ids = Vec::new(); diff --git a/integration-tests/tests/fork.rs b/integration-tests/tests/fork.rs index 278d2a9b56..8d6bf8ba3c 100644 --- a/integration-tests/tests/fork.rs +++ b/integration-tests/tests/fork.rs @@ -45,7 +45,7 @@ async fn fork_interrupted_worker(deps: &EnvBasedTestDependencies, _tracing: &Tra let http_server = run_http_server(&response, host_http_port); - let component_id = deps.store_component("http-client-2").await; + let component_id = deps.component("http-client-2").store().await; let mut env = HashMap::new(); env.insert("PORT".to_string(), host_http_port.to_string()); @@ -105,7 +105,7 @@ async fn fork_interrupted_worker(deps: &EnvBasedTestDependencies, _tracing: &Tra #[tracing::instrument] #[timeout(120000)] async fn fork_running_worker_1(deps: &EnvBasedTestDependencies, _tracing: &Tracing) { - let component_id = deps.store_component("shopping-cart").await; + let component_id = deps.component("shopping-cart").store().await; let source_worker_name = Uuid::new_v4().to_string(); @@ -189,7 +189,7 @@ async fn fork_running_worker_2(deps: &EnvBasedTestDependencies, _tracing: &Traci let host_http_port = 8587; let http_server = run_http_server(&response, host_http_port); - let component_id = deps.store_component("http-client-2").await; + let component_id = deps.component("http-client-2").store().await; let mut env = HashMap::new(); env.insert("PORT".to_string(), host_http_port.to_string()); @@ -261,7 +261,7 @@ async fn fork_running_worker_2(deps: &EnvBasedTestDependencies, _tracing: &Traci #[tracing::instrument] #[timeout(120000)] async fn fork_idle_worker(deps: &EnvBasedTestDependencies, _tracing: &Tracing) { - let component_id = deps.store_component("shopping-cart").await; + let component_id = deps.component("shopping-cart").store().await; let source_worker_name = Uuid::new_v4().to_string(); @@ -373,7 +373,7 @@ async fn fork_worker_when_target_already_exists( deps: &EnvBasedTestDependencies, _tracing: &Tracing, ) { - let component_id = deps.store_component("shopping-cart").await; + let component_id = deps.component("shopping-cart").store().await; let source_worker_name = Uuid::new_v4().to_string(); @@ -419,7 +419,7 @@ async fn fork_worker_with_invalid_oplog_index_cut_off( deps: &EnvBasedTestDependencies, _tracing: &Tracing, ) { - let component_id = deps.store_component("shopping-cart").await; + let component_id = deps.component("shopping-cart").store().await; let source_worker_name = Uuid::new_v4().to_string(); @@ -460,7 +460,7 @@ async fn fork_worker_with_invalid_oplog_index_cut_off( #[tracing::instrument] #[timeout(120000)] async fn fork_invalid_worker(deps: &EnvBasedTestDependencies, _tracing: &Tracing) { - let component_id = deps.store_component("shopping-cart").await; + let component_id = deps.component("shopping-cart").store().await; let source_worker_name = Uuid::new_v4().to_string(); @@ -497,7 +497,7 @@ async fn fork_worker_ensures_zero_divergence_until_cut_off( deps: &EnvBasedTestDependencies, _tracing: &Tracing, ) { - let component_id = deps.store_component("environment-service").await; + let component_id = deps.component("environment-service").store().await; let source_worker_name = Uuid::new_v4().to_string(); diff --git a/integration-tests/tests/plugins.rs b/integration-tests/tests/plugins.rs index 5042e6bf27..c76daf1e23 100644 --- a/integration-tests/tests/plugins.rs +++ b/integration-tests/tests/plugins.rs @@ -37,7 +37,7 @@ inherit_test_dep!(EnvBasedTestDependencies); #[test] async fn component_transformer1(deps: &EnvBasedTestDependencies, _tracing: &Tracing) { - let component_id = deps.store_unique_component("logging").await; + let component_id = deps.component("logging").unique().store().await; let port = 8999; deps.create_plugin(PluginDefinition { @@ -154,8 +154,8 @@ fn transform_component(component: Bytes) -> anyhow::Result> { #[test] async fn oplog_processor1(deps: &EnvBasedTestDependencies, _tracing: &Tracing) { - let plugin_component_id = deps.store_unique_component("oplog-processor").await; - let component_id = deps.store_unique_component("shopping-cart").await; + let plugin_component_id = deps.component("oplog-processor").unique().store().await; + let component_id = deps.component("shopping-cart").unique().store().await; deps.create_plugin(PluginDefinition { name: "oplog-processor-1".to_string(), diff --git a/integration-tests/tests/sharding.rs b/integration-tests/tests/sharding.rs index 806b76334e..e264c429a6 100644 --- a/integration-tests/tests/sharding.rs +++ b/integration-tests/tests/sharding.rs @@ -364,7 +364,7 @@ mod tests { async fn create_component_and_start_workers(&self, n: usize) -> Vec { info!("Storing component"); - let component_id = self.store_component("option-service").await; + let component_id = self.component("option-service").store().await; info!("ComponentId: {}", component_id); let mut worker_ids = Vec::new(); diff --git a/integration-tests/tests/worker.rs b/integration-tests/tests/worker.rs index 0637a54c29..55df02dfd3 100644 --- a/integration-tests/tests/worker.rs +++ b/integration-tests/tests/worker.rs @@ -31,7 +31,7 @@ use golem_common::model::oplog::{OplogIndex, WorkerResourceId}; use golem_common::model::public_oplog::{ExportedFunctionInvokedParameters, PublicOplogEntry}; use golem_common::model::{ AccountId, ComponentFilePath, ComponentFilePermissions, ComponentFileSystemNode, - ComponentFileSystemNodeDetails, ComponentId, ComponentType, FilterComparator, IdempotencyKey, + ComponentFileSystemNodeDetails, ComponentId, FilterComparator, IdempotencyKey, InitialComponentFile, ScanCursor, StringFilterComparator, TargetWorkerId, Timestamp, WorkerFilter, WorkerId, WorkerMetadata, WorkerResourceDescription, WorkerStatus, }; @@ -49,7 +49,7 @@ inherit_test_dep!(EnvBasedTestDependencies); #[tracing::instrument] #[timeout(120000)] async fn dynamic_worker_creation(deps: &EnvBasedTestDependencies, _tracing: &Tracing) { - let component_id = deps.store_component("environment-service").await; + let component_id = deps.component("environment-service").store().await; let worker_id = WorkerId { component_id: component_id.clone(), worker_name: "dynamic-worker-creation-1".to_string(), @@ -116,7 +116,7 @@ fn get_env_result(env: Vec) -> HashMap { #[tracing::instrument] #[timeout(120000)] async fn dynamic_worker_creation_without_name(deps: &EnvBasedTestDependencies, _tracing: &Tracing) { - let component_id = deps.store_component("environment-service").await; + let component_id = deps.component("environment-service").store().await; let worker_id = TargetWorkerId { component_id: component_id.clone(), worker_name: None, @@ -150,7 +150,11 @@ async fn ephemeral_worker_creation_without_name( deps: &EnvBasedTestDependencies, _tracing: &Tracing, ) { - let component_id = deps.store_ephemeral_component("environment-service").await; + let component_id = deps + .component("environment-service") + .ephemeral() + .store() + .await; let worker_id = TargetWorkerId { component_id: component_id.clone(), worker_name: None, @@ -184,7 +188,7 @@ async fn ephemeral_worker_creation_with_name_is_not_persistent( deps: &EnvBasedTestDependencies, _tracing: &Tracing, ) { - let component_id = deps.store_ephemeral_component("counters").await; + let component_id = deps.component("counters").ephemeral().store().await; let worker_id = TargetWorkerId { component_id: component_id.clone(), worker_name: Some("test".to_string()), @@ -215,7 +219,7 @@ async fn ephemeral_worker_creation_with_name_is_not_persistent( #[tracing::instrument] #[timeout(120000)] async fn counter_resource_test_1(deps: &EnvBasedTestDependencies, _tracing: &Tracing) { - let component_id = deps.store_unique_component("counters").await; + let component_id = deps.component("counters").unique().store().await; let worker_id = deps.start_worker(&component_id, "counters-1").await; deps.log_output(&worker_id).await; @@ -322,7 +326,7 @@ async fn counter_resource_test_1(deps: &EnvBasedTestDependencies, _tracing: &Tra #[tracing::instrument] #[timeout(120000)] async fn counter_resource_test_1_json(deps: &EnvBasedTestDependencies, _tracing: &Tracing) { - let component_id = deps.store_unique_component("counters").await; + let component_id = deps.component("counters").unique().store().await; let worker_id = deps.start_worker(&component_id, "counters-1j").await; deps.log_output(&worker_id).await; @@ -504,7 +508,7 @@ async fn counter_resource_test_1_json(deps: &EnvBasedTestDependencies, _tracing: #[tracing::instrument] #[timeout(120000)] async fn counter_resource_test_2(deps: &EnvBasedTestDependencies, _tracing: &Tracing) { - let component_id = deps.store_unique_component("counters").await; + let component_id = deps.component("counters").unique().store().await; let worker_id = deps.start_worker(&component_id, "counters-2").await; deps.log_output(&worker_id).await; @@ -584,7 +588,7 @@ async fn counter_resource_test_2(deps: &EnvBasedTestDependencies, _tracing: &Tra #[tracing::instrument] #[timeout(120000)] async fn counter_resource_test_2_json(deps: &EnvBasedTestDependencies, _tracing: &Tracing) { - let component_id = deps.store_unique_component("counters").await; + let component_id = deps.component("counters").unique().store().await; let worker_id = deps.start_worker(&component_id, "counters-2j").await; deps.log_output(&worker_id).await; @@ -724,7 +728,7 @@ async fn counter_resource_test_2_json(deps: &EnvBasedTestDependencies, _tracing: #[tracing::instrument] #[timeout(120000)] async fn shopping_cart_example(deps: &EnvBasedTestDependencies, _tracing: &Tracing) { - let component_id = deps.store_component("shopping-cart").await; + let component_id = deps.component("shopping-cart").store().await; let worker_id = deps.start_worker(&component_id, "shopping-cart-1").await; let _ = deps @@ -819,8 +823,8 @@ async fn shopping_cart_example(deps: &EnvBasedTestDependencies, _tracing: &Traci #[tracing::instrument] #[timeout(120000)] async fn auction_example_1(deps: &EnvBasedTestDependencies, _tracing: &Tracing) { - let registry_component_id = deps.store_component("auction_registry_composed").await; - let auction_component_id = deps.store_component("auction").await; + let registry_component_id = deps.component("auction_registry_composed").store().await; + let auction_component_id = deps.component("auction").store().await; let mut env = HashMap::new(); env.insert( @@ -882,7 +886,7 @@ fn get_worker_ids(workers: Vec<(WorkerMetadata, Option)>) -> HashSet>> = Arc::new(Mutex::new(HashSet::new())); @@ -1109,7 +1113,7 @@ async fn get_running_workers(deps: &EnvBasedTestDependencies, _tracing: &Tracing #[tracing::instrument] #[timeout(300000)] async fn auto_update_on_idle(deps: &EnvBasedTestDependencies, _tracing: &Tracing) { - let component_id = deps.store_unique_component("update-test-v1").await; + let component_id = deps.component("update-test-v1").unique().store().await; let worker_id = deps .start_worker(&component_id, "auto_update_on_idle") .await; @@ -1144,7 +1148,7 @@ async fn auto_update_on_idle_via_host_function( deps: &EnvBasedTestDependencies, _tracing: &Tracing, ) { - let component_id = deps.store_unique_component("update-test-v1").await; + let component_id = deps.component("update-test-v1").unique().store().await; let worker_id = deps .start_worker(&component_id, "auto_update_on_idle_via_host_function") .await; @@ -1153,7 +1157,7 @@ async fn auto_update_on_idle_via_host_function( let target_version = deps.update_component(&component_id, "update-test-v2").await; info!("Updated component to version {target_version}"); - let runtime_svc = deps.store_component("runtime-service").await; + let runtime_svc = deps.component("runtime-service").store().await; let runtime_svc_worker = WorkerId { component_id: runtime_svc, worker_name: "runtime-service".to_string(), @@ -1197,7 +1201,7 @@ async fn auto_update_on_idle_via_host_function( #[tracing::instrument] #[timeout(120000)] async fn get_oplog_1(deps: &EnvBasedTestDependencies, _tracing: &Tracing) { - let component_id = deps.store_component("runtime-service").await; + let component_id = deps.component("runtime-service").store().await; let worker_id = WorkerId { component_id, @@ -1257,7 +1261,7 @@ async fn get_oplog_1(deps: &EnvBasedTestDependencies, _tracing: &Tracing) { #[tracing::instrument] #[timeout(120000)] async fn search_oplog_1(deps: &EnvBasedTestDependencies, _tracing: &Tracing) { - let component_id = deps.store_component("shopping-cart").await; + let component_id = deps.component("shopping-cart").store().await; let worker_id = WorkerId { component_id, @@ -1346,7 +1350,7 @@ async fn search_oplog_1(deps: &EnvBasedTestDependencies, _tracing: &Tracing) { #[tracing::instrument] #[timeout(600000)] async fn worker_recreation(deps: &EnvBasedTestDependencies, _tracing: &Tracing) { - let component_id = deps.store_unique_component("counters").await; + let component_id = deps.component("counters").unique().store().await; let worker_id = deps .start_worker(&component_id, "counters-recreation") .await; @@ -1441,11 +1445,10 @@ async fn worker_use_initial_files(deps: &EnvBasedTestDependencies, _tracing: &Tr ]; let component_id = deps - .store_unique_component_with_files( - "initial-file-read-write", - ComponentType::Durable, - &component_files, - ) + .component("initial-file-read-write") + .unique() + .with_files(&component_files) + .store() .await; let worker_id = deps @@ -1505,11 +1508,10 @@ async fn worker_list_files(deps: &EnvBasedTestDependencies, _tracing: &Tracing) ]; let component_id = deps - .store_unique_component_with_files( - "initial-file-read-write", - ComponentType::Durable, - &component_files, - ) + .component("initial-file-read-write") + .unique() + .with_files(&component_files) + .store() .await; let worker_id = deps @@ -1587,11 +1589,10 @@ async fn worker_read_files(deps: &EnvBasedTestDependencies, _tracing: &Tracing) ]; let component_id = deps - .store_unique_component_with_files( - "initial-file-read-write", - ComponentType::Durable, - &component_files, - ) + .component("initial-file-read-write") + .unique() + .with_files(&component_files) + .store() .await; let worker_id = deps @@ -1648,11 +1649,10 @@ async fn worker_initial_files_after_automatic_worker_update( ]; let component_id = deps - .store_unique_component_with_files( - "initial-file-read-write", - ComponentType::Durable, - &component_files1, - ) + .component("initial-file-read-write") + .unique() + .with_files(&component_files1) + .store() .await; let worker_id = deps diff --git a/wasm-ast/Cargo.toml b/wasm-ast/Cargo.toml index 44542ac89d..1649d00647 100644 --- a/wasm-ast/Cargo.toml +++ b/wasm-ast/Cargo.toml @@ -4,7 +4,7 @@ version = "0.0.0" edition = "2021" license = "Apache-2.0" homepage = "https://golem.cloud" -repository = "https://github.com/golemcloud/golem-wasm-ast/" +repository = "https://github.com/golemcloud/golem/" description = "WASM AST" [dependencies] diff --git a/wasm-rpc-stubgen/Cargo.toml b/wasm-rpc-stubgen/Cargo.toml index 27f8ae35c9..183bc3e23a 100644 --- a/wasm-rpc-stubgen/Cargo.toml +++ b/wasm-rpc-stubgen/Cargo.toml @@ -4,7 +4,7 @@ version = "0.0.0" edition = "2021" license = "Apache-2.0" homepage = "https://golem.cloud" -repository = "https://github.com/golemcloud/wasm-rpc" +repository = "https://github.com/golemcloud/golem" description = "Golem WASM RPC stub generator" [lib] @@ -37,6 +37,7 @@ fs_extra = { workspace = true } glob = "0.3.1" golem-wasm-ast = { path = "../wasm-ast", version = "0.0.0" } golem-wasm-rpc = { path = "../wasm-rpc", version = "0.0.0", default-features = false, features = ["host"] } +golem-wit = "1.1.6" heck = "0.5.0" id-arena = "2.2.1" indexmap = "2.7.0" @@ -57,7 +58,7 @@ syn = "2.0.90" tempfile = { workspace = true } tokio = { workspace = true } toml = "0.8.19" -wac-graph = "=0.6.1" +wac-graph = "0.6.1" walkdir = "2.5.0" wit-bindgen-rust = "=0.26.0" wit-encoder = "=0.221.2" diff --git a/wasm-rpc-stubgen/src/wit_generate.rs b/wasm-rpc-stubgen/src/wit_generate.rs index 5d9332abaa..6ba09987d2 100644 --- a/wasm-rpc-stubgen/src/wit_generate.rs +++ b/wasm-rpc-stubgen/src/wit_generate.rs @@ -267,13 +267,13 @@ pub fn add_dependencies_to_stub_wit_dir(def: &StubDefinition) -> anyhow::Result< write_embedded_source( &target_deps.join("wasm-rpc"), "wasm-rpc.wit", - golem_wasm_rpc::WASM_RPC_WIT, + golem_wit::WASM_RPC_WIT, )?; write_embedded_source( &target_deps.join("io"), "poll.wit", - golem_wasm_rpc::WASI_POLL_WIT, + golem_wit::WASI_POLL_WIT, )?; Ok(()) diff --git a/wasm-rpc-stubgen/tests/add_dep.rs b/wasm-rpc-stubgen/tests/add_dep.rs index 20db069a0c..fadbd7c3fc 100644 --- a/wasm-rpc-stubgen/tests/add_dep.rs +++ b/wasm-rpc-stubgen/tests/add_dep.rs @@ -18,7 +18,6 @@ use test_r::test; use assert2::assert; use fs_extra::dir::CopyOptions; -use golem_wasm_rpc::{WASI_POLL_WIT, WASM_RPC_WIT}; use golem_wasm_rpc_stubgen::commands::generate::generate_client_wit_dir; use golem_wasm_rpc_stubgen::stub::{StubConfig, StubDefinition}; use golem_wasm_rpc_stubgen::wit_generate::{ @@ -26,6 +25,7 @@ use golem_wasm_rpc_stubgen::wit_generate::{ }; use golem_wasm_rpc_stubgen::wit_resolve::ResolvedWitDir; use golem_wasm_rpc_stubgen::WasmRpcOverride; +use golem_wit::{WASI_POLL_WIT, WASM_RPC_WIT}; use semver::Version; use std::path::{Path, PathBuf}; use tempfile::TempDir; diff --git a/wasm-rpc/Cargo.toml b/wasm-rpc/Cargo.toml index d51e4d3650..4abac4df26 100644 --- a/wasm-rpc/Cargo.toml +++ b/wasm-rpc/Cargo.toml @@ -4,7 +4,7 @@ version = "0.0.0" edition = "2021" license = "Apache-2.0" homepage = "https://golem.cloud" -repository = "https://github.com/golemcloud/wasm-rpc" +repository = "https://github.com/golemcloud/golem" description = "Golem WASM RPC support library" include = [] @@ -95,6 +95,8 @@ package = "golem:rpc" [package.metadata.component.target] path = "wit" +world = "wasm-rpc" [package.metadata.component.target.dependencies] +"golem:rpc" = { path = "wit/deps/wasm-rpc" } "wasi:io" = { path = "wit/deps/io" } diff --git a/wasm-rpc/src/bindings.rs b/wasm-rpc/src/bindings.rs index a3ca9b9ff7..f09d8965ad 100644 --- a/wasm-rpc/src/bindings.rs +++ b/wasm-rpc/src/bindings.rs @@ -4,7 +4,7 @@ #[rustfmt::skip] #[allow(dead_code, clippy::all)] pub mod golem { - pub mod rpc { + pub mod rpc0_1_1 { #[allow(dead_code, clippy::all)] pub mod types { #[used] @@ -3996,10 +3996,10 @@ mod _rt { extern crate alloc as alloc_crate; } #[cfg(target_arch = "wasm32")] -#[link_section = "component-type:wit-bindgen:0.36.0:golem:rpc@0.1.1:wit-value:encoded world"] +#[link_section = "component-type:wit-bindgen:0.36.0:golem:rpc:wasm-rpc:encoded world"] #[doc(hidden)] -pub static __WIT_BINDGEN_COMPONENT_TYPE: [u8; 1875] = *b"\ -\0asm\x0d\0\x01\0\0\x19\x16wit-component-encoding\x04\0\x07\xd3\x0d\x01A\x02\x01\ +pub static __WIT_BINDGEN_COMPONENT_TYPE: [u8; 1867] = *b"\ +\0asm\x0d\0\x01\0\0\x19\x16wit-component-encoding\x04\0\x07\xcc\x0d\x01A\x02\x01\ A\x05\x01B\x0a\x04\0\x08pollable\x03\x01\x01h\0\x01@\x01\x04self\x01\0\x7f\x04\0\ \x16[method]pollable.ready\x01\x02\x01@\x01\x04self\x01\x01\0\x04\0\x16[method]p\ ollable.block\x01\x03\x01p\x01\x01py\x01@\x01\x02in\x04\0\x05\x04\0\x04poll\x01\x06\ @@ -4035,9 +4035,9 @@ c.invoke\x01.\x01i&\x01@\x03\x04self)\x0dfunction-names\x0ffunction-params*\0/\x \02\x04\0&[method]future-invoke-result.subscribe\x013\x01k+\x01@\x01\x04self1\04\ \x04\0\x20[method]future-invoke-result.get\x015\x01@\x01\x03vnt\"\0\x20\x04\0\x0d\ extract-value\x016\x01@\x01\x03vnt\"\0\x15\x04\0\x0cextract-type\x017\x03\0\x15g\ -olem:rpc/types@0.1.1\x05\x02\x04\0\x19golem:rpc/wit-value@0.1.1\x04\0\x0b\x0f\x01\ -\0\x09wit-value\x03\0\0\0G\x09producers\x01\x0cprocessed-by\x02\x0dwit-component\ -\x070.220.0\x10wit-bindgen-rust\x060.36.0"; +olem:rpc/types@0.1.1\x05\x02\x04\0\x12golem:rpc/wasm-rpc\x04\0\x0b\x0e\x01\0\x08\ +wasm-rpc\x03\0\0\0G\x09producers\x01\x0cprocessed-by\x02\x0dwit-component\x070.2\ +20.0\x10wit-bindgen-rust\x060.36.0"; #[inline(never)] #[doc(hidden)] pub fn __link_custom_section_describing_imports() { diff --git a/wasm-rpc/src/lib.rs b/wasm-rpc/src/lib.rs index 2945e82db1..de0430b94c 100644 --- a/wasm-rpc/src/lib.rs +++ b/wasm-rpc/src/lib.rs @@ -76,7 +76,7 @@ pub use extractor::{WitNodePointer, WitValueExtractor}; #[cfg(not(feature = "host-bindings"))] #[cfg(feature = "stub")] -pub use bindings::golem::rpc::types::{ +pub use bindings::golem::rpc0_1_1::types::{ FutureInvokeResult, NodeIndex, ResourceMode, RpcError, Uri, WasmRpc, WitNode, WitType, WitTypeNode, WitValue, }; @@ -90,10 +90,9 @@ pub use wasmtime_wasi::Pollable; #[cfg(feature = "host-bindings")] mod generated { use ::wasmtime::component::bindgen; - bindgen!({ path: "wit", - world: "wit-value", + world: "wasm-rpc", tracing: false, async: true, trappable_imports: true, @@ -110,7 +109,7 @@ mod generated { pub use generated::golem; #[cfg(feature = "host-bindings")] -pub use generated::golem::rpc::types::{ +pub use generated::golem::rpc0_1_1::types::{ Host, HostWasmRpc, NodeIndex, ResourceMode, RpcError, Uri, WitNode, WitType, WitTypeNode, WitValue, }; @@ -432,11 +431,6 @@ impl<'a> arbitrary::Arbitrary<'a> for WitValue { } } -#[cfg(feature = "host-bindings")] -pub const WASM_RPC_WIT: &str = include_str!("../wit/wasm-rpc.wit"); -#[cfg(feature = "host-bindings")] -pub const WASI_POLL_WIT: &str = include_str!("../wit/deps/io/poll.wit"); - pub const WASM_RPC_VERSION: &str = version::lib_version!(); #[cfg(test)] diff --git a/wasm-rpc/src/value_and_type.rs b/wasm-rpc/src/value_and_type.rs index 31738d3da5..f44d641b0e 100644 --- a/wasm-rpc/src/value_and_type.rs +++ b/wasm-rpc/src/value_and_type.rs @@ -760,8 +760,8 @@ impl WitTypeBuilder { } } -impl From for ValueAndType { - fn from(value: crate::golem::rpc::types::ValueAndType) -> Self { +impl From for ValueAndType { + fn from(value: crate::golem::rpc0_1_1::types::ValueAndType) -> Self { Self { value: value.value.into(), typ: value.typ.into(), @@ -769,7 +769,7 @@ impl From for ValueAndType { } } -impl From for crate::golem::rpc::types::ValueAndType { +impl From for crate::golem::rpc0_1_1::types::ValueAndType { fn from(value: ValueAndType) -> Self { Self { value: value.value.into(), diff --git a/wasm-rpc/wit/deps/io/poll.wit b/wasm-rpc/wit/deps/io/poll.wit deleted file mode 100644 index 82fa0f312a..0000000000 --- a/wasm-rpc/wit/deps/io/poll.wit +++ /dev/null @@ -1,41 +0,0 @@ -package wasi:io@0.2.0; - -/// A poll API intended to let users wait for I/O events on multiple handles -/// at once. -interface poll { - /// `pollable` epresents a single I/O event which may be ready, or not. - resource pollable { - - /// Return the readiness of a pollable. This function never blocks. - /// - /// Returns `true` when the pollable is ready, and `false` otherwise. - ready: func() -> bool; - - /// `block` returns immediately if the pollable is ready, and otherwise - /// blocks until ready. - /// - /// This function is equivalent to calling `poll.poll` on a list - /// containing only this pollable. - block: func(); - } - - /// Poll for completion on a set of pollables. - /// - /// This function takes a list of pollables, which identify I/O sources of - /// interest, and waits until one or more of the events is ready for I/O. - /// - /// The result `list` contains one or more indices of handles in the - /// argument list that is ready for I/O. - /// - /// If the list contains more elements than can be indexed with a `u32` - /// value, this function traps. - /// - /// A timeout can be implemented by adding a pollable from the - /// wasi-clocks API to the list. - /// - /// This function does not return a `result`; polling in itself does not - /// do any I/O so it doesn't fail. If any of the I/O sources identified by - /// the pollables has an error, it is indicated by marking the source as - /// being reaedy for I/O. - poll: func(in: list>) -> list; -} diff --git a/wasm-rpc/wit/wasm-rpc.wit b/wasm-rpc/wit/wasm-rpc.wit index fda44c4559..6185fb9d83 100644 --- a/wasm-rpc/wit/wasm-rpc.wit +++ b/wasm-rpc/wit/wasm-rpc.wit @@ -1,109 +1,5 @@ -package golem:rpc@0.1.1; +package golem:rpc; -interface types { - use wasi:io/poll@0.2.0.{pollable}; - - type node-index = s32; - - record wit-value { - nodes: list, - } - - variant wit-node { - record-value(list), - variant-value(tuple>), - enum-value(u32), - flags-value(list), - tuple-value(list), - list-value(list), - option-value(option), - result-value(result, option>), - prim-u8(u8), - prim-u16(u16), - prim-u32(u32), - prim-u64(u64), - prim-s8(s8), - prim-s16(s16), - prim-s32(s32), - prim-s64(s64), - prim-float32(f32), - prim-float64(f64), - prim-char(char), - prim-bool(bool), - prim-string(string), - handle(tuple) - } - - record wit-type { - nodes: list, - } - - type resource-id = u64; - - enum resource-mode { - owned, - borrowed - } - - variant wit-type-node { - record-type(list>), - variant-type(list>>), - enum-type(list), - flags-type(list), - tuple-type(list), - list-type(node-index), - option-type(node-index), - result-type(tuple, option>), - prim-u8-type, - prim-u16-type, - prim-u32-type, - prim-u64-type, - prim-s8-type, - prim-s16-type, - prim-s32-type, - prim-s64-type, - prim-f32-type, - prim-f64-type, - prim-char-type, - prim-bool-type, - prim-string-type, - handle-type(tuple) - } - - record value-and-type { - value: wit-value, - typ: wit-type - } - - record uri { - value: string, - } - - variant rpc-error { - protocol-error(string), - denied(string), - not-found(string), - remote-internal-error(string) - } - - resource wasm-rpc { - constructor(location: uri); - - invoke-and-await: func(function-name: string, function-params: list) -> result; - invoke: func(function-name: string, function-params: list) -> result<_, rpc-error>; - - async-invoke-and-await: func(function-name: string, function-params: list) -> future-invoke-result; - } - - resource future-invoke-result { - subscribe: func() -> pollable; - get: func() -> option>; - } - - extract-value: func(vnt: value-and-type) -> wit-value; - extract-type: func(vnt: value-and-type) -> wit-type; -} - -world wit-value { - import types; +world wasm-rpc { + import golem:rpc/types@0.1.1; } diff --git a/wit/deps.lock b/wit/deps.lock new file mode 100644 index 0000000000..af60dbd9ce --- /dev/null +++ b/wit/deps.lock @@ -0,0 +1,61 @@ +[all] +url = "https://github.com/golemcloud/golem-wit/archive/v1.1.6.tar.gz" +sha256 = "6ac4f1553003474229bb40a885b48d1eaa4bea2f9d679a115da9afede30b5ad4" +sha512 = "ced8c18a35dc04123386465d98d2aeba54c486e4e22a30e7efac0548974e176528fcba36c64942d67fa35d9135806ff0fcde259eb4af051c4a29fcb397e9af1c" +deps = ["blobstore", "cli", "clocks", "filesystem", "golem", "golem-1.1", "golem-durability", "http", "io", "keyvalue", "logging", "random", "sockets", "wasm-rpc"] + +[blobstore] +sha256 = "eb50ec776eac937b2ef90b41a281d0dd7c94f0b076466d917bd0f2ed5b01d39d" +sha512 = "b40613b47a8ba236a7b3ff85cb7e3c5b8f665a6c359fc539ca01f0b2170538a47c80eb991df8f4d5ca162f4dc6e0ca4311e23ad5ce88b85b30570dedf78fcffe" + +[cli] +sha256 = "285865a31d777181b075f39e92bcfe59c89cd6bacce660be1b9a627646956258" +sha512 = "da2622210a9e3eea82b99f1a5b8a44ce5443d009cb943f7bca0bf9cf4360829b289913d7ee727c011f0f72994ea7dc8e661ebcc0a6b34b587297d80cd9b3f7e8" + +[clocks] +sha256 = "468b4d12892fe926b8eb5d398dbf579d566c93231fa44f415440572c695b7613" +sha512 = "e6b53a07221f1413953c9797c68f08b815fdaebf66419bbc1ea3e8b7dece73731062693634731f311a03957b268cf9cc509c518bd15e513c318aa04a8459b93a" + +[filesystem] +sha256 = "498c465cfd04587db40f970fff2185daa597d074c20b68a8bcbae558f261499b" +sha512 = "ead452f9b7bfb88593a502ec00d76d4228003d51c40fd0408aebc32d35c94673551b00230d730873361567cc209ec218c41fb4e95bad194268592c49e7964347" + +[golem] +sha256 = "0e9efca1e217c1072b81923393870747409f23b23f0ca4fa54c7a650065c03d9" +sha512 = "91d3b88e690cf056507f79800b27f995bc40130b391f5d6e1df6dbc649bea4bdd55688d2849feb7c96eb2824f97b854ed08ac0a5a7c4e3f4256cc32b6be66680" + +["golem-1.1"] +sha256 = "ef2aa10c130cb37c64413cd0807ea688785ac1f70c1b64c17fef8b116c681a22" +sha512 = "17a534799c7bb5a987a505c9ca3c43c632f67c4c19c0b2b2f6236e925194f0c8599d1e082d9f788b4c7ae1493daf37eebcb759e2e2599f02ffe179acace4c05b" + +[golem-durability] +sha256 = "002ce1e93698e9e9150704ac635f7806dd72160af1887d44bcdef66c8423a456" +sha512 = "e7a34d5fefa7423c056effa486f97f56bc708b92d9763df2a9e201bbef513a08c1d7ac57859bdb3ae0831736b6084b7d9a0037eb4d4e778b24b663ab0e040a75" + +[http] +sha256 = "8f44402bde16c48e28c47dc53eab0b26af5b3b3482a1852cf77673e0880ba1c1" +sha512 = "760695f9a25c25bf75a25b731cb21c3bda9e288e450edda823324ecbc73d5d798bbb5de2edad999566980836f037463ee9e57d61789d04b3f3e381475b1a9a0f" + +[io] +sha256 = "5a3b0b9774ad733d4fa144a694018b03158c21e6168234e81ed02d49a78ef9cb" +sha512 = "f88e836b2264b7a0ec8e51b79420331c42b4973ccc283c556802103e0b6a8daa5ecdf00bd984851fe52db07e703d75c1970c5fba1614b6c990d97734cad22d67" + +[keyvalue] +sha256 = "efceed91e17be50f52681ef73043f54298e2f11b48a5befdcbc92a7aea941bed" +sha512 = "b019a3333c6dadbab8c067085a5110566cfb7c3f3d9a65d482ccdac85a62eb0bdfed0ca1bbf7dafabf622ca0c6cf39b865cc234412bcbb2ff1290386cda031fa" + +[logging] +sha256 = "7fb7216c713fd329ec91bb596127a1b95eec26290290b6de0c1ef294fba858c2" +sha512 = "ceb68410bb46176051b0eee54e1a60831ca2fe820892c5f5f739e76efd4fe6ee6dc5e307947e7118b68a2a757ed3a3b471d16eb0900bb9040de47fef4bf1b56f" + +[random] +sha256 = "7371d03c037d924caba2587fb2e7c5773a0d3c5fcecbf7971e0e0ba57973c53d" +sha512 = "964c4e8925a53078e4d94ba907b54f89a0b7e154f46823a505391471466c17f53c8692682e5c85771712acd88b348686173fc07c53a3cfe3d301b8cd8ddd0de4" + +[sockets] +sha256 = "b0869b7cbbaea1410308cd379207f5841db5e30f1e1c1d9fe0582b560b668f24" +sha512 = "198a7bcf4d405b7761e1b5a0a94cdd1e144adb0c6cfbe2b64e380d377de5ac3171a4cf3f840b81ff2ef54be990c95c121f48910796228c0f54b0c3c6e973e317" + +[wasm-rpc] +sha256 = "701be51f93381116f8d66e9094278669a7c1e40f0d16eb99bf75686a2832cece" +sha512 = "99315fffd4b2c16a1d37bea31daf12fe77089922954b82e07b9ef772e5571219bb5d464307254bfc7667ad8cd4d9bb19388258b70574c536e01567ccde4d5964" diff --git a/wit/deps.toml b/wit/deps.toml new file mode 100644 index 0000000000..7371c70e90 --- /dev/null +++ b/wit/deps.toml @@ -0,0 +1 @@ +all = "https://github.com/golemcloud/golem-wit/archive/v1.1.6.tar.gz" diff --git a/wit/host.wit b/wit/host.wit new file mode 100644 index 0000000000..50a12be083 --- /dev/null +++ b/wit/host.wit @@ -0,0 +1,19 @@ +package golem:api; + +world golem { + import golem:api/host@0.2.0; + import golem:api/host@1.1.0; + import golem:api/oplog@1.1.0; + import golem:durability/durability@1.2.0; + + import wasi:blobstore/blobstore; + import wasi:blobstore/container; + import wasi:blobstore/types; + import wasi:keyvalue/atomic@0.1.0; + import wasi:keyvalue/eventual-batch@0.1.0; + import wasi:keyvalue/cache@0.1.0; + import wasi:keyvalue/eventual@0.1.0; + import wasi:keyvalue/types@0.1.0; + import wasi:keyvalue/wasi-keyvalue-error@0.1.0; + import wasi:logging/logging; +} From 4905ff5c1831a71574e25104094217bd9491af02 Mon Sep 17 00:00:00 2001 From: Daniel Vigovszky Date: Wed, 29 Jan 2025 17:43:28 +0100 Subject: [PATCH 2/6] Undo some more --- .../src/durable_host/wasm_rpc/mod.rs | 54 +++---------------- .../src/services/golem_config.rs | 2 - 2 files changed, 6 insertions(+), 50 deletions(-) diff --git a/golem-worker-executor-base/src/durable_host/wasm_rpc/mod.rs b/golem-worker-executor-base/src/durable_host/wasm_rpc/mod.rs index dd8a7c2e0a..72f56e7f6f 100644 --- a/golem-worker-executor-base/src/durable_host/wasm_rpc/mod.rs +++ b/golem-worker-executor-base/src/durable_host/wasm_rpc/mod.rs @@ -89,8 +89,8 @@ impl HostWasmRpc for DurableWorkerCtx { function_name: String, mut function_params: Vec, ) -> anyhow::Result> { - let args = persisted_get_arguments(self).await?; - let env = persisted_get_environment(self).await?; + let args = self.get_arguments().await?; + let env = self.get_environment().await?; let entry = self.table().get(&self_)?; let payload = entry.payload.downcast_ref::().unwrap(); @@ -221,8 +221,8 @@ impl HostWasmRpc for DurableWorkerCtx { function_name: String, mut function_params: Vec, ) -> anyhow::Result> { - let args = persisted_get_arguments(self).await?; - let env = persisted_get_environment(self).await?; + let args = self.get_arguments().await?; + let env = self.get_environment().await?; let entry = self.table().get(&self_)?; let payload = entry.payload.downcast_ref::().unwrap(); @@ -312,8 +312,8 @@ impl HostWasmRpc for DurableWorkerCtx { function_name: String, mut function_params: Vec, ) -> anyhow::Result> { - let args = persisted_get_arguments(self).await?; - let env = persisted_get_environment(self).await?; + let args = self.get_arguments().await?; + let env = self.get_environment().await?; let begin_index = self .state @@ -824,48 +824,6 @@ async fn try_get_typed_parameters( Vec::new() } -// For backward compatibility with Golem 1.1 - before compositional durability the RPC calls -// were calling the durable get_environment() -async fn persisted_get_environment( - ctx: &mut DurableWorkerCtx, -) -> anyhow::Result> { - let durability = Durability::, SerializableError>::new( - ctx, - "golem_environment", - "get_environment", - DurableFunctionType::ReadLocal, - ) - .await?; - - if durability.is_live() { - let result = ctx.as_wasi_view().get_environment().await; - durability.persist(ctx, (), result).await - } else { - durability.replay(ctx).await - } -} - -// For backward compatibility with Golem 1.1 - before compositional durability the RPC calls -// were calling the durable get_Arguments() -async fn persisted_get_arguments( - ctx: &mut DurableWorkerCtx, -) -> anyhow::Result> { - let durability = Durability::, SerializableError>::new( - ctx, - "golem_environment", - "get_arguments", - DurableFunctionType::ReadLocal, - ) - .await?; - - if durability.is_live() { - let result = ctx.as_wasi_view().get_arguments().await; - durability.persist(ctx, (), result).await - } else { - durability.replay(ctx).await - } -} - pub enum WasmRpcEntryPayload { Interface { #[allow(dead_code)] diff --git a/golem-worker-executor-base/src/services/golem_config.rs b/golem-worker-executor-base/src/services/golem_config.rs index f9e58a45fd..39c665682b 100644 --- a/golem-worker-executor-base/src/services/golem_config.rs +++ b/golem-worker-executor-base/src/services/golem_config.rs @@ -55,7 +55,6 @@ pub struct GolemConfig { pub port: u16, pub http_address: String, pub http_port: u16, - pub debug_worker_output: bool, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -363,7 +362,6 @@ impl Default for GolemConfig { port: 9000, http_address: "0.0.0.0".to_string(), http_port: 8082, - debug_worker_output: true, } } } From 700a724f9ca261f32c42993a443eead980d8bf11 Mon Sep 17 00:00:00 2001 From: Daniel Vigovszky Date: Wed, 29 Jan 2025 17:58:47 +0100 Subject: [PATCH 3/6] Fix --- golem-worker-executor-base/tests/wasi.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/golem-worker-executor-base/tests/wasi.rs b/golem-worker-executor-base/tests/wasi.rs index c521b9ef6d..7606b4bc2a 100644 --- a/golem-worker-executor-base/tests/wasi.rs +++ b/golem-worker-executor-base/tests/wasi.rs @@ -1857,7 +1857,8 @@ async fn wasi_incoming_request_handler( let executor = start(deps, &context).await.unwrap(); let component_id = executor - .store_component("wasi-http-incoming-request-handler") + .component("wasi-http-incoming-request-handler") + .store() .await; let worker_id = executor .start_worker(&component_id, "wasi-http-incoming-request-handler-1") @@ -1911,7 +1912,8 @@ async fn wasi_incoming_request_handler_echo( let executor = start(deps, &context).await.unwrap(); let component_id = executor - .store_component("wasi-http-incoming-request-handler-echo") + .component("wasi-http-incoming-request-handler-echo") + .store() .await; let worker_id = executor @@ -2050,7 +2052,8 @@ async fn wasi_incoming_request_handler_state( let executor = start(deps, &context).await.unwrap(); let component_id = executor - .store_component("wasi-http-incoming-request-handler-state") + .component("wasi-http-incoming-request-handler-state") + .store() .await; let worker_id = executor From 6d424809779c5ce09ecec72fcfabf47b3b1dfb61 Mon Sep 17 00:00:00 2001 From: Daniel Vigovszky Date: Wed, 29 Jan 2025 18:11:42 +0100 Subject: [PATCH 4/6] Revert test change --- golem-worker-executor-base/tests/observability.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/golem-worker-executor-base/tests/observability.rs b/golem-worker-executor-base/tests/observability.rs index 079af9f6fd..21be74cb4f 100644 --- a/golem-worker-executor-base/tests/observability.rs +++ b/golem-worker-executor-base/tests/observability.rs @@ -247,7 +247,7 @@ async fn get_oplog_with_api_changing_updates( println!("oplog\n{:#?}", oplog); check!(result[0] == Value::U64(11)); - assert_eq!(oplog.len(), 15); + assert_eq!(oplog.len(), 17); } #[test] From 52c14e052b56baa8325edb890b5ec3a4f93f6f5c Mon Sep 17 00:00:00 2001 From: Daniel Vigovszky Date: Wed, 29 Jan 2025 18:57:39 +0100 Subject: [PATCH 5/6] Fix metadata check --- golem-worker-executor-base/tests/api.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/golem-worker-executor-base/tests/api.rs b/golem-worker-executor-base/tests/api.rs index d7e3121347..61e3387c73 100644 --- a/golem-worker-executor-base/tests/api.rs +++ b/golem-worker-executor-base/tests/api.rs @@ -1273,7 +1273,7 @@ async fn get_worker_metadata( ); check!(metadata2.last_known_status.component_size == expected_component_size); - check!(metadata2.last_known_status.total_linear_memory_size == 2293760); + check!(metadata2.last_known_status.total_linear_memory_size == 1245184); } #[test] From cd4cb736fae8e84eb190245bb144933c57e1333e Mon Sep 17 00:00:00 2001 From: Daniel Vigovszky Date: Thu, 30 Jan 2025 09:16:16 +0100 Subject: [PATCH 6/6] Fix --- .../tests/rust_rpc_stubless.rs | 53 +++++++++---------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/golem-worker-executor-base/tests/rust_rpc_stubless.rs b/golem-worker-executor-base/tests/rust_rpc_stubless.rs index d27f2c0db8..84126b9517 100644 --- a/golem-worker-executor-base/tests/rust_rpc_stubless.rs +++ b/golem-worker-executor-base/tests/rust_rpc_stubless.rs @@ -1141,34 +1141,33 @@ async fn golem_bug_1265_test( let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await.unwrap(); - let counters_component_id = executor.store_component("counters").await; + let counters_component_id = executor.component("counters").store().await; let caller_component_id = executor - .store_component_with_dynamic_linking( - "caller", - &[ - ( - "rpc:counters-client/counters-client", - DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { - target_interface_name: HashMap::from_iter(vec![ - ("api".to_string(), "rpc:counters-exports/api".to_string()), - ( - "counter".to_string(), - "rpc:counters-exports/api".to_string(), - ), - ]), - }), - ), - ( - "rpc:ephemeral-client/ephemeral-client", - DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { - target_interface_name: HashMap::from_iter(vec![( - "api".to_string(), - "rpc:ephemeral-exports/api".to_string(), - )]), - }), - ), - ], - ) + .component("caller") + .with_dynamic_linking(&[ + ( + "rpc:counters-client/counters-client", + DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { + target_interface_name: HashMap::from_iter(vec![ + ("api".to_string(), "rpc:counters-exports/api".to_string()), + ( + "counter".to_string(), + "rpc:counters-exports/api".to_string(), + ), + ]), + }), + ), + ( + "rpc:ephemeral-client/ephemeral-client", + DynamicLinkedInstance::WasmRpc(DynamicLinkedWasmRpc { + target_interface_name: HashMap::from_iter(vec![( + "api".to_string(), + "rpc:ephemeral-exports/api".to_string(), + )]), + }), + ), + ]) + .store() .await; let mut env = HashMap::new();