diff --git a/Cargo.lock b/Cargo.lock index 759a94a031..b0775d6a56 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7978,6 +7978,7 @@ dependencies = [ "spin-trigger", "spin-trigger-http", "spin-trigger-redis", + "spin-variables", "subprocess", "tempfile", "terminal", @@ -8352,6 +8353,7 @@ name = "spin-factors-test" version = "3.4.0-pre0" dependencies = [ "spin-app", + "spin-common", "spin-factors", "spin-loader", "spin-telemetry", @@ -8500,6 +8502,7 @@ dependencies = [ "spin-manifest", "spin-outbound-networking-config", "spin-serde", + "spin-variables", "tempfile", "terminal", "tokio", @@ -8797,9 +8800,11 @@ dependencies = [ "spin-factors", "spin-factors-executor", "spin-telemetry", + "spin-variables", "spin-world", "tempfile", "tokio", + "toml", "tracing", ] @@ -8862,9 +8867,11 @@ dependencies = [ "azure_security_keyvault", "dotenvy", "serde", + "spin-common", "spin-expressions", "spin-factor-variables", "spin-factors", + "spin-locked-app", "spin-world", "tokio", "tracing", diff --git a/Cargo.toml b/Cargo.toml index 576bf6f06f..50ae9f4a32 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,12 +64,13 @@ spin-oci = { path = "crates/oci" } spin-plugins = { path = "crates/plugins" } spin-runtime-factors = { path = "crates/runtime-factors" } spin-telemetry = { path = "crates/telemetry", features = [ - "tracing-log-compat", + "tracing-log-compat", ] } spin-templates = { path = "crates/templates" } spin-trigger = { path = "crates/trigger" } spin-trigger-http = { path = "crates/trigger-http" } spin-trigger-redis = { path = "crates/trigger-redis" } +spin-variables = { path = "crates/variables" } terminal = { path = "crates/terminal" } [target.'cfg(target_os = "linux")'.dependencies] @@ -95,10 +96,10 @@ testing-framework = { path = "tests/testing-framework" } [build-dependencies] cargo-target-dep = { git = "https://github.com/fermyon/cargo-target-dep", rev = "482f269eceb7b1a7e8fc618bf8c082dd24979cf1" } vergen = { version = "^8.2.1", default-features = false, features = [ - "build", - "git", - "gitcl", - "cargo", + "build", + "git", + "gitcl", + "cargo", ] } [features] @@ -111,10 +112,10 @@ llm-cublas = ["llm", "spin-runtime-factors/llm-cublas"] [workspace] members = [ - "crates/*", - "tests/conformance-tests", - "tests/runtime-tests", - "tests/testing-framework", + "crates/*", + "tests/conformance-tests", + "tests/runtime-tests", + "tests/testing-framework", ] [workspace.dependencies] @@ -186,4 +187,4 @@ blocks_in_conditions = "allow" [[bin]] name = "spin" -path = "src/bin/spin.rs" \ No newline at end of file +path = "src/bin/spin.rs" diff --git a/crates/common/src/env.rs b/crates/common/src/env.rs new file mode 100644 index 0000000000..6bca6eb0eb --- /dev/null +++ b/crates/common/src/env.rs @@ -0,0 +1,12 @@ +//! Environment variable utilities + +/// Defines the default environment variable prefix used by Spin. +pub const DEFAULT_ENV_PREFIX: &str = "SPIN_VARIABLE"; + +/// Creates an environment variable key based on the given prefix and key. +pub fn env_key(prefix: Option, key: &str) -> String { + let prefix = prefix.unwrap_or_else(|| DEFAULT_ENV_PREFIX.to_string()); + let upper_key = key.to_ascii_uppercase(); + let key = format!("{prefix}_{upper_key}"); + key +} diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index f747d1544f..a7f5a687b7 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -10,6 +10,7 @@ pub mod arg_parser; pub mod data_dir; +pub mod env; pub mod paths; pub mod sha256; pub mod sloth; diff --git a/crates/factors-test/Cargo.toml b/crates/factors-test/Cargo.toml index 80ab8aa655..094bc0b53c 100644 --- a/crates/factors-test/Cargo.toml +++ b/crates/factors-test/Cargo.toml @@ -6,6 +6,7 @@ edition = { workspace = true } [dependencies] spin-app = { path = "../app" } +spin-common = { path = "../common" } spin-factors = { path = "../factors" } spin-loader = { path = "../loader" } spin-telemetry = { path = "../telemetry", features = ["testing"] } diff --git a/crates/factors-test/src/lib.rs b/crates/factors-test/src/lib.rs index 449b95bcf6..f7d3d3428b 100644 --- a/crates/factors-test/src/lib.rs +++ b/crates/factors-test/src/lib.rs @@ -1,4 +1,5 @@ use spin_app::locked::LockedApp; +use spin_common::env::env_key; use spin_factors::{ anyhow::{self, Context}, wasmtime::{component::Linker, Config, Engine}, @@ -100,6 +101,8 @@ impl TestEnvironment { pub async fn build_locked_app(manifest: &toml::Table) -> anyhow::Result { let toml_str = toml::to_string(manifest).context("failed serializing manifest")?; let dir = tempfile::tempdir().context("failed creating tempdir")?; + // `foo` variable is set to require. As we're not providing a default value, env is checked. + std::env::set_var(env_key(None, "foo"), "baz"); let path = dir.path().join("spin.toml"); std::fs::write(&path, toml_str).context("failed writing manifest")?; spin_loader::from_file(&path, FilesMountStrategy::Direct, None).await diff --git a/crates/loader/Cargo.toml b/crates/loader/Cargo.toml index c010f62a25..c484a8967e 100644 --- a/crates/loader/Cargo.toml +++ b/crates/loader/Cargo.toml @@ -20,6 +20,7 @@ spin-locked-app = { path = "../locked-app" } spin-manifest = { path = "../manifest" } spin-outbound-networking-config = { path = "../outbound-networking-config" } spin-serde = { path = "../serde" } +spin-variables = { path = "../variables" } tempfile = { workspace = true } terminal = { path = "../terminal" } tokio = { workspace = true } diff --git a/crates/runtime-factors/src/build.rs b/crates/runtime-factors/src/build.rs index 16716b261c..007438e12e 100644 --- a/crates/runtime-factors/src/build.rs +++ b/crates/runtime-factors/src/build.rs @@ -8,7 +8,7 @@ use spin_runtime_config::ResolvedRuntimeConfig; use spin_trigger::cli::{ FactorsConfig, InitialKvSetterHook, KeyValueDefaultStoreSummaryHook, MaxInstanceMemoryHook, RuntimeFactorsBuilder, SqlStatementExecutorHook, SqliteDefaultStoreSummaryHook, - StdioLoggingExecutorHooks, + StdioLoggingExecutorHooks, VariableSorterExecutorHooks, }; /// A [`RuntimeFactorsBuilder`] for [`TriggerFactors`]. @@ -58,6 +58,9 @@ impl RuntimeFactorsBuilder for FactorsBuilder { executor.add_hooks(InitialKvSetterHook::new(args.key_values.clone())); executor.add_hooks(SqliteDefaultStoreSummaryHook); executor.add_hooks(KeyValueDefaultStoreSummaryHook); + executor.add_hooks(VariableSorterExecutorHooks::new( + runtime_config.toml.clone(), + )); let max_instance_memory = args .max_instance_memory diff --git a/crates/trigger/Cargo.toml b/crates/trigger/Cargo.toml index 44db6ff910..ce5ee98230 100644 --- a/crates/trigger/Cargo.toml +++ b/crates/trigger/Cargo.toml @@ -32,7 +32,9 @@ spin-factor-wasi = { path = "../factor-wasi" } spin-factors = { path = "../factors" } spin-factors-executor = { path = "../factors-executor" } spin-telemetry = { path = "../telemetry" } +spin-variables = { path = "../variables" } tokio = { workspace = true, features = ["fs", "rt"] } +toml = { workspace = true } tracing = { workspace = true } [dev-dependencies] diff --git a/crates/trigger/src/cli.rs b/crates/trigger/src/cli.rs index d2cfab22f6..67913ff241 100644 --- a/crates/trigger/src/cli.rs +++ b/crates/trigger/src/cli.rs @@ -4,6 +4,7 @@ mod max_instance_memory; mod sqlite_statements; mod stdio; mod summary; +mod variables; use std::path::PathBuf; use std::{future::Future, sync::Arc}; @@ -25,6 +26,7 @@ pub use sqlite_statements::SqlStatementExecutorHook; use stdio::FollowComponents; pub use stdio::StdioLoggingExecutorHooks; pub use summary::{KeyValueDefaultStoreSummaryHook, SqliteDefaultStoreSummaryHook}; +pub use variables::VariableSorterExecutorHooks; pub const APP_LOG_DIR: &str = "APP_LOG_DIR"; pub const SPIN_TRUNCATE_LOGS: &str = "SPIN_TRUNCATE_LOGS"; diff --git a/crates/trigger/src/cli/variables.rs b/crates/trigger/src/cli/variables.rs new file mode 100644 index 0000000000..cb49bae208 --- /dev/null +++ b/crates/trigger/src/cli/variables.rs @@ -0,0 +1,44 @@ +use spin_core::async_trait; +use spin_factors::RuntimeFactors; +use spin_factors_executor::ExecutorHooks; +use spin_variables::{VariableProviderConfiguration, VariableSourcer}; + +/// Implements TriggerHooks, sorting required variables +pub struct VariableSorterExecutorHooks { + table: toml::Table, +} + +impl VariableSorterExecutorHooks { + pub fn new(table: toml::Table) -> Self { + Self { table } + } +} + +#[async_trait] +impl ExecutorHooks for VariableSorterExecutorHooks { + async fn configure_app( + &self, + configured_app: &spin_factors::ConfiguredApp, + ) -> anyhow::Result<()> { + for (key, variable) in configured_app.app().variables() { + self.variable_env_checker(key.clone(), variable.clone())?; + } + Ok(()) + } +} + +impl VariableSourcer for VariableSorterExecutorHooks { + fn variable_env_checker(&self, key: String, val: spin_app::Variable) -> anyhow::Result<()> { + let configs = spin_variables::variable_provider_config_from_toml(&self.table)?; + + if let Some(config) = configs.into_iter().next() { + let (dotenv_path, prefix) = match config { + VariableProviderConfiguration::Env(env) => (env.dotenv_path, env.prefix), + _ => (None, None), + }; + return self.check(key, val, dotenv_path, prefix); + } + + Err(anyhow::anyhow!("No environment variable provider found")) + } +} diff --git a/crates/variables/Cargo.toml b/crates/variables/Cargo.toml index 2ed71d383b..7a3a1f6852 100644 --- a/crates/variables/Cargo.toml +++ b/crates/variables/Cargo.toml @@ -14,9 +14,11 @@ azure_identity = { git = "https://github.com/azure/azure-sdk-for-rust", rev = "8 azure_security_keyvault = { git = "https://github.com/azure/azure-sdk-for-rust", rev = "8c4caa251c3903d5eae848b41bb1d02a4d65231c" } dotenvy = "0.15" serde = { workspace = true } +spin-common = { path = "../common" } spin-expressions = { path = "../expressions" } spin-factor-variables = { path = "../factor-variables" } spin-factors = { path = "../factors" } +spin-locked-app = { path = "../locked-app" } spin-world = { path = "../world" } tokio = { workspace = true, features = ["rt-multi-thread"] } tracing = { workspace = true } diff --git a/crates/variables/src/env.rs b/crates/variables/src/env.rs index 6976b00ca3..d8c647d435 100644 --- a/crates/variables/src/env.rs +++ b/crates/variables/src/env.rs @@ -6,6 +6,7 @@ use std::{ }; use serde::Deserialize; +use spin_common::env::env_key; use spin_expressions::{Key, Provider}; use spin_factors::anyhow::{self, Context as _}; use spin_world::async_trait; @@ -25,8 +26,6 @@ pub struct EnvVariablesConfig { pub dotenv_path: Option, } -const DEFAULT_ENV_PREFIX: &str = "SPIN_VARIABLE"; - type EnvFetcherFn = Box Result + Send + Sync>; /// A [`Provider`] that uses environment variables. @@ -71,14 +70,7 @@ impl EnvVariablesProvider { /// Gets the value of a variable from the environment. fn get_sync(&self, key: &Key) -> anyhow::Result> { - let prefix = self - .prefix - .clone() - .unwrap_or_else(|| DEFAULT_ENV_PREFIX.to_string()); - - let upper_key = key.as_ref().to_ascii_uppercase(); - let env_key = format!("{prefix}_{upper_key}"); - + let env_key = env_key(self.prefix.clone(), key.as_ref()); self.query_env(&env_key) } diff --git a/crates/variables/src/lib.rs b/crates/variables/src/lib.rs index 79130aeafa..de58dc0d09 100644 --- a/crates/variables/src/lib.rs +++ b/crates/variables/src/lib.rs @@ -5,8 +5,12 @@ mod env; mod statik; mod vault; +use std::path::PathBuf; + pub use azure_key_vault::*; pub use env::*; +use spin_common::{env::env_key, ui::quoted_path}; +use spin_locked_app::Variable; pub use statik::*; pub use vault::*; @@ -38,6 +42,24 @@ pub fn runtime_config_from_toml(table: &impl GetTomlValue) -> anyhow::Result anyhow::Result> { + if let Some(array) = table + .get("variables_provider") + .or_else(|| table.get("config_provider")) + { + array + .clone() + .try_into::>() + .map_err(|e| anyhow::anyhow!("Failed to parse variable provider configuration: {}", e)) + } else { + Ok(vec![VariableProviderConfiguration::Env( + EnvVariablesConfig::default(), + )]) + } +} + /// A runtime configuration used in the Spin CLI for one type of variable provider. #[derive(Debug, Deserialize)] #[serde(rename_all = "snake_case", tag = "type")] @@ -70,3 +92,34 @@ impl VariableProviderConfiguration { Ok(provider) } } + +pub trait VariableSourcer { + fn variable_env_checker(&self, key: String, val: Variable) -> anyhow::Result<()>; + + fn check( + &self, + key: String, + mut val: Variable, + dotenv_path: Option, + prefix: Option, + ) -> anyhow::Result<()> { + if val.default.is_some() { + return Ok(()); + } + + if let Some(path) = dotenv_path { + _ = std::env::set_current_dir(path); + } + + match std::env::var(env_key(prefix, &key)) { + Ok(v) => { + val.default = Some(v); + Ok(()) + } + Err(_) => Err(anyhow::anyhow!( + "Variable data not provided for {}", + quoted_path(key) + )), + } + } +} diff --git a/tests/integration.rs b/tests/integration.rs index a2e358195c..006e56b3f2 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -544,7 +544,13 @@ mod integration_tests { use std::collections::HashMap; use crate::testcases::run_test_inited; + use spin_common::env::env_key; + const VAULT_ROOT_TOKEN: &str = "root"; + + // `password` variable is set to require. As we're not providing a default value, env is checked. + std::env::set_var(env_key(None, "password"), "password"); + run_test_inited( "vault-variables-test", SpinConfig {