diff --git a/changelog.d/add_disable_interpolate_env_var_switch.feat.md b/changelog.d/add_disable_interpolate_env_var_switch.feat.md new file mode 100644 index 0000000000000..e608e0fc984ce --- /dev/null +++ b/changelog.d/add_disable_interpolate_env_var_switch.feat.md @@ -0,0 +1,3 @@ +Added `--disable-env-var-interpolation` CLI option to prevent env var interpolation. The `VECTOR_DISABLE_ENV_VAR_INTERPOLATION` can also be used to disable interpolation. + +authors: graphcareful diff --git a/src/app.rs b/src/app.rs index a89349285278a..2f70210530858 100644 --- a/src/app.rs +++ b/src/app.rs @@ -82,6 +82,7 @@ impl ApplicationConfig { watcher_conf, opts.require_healthy, opts.allow_empty_config, + opts.disable_env_var_interpolation, graceful_shutdown_duration, signal_handler, ) @@ -268,6 +269,7 @@ impl Application { signals, topology_controller, allow_empty_config: root_opts.allow_empty_config, + interpolate_env: !root_opts.disable_env_var_interpolation, }) } } @@ -279,6 +281,7 @@ pub struct StartedApplication { pub signals: SignalPair, pub topology_controller: SharedTopologyController, pub allow_empty_config: bool, + pub interpolate_env: bool, } impl StartedApplication { @@ -294,6 +297,7 @@ impl StartedApplication { topology_controller, internal_topologies, allow_empty_config, + interpolate_env, } = self; let mut graceful_crash = UnboundedReceiverStream::new(graceful_crash_receiver); @@ -310,6 +314,7 @@ impl StartedApplication { &config_paths, &mut signal_handler, allow_empty_config, + interpolate_env, ).await { break signal; }, @@ -338,6 +343,7 @@ async fn handle_signal( config_paths: &[ConfigPath], signal_handler: &mut SignalHandler, allow_empty_config: bool, + interpolate_env: bool, ) -> Option { match signal { Ok(SignalTo::ReloadComponents(components_to_reload)) => { @@ -356,6 +362,7 @@ async fn handle_signal( &topology_controller.config_paths, signal_handler, allow_empty_config, + interpolate_env, ) .await; @@ -378,6 +385,7 @@ async fn handle_signal( &topology_controller.config_paths, signal_handler, allow_empty_config, + interpolate_env, ) .await; @@ -520,6 +528,7 @@ pub async fn load_configs( watcher_conf: Option, require_healthy: Option, allow_empty_config: bool, + interpolate_env: bool, graceful_shutdown_duration: Option, signal_handler: &mut SignalHandler, ) -> Result { @@ -539,6 +548,7 @@ pub async fn load_configs( &config_paths, signal_handler, allow_empty_config, + interpolate_env, ) .await .map_err(handle_config_errors)?; diff --git a/src/cli.rs b/src/cli.rs index 2b2ebd621bff9..ebb73fc77cfa2 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -136,6 +136,14 @@ pub struct RootOpts { #[arg(short, long, action = ArgAction::Count)] pub quiet: u8, + /// Disable interpolation of environment variables in configuration files. + #[arg( + long, + env = "VECTOR_DISABLE_ENV_VAR_INTERPOLATION", + default_value = "false" + )] + pub disable_env_var_interpolation: bool, + /// Set the logging format #[arg(long, default_value = "text", env = "VECTOR_LOG_FORMAT")] pub log_format: LogFormat, diff --git a/src/config/cmd.rs b/src/config/cmd.rs index 8065b496b5848..4607b5a708249 100644 --- a/src/config/cmd.rs +++ b/src/config/cmd.rs @@ -3,7 +3,9 @@ use std::path::PathBuf; use clap::Parser; use serde_json::Value; -use super::{ConfigBuilder, load_builder_from_paths, load_source_from_paths, process_paths}; +use super::{ + ConfigBuilder, load_builder_from_paths_with_opts, load_source_from_paths, process_paths, +}; use crate::{cli::handle_config_errors, config}; #[derive(Parser, Debug, Clone)] @@ -54,6 +56,14 @@ pub struct Opts { value_delimiter(',') )] pub config_dirs: Vec, + + /// Disable interpolation of environment variables in configuration files. + #[arg( + long, + env = "VECTOR_DISABLE_ENV_VAR_INTERPOLATION", + default_value = "false" + )] + pub disable_env_var_interpolation: bool, } impl Opts { @@ -166,10 +176,12 @@ pub fn cmd(opts: &Opts) -> exitcode::ExitCode { // Start by serializing to a `ConfigBuilder`. This will leverage validation in config // builder fields which we'll use to error out if required. let (paths, builder) = match process_paths(&paths) { - Some(paths) => match load_builder_from_paths(&paths) { - Ok(builder) => (paths, builder), - Err(errs) => return handle_config_errors(errs), - }, + Some(paths) => { + match load_builder_from_paths_with_opts(&paths, !opts.disable_env_var_interpolation) { + Ok(builder) => (paths, builder), + Err(errs) => return handle_config_errors(errs), + } + } None => return exitcode::CONFIG, }; diff --git a/src/config/loading/config_builder.rs b/src/config/loading/config_builder.rs index 17aea4ab12515..ad242e8cb9d38 100644 --- a/src/config/loading/config_builder.rs +++ b/src/config/loading/config_builder.rs @@ -12,28 +12,39 @@ use crate::config::{ pub struct ConfigBuilderLoader { builder: ConfigBuilder, secrets: Option>, + interpolate_env: bool, } impl ConfigBuilderLoader { - pub fn new() -> Self { + pub fn new_with_opts(interpolate_env: bool) -> Self { Self { builder: ConfigBuilder::default(), secrets: None, + interpolate_env, } } - pub fn with_secrets(secrets: HashMap) -> Self { + pub fn with_secrets_and_opts(secrets: HashMap, interpolate_env: bool) -> Self { Self { builder: ConfigBuilder::default(), secrets: Some(secrets), + interpolate_env, } } } impl Process for ConfigBuilderLoader { /// Prepares input for a `ConfigBuilder` by interpolating environment variables. - fn prepare(&mut self, input: R) -> Result> { - let prepared_input = prepare_input(input)?; + fn prepare(&mut self, mut input: R) -> Result> { + let prepared_input = if self.interpolate_env { + prepare_input(input)? + } else { + let mut s = String::new(); + input + .read_to_string(&mut s) + .map_err(|e| vec![e.to_string()])?; + s + }; let prepared_input = self .secrets .as_ref() diff --git a/src/config/loading/mod.rs b/src/config/loading/mod.rs index 47848889d197a..8f8528e49f710 100644 --- a/src/config/loading/mod.rs +++ b/src/config/loading/mod.rs @@ -119,8 +119,11 @@ pub fn process_paths(config_paths: &[ConfigPath]) -> Option> { Some(paths) } -pub fn load_from_paths(config_paths: &[ConfigPath]) -> Result> { - let builder = load_builder_from_paths(config_paths)?; +pub fn load_from_paths( + config_paths: &[ConfigPath], + interpolate_env: bool, +) -> Result> { + let builder = load_builder_from_paths_with_opts(config_paths, interpolate_env)?; let (config, build_warnings) = builder.build_with_warnings()?; for warning in build_warnings { @@ -137,9 +140,11 @@ pub async fn load_from_paths_with_provider_and_secrets( config_paths: &[ConfigPath], signal_handler: &mut signal::SignalHandler, allow_empty: bool, + interpolate_env: bool, ) -> Result> { // Load secret backends first - let mut secrets_backends_loader = load_secret_backends_from_paths(config_paths)?; + let mut secrets_backends_loader = + load_secret_backends_from_paths_with_opts(config_paths, interpolate_env)?; // And then, if needed, retrieve secrets from configured backends let mut builder = if secrets_backends_loader.has_secrets_to_retrieve() { debug!(message = "Secret placeholders found, retrieving secrets from configured backends."); @@ -147,10 +152,14 @@ pub async fn load_from_paths_with_provider_and_secrets( .retrieve(&mut signal_handler.subscribe()) .await .map_err(|e| vec![e])?; - load_builder_from_paths_with_secrets(config_paths, resolved_secrets)? + load_builder_from_paths_with_opts_with_secrets_and_opts( + config_paths, + resolved_secrets, + interpolate_env, + )? } else { debug!(message = "No secret placeholder found, skipping secret resolution."); - load_builder_from_paths(config_paths)? + load_builder_from_paths_with_opts(config_paths, interpolate_env)? }; builder.allow_empty = allow_empty; @@ -180,9 +189,11 @@ pub async fn load_from_str_with_secrets( format: Format, signal_handler: &mut signal::SignalHandler, allow_empty: bool, + interpolate_env: bool, ) -> Result> { // Load secret backends first - let mut secrets_backends_loader = load_secret_backends_from_input(input.as_bytes(), format)?; + let mut secrets_backends_loader = + load_secret_backends_from_input_with_opts(input.as_bytes(), format, interpolate_env)?; // And then, if needed, retrieve secrets from configured backends let mut builder = if secrets_backends_loader.has_secrets_to_retrieve() { debug!(message = "Secret placeholders found, retrieving secrets from configured backends."); @@ -190,10 +201,15 @@ pub async fn load_from_str_with_secrets( .retrieve(&mut signal_handler.subscribe()) .await .map_err(|e| vec![e])?; - load_builder_from_input_with_secrets(input.as_bytes(), format, resolved_secrets)? + load_builder_from_input_with_secrets_and_opts( + input.as_bytes(), + format, + resolved_secrets, + interpolate_env, + )? } else { debug!(message = "No secret placeholder found, skipping secret resolution."); - load_builder_from_input(input.as_bytes(), format)? + load_builder_from_input_with_opts(input.as_bytes(), format, interpolate_env)? }; builder.allow_empty = allow_empty; @@ -256,31 +272,51 @@ where } /// Uses `ConfigBuilderLoader` to process `ConfigPaths`, deserializing to a `ConfigBuilder`. -pub fn load_builder_from_paths(config_paths: &[ConfigPath]) -> Result> { - loader_from_paths(ConfigBuilderLoader::new(), config_paths) +pub fn load_builder_from_paths_with_opts( + config_paths: &[ConfigPath], + interpolate_env: bool, +) -> Result> { + loader_from_paths( + ConfigBuilderLoader::new_with_opts(interpolate_env), + config_paths, + ) } -fn load_builder_from_input( +fn load_builder_from_input_with_opts( input: R, format: Format, + interpolate_env: bool, ) -> Result> { - loader_from_input(ConfigBuilderLoader::new(), input, format) + loader_from_input( + ConfigBuilderLoader::new_with_opts(interpolate_env), + input, + format, + ) } /// Uses `ConfigBuilderLoader` to process `ConfigPaths`, performing secret replacement and deserializing to a `ConfigBuilder` -pub fn load_builder_from_paths_with_secrets( +pub fn load_builder_from_paths_with_opts_with_secrets_and_opts( config_paths: &[ConfigPath], secrets: HashMap, + interpolate_env: bool, ) -> Result> { - loader_from_paths(ConfigBuilderLoader::with_secrets(secrets), config_paths) + loader_from_paths( + ConfigBuilderLoader::with_secrets_and_opts(secrets, interpolate_env), + config_paths, + ) } -fn load_builder_from_input_with_secrets( +fn load_builder_from_input_with_secrets_and_opts( input: R, format: Format, secrets: HashMap, + interpolate_env: bool, ) -> Result> { - loader_from_input(ConfigBuilderLoader::with_secrets(secrets), input, format) + loader_from_input( + ConfigBuilderLoader::with_secrets_and_opts(secrets, interpolate_env), + input, + format, + ) } /// Uses `SourceLoader` to process `ConfigPaths`, deserializing to a toml `SourceMap`. @@ -291,17 +327,26 @@ pub fn load_source_from_paths( } /// Uses `SecretBackendLoader` to process `ConfigPaths`, deserializing to a `SecretBackends`. -pub fn load_secret_backends_from_paths( +pub fn load_secret_backends_from_paths_with_opts( config_paths: &[ConfigPath], + interpolate_env: bool, ) -> Result> { - loader_from_paths(SecretBackendLoader::new(), config_paths) + loader_from_paths( + SecretBackendLoader::new_with_opts(interpolate_env), + config_paths, + ) } -fn load_secret_backends_from_input( +fn load_secret_backends_from_input_with_opts( input: R, format: Format, + interpolate_env: bool, ) -> Result> { - loader_from_input(SecretBackendLoader::new(), input, format) + loader_from_input( + SecretBackendLoader::new_with_opts(interpolate_env), + input, + format, + ) } pub fn load_from_str(input: &str, format: Format) -> Result> { @@ -396,7 +441,7 @@ fn default_config_paths() -> Vec { mod tests { use std::path::PathBuf; - use super::load_builder_from_paths; + use super::load_builder_from_paths_with_opts; use crate::config::{ComponentKey, ConfigPath}; #[test] @@ -406,7 +451,7 @@ mod tests { .join("namespacing") .join("success"); let configs = vec![ConfigPath::Dir(path)]; - let builder = load_builder_from_paths(&configs).unwrap(); + let builder = load_builder_from_paths_with_opts(&configs, true).unwrap(); assert!( builder .transforms @@ -432,7 +477,7 @@ mod tests { .join("namespacing") .join("ignore-invalid"); let configs = vec![ConfigPath::Dir(path)]; - load_builder_from_paths(&configs).unwrap(); + load_builder_from_paths_with_opts(&configs, true).unwrap(); } #[test] @@ -442,7 +487,7 @@ mod tests { .join("config-dir") .join("ignore-unknown"); let configs = vec![ConfigPath::Dir(path)]; - load_builder_from_paths(&configs).unwrap(); + load_builder_from_paths_with_opts(&configs, true).unwrap(); } #[test] @@ -452,7 +497,7 @@ mod tests { .join("config-dir") .join("globals"); let configs = vec![ConfigPath::Dir(path)]; - load_builder_from_paths(&configs).unwrap(); + load_builder_from_paths_with_opts(&configs, true).unwrap(); } #[test] @@ -462,6 +507,6 @@ mod tests { .join("config-dir") .join("globals-duplicate"); let configs = vec![ConfigPath::Dir(path)]; - load_builder_from_paths(&configs).unwrap(); + load_builder_from_paths_with_opts(&configs, true).unwrap(); } } diff --git a/src/config/loading/secret.rs b/src/config/loading/secret.rs index 57fac29238ffd..6d0200d9913e6 100644 --- a/src/config/loading/secret.rs +++ b/src/config/loading/secret.rs @@ -42,13 +42,15 @@ pub(crate) struct SecretBackendOuter { pub struct SecretBackendLoader { backends: IndexMap, pub(crate) secret_keys: HashMap>, + interpolate_env: bool, } impl SecretBackendLoader { - pub(crate) fn new() -> Self { + pub(crate) fn new_with_opts(interpolate_env: bool) -> Self { Self { backends: IndexMap::new(), secret_keys: HashMap::new(), + interpolate_env, } } @@ -88,8 +90,16 @@ impl SecretBackendLoader { } impl Process for SecretBackendLoader { - fn prepare(&mut self, input: R) -> Result> { - let config_string = prepare_input(input)?; + fn prepare(&mut self, mut input: R) -> Result> { + let config_string = if self.interpolate_env { + prepare_input(input)? + } else { + let mut s = String::new(); + input + .read_to_string(&mut s) + .map_err(|e| vec![e.to_string()])?; + s + }; // Collect secret placeholders just after env var processing collect_secret_keys(&config_string, &mut self.secret_keys); Ok(config_string) diff --git a/src/config/mod.rs b/src/config/mod.rs index 20d06d59de568..6ae3ba63b0609 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -54,7 +54,7 @@ pub use diff::ConfigDiff; pub use enrichment_table::{EnrichmentTableConfig, EnrichmentTableOuter}; pub use format::{Format, FormatHint}; pub use loading::{ - COLLECTOR, CONFIG_PATHS, load, load_builder_from_paths, load_from_paths, + COLLECTOR, CONFIG_PATHS, load, load_builder_from_paths_with_opts, load_from_paths, load_from_paths_with_provider_and_secrets, load_from_str, load_from_str_with_secrets, load_source_from_paths, merge_path_lists, process_paths, }; diff --git a/src/config/unit_test/mod.rs b/src/config/unit_test/mod.rs index 1c5aa2d534bd0..5c26d03cfca92 100644 --- a/src/config/unit_test/mod.rs +++ b/src/config/unit_test/mod.rs @@ -90,7 +90,7 @@ fn init_log_schema_from_paths( config_paths: &[ConfigPath], deny_if_set: bool, ) -> Result<(), Vec> { - let builder = config::loading::load_builder_from_paths(config_paths)?; + let builder = config::loading::load_builder_from_paths_with_opts(config_paths, true)?; vector_lib::config::init_log_schema(builder.global.log_schema, deny_if_set); Ok(()) } @@ -100,15 +100,15 @@ pub async fn build_unit_tests_main( signal_handler: &mut signal::SignalHandler, ) -> Result, Vec> { init_log_schema_from_paths(paths, false)?; - let mut secrets_backends_loader = loading::load_secret_backends_from_paths(paths)?; + let mut secrets_backends_loader = loading::load_secret_backends_from_paths_with_opts(paths, true)?; let config_builder = if secrets_backends_loader.has_secrets_to_retrieve() { let resolved_secrets = secrets_backends_loader .retrieve(&mut signal_handler.subscribe()) .await .map_err(|e| vec![e])?; - loading::load_builder_from_paths_with_secrets(paths, resolved_secrets)? + loading::load_builder_from_paths_with_opts_with_secrets_and_opts(paths, resolved_secrets, true)? } else { - loading::load_builder_from_paths(paths)? + loading::load_builder_from_paths_with_opts(paths, true)? }; build_unit_tests(config_builder).await diff --git a/src/graph.rs b/src/graph.rs index 1b4856036cd2c..cfdcfffb3c6f5 100644 --- a/src/graph.rs +++ b/src/graph.rs @@ -52,6 +52,14 @@ pub struct Opts { /// information on the `mermaid` format. #[arg(id = "format", long, default_value = "dot")] pub format: OutputFormat, + + /// Disable interpolation of environment variables in configuration files. + #[arg( + long, + env = "VECTOR_DISABLE_ENV_VAR_INTERPOLATION", + default_value = "false" + )] + pub disable_env_var_interpolation: bool, } #[derive(clap::ValueEnum, Debug, Clone, Copy, PartialEq, Eq)] @@ -93,7 +101,7 @@ pub(crate) fn cmd(opts: &Opts) -> exitcode::ExitCode { None => return exitcode::CONFIG, }; - let config = match config::load_from_paths(&paths) { + let config = match config::load_from_paths(&paths, !opts.disable_env_var_interpolation) { Ok(config) => config, Err(errs) => { #[allow(clippy::print_stderr)] diff --git a/src/validate.rs b/src/validate.rs index 320450f642c42..a1b231d9a54ba 100644 --- a/src/validate.rs +++ b/src/validate.rs @@ -141,7 +141,7 @@ pub fn validate_config(opts: &Opts, fmt: &mut Formatter) -> Option { fmt.title(format!("Failed to load {:?}", &paths_list)); fmt.sub_error(errors); }; - let builder = config::load_builder_from_paths(&paths) + let builder = config::load_builder_from_paths_with_opts(&paths, true) .map_err(&mut report_error) .ok()?; config::init_log_schema(builder.global.log_schema.clone(), true);