feat(config): Add an option to prevent interpolation of env vars within config loading process

changelog.d/add_disable_interpolate_env_var_switch.feat.md

@@ -0,0 +1,3 @@
+Added `--disable-env-var-interpolation` CLI option to prevent env var interpolation. The `VECTOR_DISABLE_ENV_VAR_INTERPOLATION` can also be used to disable interpolation.
+
+authors: graphcareful

src/app.rs

@@ -82,6 +82,7 @@ impl ApplicationConfig {
             watcher_conf,
             opts.require_healthy,
             opts.allow_empty_config,
+            opts.disable_env_var_interpolation,
             graceful_shutdown_duration,
             signal_handler,
         )
@@ -268,6 +269,7 @@ impl Application {
             signals,
             topology_controller,
             allow_empty_config: root_opts.allow_empty_config,
+            interpolate_env: !root_opts.disable_env_var_interpolation,
         })
     }
 }
@@ -279,6 +281,7 @@ pub struct StartedApplication {
     pub signals: SignalPair,
     pub topology_controller: SharedTopologyController,
     pub allow_empty_config: bool,
+    pub interpolate_env: bool,
 }
 
 impl StartedApplication {
@@ -294,6 +297,7 @@ impl StartedApplication {
             topology_controller,
             internal_topologies,
             allow_empty_config,
+            interpolate_env,
         } = self;
 
         let mut graceful_crash = UnboundedReceiverStream::new(graceful_crash_receiver);
@@ -310,6 +314,7 @@ impl StartedApplication {
                     &config_paths,
                     &mut signal_handler,
                     allow_empty_config,
+                    interpolate_env,
                 ).await {
                     break signal;
                 },
@@ -338,6 +343,7 @@ async fn handle_signal(
     config_paths: &[ConfigPath],
     signal_handler: &mut SignalHandler,
     allow_empty_config: bool,
+    interpolate_env: bool,
 ) -> Option<SignalTo> {
     match signal {
         Ok(SignalTo::ReloadComponents(components_to_reload)) => {
@@ -356,6 +362,7 @@ async fn handle_signal(
                 &topology_controller.config_paths,
                 signal_handler,
                 allow_empty_config,
+                interpolate_env,
             )
             .await;
 
@@ -378,6 +385,7 @@ async fn handle_signal(
                 &topology_controller.config_paths,
                 signal_handler,
                 allow_empty_config,
+                interpolate_env,
             )
             .await;
 
@@ -520,6 +528,7 @@ pub async fn load_configs(
     watcher_conf: Option<config::watcher::WatcherConfig>,
     require_healthy: Option<bool>,
     allow_empty_config: bool,
+    interpolate_env: bool,
     graceful_shutdown_duration: Option<Duration>,
     signal_handler: &mut SignalHandler,
 ) -> Result<Config, ExitCode> {
@@ -539,6 +548,7 @@ pub async fn load_configs(
         &config_paths,
         signal_handler,
         allow_empty_config,
+        interpolate_env,
     )
     .await
     .map_err(handle_config_errors)?;

src/cli.rs

@@ -136,6 +136,14 @@ pub struct RootOpts {
     #[arg(short, long, action = ArgAction::Count)]
     pub quiet: u8,
 
+    /// Disable interpolation of environment variables in configuration files.
+    #[arg(
+        long,
+        env = "VECTOR_DISABLE_ENV_VAR_INTERPOLATION",
+        default_value = "false"
+    )]
+    pub disable_env_var_interpolation: bool,
+
     /// Set the logging format
     #[arg(long, default_value = "text", env = "VECTOR_LOG_FORMAT")]
     pub log_format: LogFormat,

src/config/cmd.rs

@@ -3,7 +3,9 @@ use std::path::PathBuf;
 use clap::Parser;
 use serde_json::Value;
 
-use super::{ConfigBuilder, load_builder_from_paths, load_source_from_paths, process_paths};
+use super::{
+    ConfigBuilder, load_builder_from_paths_with_opts, load_source_from_paths, process_paths,
+};
 use crate::{cli::handle_config_errors, config};
 
 #[derive(Parser, Debug, Clone)]
@@ -54,6 +56,14 @@ pub struct Opts {
         value_delimiter(',')
     )]
     pub config_dirs: Vec<PathBuf>,
+
+    /// Disable interpolation of environment variables in configuration files.
+    #[arg(
+        long,
+        env = "VECTOR_DISABLE_ENV_VAR_INTERPOLATION",
+        default_value = "false"
+    )]
+    pub disable_env_var_interpolation: bool,
 }
 
 impl Opts {
@@ -166,10 +176,12 @@ pub fn cmd(opts: &Opts) -> exitcode::ExitCode {
     // Start by serializing to a `ConfigBuilder`. This will leverage validation in config
     // builder fields which we'll use to error out if required.
     let (paths, builder) = match process_paths(&paths) {
-        Some(paths) => match load_builder_from_paths(&paths) {
-            Ok(builder) => (paths, builder),
-            Err(errs) => return handle_config_errors(errs),
-        },
+        Some(paths) => {
+            match load_builder_from_paths_with_opts(&paths, !opts.disable_env_var_interpolation) {
+                Ok(builder) => (paths, builder),
+                Err(errs) => return handle_config_errors(errs),
+            }
+        }
         None => return exitcode::CONFIG,
     };
 

src/config/loading/config_builder.rs

@@ -12,28 +12,39 @@ use crate::config::{
 pub struct ConfigBuilderLoader {
     builder: ConfigBuilder,
     secrets: Option<HashMap<String, String>>,
+    interpolate_env: bool,
 }
 
 impl ConfigBuilderLoader {
-    pub fn new() -> Self {
+    pub fn new_with_opts(interpolate_env: bool) -> Self {
         Self {
             builder: ConfigBuilder::default(),
             secrets: None,
+            interpolate_env,
         }
     }
 
-    pub fn with_secrets(secrets: HashMap<String, String>) -> Self {
+    pub fn with_secrets_and_opts(secrets: HashMap<String, String>, interpolate_env: bool) -> Self {
         Self {
             builder: ConfigBuilder::default(),
             secrets: Some(secrets),
+            interpolate_env,
         }
     }
 }
 
 impl Process for ConfigBuilderLoader {
     /// Prepares input for a `ConfigBuilder` by interpolating environment variables.
-    fn prepare<R: Read>(&mut self, input: R) -> Result<String, Vec<String>> {
-        let prepared_input = prepare_input(input)?;
+    fn prepare<R: Read>(&mut self, mut input: R) -> Result<String, Vec<String>> {
+        let prepared_input = if self.interpolate_env {
+            prepare_input(input)?
+        } else {
+            let mut s = String::new();
+            input
+                .read_to_string(&mut s)
+                .map_err(|e| vec![e.to_string()])?;
+            s
+        };
         let prepared_input = self
             .secrets
             .as_ref()