Ablity to use own enviornment variables in config files

.dockerignore

@@ -3,3 +3,4 @@ target/
 .gitignore
 .github/
 .vscode/
+.env

.gitignore

@@ -1,6 +1,7 @@
 /target/
 .envrc
 indexer.toml
+.env
 # These are backup files generated by rustfmt
 **/*.rs.bk
 **/*.idea

config/Cargo.toml

@@ -15,6 +15,7 @@ serde_with = { version = "3.8.1", default-features = false }
 serde_repr = "0.1.19"
 serde_ignored = "0.1.10"
 url = { version = "2.5.0", features = ["serde"] }
+regex = "1.11.0"
 
 [dev-dependencies]
 sealed_test = "1.1.0"

config/src/config.rs

@@ -13,8 +13,10 @@ use tracing::warn;
 
 use alloy::primitives::Address;
 use bip39::Mnemonic;
+use regex::Regex;
 use serde::Deserialize;
 use serde_with::serde_as;
+use std::env;
 use thegraph_core::DeploymentId;
 use url::Url;
 
@@ -71,17 +73,36 @@ impl Config {
     pub fn parse(prefix: ConfigPrefix, filename: &PathBuf) -> Result<Self, String> {
         let config_defaults = include_str!("../default_values.toml");
 
+        let mut config_content = std::fs::read_to_string(filename)
+            .map_err(|e| format!("Failed to read config file: {}", e))?;
+
+        config_content = Self::substitute_env_vars(config_content)?;
+
         let config: ConfigWrapper = Figment::new()
             .merge(Toml::string(config_defaults))
-            .merge(Toml::file(filename))
+            .merge(Toml::string(&config_content))
             .merge(Env::prefixed(prefix.get_prefix()).split("__"))
             .extract()
             .map_err(|e| e.to_string())?;
-        config.0.validate()?;
 
+        config.0.validate()?;
         Ok(config.0)
     }
 
+    fn substitute_env_vars(content: String) -> Result<String, String> {
+        let re = Regex::new(r"\$\{([A-Z_][A-Z0-9_]*)\}").map_err(|e| e.to_string())?;
+
+        let result = re.replace_all(&content, |caps: &regex::Captures| {
+            let var_name = &caps[1];
+            // only substitues the value if the env variable is found else the same is kept
+            match env::var(var_name) {
+                Ok(value) => value,
+                Err(_) => format!("${{{}}}", var_name),
+            }
+        });
+        Ok(result.into_owned())
+    }
+
     // custom validation of the values
     fn validate(&self) -> Result<(), String> {
         match &self.tap.rav_request.trigger_value_divisor {
@@ -279,9 +300,8 @@ pub struct RavRequestConfig {
 
 #[cfg(test)]
 mod tests {
-    use std::{fs, path::PathBuf};
-
     use sealed_test::prelude::*;
+    use std::{env, fs, path::PathBuf};
     use tracing_test::traced_test;
 
     use crate::{Config, ConfigPrefix};
@@ -398,4 +418,45 @@ mod tests {
             test_value
         );
     }
+
+    // Test to check substitute_env_vars function is substituting env variables
+    // indexers can use ${ENV_VAR_NAME} to point to the required env variable
+    #[test]
+    fn test_substitute_env_vars() {
+        // Set up environment variables
+        env::set_var("TEST_VAR1", "hello");
+        env::set_var("TEST_VAR2", "world");
+        // TEST_VAR3 not setup and should not be replaced by the function
+
+        let input = r#"
+            [section1]
+            key1 = "${TEST_VAR1}"
+            key2 = "${TEST_VAR2}"
+            key3 = "${TEST_VAR3}"
+            key4 = "prefix_${TEST_VAR1}_${TEST_VAR2}_suffix"
+            key5 = "a_key_without_substitution"
+        "#
+        .to_string();
+
+        let expected_output = r#"
+            [section1]
+            key1 = "hello"
+            key2 = "world"
+            key3 = "${TEST_VAR3}"
+            key4 = "prefix_hello_world_suffix"
+            key5 = "a_key_without_substitution"
+        "#
+        .to_string();
+
+        let result = Config::substitute_env_vars(input).expect("Failed to substitute env vars ");
+
+        assert_eq!(
+            result, expected_output,
+            "Environment variable substitution failed"
+        );
+
+        // Clean up environment variables
+        env::remove_var("TEST_VAR1");
+        env::remove_var("TEST_VAR2");
+    }
 }