diff --git a/.dockerignore b/.dockerignore index c6aa8145d..50369eb7a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,3 +3,4 @@ target/ .gitignore .github/ .vscode/ +.env \ No newline at end of file diff --git a/.gitignore b/.gitignore index f222c4a49..95fc237d0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ /target/ .envrc indexer.toml +.env # These are backup files generated by rustfmt **/*.rs.bk **/*.idea diff --git a/Cargo.lock b/Cargo.lock index ba87df1f8..a03cddb52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2504,8 +2504,8 @@ dependencies = [ "aho-corasick", "bstr", "log", - "regex-automata 0.4.7", - "regex-syntax 0.8.4", + "regex-automata 0.4.8", + "regex-syntax 0.8.5", ] [[package]] @@ -2608,9 +2608,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ff8ae62cd3a9102e5637afc8452c55acf3844001bd5374e0b0bd7b6616c038" +checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" dependencies = [ "ahash 0.8.11", ] @@ -3007,6 +3007,7 @@ dependencies = [ "bigdecimal", "bip39", "figment", + "regex", "sealed_test", "serde", "serde_ignored", @@ -3556,13 +3557,13 @@ dependencies = [ [[package]] name = "metrics-util" -version = "0.15.1" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4de2ed6e491ed114b40b732e4d1659a9d53992ebd87490c44a6ffe23739d973e" +checksum = "111cb375987443c3de8d503580b536f77dc8416d32db62d9456db5d93bd7ac47" dependencies = [ "crossbeam-epoch", "crossbeam-utils", - "hashbrown 0.13.1", + "hashbrown 0.13.2", "metrics", "num_cpus", "quanta 0.11.1", @@ -4257,7 +4258,7 @@ dependencies = [ "rand 0.8.5", "rand_chacha 0.3.1", "rand_xorshift", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", "rusty-fork", "tempfile", "unarray", @@ -4480,14 +4481,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.6" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.7", - "regex-syntax 0.8.4", + "regex-automata 0.4.8", + "regex-syntax 0.8.5", ] [[package]] @@ -4501,13 +4502,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", ] [[package]] @@ -4518,9 +4519,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "rend" diff --git a/config/Cargo.toml b/config/Cargo.toml index 50bfbe253..d4a60abcd 100644 --- a/config/Cargo.toml +++ b/config/Cargo.toml @@ -15,6 +15,7 @@ serde_with = { version = "3.8.1", default-features = false } serde_repr = "0.1.19" serde_ignored = "0.1.10" url = { version = "2.5.0", features = ["serde"] } +regex = "1.11.0" [dev-dependencies] sealed_test = "1.1.0" diff --git a/config/src/config.rs b/config/src/config.rs index 359a722c2..1940c32e7 100644 --- a/config/src/config.rs +++ b/config/src/config.rs @@ -13,8 +13,10 @@ use tracing::warn; use alloy::primitives::Address; use bip39::Mnemonic; +use regex::Regex; use serde::Deserialize; use serde_with::serde_as; +use std::env; use thegraph_core::DeploymentId; use url::Url; @@ -71,17 +73,53 @@ impl Config { pub fn parse(prefix: ConfigPrefix, filename: &PathBuf) -> Result { let config_defaults = include_str!("../default_values.toml"); + let mut config_content = std::fs::read_to_string(filename) + .map_err(|e| format!("Failed to read config file: {}", e))?; + + config_content = Self::substitute_env_vars(config_content)?; + let config: ConfigWrapper = Figment::new() .merge(Toml::string(config_defaults)) - .merge(Toml::file(filename)) + .merge(Toml::string(&config_content)) .merge(Env::prefixed(prefix.get_prefix()).split("__")) .extract() .map_err(|e| e.to_string())?; config.0.validate()?; - Ok(config.0) } + fn substitute_env_vars(content: String) -> Result { + let reg = Regex::new(r"\$\{([A-Z_][A-Z0-9_]*)\}").map_err(|e| e.to_string())?; + let mut missing_vars = Vec::new(); + let mut result = String::new(); + + for line in content.lines() { + if !line.trim_start().starts_with('#') { + let processed_line = reg.replace_all(line, |caps: ®ex::Captures| { + let var_name = &caps[1]; + match env::var(var_name) { + Ok(value) => value, + Err(_) => { + missing_vars.push(var_name.to_string()); + format!("${{{}}}", var_name) + } + } + }); + result.push_str(&processed_line); + result.push('\n'); + } + } + + if !missing_vars.is_empty() { + return Err(format!( + "Missing environment variables: {}", + missing_vars.join(", ") + )); + } + + Ok(result.trim_end().to_string()) + } + // custom validation of the values fn validate(&self) -> Result<(), String> { match &self.tap.rav_request.trigger_value_divisor { @@ -336,9 +374,8 @@ pub struct RavRequestConfig { #[cfg(test)] mod tests { - use std::{fs, path::PathBuf}; - use sealed_test::prelude::*; + use std::{env, fs, path::PathBuf}; use tracing_test::traced_test; use crate::{Config, ConfigPrefix}; @@ -426,7 +463,7 @@ mod tests { .unwrap_err(); let test_value = "http://localhost:8000/testvalue"; - std::env::set_var("INDEXER_SERVICE_SUBGRAPHS__NETWORK__QUERY_URL", test_value); + env::set_var("INDEXER_SERVICE_SUBGRAPHS__NETWORK__QUERY_URL", test_value); let config = Config::parse( ConfigPrefix::Service, @@ -444,7 +481,7 @@ mod tests { #[sealed_test(files = ["minimal-config-example.toml"])] fn test_override_with_env() { let test_value = "http://localhost:8000/testvalue"; - std::env::set_var("INDEXER_SERVICE_SUBGRAPHS__NETWORK__QUERY_URL", test_value); + env::set_var("INDEXER_SERVICE_SUBGRAPHS__NETWORK__QUERY_URL", test_value); let config = Config::parse( ConfigPrefix::Service, @@ -457,6 +494,99 @@ mod tests { test_value ); } + + // Test to check substitute_env_vars function is substituting env variables + // indexers can use ${ENV_VAR_NAME} to point to the required env variable + #[test] + fn test_substitution_using_regex() { + // Set up environment variables + env::set_var("TEST_VAR1", "changed_value_1"); + + let input = r#" + [section1] + key1 = "${TEST_VAR1}" + key2 = "${TEST_VAR-default}" + key3 = "{{TEST_VAR3}}" + + [section2] + key4 = "prefix_${TEST_VAR1}_${TEST_VAR-default}_suffix" + key5 = "a_key_without_substitution" + "# + .to_string(); + + let expected_output = r#" + [section1] + key1 = "changed_value_1" + key2 = "${TEST_VAR-default}" + key3 = "{{TEST_VAR3}}" + + [section2] + key4 = "prefix_changed_value_1_${TEST_VAR-default}_suffix" + key5 = "a_key_without_substitution" + "# + .to_string(); + + let result = Config::substitute_env_vars(input).expect("error substiting env variables"); + + assert_eq!( + result.trim(), + expected_output.trim(), + "Environment variable substitution failed" + ); + + // Clean up environment variables + env::remove_var("TEST_VAR1"); + } + #[sealed_test(files = ["minimal-config-example.toml"])] + fn test_parse_with_env_substitution_and_overrides() { + let mut minimal_config: toml::Value = toml::from_str( + fs::read_to_string("minimal-config-example.toml") + .unwrap() + .as_str(), + ) + .unwrap(); + // Change the subgraphs query_url to an env variable + minimal_config + .get_mut("subgraphs") + .unwrap() + .get_mut("network") + .unwrap() + .as_table_mut() + .unwrap() + .insert( + String::from("query_url"), + toml::Value::String("${QUERY_URL}".to_string()), + ); + + // Save the modified minimal config to a named temporary file using tempfile + let temp_minimal_config_path = tempfile::NamedTempFile::new().unwrap(); + fs::write( + temp_minimal_config_path.path(), + toml::to_string(&minimal_config).unwrap(), + ) + .unwrap(); + + // This should fail because the QUERY_URL env variable is not setup + Config::parse( + ConfigPrefix::Service, + &PathBuf::from(temp_minimal_config_path.path()), + ) + .unwrap_err(); + + let test_value = "http://localhost:8000/testvalue"; + env::set_var("QUERY_URL", test_value); + + let config = Config::parse( + ConfigPrefix::Service, + &PathBuf::from(temp_minimal_config_path.path()), + ) + .unwrap(); + + assert_eq!( + config.subgraphs.network.config.query_url.as_str(), + test_value + ); + } #[test] fn test_url_format() { let data = DatabaseConfig::PostgresVars {