feat(config): Allow environment references in Relay configuration (#4750)

> Allow ${VAR} references in Relay's config. Any leaf value can now be a
reference to an environment variable.

Another attempt at #4748 for #4647, now with version 0.2 (instead of
0.1).

serde-vars 0.2 is now more lenient in deserialization when it can (for
strings and bytes) and the underlying deserializer supports it. This
will fix the config issue with YAML configs, which broke self hosted
CI/the self hosted default Kafka config.

Author: Dav1dde

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## Unreleased
+
+**Features**:
+
+- Allow environment references in Relay configuration. ([#4750](https://github.com/getsentry/relay/pull/4750))
+
 ## 25.5.0
 
 **Features**:

Cargo.lock

@@ -172,6 +172,7 @@ sentry-types = "0.36.0"
 sentry_protos = "0.2.0"
 serde = { version = "1.0.215", features = ["derive", "rc"] }
 serde-transcode = "1.1.1"
+serde-vars = "0.2"
 serde_bytes = "0.11"
 serde_json = "1.0.133"
 serde_path_to_error = "0.1.16"

Cargo.toml

@@ -27,6 +27,7 @@ relay-log = { workspace = true, features = ["init"] }
 relay-metrics = { workspace = true }
 relay-redis = { workspace = true }
 serde = { workspace = true }
+serde-vars = { workspace = true }
 serde_json = { workspace = true }
 serde_yaml = { workspace = true }
 thiserror = { workspace = true }

relay-config/Cargo.toml

@@ -174,12 +174,18 @@ trait ConfigObject: DeserializeOwned + Serialize {
 
         let f = fs::File::open(&path)
             .with_context(|| ConfigError::file(ConfigErrorKind::CouldNotOpenFile, &path))?;
+        let f = io::BufReader::new(f);
 
+        let mut source = serde_vars::EnvSource::default();
         match Self::format() {
-            ConfigFormat::Yaml => serde_yaml::from_reader(io::BufReader::new(f))
-                .with_context(|| ConfigError::file(ConfigErrorKind::BadYaml, &path)),
-            ConfigFormat::Json => serde_json::from_reader(io::BufReader::new(f))
-                .with_context(|| ConfigError::file(ConfigErrorKind::BadJson, &path)),
+            ConfigFormat::Yaml => {
+                serde_vars::deserialize(serde_yaml::Deserializer::from_reader(f), &mut source)
+                    .with_context(|| ConfigError::file(ConfigErrorKind::BadYaml, &path))
+            }
+            ConfigFormat::Json => {
+                serde_vars::deserialize(&mut serde_json::Deserializer::from_reader(f), &mut source)
+                    .with_context(|| ConfigError::file(ConfigErrorKind::BadJson, &path))
+            }
         }
     }
 

relay-config/src/config.rs

@@ -28,3 +28,19 @@ def test_invalid_topics_raise_error(mini_sentry, relay_with_processing):
 
     error = str(mini_sentry.test_failures.get_nowait())
     assert "failed to validate the topic with name" in error
+
+
+def test_missing_env_var_in_config(mini_sentry, relay, relay_credentials):
+    credentials = relay_credentials()
+    relay = relay(
+        mini_sentry,
+        credentials=credentials,
+        wait_health_check=False,
+        options={
+            "http": {
+                "encoding": "${THIS_DOES_NOT_EXIST_OTHER_WISE_THE_TEST_WILL_PASS}",
+            }
+        },
+    )
+
+    assert relay.wait_for_exit() != 0