Load the configuration from a common Figment instance

This should avoid loading the same files multiple times.
It should also make it easier to do post-processing on the
configuration, like validation.

This does deprecate one undocumented feature: the ability to override
some fields during the configuration generation using environment
variables.

Author: sandhose

Cargo.lock

@@ -78,6 +78,11 @@ features = ["serde", "clock"]
 version = "4.5.3"
 features = ["derive"]
 
+# Configuration loading
+[workspace.dependencies.figment]
+version = "0.10.15"
+features = ["env", "yaml", "test"]
+
 # HTTP headers
 [workspace.dependencies.headers]
 version = "0.3.9"

Cargo.toml

@@ -17,6 +17,7 @@ axum = "0.6.20"
 camino.workspace = true
 clap.workspace = true
 dotenvy = "0.15.7"
+figment.workspace = true
 httpdate = "1.0.3"
 hyper.workspace = true
 ipnetwork = "0.20.0"

crates/cli/Cargo.toml

@@ -15,6 +15,7 @@
 use anyhow::Context;
 use camino::Utf8PathBuf;
 use clap::Parser;
+use figment::Figment;
 use mas_config::{ConfigurationSection, RootConfig, SyncConfig};
 use mas_storage::SystemClock;
 use mas_storage_pg::MIGRATOR;
@@ -67,13 +68,13 @@ enum Subcommand {
 }
 
 impl Options {
-    pub async fn run(self, root: &super::Options) -> anyhow::Result<()> {
+    pub async fn run(self, figment: &Figment) -> anyhow::Result<()> {
         use Subcommand as SC;
         match self.subcommand {
             SC::Dump { output } => {
                 let _span = info_span!("cli.config.dump").entered();
 
-                let config: RootConfig = root.load_config()?;
+                let config = RootConfig::extract(figment)?;
                 let config = serde_yaml::to_string(&config)?;
 
                 if let Some(output) = output {
@@ -89,16 +90,16 @@ impl Options {
             SC::Check => {
                 let _span = info_span!("cli.config.check").entered();
 
-                let _config: RootConfig = root.load_config()?;
-                info!(path = ?root.config, "Configuration file looks good");
+                let _config = RootConfig::extract(figment)?;
+                info!("Configuration file looks good");
             }
 
             SC::Generate { output } => {
                 let _span = info_span!("cli.config.generate").entered();
 
                 // XXX: we should disallow SeedableRng::from_entropy
                 let rng = rand_chacha::ChaChaRng::from_entropy();
-                let config = RootConfig::load_and_generate(rng).await?;
+                let config = RootConfig::generate(rng).await?;
                 let config = serde_yaml::to_string(&config)?;
 
                 if let Some(output) = output {
@@ -112,7 +113,7 @@ impl Options {
             }
 
             SC::Sync { prune, dry_run } => {
-                let config: SyncConfig = root.load_config()?;
+                let config = SyncConfig::extract(figment)?;
                 let clock = SystemClock::default();
                 let encrypter = config.secrets.encrypter();
 

crates/cli/src/commands/config.rs

@@ -14,7 +14,8 @@
 
 use anyhow::Context;
 use clap::Parser;
-use mas_config::DatabaseConfig;
+use figment::Figment;
+use mas_config::{ConfigurationSection, DatabaseConfig};
 use mas_storage_pg::MIGRATOR;
 use tracing::{info_span, Instrument};
 
@@ -33,9 +34,9 @@ enum Subcommand {
 }
 
 impl Options {
-    pub async fn run(self, root: &super::Options) -> anyhow::Result<()> {
+    pub async fn run(self, figment: &Figment) -> anyhow::Result<()> {
         let _span = info_span!("cli.database.migrate").entered();
-        let config: DatabaseConfig = root.load_config()?;
+        let config = DatabaseConfig::extract(figment)?;
         let mut conn = database_connection_from_config(&config).await?;
 
         // Run pending migrations

crates/cli/src/commands/database.rs

@@ -13,8 +13,9 @@
 // limitations under the License.
 
 use clap::Parser;
+use figment::Figment;
 use hyper::{Response, Uri};
-use mas_config::PolicyConfig;
+use mas_config::{ConfigurationSection, PolicyConfig};
 use mas_handlers::HttpClientFactory;
 use mas_http::HttpServiceExt;
 use tokio::io::AsyncWriteExt;
@@ -65,7 +66,7 @@ fn print_headers(parts: &hyper::http::response::Parts) {
 
 impl Options {
     #[tracing::instrument(skip_all)]
-    pub async fn run(self, root: &super::Options) -> anyhow::Result<()> {
+    pub async fn run(self, figment: &Figment) -> anyhow::Result<()> {
         use Subcommand as SC;
         let http_client_factory = HttpClientFactory::new();
         match self.subcommand {
@@ -120,7 +121,7 @@ impl Options {
 
             SC::Policy => {
                 let _span = info_span!("cli.debug.policy").entered();
-                let config: PolicyConfig = root.load_config()?;
+                let config = PolicyConfig::extract(figment)?;
                 info!("Loading and compiling the policy module");
                 let policy_factory = policy_factory_from_config(&config).await?;
 

crates/cli/src/commands/debug.rs

@@ -19,7 +19,8 @@
 
 use anyhow::Context;
 use clap::Parser;
-use mas_config::RootConfig;
+use figment::Figment;
+use mas_config::{ConfigurationSection, RootConfig};
 use mas_handlers::HttpClientFactory;
 use mas_http::HttpServiceExt;
 use tower::{Service, ServiceExt};
@@ -34,11 +35,11 @@ pub(super) struct Options {}
 
 impl Options {
     #[allow(clippy::too_many_lines)]
-    pub async fn run(self, root: &super::Options) -> anyhow::Result<()> {
+    pub async fn run(self, figment: &Figment) -> anyhow::Result<()> {
         let _span = info_span!("cli.doctor").entered();
         info!("💡 Running diagnostics, make sure that both MAS and Synapse are running, and that MAS is using the same configuration files as this tool.");
 
-        let config: RootConfig = root.load_config()?;
+        let config = RootConfig::extract(figment)?;
 
         // We'll need an HTTP client
         let http_client_factory = HttpClientFactory::new();

crates/cli/src/commands/doctor.rs

@@ -14,7 +14,8 @@
 
 use anyhow::Context;
 use clap::Parser;
-use mas_config::{DatabaseConfig, PasswordsConfig};
+use figment::Figment;
+use mas_config::{ConfigurationSection, DatabaseConfig, PasswordsConfig};
 use mas_data_model::{Device, TokenType};
 use mas_storage::{
     compat::{CompatAccessTokenRepository, CompatSessionRepository},
@@ -89,7 +90,7 @@ enum Subcommand {
 
 impl Options {
     #[allow(clippy::too_many_lines)]
-    pub async fn run(self, root: &super::Options) -> anyhow::Result<()> {
+    pub async fn run(self, figment: &Figment) -> anyhow::Result<()> {
         use Subcommand as SC;
         let clock = SystemClock::default();
         // XXX: we should disallow SeedableRng::from_entropy
@@ -100,8 +101,8 @@ impl Options {
                 let _span =
                     info_span!("cli.manage.set_password", user.username = %username).entered();
 
-                let database_config: DatabaseConfig = root.load_config()?;
-                let passwords_config: PasswordsConfig = root.load_config()?;
+                let database_config = DatabaseConfig::extract(figment)?;
+                let passwords_config = PasswordsConfig::extract(figment)?;
 
                 let mut conn = database_connection_from_config(&database_config).await?;
                 let password_manager = password_manager_from_config(&passwords_config).await?;
@@ -136,7 +137,7 @@ impl Options {
                 )
                 .entered();
 
-                let database_config: DatabaseConfig = root.load_config()?;
+                let database_config = DatabaseConfig::extract(figment)?;
                 let mut conn = database_connection_from_config(&database_config).await?;
                 let txn = conn.begin().await?;
                 let mut repo = PgRepository::from_conn(txn);
@@ -170,7 +171,7 @@ impl Options {
                 admin,
                 device_id,
             } => {
-                let database_config: DatabaseConfig = root.load_config()?;
+                let database_config = DatabaseConfig::extract(figment)?;
                 let mut conn = database_connection_from_config(&database_config).await?;
                 let txn = conn.begin().await?;
                 let mut repo = PgRepository::from_conn(txn);
@@ -215,7 +216,7 @@ impl Options {
 
             SC::ProvisionAllUsers => {
                 let _span = info_span!("cli.manage.provision_all_users").entered();
-                let database_config: DatabaseConfig = root.load_config()?;
+                let database_config = DatabaseConfig::extract(figment)?;
                 let mut conn = database_connection_from_config(&database_config).await?;
                 let mut txn = conn.begin().await?;
 
@@ -241,7 +242,7 @@ impl Options {
             SC::KillSessions { username, dry_run } => {
                 let _span =
                     info_span!("cli.manage.kill_sessions", user.username = username).entered();
-                let database_config: DatabaseConfig = root.load_config()?;
+                let database_config = DatabaseConfig::extract(figment)?;
                 let mut conn = database_connection_from_config(&database_config).await?;
                 let txn = conn.begin().await?;
                 let mut repo = PgRepository::from_conn(txn);
@@ -361,7 +362,7 @@ impl Options {
                 deactivate,
             } => {
                 let _span = info_span!("cli.manage.lock_user", user.username = username).entered();
-                let config: DatabaseConfig = root.load_config()?;
+                let config = DatabaseConfig::extract(figment)?;
                 let mut conn = database_connection_from_config(&config).await?;
                 let txn = conn.begin().await?;
                 let mut repo = PgRepository::from_conn(txn);
@@ -393,7 +394,7 @@ impl Options {
 
             SC::UnlockUser { username } => {
                 let _span = info_span!("cli.manage.lock_user", user.username = username).entered();
-                let config: DatabaseConfig = root.load_config()?;
+                let config = DatabaseConfig::extract(figment)?;
                 let mut conn = database_connection_from_config(&config).await?;
                 let txn = conn.begin().await?;
                 let mut repo = PgRepository::from_conn(txn);

crates/cli/src/commands/manage.rs

@@ -12,10 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use anyhow::Context;
 use camino::Utf8PathBuf;
 use clap::Parser;
-use mas_config::ConfigurationSection;
+use figment::{
+    providers::{Env, Format, Yaml},
+    Figment,
+};
 
 mod config;
 mod database;
@@ -65,22 +67,23 @@ pub struct Options {
 }
 
 impl Options {
-    pub async fn run(mut self) -> anyhow::Result<()> {
+    pub async fn run(self, figment: &Figment) -> anyhow::Result<()> {
         use Subcommand as S;
-        match self.subcommand.take() {
-            Some(S::Config(c)) => c.run(&self).await,
-            Some(S::Database(c)) => c.run(&self).await,
-            Some(S::Server(c)) => c.run(&self).await,
-            Some(S::Worker(c)) => c.run(&self).await,
-            Some(S::Manage(c)) => c.run(&self).await,
-            Some(S::Templates(c)) => c.run(&self).await,
-            Some(S::Debug(c)) => c.run(&self).await,
-            Some(S::Doctor(c)) => c.run(&self).await,
-            None => self::server::Options::default().run(&self).await,
+        match self.subcommand {
+            Some(S::Config(c)) => c.run(figment).await,
+            Some(S::Database(c)) => c.run(figment).await,
+            Some(S::Server(c)) => c.run(figment).await,
+            Some(S::Worker(c)) => c.run(figment).await,
+            Some(S::Manage(c)) => c.run(figment).await,
+            Some(S::Templates(c)) => c.run(figment).await,
+            Some(S::Debug(c)) => c.run(figment).await,
+            Some(S::Doctor(c)) => c.run(figment).await,
+            None => self::server::Options::default().run(figment).await,
         }
     }
 
-    pub fn load_config<T: ConfigurationSection>(&self) -> anyhow::Result<T> {
+    /// Get a [`Figment`] instance with the configuration loaded
+    pub fn figment(&self) -> Figment {
         let configs = if self.config.is_empty() {
             // Read the MAS_CONFIG environment variable
             std::env::var("MAS_CONFIG")
@@ -93,7 +96,10 @@ impl Options {
         } else {
             self.config.clone()
         };
+        let base = Figment::new().merge(Env::prefixed("MAS_").split("_"));
 
-        T::load_from_files(&configs).context("could not load configuration")
+        configs
+            .into_iter()
+            .fold(base, |f, path| f.merge(Yaml::file(path)))
     }
 }

crates/cli/src/commands/mod.rs

@@ -16,8 +16,9 @@ use std::{collections::BTreeSet, sync::Arc, time::Duration};
 
 use anyhow::Context;
 use clap::Parser;
+use figment::Figment;
 use itertools::Itertools;
-use mas_config::AppConfig;
+use mas_config::{AppConfig, ClientsConfig, ConfigurationSection, UpstreamOAuth2Config};
 use mas_handlers::{ActivityTracker, CookieManager, HttpClientFactory, MetadataCache, SiteConfig};
 use mas_listener::{server::Server, shutdown::ShutdownStream};
 use mas_matrix_synapse::SynapseConnection;
@@ -63,9 +64,9 @@ pub(super) struct Options {
 
 impl Options {
     #[allow(clippy::too_many_lines)]
-    pub async fn run(self, root: &super::Options) -> anyhow::Result<()> {
+    pub async fn run(self, figment: &Figment) -> anyhow::Result<()> {
         let span = info_span!("cli.run.init").entered();
-        let config: AppConfig = root.load_config()?;
+        let config = AppConfig::extract(figment)?;
 
         if self.migrate {
             warn!("The `--migrate` flag is deprecated and will be removed in a future release. Please use `--no-migrate` to disable automatic migrations on startup.");
@@ -101,8 +102,8 @@ impl Options {
         } else {
             // Sync the configuration with the database
             let mut conn = pool.acquire().await?;
-            let clients_config = root.load_config()?;
-            let upstream_oauth2_config = root.load_config()?;
+            let clients_config = ClientsConfig::extract(figment)?;
+            let upstream_oauth2_config = UpstreamOAuth2Config::extract(figment)?;
 
             crate::sync::config_sync(
                 upstream_oauth2_config,