Make the new migration tool generally available

crates/cli/src/commands/config.rs

@@ -11,7 +11,7 @@ use camino::Utf8PathBuf;
 use clap::Parser;
 use figment::Figment;
 use mas_config::{ConfigurationSection, RootConfig, SyncConfig};
-use mas_storage::SystemClock;
+use mas_storage::{Clock as _, SystemClock};
 use mas_storage_pg::MIGRATOR;
 use rand::SeedableRng;
 use tokio::io::AsyncWriteExt;
@@ -46,6 +46,10 @@ enum Subcommand {
         /// If not specified, the config will be written to stdout
         #[clap(short, long)]
         output: Option<Utf8PathBuf>,
+
+        /// Existing Synapse configuration used to generate the MAS config
+        #[arg(short, long, action = clap::ArgAction::Append)]
+        synapse_config: Vec<Utf8PathBuf>,
     },
 
     /// Sync the clients and providers from the config file to the database
@@ -88,14 +92,24 @@ impl Options {
                 info!("Configuration file looks good");
             }
 
-            SC::Generate { output } => {
+            SC::Generate {
+                output,
+                synapse_config,
+            } => {
                 let _span = info_span!("cli.config.generate").entered();
+                let clock = SystemClock::default();
 
                 // XXX: we should disallow SeedableRng::from_entropy
-                let rng = rand_chacha::ChaChaRng::from_entropy();
-                let config = RootConfig::generate(rng).await?;
-                let config = serde_yaml::to_string(&config)?;
+                let mut rng = rand_chacha::ChaChaRng::from_entropy();
+                let mut config = RootConfig::generate(&mut rng).await?;
 
+                if !synapse_config.is_empty() {
+                    info!("Adjusting MAS config to match Synapse config from {synapse_config:?}");
+                    let synapse_config = syn2mas::synapse_config::Config::load(&synapse_config)?;
+                    config = synapse_config.adjust_mas_config(config, &mut rng, clock.now());
+                }
+
+                let config = serde_yaml::to_string(&config)?;
                 if let Some(output) = output {
                     info!("Writing configuration to {output:?}");
                     let mut file = tokio::fs::File::create(output).await?;

crates/cli/src/commands/syn2mas.rs

@@ -1,3 +1,8 @@
+// Copyright 2024, 2025 New Vector Ltd.
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+// Please see LICENSE in the repository root for full details.
+
 use std::{collections::HashMap, process::ExitCode, time::Duration};
 
 use anyhow::Context;
@@ -15,7 +20,7 @@ use sqlx::{Connection, Either, PgConnection, postgres::PgConnectOptions, types::
 use syn2mas::{
     LockedMasDatabase, MasWriter, Progress, ProgressStage, SynapseReader, synapse_config,
 };
-use tracing::{Instrument, error, info, info_span, warn};
+use tracing::{Instrument, error, info, info_span};
 
 use crate::util::{DatabaseConnectOptions, database_connection_from_config_with_options};
 
@@ -32,19 +37,10 @@ pub(super) struct Options {
     #[command(subcommand)]
     subcommand: Subcommand,
 
-    /// This version of the syn2mas tool is EXPERIMENTAL and INCOMPLETE. It is
-    /// only suitable for TESTING. If you want to use this tool anyway,
-    /// please pass this argument.
-    ///
-    /// If you want to migrate from Synapse to MAS today, please use the
-    /// Node.js-based tool in the MAS repository.
-    #[clap(long = "i-swear-i-am-just-testing-in-a-staging-environment")]
-    experimental_accepted: bool,
-
     /// Path to the Synapse configuration (in YAML format).
     /// May be specified multiple times if multiple Synapse configuration files
     /// are in use.
-    #[clap(long = "synapse-config")]
+    #[clap(long = "synapse-config", global = true)]
     synapse_configuration_files: Vec<Utf8PathBuf>,
 
     /// Override the Synapse database URI.
@@ -64,7 +60,7 @@ pub(super) struct Options {
     /// environment variables `PGHOST`, `PGPORT`, `PGUSER`, `PGDATABASE`,
     /// `PGPASSWORD`, etc. It is valid to specify the URL `postgresql:` and
     /// configure all values through those environment variables.
-    #[clap(long = "synapse-database-uri")]
+    #[clap(long = "synapse-database-uri", global = true)]
     synapse_database_uri: Option<PgConnectOptions>,
 }
 
@@ -74,8 +70,17 @@ enum Subcommand {
     ///
     /// It is OK for Synapse to be online during these checks.
     Check,
+
     /// Perform a migration. Synapse must be offline during this process.
-    Migrate,
+    Migrate {
+        /// Perform a dry-run migration, which is safe to run with Synapse
+        /// running, and will restore the MAS database to an empty state.
+        ///
+        /// This still *does* write to the MAS database, making it more
+        /// realistic compared to the final migration.
+        #[clap(long)]
+        dry_run: bool,
+    },
 }
 
 /// The number of parallel writing transactions active against the MAS database.
@@ -85,14 +90,6 @@ impl Options {
     #[tracing::instrument("cli.syn2mas.run", skip_all)]
     #[allow(clippy::too_many_lines)]
     pub async fn run(self, figment: &Figment) -> anyhow::Result<ExitCode> {
-        warn!(
-            "This version of the syn2mas tool is EXPERIMENTAL and INCOMPLETE. Do not use it, except for TESTING."
-        );
-        if !self.experimental_accepted {
-            error!("Please agree that you can only use this tool for testing.");
-            return Ok(ExitCode::FAILURE);
-        }
-
         if self.synapse_configuration_files.is_empty() {
             error!("Please specify the path to the Synapse configuration file(s).");
             return Ok(ExitCode::FAILURE);
@@ -130,11 +127,10 @@ impl Options {
             .await
             .context("could not run migrations")?;
 
-        if matches!(&self.subcommand, Subcommand::Migrate) {
+        if matches!(&self.subcommand, Subcommand::Migrate { .. }) {
             // First perform a config sync
             // This is crucial to ensure we register upstream OAuth providers
             // in the MAS database
-            //
             let config = SyncConfig::extract(figment)?;
             let clock = SystemClock::default();
             let encrypter = config.secrets.encrypter();
@@ -213,7 +209,8 @@ impl Options {
 
                 Ok(ExitCode::SUCCESS)
             }
-            Subcommand::Migrate => {
+
+            Subcommand::Migrate { dry_run } => {
                 let provider_id_mappings: HashMap<String, Uuid> = {
                     let mas_oauth2 = UpstreamOAuth2Config::extract_or_default(figment)?;
 
@@ -229,21 +226,20 @@ impl Options {
 
                 // TODO how should we handle warnings at this stage?
 
-                // TODO this dry-run flag should be set to false in real circumstances !!!
-                let reader = SynapseReader::new(&mut syn_conn, true).await?;
-                let mut writer_mas_connections = Vec::with_capacity(NUM_WRITER_CONNECTIONS);
-                for _ in 0..NUM_WRITER_CONNECTIONS {
-                    writer_mas_connections.push(
+                let reader = SynapseReader::new(&mut syn_conn, dry_run).await?;
+                let writer_mas_connections =
+                    futures_util::future::try_join_all((0..NUM_WRITER_CONNECTIONS).map(|_| {
                         database_connection_from_config_with_options(
                             &config,
                             &DatabaseConnectOptions {
                                 log_slow_statements: false,
                             },
                         )
-                        .await?,
-                    );
-                }
-                let writer = MasWriter::new(mas_connection, writer_mas_connections).await?;
+                    }))
+                    .instrument(tracing::info_span!("syn2mas.mas_writer_connections"))
+                    .await?;
+                let writer =
+                    MasWriter::new(mas_connection, writer_mas_connections, dry_run).await?;
 
                 let clock = SystemClock::default();
                 // TODO is this rng ok?
@@ -256,7 +252,6 @@ impl Options {
                     tokio::spawn(occasional_progress_logger(progress.clone()));
 
                 let mas_matrix = MatrixConfig::extract(figment)?;
-                eprintln!("\n\n");
                 syn2mas::migrate(
                     reader,
                     writer,
@@ -276,13 +271,13 @@ impl Options {
     }
 }
 
-/// Logs progress every 30 seconds, as a lightweight alternative to a progress
-/// bar. For most deployments, the migration will not take 30 seconds so this
+/// Logs progress every 5 seconds, as a lightweight alternative to a progress
+/// bar. For most deployments, the migration will not take 5 seconds so this
 /// will not be relevant. In other cases, this will give the operator an idea of
 /// what's going on.
 async fn occasional_progress_logger(progress: Progress) {
     loop {
-        tokio::time::sleep(Duration::from_secs(30)).await;
+        tokio::time::sleep(Duration::from_secs(5)).await;
         match &**progress.get_current_stage() {
             ProgressStage::SettingUp => {
                 info!(name: "progress", "still setting up");

crates/config/src/sections/mod.rs

@@ -38,7 +38,9 @@ pub use self::{
         Resource as HttpResource, TlsConfig as HttpTlsConfig, UnixOrTcp,
     },
     matrix::{HomeserverKind, MatrixConfig},
-    passwords::{Algorithm as PasswordAlgorithm, PasswordsConfig},
+    passwords::{
+        Algorithm as PasswordAlgorithm, HashingScheme as PasswordHashingScheme, PasswordsConfig,
+    },
     policy::PolicyConfig,
     rate_limiting::RateLimitingConfig,
     secrets::SecretsConfig,

crates/config/src/sections/passwords.rs

@@ -16,7 +16,7 @@ use crate::ConfigurationSection;
 fn default_schemes() -> Vec<HashingScheme> {
     vec![HashingScheme {
         version: 1,
-        algorithm: Algorithm::Argon2id,
+        algorithm: Algorithm::default(),
         cost: None,
         secret: None,
         secret_file: None,
@@ -36,10 +36,14 @@ fn default_minimum_complexity() -> u8 {
 pub struct PasswordsConfig {
     /// Whether password-based authentication is enabled
     #[serde(default = "default_enabled")]
-    enabled: bool,
+    pub enabled: bool,
 
+    /// The hashing schemes to use for hashing and validating passwords
+    ///
+    /// The hashing scheme with the highest version number will be used for
+    /// hashing new passwords.
     #[serde(default = "default_schemes")]
-    schemes: Vec<HashingScheme>,
+    pub schemes: Vec<HashingScheme>,
 
     /// Score between 0 and 4 determining the minimum allowed password
     /// complexity. Scores are based on the ESTIMATED number of guesses
@@ -154,23 +158,30 @@ impl PasswordsConfig {
     }
 }
 
+/// Parameters for a password hashing scheme
 #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
 pub struct HashingScheme {
-    version: u16,
+    /// The version of the hashing scheme. They must be unique, and the highest
+    /// version will be used for hashing new passwords.
+    pub version: u16,
 
-    algorithm: Algorithm,
+    /// The hashing algorithm to use
+    pub algorithm: Algorithm,
 
     /// Cost for the bcrypt algorithm
     #[serde(skip_serializing_if = "Option::is_none")]
     #[schemars(default = "default_bcrypt_cost")]
-    cost: Option<u32>,
+    pub cost: Option<u32>,
 
+    /// An optional secret to use when hashing passwords. This makes it harder
+    /// to brute-force the passwords in case of a database leak.
     #[serde(skip_serializing_if = "Option::is_none")]
-    secret: Option<String>,
+    pub secret: Option<String>,
 
+    /// Same as `secret`, but read from a file.
     #[serde(skip_serializing_if = "Option::is_none")]
     #[schemars(with = "Option<String>")]
-    secret_file: Option<Utf8PathBuf>,
+    pub secret_file: Option<Utf8PathBuf>,
 }
 
 #[allow(clippy::unnecessary_wraps)]
@@ -179,13 +190,14 @@ fn default_bcrypt_cost() -> Option<u32> {
 }
 
 /// A hashing algorithm
-#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, PartialEq, Eq)]
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, PartialEq, Eq, Default)]
 #[serde(rename_all = "lowercase")]
 pub enum Algorithm {
     /// bcrypt
     Bcrypt,
 
     /// argon2id
+    #[default]
     Argon2id,
 
     /// PBKDF2

crates/config/src/sections/upstream_oauth2.rs

@@ -418,6 +418,23 @@ pub struct Provider {
     )]
     pub id: Ulid,
 
+    /// The ID of the provider that was used by Synapse.
+    /// In order to perform a Synapse-to-MAS migration, this must be specified.
+    ///
+    /// ## For providers that used OAuth 2.0 or OpenID Connect in Synapse
+    ///
+    /// ### For `oidc_providers`:
+    /// This should be specified as `oidc-` followed by the ID that was
+    /// configured as `idp_id` in one of the `oidc_providers` in the Synapse
+    /// configuration.
+    /// For example, if Synapse's configuration contained `idp_id: wombat` for
+    /// this provider, then specify `oidc-wombat` here.
+    ///
+    /// ### For `oidc_config` (legacy):
+    /// Specify `oidc` here.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub synapse_idp_id: Option<String>,
+
     /// The OIDC issuer URL
     ///
     /// This is required if OIDC discovery is enabled (which is the default)
@@ -548,21 +565,4 @@ pub struct Provider {
     /// Orders of the keys are not preserved.
     #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
     pub additional_authorization_parameters: BTreeMap<String, String>,
-
-    /// The ID of the provider that was used by Synapse.
-    /// In order to perform a Synapse-to-MAS migration, this must be specified.
-    ///
-    /// ## For providers that used OAuth 2.0 or OpenID Connect in Synapse
-    ///
-    /// ### For `oidc_providers`:
-    /// This should be specified as `oidc-` followed by the ID that was
-    /// configured as `idp_id` in one of the `oidc_providers` in the Synapse
-    /// configuration.
-    /// For example, if Synapse's configuration contained `idp_id: wombat` for
-    /// this provider, then specify `oidc-wombat` here.
-    ///
-    /// ### For `oidc_config` (legacy):
-    /// Specify `oidc` here.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub synapse_idp_id: Option<String>,
 }