Foundation of new syn2mas tool (#3636)

Author: reivilibre

Cargo.lock

@@ -54,6 +54,7 @@ mas-tasks = { path = "./crates/tasks/", version = "=0.13.0-rc.1" }
 mas-templates = { path = "./crates/templates/", version = "=0.13.0-rc.1" }
 mas-tower = { path = "./crates/tower/", version = "=0.13.0-rc.1" }
 oauth2-types = { path = "./crates/oauth2-types/", version = "=0.13.0-rc.1" }
+syn2mas = { path = "./crates/syn2mas", version = "=0.13.0-rc.1" }
 
 # OpenAPI schema generation and validation
 [workspace.dependencies.aide]
@@ -65,6 +66,9 @@ features = ["axum", "axum-headers", "macros"]
 version = "7.0.14"
 features = ["chrono", "url", "tracing"]
 
+[workspace.dependencies.async-stream]
+version = "0.3.6"
+
 # Utility to write and implement async traits
 [workspace.dependencies.async-trait]
 version = "0.1.85"
@@ -94,6 +98,10 @@ version = "1.9.0"
 [workspace.dependencies.camino]
 version = "1.1.9"
 
+# Memory optimisation for short strings
+[workspace.dependencies.compact_str]
+version = "0.8.0"
+
 # Time utilities
 [workspace.dependencies.chrono]
 version = "0.4.39"
@@ -312,11 +320,17 @@ features = [
 [workspace.dependencies.thiserror]
 version = "2.0.11"
 
+[workspace.dependencies.thiserror-ext]
+version = "0.2.0"
+
 # Async runtime
 [workspace.dependencies.tokio]
 version = "1.43.0"
 features = ["full"]
 
+[workspace.dependencies.tokio-stream]
+version = "0.1.16"
+
 # Useful async utilities
 [workspace.dependencies.tokio-util]
 version = "0.7.13"

Cargo.toml

@@ -1,4 +1,4 @@
-doc-valid-idents = ["OpenID", "OAuth", "..", "PostgreSQL"]
+doc-valid-idents = ["OpenID", "OAuth", "..", "PostgreSQL", "SQLite"]
 
 disallowed-methods = [
     { path = "rand::thread_rng", reason = "do not create rngs on the fly, pass them as parameters" },

clippy.toml

@@ -37,6 +37,7 @@ serde_yaml = "0.9.34"
 sqlx.workspace = true
 tokio.workspace = true
 tokio-util.workspace = true
+tokio-stream.workspace = true
 tower.workspace = true
 tower-http.workspace = true
 url.workspace = true
@@ -78,6 +79,9 @@ mas-tasks.workspace = true
 mas-templates.workspace = true
 mas-tower.workspace = true
 
+oauth2-types.workspace = true
+syn2mas.workspace = true
+
 [build-dependencies]
 anyhow.workspace = true
 vergen-gitcl = { version = "1.0.5", features = ["rustc"] }

crates/cli/Cargo.toml

@@ -19,6 +19,7 @@ mod debug;
 mod doctor;
 mod manage;
 mod server;
+mod syn2mas;
 mod templates;
 mod worker;
 
@@ -48,6 +49,10 @@ enum Subcommand {
 
     /// Run diagnostics on the deployment
     Doctor(self::doctor::Options),
+
+    /// Migrate from Synapse's built-in auth system to MAS.
+    #[clap(name = "syn2mas")]
+    Syn2Mas(self::syn2mas::Options),
 }
 
 #[derive(Parser, Debug)]
@@ -75,6 +80,7 @@ impl Options {
             Some(S::Templates(c)) => Box::pin(c.run(figment)).await,
             Some(S::Debug(c)) => Box::pin(c.run(figment)).await,
             Some(S::Doctor(c)) => Box::pin(c.run(figment)).await,
+            Some(S::Syn2Mas(c)) => Box::pin(c.run(figment)).await,
             None => Box::pin(self::server::Options::default().run(figment)).await,
         }
     }

crates/cli/src/commands/mod.rs

@@ -0,0 +1,83 @@
+use std::process::ExitCode;
+
+use anyhow::Context;
+use clap::Parser;
+use figment::Figment;
+use mas_config::{ConfigurationSectionExt, DatabaseConfig};
+use rand::thread_rng;
+use sqlx::{Connection, Either, PgConnection};
+use syn2mas::{LockedMasDatabase, MasWriter, SynapseReader};
+use tracing::{error, warn};
+
+use crate::util::database_connection_from_config;
+
+#[derive(Parser, Debug)]
+pub(super) struct Options {
+    #[command(subcommand)]
+    subcommand: Subcommand,
+
+    /// This version of the syn2mas tool is EXPERIMENTAL and INCOMPLETE. It is only suitable for TESTING.
+    /// If you want to use this tool anyway, please pass this argument.
+    ///
+    /// If you want to migrate from Synapse to MAS today, please use the Node.js-based tool in the MAS repository.
+    #[clap(long = "i-swear-i-am-just-testing-in-a-staging-environment")]
+    experimental_accepted: bool,
+}
+
+#[derive(Parser, Debug)]
+enum Subcommand {
+    Check,
+    Migrate,
+}
+
+/// The number of parallel writing transactions active against the MAS database.
+const NUM_WRITER_CONNECTIONS: usize = 8;
+
+impl Options {
+    pub async fn run(self, figment: &Figment) -> anyhow::Result<ExitCode> {
+        warn!("This version of the syn2mas tool is EXPERIMENTAL and INCOMPLETE. Do not use it, except for TESTING.");
+        if !self.experimental_accepted {
+            error!("Please agree that you can only use this tool for testing.");
+            return Ok(ExitCode::FAILURE);
+        }
+
+        // TODO allow configuring the synapse database location
+        let mut syn_conn = PgConnection::connect("postgres:///fakesyn").await.unwrap();
+
+        let config = DatabaseConfig::extract_or_default(figment)?;
+
+        let mut mas_connection = database_connection_from_config(&config).await?;
+
+        let Either::Left(mut mas_connection) = LockedMasDatabase::try_new(&mut mas_connection)
+            .await
+            .context("failed to issue query to lock database")?
+        else {
+            error!("Failed to acquire syn2mas lock on the database.");
+            error!("This likely means that another syn2mas instance is already running!");
+            return Ok(ExitCode::FAILURE);
+        };
+
+        syn2mas::mas_pre_migration_checks(&mut mas_connection).await?;
+        syn2mas::synapse_pre_migration_checks(&mut syn_conn).await?;
+
+        let mut reader = SynapseReader::new(&mut syn_conn, true).await?;
+        let mut writer_mas_connections = Vec::with_capacity(NUM_WRITER_CONNECTIONS);
+        for _ in 0..NUM_WRITER_CONNECTIONS {
+            writer_mas_connections.push(database_connection_from_config(&config).await?);
+        }
+        let mut writer = MasWriter::new(mas_connection, writer_mas_connections).await?;
+
+        // TODO is this rng ok?
+        #[allow(clippy::disallowed_methods)]
+        let mut rng = thread_rng();
+
+        // TODO progress reporting
+        // TODO allow configuring the server name
+        syn2mas::migrate(&mut reader, &mut writer, "matrix.org", &mut rng).await?;
+
+        reader.finish().await?;
+        writer.finish().await?;
+
+        Ok(ExitCode::SUCCESS)
+    }
+}