syn2mas: Migrate threepids to MAS (#3878)

* Add a table to hold unsupported threepids

* Migrate threepids from Synapse to MAS

Author: reivilibre

crates/cli/src/commands/syn2mas.rs

@@ -12,43 +12,51 @@ use tracing::{error, warn};
 
 use crate::util::database_connection_from_config;
 
-/// The exit code used by `syn2mas check` and `syn2mas migrate` when there are errors preventing migration.
+/// The exit code used by `syn2mas check` and `syn2mas migrate` when there are
+/// errors preventing migration.
 const EXIT_CODE_CHECK_ERRORS: u8 = 10;
 
-/// The exit code used by `syn2mas check` when there are warnings which should be considered prior to migration.
+/// The exit code used by `syn2mas check` when there are warnings which should
+/// be considered prior to migration.
 const EXIT_CODE_CHECK_WARNINGS: u8 = 11;
 
 #[derive(Parser, Debug)]
 pub(super) struct Options {
     #[command(subcommand)]
     subcommand: Subcommand,
 
-    /// This version of the syn2mas tool is EXPERIMENTAL and INCOMPLETE. It is only suitable for TESTING.
-    /// If you want to use this tool anyway, please pass this argument.
+    /// This version of the syn2mas tool is EXPERIMENTAL and INCOMPLETE. It is
+    /// only suitable for TESTING. If you want to use this tool anyway,
+    /// please pass this argument.
     ///
-    /// If you want to migrate from Synapse to MAS today, please use the Node.js-based tool in the MAS repository.
+    /// If you want to migrate from Synapse to MAS today, please use the
+    /// Node.js-based tool in the MAS repository.
     #[clap(long = "i-swear-i-am-just-testing-in-a-staging-environment")]
     experimental_accepted: bool,
 
     /// Path to the Synapse configuration (in YAML format).
-    /// May be specified multiple times if multiple Synapse configuration files are in use.
+    /// May be specified multiple times if multiple Synapse configuration files
+    /// are in use.
     #[clap(long = "synapse-config")]
     synapse_configuration_files: Vec<Utf8PathBuf>,
 
     /// Override the Synapse database URI.
-    /// syn2mas normally loads the Synapse database connection details from the Synapse configuration.
-    /// However, it may sometimes be necessary to override the database URI and in that case this flag can be used.
+    /// syn2mas normally loads the Synapse database connection details from the
+    /// Synapse configuration. However, it may sometimes be necessary to
+    /// override the database URI and in that case this flag can be used.
     ///
     /// Should be a connection URI of the following general form:
     /// ```text
     /// postgresql://[user[:password]@][host][:port][/dbname][?param1=value1&...]
     /// ```
-    /// To use a UNIX socket at a custom path, the host should be a path to a socket, but in the URI string
-    /// it must be URI-encoded by replacing `/` with `%2F`.
+    /// To use a UNIX socket at a custom path, the host should be a path to a
+    /// socket, but in the URI string it must be URI-encoded by replacing
+    /// `/` with `%2F`.
     ///
-    /// Finally, any missing values will be loaded from the libpq-compatible environment variables
-    /// `PGHOST`, `PGPORT`, `PGUSER`, `PGDATABASE`, `PGPASSWORD`, etc.
-    /// It is valid to specify the URL `postgresql:` and configure all values through those environment variables.
+    /// Finally, any missing values will be loaded from the libpq-compatible
+    /// environment variables `PGHOST`, `PGPORT`, `PGUSER`, `PGDATABASE`,
+    /// `PGPASSWORD`, etc. It is valid to specify the URL `postgresql:` and
+    /// configure all values through those environment variables.
     #[clap(long = "synapse-database-uri")]
     synapse_database_uri: Option<PgConnectOptions>,
 }

crates/storage-pg/migrations/20250124151529_unsupported_threepids_table.sql

@@ -0,0 +1,30 @@
+-- Copyright 2025 New Vector Ltd.
+--
+-- SPDX-License-Identifier: AGPL-3.0-only
+-- Please see LICENSE in the repository root for full details.
+
+
+
+-- Tracks third-party ID associations that have been verified but are
+-- not currently supported by MAS.
+-- This is currently used when importing third-party IDs from Synapse,
+-- which historically could verify at least phone numbers.
+-- E-mail associations will not be stored in this table because those are natively
+-- supported by MAS; see the `user_emails` table.
+
+CREATE TABLE user_unsupported_third_party_ids(
+    -- The owner of the third-party ID assocation
+    user_id UUID NOT NULL
+      REFERENCES users(user_id) ON DELETE CASCADE,
+
+    -- What type of association is this?
+    medium TEXT NOT NULL,
+
+    -- The address of the associated ID, e.g. a phone number or other identifier.
+    address TEXT NOT NULL,
+
+    -- When the association was created
+    created_at TIMESTAMP WITH TIME ZONE NOT NULL,
+
+    PRIMARY KEY (user_id, medium, address)
+);

crates/syn2mas/.sqlx/query-b11590549fdd4cdcd36c937a353b5b37ab50db3505712c35610b822cda322b5b.json

@@ -27,14 +27,14 @@ rand.workspace = true
 uuid = "1.10.0"
 ulid = { workspace = true, features = ["uuid"] }
 
+mas-config.workspace = true
+
 [dev-dependencies]
 mas-storage-pg.workspace = true
 
 anyhow.workspace = true
 insta.workspace = true
 serde.workspace = true
 
-mas-config.workspace = true
-
 [lints]
 workspace = true

crates/syn2mas/.sqlx/query-dfbd462f7874d3dae551f2a0328a853a8a7efccdc20b968d99d8c18deda8dd00.json

@@ -8,11 +8,13 @@ mod synapse_reader;
 
 mod migration;
 
-pub use self::mas_writer::locking::LockedMasDatabase;
-pub use self::mas_writer::{checks::mas_pre_migration_checks, MasWriter};
-pub use self::migration::migrate;
-pub use self::synapse_reader::checks::{
-    synapse_config_check, synapse_config_check_against_mas_config, synapse_database_check,
+pub use self::{
+    mas_writer::{checks::mas_pre_migration_checks, locking::LockedMasDatabase, MasWriter},
+    migration::migrate,
+    synapse_reader::{
+        checks::{
+            synapse_config_check, synapse_config_check_against_mas_config, synapse_database_check,
+        },
+        config as synapse_config, SynapseReader,
+    },
 };
-pub use self::synapse_reader::config as synapse_config;
-pub use self::synapse_reader::SynapseReader;

crates/syn2mas/Cargo.toml

@@ -5,7 +5,8 @@
 
 //! # MAS Database Checks
 //!
-//! This module provides safety checks to run against a MAS database before running the Synapse-to-MAS migration.
+//! This module provides safety checks to run against a MAS database before
+//! running the Synapse-to-MAS migration.
 
 use thiserror::Error;
 use thiserror_ext::ContextInto;
@@ -43,7 +44,8 @@ pub enum Error {
 ///
 /// - If any database access error occurs.
 /// - If any MAS tables involved in the migration are not empty.
-/// - If we can't check whether syn2mas is already in progress on this database or not.
+/// - If we can't check whether syn2mas is already in progress on this database
+///   or not.
 #[tracing::instrument(skip_all)]
 pub async fn mas_pre_migration_checks<'a>(
     mas_connection: &mut LockedMasDatabase<'a>,
@@ -56,7 +58,8 @@ pub async fn mas_pre_migration_checks<'a>(
         return Ok(());
     }
 
-    // Check that the database looks like a MAS database and that it is also an empty database.
+    // Check that the database looks like a MAS database and that it is also an
+    // empty database.
 
     for &table in MAS_TABLES_AFFECTED_BY_MIGRATION {
         let row_present = sqlx::query(&format!("SELECT 1 AS dummy FROM {table} LIMIT 1"))

crates/syn2mas/src/lib.rs

@@ -13,17 +13,19 @@ use sqlx::{
 static SYN2MAS_ADVISORY_LOCK: LazyLock<PgAdvisoryLock> =
     LazyLock::new(|| PgAdvisoryLock::new("syn2mas-maswriter"));
 
-/// A wrapper around a Postgres connection which holds a session-wide advisory lock
-/// preventing concurrent access by other syn2mas instances.
+/// A wrapper around a Postgres connection which holds a session-wide advisory
+/// lock preventing concurrent access by other syn2mas instances.
 pub struct LockedMasDatabase<'conn> {
     inner: PgAdvisoryLockGuard<'static, &'conn mut PgConnection>,
 }
 
 impl<'conn> LockedMasDatabase<'conn> {
-    /// Attempts to lock the MAS database against concurrent access by other syn2mas instances.
+    /// Attempts to lock the MAS database against concurrent access by other
+    /// syn2mas instances.
     ///
     /// If the lock can be acquired, returns a `LockedMasDatabase`.
-    /// If the lock cannot be acquired, returns the connection back to the caller wrapped in `Either::Right`.
+    /// If the lock cannot be acquired, returns the connection back to the
+    /// caller wrapped in `Either::Right`.
     ///
     /// # Errors
     ///