syn2mas: Track skipped entities in the syn2mas progress (#4225)

Author: sandhose

crates/cli/src/commands/syn2mas.rs

@@ -1,4 +1,4 @@
-use std::{collections::HashMap, process::ExitCode, sync::atomic::Ordering, time::Duration};
+use std::{collections::HashMap, process::ExitCode, time::Duration};
 
 use anyhow::Context;
 use camino::Utf8PathBuf;
@@ -289,13 +289,14 @@ async fn occasional_progress_logger(progress: Progress) {
             }
             ProgressStage::MigratingData {
                 entity,
-                migrated,
+                counter,
                 approx_count,
             } => {
-                let migrated = migrated.load(Ordering::Relaxed);
+                let migrated = counter.migrated();
+                let skipped = counter.skipped();
                 #[allow(clippy::cast_precision_loss)]
-                let percent = (f64::from(migrated) / *approx_count as f64) * 100.0;
-                info!(name: "progress", "migrating {entity}: {migrated}/~{approx_count} (~{percent:.1}%)");
+                let percent = (f64::from(migrated + skipped) / *approx_count as f64) * 100.0;
+                info!(name: "progress", "migrating {entity}: {migrated} ({skipped} skipped) /~{approx_count} (~{percent:.1}%)");
             }
             ProgressStage::RebuildIndex { index_name } => {
                 info!(name: "progress", "still waiting for rebuild of index {index_name}");

crates/syn2mas/src/lib.rs

@@ -16,7 +16,7 @@ type HashMap<K, V> = rustc_hash::FxHashMap<K, V>;
 pub use self::{
     mas_writer::{MasWriter, checks::mas_pre_migration_checks, locking::LockedMasDatabase},
     migration::migrate,
-    progress::{Progress, ProgressStage},
+    progress::{Progress, ProgressCounter, ProgressStage},
     synapse_reader::{
         SynapseReader,
         checks::{

crates/syn2mas/src/mas_writer/mod.rs

@@ -29,6 +29,7 @@ use self::{
     constraint_pausing::{ConstraintDescription, IndexDescription},
     locking::LockedMasDatabase,
 };
+use crate::Progress;
 
 pub mod checks;
 pub mod locking;
@@ -550,16 +551,19 @@ impl MasWriter {
         conn: &mut LockedMasDatabase,
         indices_to_restore: &[IndexDescription],
         constraints_to_restore: &[ConstraintDescription],
+        progress: &Progress,
     ) -> Result<(), Error> {
         // First restore all indices. The order is not important as far as I know.
         // However the indices are needed before constraints.
         for index in indices_to_restore.iter().rev() {
+            progress.rebuild_index(index.name.clone());
             constraint_pausing::restore_index(conn.as_mut(), index).await?;
         }
         // Then restore all constraints.
         // The order here is the reverse of drop order, since some constraints may rely
         // on other constraints to work.
         for constraint in constraints_to_restore.iter().rev() {
+            progress.rebuild_constraint(constraint.name.clone());
             constraint_pausing::restore_constraint(conn.as_mut(), constraint).await?;
         }
         Ok(())
@@ -574,7 +578,7 @@ impl MasWriter {
     ///
     /// - If the database connection experiences an error.
     #[tracing::instrument(skip_all)]
-    pub async fn finish(mut self) -> Result<PgConnection, Error> {
+    pub async fn finish(mut self, progress: &Progress) -> Result<PgConnection, Error> {
         self.write_buffer_finish_checker.check_all_finished()?;
 
         // Commit all writer transactions to the database.
@@ -595,6 +599,7 @@ impl MasWriter {
             &mut self.conn,
             &self.indices_to_restore,
             &self.constraints_to_restore,
+            progress,
         )
         .await?;
 
@@ -1145,7 +1150,7 @@ mod test {
     use uuid::{NonNilUuid, Uuid};
 
     use crate::{
-        LockedMasDatabase, MasWriter,
+        LockedMasDatabase, MasWriter, Progress,
         mas_writer::{
             MasNewCompatAccessToken, MasNewCompatRefreshToken, MasNewCompatSession,
             MasNewEmailThreepid, MasNewUnsupportedThreepid, MasNewUpstreamOauthLink, MasNewUser,
@@ -1275,7 +1280,10 @@ mod test {
             .await
             .expect("failed to write user");
 
-        let mut conn = writer.finish().await.expect("failed to finish MasWriter");
+        let mut conn = writer
+            .finish(&Progress::default())
+            .await
+            .expect("failed to finish MasWriter");
 
         assert_db_snapshot!(&mut conn);
     }
@@ -1309,7 +1317,10 @@ mod test {
             .await
             .expect("failed to write password");
 
-        let mut conn = writer.finish().await.expect("failed to finish MasWriter");
+        let mut conn = writer
+            .finish(&Progress::default())
+            .await
+            .expect("failed to finish MasWriter");
 
         assert_db_snapshot!(&mut conn);
     }
@@ -1342,7 +1353,10 @@ mod test {
             .await
             .expect("failed to write e-mail");
 
-        let mut conn = writer.finish().await.expect("failed to finish MasWriter");
+        let mut conn = writer
+            .finish(&Progress::default())
+            .await
+            .expect("failed to finish MasWriter");
 
         assert_db_snapshot!(&mut conn);
     }
@@ -1376,7 +1390,10 @@ mod test {
             .await
             .expect("failed to write phone number (unsupported threepid)");
 
-        let mut conn = writer.finish().await.expect("failed to finish MasWriter");
+        let mut conn = writer
+            .finish(&Progress::default())
+            .await
+            .expect("failed to finish MasWriter");
 
         assert_db_snapshot!(&mut conn);
     }
@@ -1412,7 +1429,10 @@ mod test {
             .await
             .expect("failed to write link");
 
-        let mut conn = writer.finish().await.expect("failed to finish MasWriter");
+        let mut conn = writer
+            .finish(&Progress::default())
+            .await
+            .expect("failed to finish MasWriter");
 
         assert_db_snapshot!(&mut conn);
     }
@@ -1450,7 +1470,10 @@ mod test {
             .await
             .expect("failed to write compat session");
 
-        let mut conn = writer.finish().await.expect("failed to finish MasWriter");
+        let mut conn = writer
+            .finish(&Progress::default())
+            .await
+            .expect("failed to finish MasWriter");
 
         assert_db_snapshot!(&mut conn);
     }
@@ -1499,7 +1522,10 @@ mod test {
             .await
             .expect("failed to write access token");
 
-        let mut conn = writer.finish().await.expect("failed to finish MasWriter");
+        let mut conn = writer
+            .finish(&Progress::default())
+            .await
+            .expect("failed to finish MasWriter");
 
         assert_db_snapshot!(&mut conn);
     }
@@ -1560,7 +1586,10 @@ mod test {
             .await
             .expect("failed to write refresh token");
 
-        let mut conn = writer.finish().await.expect("failed to finish MasWriter");
+        let mut conn = writer
+            .finish(&Progress::default())
+            .await
+            .expect("failed to finish MasWriter");
 
         assert_db_snapshot!(&mut conn);
     }