Merge pull request #115 from movementlabsxyz/l-monninger/pre-l1-merge-redux

feat: pre-l1-merge migration

Author: l-monninger

.github/workflows/tests.yml

@@ -11,7 +11,7 @@ jobs:
         command:
           # cargo checks and tests
           - cargo check --all-targets
-          - cargo test
+          - RUST_MIN_STACK=8388608 cargo test -- --nocapture
 
     with:
       command: ${{ matrix.command }}

Cargo.lock

@@ -104,13 +104,16 @@ portpicker = "0.1.0"
 
 
 # movement
-maptos-opt-executor = { git = "https://github.com/movementlabsxyz/movement.git", rev = "8f820a5c23ce2b632200043acdabb3e6582c078a" }
-movement-client = { git = "https://github.com/movementlabsxyz/movement.git", rev = "8f820a5c23ce2b632200043acdabb3e6582c078a" }
-syncador = { git = "https://github.com/movementlabsxyz/movement.git", rev = "8f820a5c23ce2b632200043acdabb3e6582c078a" }
-movement-types = { git = "https://github.com/movementlabsxyz/movement.git", rev = "8f820a5c23ce2b632200043acdabb3e6582c078a" }
-maptos-execution-util = { git = "https://github.com/movementlabsxyz/movement.git", rev = "8f820a5c23ce2b632200043acdabb3e6582c078a" }
-movement-util = { git = "https://github.com/movementlabsxyz/movement.git", rev = "8f820a5c23ce2b632200043acdabb3e6582c078a" }
-
+maptos-opt-executor = { git = "https://github.com/movementlabsxyz/movement.git", rev = "07d5fe82f2c824a7f7e032c9dce339afdbbcb5a0" }
+movement-client = { git = "https://github.com/movementlabsxyz/movement.git", rev = "07d5fe82f2c824a7f7e032c9dce339afdbbcb5a0" }
+syncador = { git = "https://github.com/movementlabsxyz/movement.git", rev = "07d5fe82f2c824a7f7e032c9dce339afdbbcb5a0" }
+movement-types = { git = "https://github.com/movementlabsxyz/movement.git", rev = "07d5fe82f2c824a7f7e032c9dce339afdbbcb5a0" }
+maptos-execution-util = { git = "https://github.com/movementlabsxyz/movement.git", rev = "07d5fe82f2c824a7f7e032c9dce339afdbbcb5a0" }
+movement-util = { git = "https://github.com/movementlabsxyz/movement.git", rev = "07d5fe82f2c824a7f7e032c9dce339afdbbcb5a0" }
+aptos-framework-pre-l1-merge-release = { git = "https://github.com/movementlabsxyz/movement.git", rev = "07d5fe82f2c824a7f7e032c9dce339afdbbcb5a0" }
+movement-signer = { git = "https://github.com/movementlabsxyz/movement.git", rev = "07d5fe82f2c824a7f7e032c9dce339afdbbcb5a0" }
+movement-signer-loader = { git = "https://github.com/movementlabsxyz/movement.git", rev = "07d5fe82f2c824a7f7e032c9dce339afdbbcb5a0" }
+movement-config = { git = "https://github.com/movementlabsxyz/movement.git", rev = "07d5fe82f2c824a7f7e032c9dce339afdbbcb5a0" }
 
 # aptos-core
 aptos-executor = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "cb3b7b02f68c7d6982257e4c59955d0e63b513db" }
@@ -130,11 +133,6 @@ aptos-executor-types = { git = "https://github.com/movementlabsxyz/aptos-core.gi
 aptos-vm-genesis = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "cb3b7b02f68c7d6982257e4c59955d0e63b513db" }
 
 
-# secure-signing
-secure-signer = { git = "https://github.com/movementlabsxyz/secure-signing.git", rev = "f37eebd6d494d21b26b2faa44e1188de42ff1007" }
-secure-signer-loader = { git = "https://github.com/movementlabsxyz/secure-signing.git", rev = "f37eebd6d494d21b26b2faa44e1188de42ff1007" }
-secure-signer-aws-kms = { git = "https://github.com/movementlabsxyz/secure-signing.git", rev = "f37eebd6d494d21b26b2faa44e1188de42ff1007" }
-
 # kestrel 
 kestrel = { git = "https://github.com/movementlabsxyz/kestrel.git", rev = "3220d704df7e06d1dcc5266e15eaf05db86fdb07" }
 jsonlvar = { git = "https://github.com/movementlabsxyz/kestrel.git", rev = "3220d704df7e06d1dcc5266e15eaf05db86fdb07" }
@@ -169,8 +167,11 @@ mtma-track-dev = { path = "migration/cli/track-dev" }
 mtma-core = { path = "migration/core/mtma" }
 mtma-null-core = { path = "migration/core/mtma-null" }
 mtma-node-replay-core = { path = "migration/core/node/mtma-replay" }
+
 #### migrator
 mtma-migrator-null-core = { path = "migration/core/migrator/mtma-null" }
+mtma-migrator-pre-l1-merge-core = { path = "migration/core/migrator/pre-l1-merge" }
+mtma-migrate-chain-core = { path = "migration/core/mtma-migrate-chain" }
 
 #### node
 mtma-node-null-core = { path = "migration/core/node/mtma-null" }
@@ -253,12 +254,4 @@ debug = true
 [patch.crates-io]
 serde-reflection = { git = "https://github.com/aptos-labs/serde-reflection", rev = "73b6bbf748334b71ff6d7d09d06a29e3062ca075" }
 merlin = { git = "https://github.com/aptos-labs/merlin" }
-x25519-dalek = { git = "https://github.com/aptos-labs/x25519-dalek", branch = "zeroize_v1" }
-futures = { git = "https://github.com/aptos-labs/futures-rs", branch = "backport" }
-futures-channel = { git = "https://github.com/aptos-labs/futures-rs", branch = "backport" }
-futures-executor = { git = "https://github.com/aptos-labs/futures-rs", branch = "backport" }
-futures-core = { git = "https://github.com/aptos-labs/futures-rs", branch = "backport" }
-futures-util = { git = "https://github.com/aptos-labs/futures-rs", branch = "backport" }
-futures-sink = { git = "https://github.com/aptos-labs/futures-rs", branch = "backport" }
-futures-io = { git = "https://github.com/aptos-labs/futures-rs", branch = "backport" }
-futures-task = { git = "https://github.com/aptos-labs/futures-rs", branch = "backport" }
+x25519-dalek = { git = "https://github.com/aptos-labs/x25519-dalek", branch = "zeroize_v1" }

Cargo.toml

@@ -0,0 +1,37 @@
+[package]
+name = "mtma-migrator-checks-pre-l1-merge"
+version = { workspace = true }
+edition = { workspace = true }
+license = { workspace = true }
+authors = { workspace = true }
+homepage = { workspace = true }
+publish = { workspace = true }
+rust-version = { workspace = true }
+
+[dependencies]
+mtma-types = { workspace = true }
+mtma-migrator-types = { workspace = true }
+mtma-migrator-test-types = { workspace = true }
+anyhow = { workspace = true }
+mtma-node-replay-core = { workspace = true }
+mtma-node-test-global-storage-injective-criterion = { workspace = true }
+mtma-node-test-global-storage-includes-criterion = { workspace = true }
+mtma-node-test-global-storage-not-empty-criterion = { workspace = true }
+mtma-migrator-test-accounts-equal-criterion = { workspace = true }
+mtma-node-preludes = { workspace = true }
+mtma-node-null-core = { workspace = true }
+tokio = { workspace = true }
+chrono = { workspace = true }
+rand = { workspace = true }
+kestrel = { workspace = true }
+mtma-node-test-types = { workspace = true }
+mtma-migrator-pre-l1-merge-core = { workspace = true }
+movement-core = { workspace = true }
+
+[dev-dependencies]
+tracing-test = { workspace = true }
+tracing = { workspace = true }
+tempfile = { workspace = true }
+
+[lints]
+workspace = true

checks/migrator/checks/pre-l1-merge/Cargo.toml

@@ -0,0 +1,2 @@
+# Migration Executor Checks Pre-L1 Merge
+A basic testing crate for the pre-l1-merge portion of the migration. 

checks/migrator/checks/pre-l1-merge/README.md

@@ -0,0 +1 @@
+pub mod pre_l1_merge;

checks/migrator/checks/pre-l1-merge/src/lib.rs

@@ -0,0 +1,121 @@
+#[cfg(test)]
+pub mod test {
+
+	use anyhow::Context;
+	use movement_core::Movement;
+	use mtma_migrator_pre_l1_merge_core::config::Config as PreL1MergeConfig;
+	use mtma_migrator_test_accounts_equal_criterion::AccountsEqual;
+	use mtma_migrator_test_types::check::checked_migration;
+	use mtma_migrator_types::migrator::movement_migrator::{MovementMigrator, Overlays, Runner};
+	use mtma_node_test_types::prelude::Prelude;
+	use mtma_types::movement::aptos_sdk::types::account_config::aptos_test_root_address;
+	use mtma_types::movement::movement_config::Config as MovementConfig;
+	use tracing::info;
+
+	#[tokio::test(flavor = "multi_thread")]
+	#[tracing_test::traced_test]
+	async fn test_pre_l1_merge() -> Result<(), anyhow::Error> {
+		// use a scope to ensure everything is dropped
+		{
+			// Create a MovementConfig with the correct port
+			let mut movement_config = MovementConfig::default();
+			movement_config
+				.execution_config
+				.maptos_config
+				.client
+				.maptos_rest_connection_port = 30731;
+
+			// Get the account address from the same MovementConfig
+			let account_address = aptos_test_root_address();
+
+			// Create a Movement instance with the config
+			let movement = Movement::new(
+				movement_config.clone(),
+				movement_core::MovementWorkspace::try_temp()?,
+				Overlays::default(),
+				true,
+				true,
+			);
+
+			// Form the migrator with the configured Movement instance
+			let mut movement_migrator = MovementMigrator::new(Runner::Movement(movement));
+
+			// Start the migrator so that it's running in the background.
+			// In the future, some migrators may be for already running nodes.
+			let movement_migrator_for_task = movement_migrator.clone();
+			let movement_migrator_task = kestrel::task(async move {
+				movement_migrator_for_task.run().await?;
+				Ok::<_, anyhow::Error>(())
+			});
+
+			// Wait for the Celestia light node to be ready
+			tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
+
+			// wait for the rest client to be ready
+			info!("Waiting for REST client to be ready (timeout: 600s)");
+			let rest_api_url = movement_migrator
+				.wait_for_rest_api_url(tokio::time::Duration::from_secs(600))
+				.await?;
+			info!("REST API URL: {}", rest_api_url);
+
+			movement_migrator
+				.wait_for_rest_client_ready(tokio::time::Duration::from_secs(600))
+				.await
+				.context(
+					"failed to wait for movement migrator rest client while running accounts equal manual prelude",
+				)?;
+			info!("REST client is ready");
+
+			// Wait for the account to be created
+			info!("Getting REST client for account check (timeout: 30s)");
+			let rest_client = movement_migrator
+				.wait_for_rest_client_ready(tokio::time::Duration::from_secs(30))
+				.await
+				.context("failed to get rest client")?;
+			info!("Checking for account existence, address: {}", account_address);
+			match rest_client.get_account(account_address).await {
+				Ok(_) => {
+					info!("Account found");
+				}
+				Err(e) => {
+					info!("Account not found: {:?}", e);
+				}
+			}
+
+			// Form the prelude.
+			info!("Creating empty prelude");
+			let prelude = Prelude::new_empty();
+
+			// Form the migration.
+			let mut migration_config = PreL1MergeConfig::default();
+			migration_config.account_address = Some(account_address);
+			let migration = migration_config.build()?;
+
+			// Run the checked migration (skipping the accounts equal check).
+			info!("Running migration");
+			match checked_migration(
+				&mut movement_migrator,
+				&prelude,
+				&migration,
+				Vec::<AccountsEqual>::new(),
+			)
+			.await
+			{
+				Ok(()) => {}
+				Err(e) => {
+					info!("Migration failed: {:?}", e);
+					return Err(anyhow::anyhow!("Migration failed: {:?}", e));
+				}
+			}
+			info!("Migration succeeded");
+
+			kestrel::end!(movement_migrator_task)?;
+		}
+
+		// exit the test is fine when you only have one test per crate because when cargo test is run across a workspace, it actually multi-processes the tests by crate
+		std::process::exit(0);
+
+		#[allow(unreachable_code)]
+		Ok(())
+	}
+}

checks/migrator/checks/pre-l1-merge/src/pre_l1_merge.rs

@@ -25,6 +25,7 @@ chrono = { workspace = true }
 rand = { workspace = true }
 kestrel = { workspace = true }
 mtma-node-test-types = { workspace = true }
+mtma-migrator-pre-l1-merge-core = { workspace = true }
 
 [dev-dependencies]
 tracing-test = { workspace = true }

checks/migrator/checks/sketchpad/Cargo.toml

@@ -17,6 +17,7 @@ anyhow = { workspace = true }
 clap-markdown-ext = { workspace = true }
 orfile = { workspace = true }
 mtma-core = { workspace = true }
+mtma-migrate-chain-core = { workspace = true }
 jemallocator = { workspace = true }
 mtma-node-null-core = { workspace = true }
 mtma-node-test-global-storage-injective-criterion = { workspace = true }

migration/cli/migrate-chain-dev/Cargo.toml

@@ -103,12 +103,16 @@ Migrate from Movement to MovementAptos
 
 Run migrate with all parameters passed explicitly as CLI flags. See Orfile documentation for more details: <https://github.com/movementlabsxyz/orfile>
 
-**Usage:** `mtma-migrate-chain-dev migrate where --movement-state-db-path <MOVEMENT_STATE_DB_PATH> --movement-aptos-state-db-path <MOVEMENT_APTOS_STATE_DB_PATH>`
+**Usage:** `mtma-migrate-chain-dev migrate where --from <FROM> --to <TO> --maptos-signer <MAPTOS_SIGNER> --da-signer <DA_SIGNER> --mcr-signer <MCR_SIGNER> --movement-args <MOVEMENT_ARGS>`
 
 ###### **Options:**
 
-* `--movement-state-db-path <MOVEMENT_STATE_DB_PATH>` — The path to the input Movement state database
-* `--movement-aptos-state-db-path <MOVEMENT_APTOS_STATE_DB_PATH>` — The path to the output MovementAptos state database
+* `--from <FROM>` — The known release you are migrating from, eg elsa or biarritz-rc1
+* `--to <TO>` — The known release you are migrating to, eg biarritz-rc1 or pre-l1-merge
+* `--maptos-signer <MAPTOS_SIGNER>` — The canonical string for the Maptos signer used in the migration
+* `--da-signer <DA_SIGNER>` — The canonical string for the DA signer used in the migration
+* `--mcr-signer <MCR_SIGNER>` — The canonical string for the MCR signer used in the migration
+* `--movement-args <MOVEMENT_ARGS>` — Movement configuration arguments