Merge pull request #77 from movementlabsxyz/l-monninger/aptos-runner-rewrite

feat: Embedded Aptos Runnner

Author: andygolay

Cargo.lock

@@ -6,18 +6,22 @@ members = [
 
     # migration
     "migration/cli/*",
-    "migration/core/*",
+    "migration/core/mtma",
+    "migration/core/node/*",
+    "migration/core/migrator/*",
     "migration/util/*",
     # executor
-    "checks/executor/util/*",
-    "checks/executor/citeria/*",
-    "checks/executor/checks/*",
-    "checks/executor/preludes",
+    "checks/node/util/*",
+    "checks/node/citeria/*",
+    "checks/node/checks/*",
+    "checks/node/preludes",
     # e2e
-    "checks/e2e/util/*",
-    "checks/e2e/citeria/*",
+    "checks/migrator/util/*",
+    "checks/migrator/citeria/*",
+    "checks/migrator/checks/*",
     # util
     "util/movement/*",
+    "util/movement-aptos/*",
 
 ]
 
@@ -38,6 +42,7 @@ async-trait = "0.1.71"
 clap = { version = "4.4.10", features = ["derive"] }
 dotenv = "0.15.0"
 futures = "0.3.17"
+futures-channel = "0.3.17"
 serde = "1.0"
 serde_json = "1.0.140"
 serde_derive = "1.0"
@@ -108,7 +113,8 @@ aptos-db = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "2
 aptos_schemadb = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "2941ee2b5876ab93cfb8010154c9466d7d593ed9" }
 aptos-config = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "2941ee2b5876ab93cfb8010154c9466d7d593ed9" }
 aptos-db-indexer = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "2941ee2b5876ab93cfb8010154c9466d7d593ed9" }
-# model checking and verification
+aptos-node = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "2941ee2b5876ab93cfb8010154c9466d7d593ed9" }
+aptos-cached-packages = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "2941ee2b5876ab93cfb8010154c9466d7d593ed9" }
 
 
 # secure-signing
@@ -148,29 +154,34 @@ mtma-track-dev = { path = "migration/cli/track-dev" }
 
 ### core
 mtma-core = { path = "migration/core/mtma" }
-mtma-null-core = { path = "migration/core/mtma-null" }
+#### migrator
+mtma-migrator-null-core = { path = "migration/core/migrator/mtma-null" }
+
+#### node
+mtma-node-null-core = { path = "migration/core/node/mtma-null" }
 
 ### preludes
-migration-executor-preludes = { path = "checks/executor/preludes" }
+mtma-node-preludes = { path = "checks/node/preludes" }
 
 ### util
-migration-executor-types = { path = "migration/util/executor-types" }
-
+mtma-node-types = { path = "migration/util/node-types" }
+mtma-migrator-types = { path = "migration/util/migrator-types" }
 ## checks
 ### executor
-migration-executor-test-types = { path = "checks/executor/util/types" }
-migration-executor-test-global-storage-injective-criterion = { path = "checks/executor/citeria/global-storage-injective" }
-migration-executor-test-global-storage-includes-criterion = { path = "checks/executor/citeria/global-storage-includes" }
-migration-executor-test-global-storage-not-empty-criterion = { path = "checks/executor/citeria/global-storage-not-empty" }
+mtma-node-test-types = { path = "checks/node/util/types" }
+mtma-node-test-global-storage-injective-criterion = { path = "checks/node/citeria/global-storage-injective" }
+mtma-node-test-global-storage-includes-criterion = { path = "checks/node/citeria/global-storage-includes" }
+mtma-node-test-global-storage-not-empty-criterion = { path = "checks/node/citeria/global-storage-not-empty" }
 
 
 ### e2e
-migration-e2e-test-types = { path = "checks/e2e/util/types" }
+mtma-migrator-test-types = { path = "checks/migrator/util/types" }
 
 ## util
 movement-syncing = { path = "util/movement/movement-syncing" }
 movement-core = { path = "util/movement/movement-core" }
 movement-util = { path = "util/movement/movement-util" }
+movement-aptos-core = { path = "util/movement-aptos/movement-aptos-core" }
 
 [workspace.lints.clippy]
 debug_assert_with_mut_call = "deny"

Cargo.toml

@@ -23,6 +23,13 @@ We otherwise recommend reading the [checks](./checks/README.md) and working down
 
 Please see [CONTRIBUTING.md](CONTRIBUTING.md) file for additional contribution guidelines.
 
+## Strategies
+The migration is organized into passes, which are categorized as follows:
+
+- [`node`](/migration/core/node) for migration passes that take place with access to node processes, memory, and disk.
+- `transaction` is suggested but not broken out for all over-the-wire/transaction-based migration passes. 
+- [`migrator`](migration/core/migrator) for migration passes that take place with direct or indirect access to all other levels of abstraction--everything that is needed to migrate. 
+
 ## Organization
 
 There are five subdirectories which progressively build on one another for node logic.

README.md

@@ -1,5 +1,5 @@
 [package]
-name = "migration-executor-checks-sketchpad"
+name = "mtma-migrator-checks-sketchpad"
 version = { workspace = true }
 edition = { workspace = true }
 license = { workspace = true }
@@ -9,16 +9,16 @@ publish = { workspace = true }
 rust-version = { workspace = true }
 
 [dependencies]
-migration-executor-test-types = { workspace = true }
+mtma-migrator-types = { workspace = true }
+mtma-migrator-test-types = { workspace = true }
 anyhow = { workspace = true }
-mtma-null-core = { workspace = true }
-migration-executor-test-global-storage-injective-criterion = { workspace = true }
-migration-executor-test-global-storage-includes-criterion = { workspace = true }
-migration-executor-test-global-storage-not-empty-criterion = { workspace = true }
-migration-executor-preludes = { workspace = true }
+mtma-node-null-core = { workspace = true }
 tokio = { workspace = true }
 chrono = { workspace = true }
 rand = { workspace = true }
+kestrel = { workspace = true }
+mtma-node-preludes = { workspace = true }
+mtma-node-test-types = { workspace = true }
 
 [dev-dependencies]
 tracing-test = { workspace = true }

checks/e2e/README.md renamed to checks/migrator/README.md

@@ -0,0 +1,50 @@
+#[cfg(test)]
+pub mod test {
+
+	use mtma_migrator_test_types::check::checked_migration;
+	use mtma_migrator_types::migrator::MovementMigrator;
+	use mtma_node_null_core::config::Config as MtmaNullConfig;
+	use mtma_node_test_types::prelude::{Prelude, PreludeError};
+
+	#[tokio::test]
+	#[tracing_test::traced_test]
+	async fn test_global_storage_includes_null() -> Result<(), anyhow::Error> {
+		// Form the migrator.
+		let mut movement_migrator = MovementMigrator::try_temp()?;
+
+		// Start the migrator so that it's running in the background.
+		// In the future, some migrators may be for already running nodes.
+		let movement_migrator_for_task = movement_migrator.clone();
+		let movement_migrator_task = kestrel::task(async move {
+			movement_migrator_for_task.run().await?;
+			Ok::<_, anyhow::Error>(())
+		});
+
+		// Form the prelude.
+		// todo: this needs to be updated to use the prelude generator
+		let prelude = Prelude::new_empty();
+
+		// Form the migration.
+		let migration_config = MtmaNullConfig::default();
+		let migration = migration_config.build()?;
+
+		// Run the checked migration.
+		checked_migration(&mut movement_migrator, &prelude, &migration, vec![]).await?;
+
+		// WITHIN THE CRITERION, you'd be able to do similar background startup and processing to the above. The [Migrationish] trait will give you a [MovementAptosMigrator] which can spawn the runner in the background.
+		//
+		// Note: WITHIN THE CRITERION, you'll be able to do something like the below.
+		// let movement_aptos_migrator_for_task = movement_aptos_migrator.clone();
+		// let movement_aptos_migrator_task = kestrel::task(async move {
+		// 	movement_aptos_migrator_for_task.run().await?;
+		// 	Ok::<_, anyhow::Error>(())
+		// });
+		// // TODO: SOMETHING
+		// kestrel::end!(movement_aptos_migrator_task)?;
+
+		// end the running migrators
+		kestrel::end!(movement_migrator_task)?;
+
+		Ok(())
+	}
+}

checks/executor/checks/sketchpad/Cargo.toml renamed to checks/migrator/checks/sketchpad/Cargo.toml

@@ -0,0 +1 @@
+pub mod example;

checks/executor/checks/sketchpad/README.md renamed to checks/migrator/checks/sketchpad/README.md

@@ -1,5 +1,5 @@
 [package]
-name = "migration-e2e-test-empty-criterion"
+name = "mtma-migrator-test-empty-criterion"
 version = { workspace = true }
 edition = { workspace = true }
 license = { workspace = true }
@@ -9,7 +9,7 @@ publish = { workspace = true }
 rust-version = { workspace = true }
 
 [dependencies]
-migration-executor-test-types = { workspace = true }
+mtma-node-test-types = { workspace = true }
 
 
 [lints]