Merge pull request #33 from movementlabsxyz/l-monninger/prelude

feat: introduce prelude and provides sketchpad migration test

Author: l-monninger

.github/workflows/nix-command.yml

@@ -19,7 +19,7 @@ on:
 
 jobs:
   run-command:
-    runs-on: ubuntu-latest
+    runs-on: buildjet-16vcpu-ubuntu-2204
     steps:
       - name: Free Disk Space (Ubuntu)
         uses: jlumbroso/free-disk-space@main

.gitignore

@@ -1,4 +1,6 @@
 target
 *.pem
 .debug
-.env*
+.env*
+.idea
+.direnv

Cargo.lock

@@ -4,20 +4,21 @@ resolver = "2"
 
 members = [
 
-"migration/cli/*",
-"migration/core/*", 
-
-# executor
-"checks/executor/util/*",
-"checks/executor/citeria/*",
-"checks/executor/cli/*",
-
-# e2e
-"checks/e2e/util/*",
-"checks/e2e/citeria/*",
-
-# util
-"util/include-dir"
+    # migration
+    "migration/cli/*",
+    "migration/core/*",
+    "migration/util/*",
+    # executor
+    "checks/executor/util/*",
+    "checks/executor/citeria/*",
+    "checks/executor/checks/*",
+    "checks/executor/preludes",
+    # e2e
+    "checks/e2e/util/*",
+    "checks/e2e/citeria/*",
+    # util
+    "util/include-dir",
+    "util/movement-syncing",
 
 ]
 
@@ -49,8 +50,8 @@ tracing-test = "0.2.5"
 tokio = { version = "1.35.1", features = ["full", "tracing"] }
 tokio-stream = "0.1.15"
 hex = { version = "0.4.3", default-features = false, features = [
-  "alloc",
-  "serde",
+    "alloc",
+    "serde",
 ] }
 zip = { version = "2.4.2", features = ["flate2"] }
 walkdir = "2.3.2"
@@ -72,6 +73,9 @@ tonic-reflection = "0.12.3"
 tonic-web = "0.12.3"
 prost = "0.13.3"
 libp2p = { version = "0.55.0", features = ["tcp", "quic"] }
+chrono = "0.4.31"
+rand = "0.7.3"
+uuid = "1.10.0"
 
 poem = { version = "=3.1.3", features = ["anyhow", "compression", "rustls"] }
 poem-openapi = { version = "=5.1.2", features = ["swagger-ui", "url"] }
@@ -83,25 +87,33 @@ jemallocator = { version = "0.5.0", features = [
 jemalloc-sys = "0.5.4"
 
 either = "1.8.1"
+tempdir = "0.3.7"
+sysinfo = "0.34.2"
 
 # movement
-maptos-opt-executor = { git = "https://github.com/movementlabsxyz/movement.git", rev = "f6e229e53066f068302dd78eb57d447bd2fa4d3e" }
-movement-client = { git = "https://github.com/movementlabsxyz/movement.git", rev = "f6e229e53066f068302dd78eb57d447bd2fa4d3e" }
-
-# maptos-core
-aptos-executor= { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "4e1871804307948d43002fc81d8ea723caeb5c31" }
-aptos-vm = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "4e1871804307948d43002fc81d8ea723caeb5c31" }
-aptos-rest-client = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "4e1871804307948d43002fc81d8ea723caeb5c31" }
-aptos-storage-interface = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "4e1871804307948d43002fc81d8ea723caeb5c31" }
-aptos-types = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "4e1871804307948d43002fc81d8ea723caeb5c31" }
-
+maptos-opt-executor = { git = "https://github.com/movementlabsxyz/movement.git", rev = "aa1ffed1a113441a65662792d15682ad52406108" }
+movement-client = { git = "https://github.com/movementlabsxyz/movement.git", rev = "aa1ffed1a113441a65662792d15682ad52406108" }
+syncador =  { git = "https://github.com/movementlabsxyz/movement.git", rev = "aa1ffed1a113441a65662792d15682ad52406108" }
+movement-types = { git = "https://github.com/movementlabsxyz/movement.git", rev = "aa1ffed1a113441a65662792d15682ad52406108" }
+maptos-execution-util = { git = "https://github.com/movementlabsxyz/movement.git", rev = "aa1ffed1a113441a65662792d15682ad52406108" }
+
+# aptos-core
+aptos-executor = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "2941ee2b5876ab93cfb8010154c9466d7d593ed9" }
+aptos-vm = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "2941ee2b5876ab93cfb8010154c9466d7d593ed9" }
+aptos-rest-client = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "2941ee2b5876ab93cfb8010154c9466d7d593ed9" }
+aptos-storage-interface = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "2941ee2b5876ab93cfb8010154c9466d7d593ed9" }
+aptos-types = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "2941ee2b5876ab93cfb8010154c9466d7d593ed9" }
+aptos-db = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "2941ee2b5876ab93cfb8010154c9466d7d593ed9" }
+aptos_schemadb = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "2941ee2b5876ab93cfb8010154c9466d7d593ed9" }
+aptos-config = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "2941ee2b5876ab93cfb8010154c9466d7d593ed9" }
+aptos-db-indexer = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "2941ee2b5876ab93cfb8010154c9466d7d593ed9" }
 # model checking and verification
 
 
 # secure-signing
 secure-signer = { git = "https://github.com/movementlabsxyz/secure-signing.git", rev = "f37eebd6d494d21b26b2faa44e1188de42ff1007" }
-secure-signer-loader = { git = "https://github.com/movementlabsxyz/secure-signing.git", rev = "f37eebd6d494d21b26b2faa44e1188de42ff1007"  }
-secure-signer-aws-kms = { git = "https://github.com/movementlabsxyz/secure-signing.git", rev = "f37eebd6d494d21b26b2faa44e1188de42ff1007"  }
+secure-signer-loader = { git = "https://github.com/movementlabsxyz/secure-signing.git", rev = "f37eebd6d494d21b26b2faa44e1188de42ff1007" }
+secure-signer-aws-kms = { git = "https://github.com/movementlabsxyz/secure-signing.git", rev = "f37eebd6d494d21b26b2faa44e1188de42ff1007" }
 
 # kestrel 
 kestrel = { git = "https://github.com/movementlabsxyz/kestrel.git", rev = "85e2f681ae3e54b29bb18d4ed7ca750116173321" }
@@ -112,20 +124,35 @@ commander = { git = "https://github.com/movementlabsxyz/kestrel.git", rev = "85e
 orfile = { git = "https://github.com/movementlabsxyz/orfile.git", rev = "d02299214e5aa202130cd1eb9593ce8064d1ec69" }
 
 # docs
-clap-markdown-ext = { git = "https://github.com/movementlabsxyz/clap-markdown-ext.git", rev = "8f54fe424504bf37fb01dc69aaed8166e429fe6a"}
+clap-markdown-ext = { git = "https://github.com/movementlabsxyz/clap-markdown-ext.git", rev = "8f54fe424504bf37fb01dc69aaed8166e429fe6a" }
 
 # internal
-movement-to-movement-aptos-core = { path = "migration/core/movement-to-movement-aptos" }
+## migration
+
+### core
+mtma-core = { path = "migration/core/mtma" }
+mtma-null-core = { path = "migration/core/mtma-null" }
+
+### preludes
+migration-executor-preludes = { path = "checks/executor/preludes" }
+
+### util
+migration-executor-types = { path = "migration/util/executor-types" }
 
 ## checks
 ### executor
 migration-executor-test-types = { path = "checks/executor/util/types" }
+migration-executor-test-global-storage-injective-criterion = { path = "checks/executor/citeria/global-storage-injective" }
+migration-executor-test-global-storage-includes-criterion = { path = "checks/executor/citeria/global-storage-includes" }
+migration-executor-test-global-storage-not-empty-criterion = { path = "checks/executor/citeria/global-storage-not-empty" }
+
 
 ### e2e
 migration-e2e-test-types = { path = "checks/e2e/util/types" }
 
 ## util
 include-dir = { path = "util/include-dir" }
+movement-syncing = { path = "util/movement-syncing" }
 
 [workspace.lints.clippy]
 debug_assert_with_mut_call = "deny"

Cargo.toml

@@ -0,0 +1,32 @@
+[package]
+name = "migration-executor-checks-mainnet"
+version = { workspace = true }
+edition = { workspace = true }
+license = { workspace = true }
+authors = { workspace = true }
+homepage = { workspace = true }
+publish = { workspace = true }
+rust-version = { workspace = true }
+
+[dependencies]
+migration-executor-test-types = { workspace = true }
+anyhow = { workspace = true }
+mtma-null-core = { workspace = true }
+migration-executor-test-global-storage-injective-criterion = { workspace = true }
+migration-executor-test-global-storage-includes-criterion = { workspace = true }
+migration-executor-test-global-storage-not-empty-criterion = { workspace = true }
+migration-executor-preludes = { workspace = true }
+movement-syncing = { workspace = true }
+maptos-execution-util = { workspace = true }
+tokio = { workspace = true }
+chrono = { workspace = true }
+rand = { workspace = true }
+futures = { workspace = true }
+sysinfo = { workspace = true }
+
+[dev-dependencies]
+tracing-test = { workspace = true }
+tracing = { workspace = true }
+
+[lints]
+workspace = true

checks/e2e/util/types/src/criterion/mod.rs renamed to checks/e2e/util/types/src/criterion.rs

@@ -0,0 +1,67 @@
+#[cfg(test)]
+pub mod test {
+
+	use futures::channel::mpsc as futures_mpsc;
+	use maptos_execution_util::config::Config as MaptosConfig;
+	use migration_executor_test_global_storage_includes_criterion::GlobalStorageIncludes;
+	use migration_executor_test_types::{
+		check::checked_migration,
+		criterion::movement_executor::{MovementExecutor, MovementOptExecutor},
+		prelude::Prelude,
+	};
+	use movement_syncing::db::DbSync;
+	use mtma_null_core::config::Config as MtmaNullConfig;
+	use sysinfo::Disks;
+
+	#[tokio::test]
+	#[tracing_test::traced_test]
+	async fn test_global_storage_includess_null() -> Result<(), anyhow::Error> {
+		let disks = Disks::new_with_refreshed_list();
+
+		let total_disk_space: u64 = disks.iter().map(|disk| disk.total_space()).sum();
+
+		let total_free_space: u64 = disks.iter().map(|disk| disk.available_space()).sum();
+
+		const REQUIRED_SPACE: u64 = 1_000_000_000_000; // 1 TB
+
+		if total_disk_space < REQUIRED_SPACE {
+			println!("Device has less than 1 TB of *total* disk space — skipping test (not a viable test platform).");
+			return Ok(()); // soft pass
+		}
+
+		if total_free_space < REQUIRED_SPACE {
+			println!("Device has enough total disk space, but less than 1 TB is free.");
+			return Err(anyhow::anyhow!("not enough free disk space"));
+		}
+
+		println!("Sufficient disk space available.");
+
+		// sync the db
+		let db_sync = DbSync::mainnet_debug();
+		db_sync.pull().await?;
+
+		// form the executor
+		let (sender, _receiver) = futures_mpsc::channel(100);
+		let mut config = MaptosConfig::default();
+		config.chain.maptos_db_path = Some(db_sync.destination_db_path().clone());
+		let movement_opt_executor = MovementOptExecutor::try_from_config(config, sender).await?;
+		let mut movement_executor = MovementExecutor::new(movement_opt_executor);
+
+		// empty prelude
+		let prelude = Prelude::new_empty();
+
+		// form the migration
+		let migration_config = MtmaNullConfig::default();
+		let migration = migration_config.build()?;
+
+		// run the checked migration
+		checked_migration(
+			&mut movement_executor,
+			&prelude,
+			&migration,
+			vec![Box::new(GlobalStorageIncludes::new())],
+		)
+		.await?;
+		Ok(())
+	}
+}

checks/executor/checks/movement-mainnet/Cargo.toml

@@ -0,0 +1 @@
+pub mod global_storage_includes;

checks/executor/checks/movement-mainnet/src/global_storage_includes.rs

@@ -0,0 +1,28 @@
+[package]
+name = "migration-executor-checks-sketchpad"
+version = { workspace = true }
+edition = { workspace = true }
+license = { workspace = true }
+authors = { workspace = true }
+homepage = { workspace = true }
+publish = { workspace = true }
+rust-version = { workspace = true }
+
+[dependencies]
+migration-executor-test-types = { workspace = true }
+anyhow = { workspace = true }
+mtma-null-core = { workspace = true }
+migration-executor-test-global-storage-injective-criterion = { workspace = true }
+migration-executor-test-global-storage-includes-criterion = { workspace = true }
+migration-executor-test-global-storage-not-empty-criterion = { workspace = true }
+migration-executor-preludes = { workspace = true }
+tokio = { workspace = true }
+chrono = { workspace = true }
+rand = { workspace = true }
+
+[dev-dependencies]
+tracing-test = { workspace = true }
+tracing = { workspace = true }
+
+[lints]
+workspace = true

checks/executor/checks/movement-mainnet/src/lib.rs

@@ -0,0 +1,2 @@
+# Migration Executor Checks Sketchpad
+A sketchpad for playing around with checks on the testpad.