Merge pull request #91 from movementlabsxyz/l-monninger/fix-movement-aptos-core

l-monninger · web-flow · commit 8f3e0d8b2276 · 2025-05-25T15:15:19.000-10:00
fix: movement-aptos-core
diff --git a/.gitignore b/.gitignore
@@ -1,6 +1,7 @@
 target
 *.pem
 .debug
+debug
 .env*
 .idea
 .direnv
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -45,6 +45,7 @@ futures = "0.3.17"
 futures-channel = "0.3.17"
 serde = "1.0"
 serde_json = "1.0.140"
+serde_yaml = "0.8.24"
 serde_derive = "1.0"
 thiserror = "1.0.50"
 tracing = "0.1.40"
@@ -94,6 +95,7 @@ reqwest = { version = "0.12.11", features = ["json"] }
 either = "1.8.1"
 tempdir = "0.3.7"
 sysinfo = "0.34.2"
+portpicker = "0.1.0"
 
 
 # movement
@@ -104,17 +106,17 @@ movement-types = { git = "https://github.com/movementlabsxyz/movement.git", rev
 maptos-execution-util = { git = "https://github.com/movementlabsxyz/movement.git", rev = "aa1ffed1a113441a65662792d15682ad52406108" }
 
 # aptos-core
-aptos-executor = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "2941ee2b5876ab93cfb8010154c9466d7d593ed9" }
-aptos-vm = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "2941ee2b5876ab93cfb8010154c9466d7d593ed9" }
-aptos-rest-client = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "2941ee2b5876ab93cfb8010154c9466d7d593ed9" }
-aptos-storage-interface = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "2941ee2b5876ab93cfb8010154c9466d7d593ed9" }
-aptos-types = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "2941ee2b5876ab93cfb8010154c9466d7d593ed9" }
-aptos-db = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "2941ee2b5876ab93cfb8010154c9466d7d593ed9" }
-aptos_schemadb = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "2941ee2b5876ab93cfb8010154c9466d7d593ed9" }
-aptos-config = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "2941ee2b5876ab93cfb8010154c9466d7d593ed9" }
-aptos-db-indexer = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "2941ee2b5876ab93cfb8010154c9466d7d593ed9" }
-aptos-node = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "2941ee2b5876ab93cfb8010154c9466d7d593ed9" }
-aptos-cached-packages = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "2941ee2b5876ab93cfb8010154c9466d7d593ed9" }
+aptos-executor = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "cb3b7b02f68c7d6982257e4c59955d0e63b513db" }
+aptos-vm = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "cb3b7b02f68c7d6982257e4c59955d0e63b513db" }
+aptos-rest-client = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "cb3b7b02f68c7d6982257e4c59955d0e63b513db" }
+aptos-storage-interface = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "cb3b7b02f68c7d6982257e4c59955d0e63b513db" }
+aptos-types = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "cb3b7b02f68c7d6982257e4c59955d0e63b513db" }
+aptos-db = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "cb3b7b02f68c7d6982257e4c59955d0e63b513db" }
+aptos_schemadb = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "cb3b7b02f68c7d6982257e4c59955d0e63b513db" }
+aptos-config = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "cb3b7b02f68c7d6982257e4c59955d0e63b513db" }
+aptos-db-indexer = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "cb3b7b02f68c7d6982257e4c59955d0e63b513db" }
+aptos-node = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "cb3b7b02f68c7d6982257e4c59955d0e63b513db" }
+aptos-cached-packages = { git = "https://github.com/movementlabsxyz/aptos-core.git", rev = "cb3b7b02f68c7d6982257e4c59955d0e63b513db" }
 
 
 # secure-signing
@@ -123,12 +125,12 @@ secure-signer-loader = { git = "https://github.com/movementlabsxyz/secure-signin
 secure-signer-aws-kms = { git = "https://github.com/movementlabsxyz/secure-signing.git", rev = "f37eebd6d494d21b26b2faa44e1188de42ff1007" }
 
 # kestrel 
-kestrel = { git = "https://github.com/movementlabsxyz/kestrel.git", rev = "96fa1a31088af992748c8dbe6b8fe4ccfba771fa" }
-jsonlvar = { git = "https://github.com/movementlabsxyz/kestrel.git", rev = "96fa1a31088af992748c8dbe6b8fe4ccfba771fa" }
-commander = { git = "https://github.com/movementlabsxyz/kestrel.git", rev = "96fa1a31088af992748c8dbe6b8fe4ccfba771fa" }
-include-dir = { git = "https://github.com/movementlabsxyz/kestrel.git", rev = "96fa1a31088af992748c8dbe6b8fe4ccfba771fa" }
-include-vendor = { git = "https://github.com/movementlabsxyz/kestrel.git", rev = "96fa1a31088af992748c8dbe6b8fe4ccfba771fa" }
-ready-docker = { git = "https://github.com/movementlabsxyz/kestrel.git", rev = "96fa1a31088af992748c8dbe6b8fe4ccfba771fa" }
+kestrel = { git = "https://github.com/movementlabsxyz/kestrel.git", rev = "9c56eb9f6dde1564493835c7e2c61b9fd2b00a6c" }
+jsonlvar = { git = "https://github.com/movementlabsxyz/kestrel.git", rev = "9c56eb9f6dde1564493835c7e2c61b9fd2b00a6c" }
+commander = { git = "https://github.com/movementlabsxyz/kestrel.git", rev = "9c56eb9f6dde1564493835c7e2c61b9fd2b00a6c" }
+include-dir = { git = "https://github.com/movementlabsxyz/kestrel.git", rev = "9c56eb9f6dde1564493835c7e2c61b9fd2b00a6c" }
+include-vendor = { git = "https://github.com/movementlabsxyz/kestrel.git", rev = "9c56eb9f6dde1564493835c7e2c61b9fd2b00a6c" }
+ready-docker = { git = "https://github.com/movementlabsxyz/kestrel.git", rev = "9c56eb9f6dde1564493835c7e2c61b9fd2b00a6c" }
 
 # orfile
 orfile = { git = "https://github.com/movementlabsxyz/orfile.git", rev = "b49cb42495816fa00a1107be5bbf41aff3a8255d" }
@@ -181,6 +183,7 @@ mtma-migrator-test-types = { path = "checks/migrator/util/types" }
 movement-syncing = { path = "util/movement/movement-syncing" }
 movement-core = { path = "util/movement/movement-core" }
 movement-util = { path = "util/movement/movement-util" }
+movement-aptos = { path = "util/movement-aptos/movement-aptos" }
 movement-aptos-core = { path = "util/movement-aptos/movement-aptos-core" }
 
 [workspace.lints.clippy]
diff --git a/checks/migrator/checks/sketchpad/src/example.rs b/checks/migrator/checks/sketchpad/src/example.rs
@@ -6,6 +6,7 @@ pub mod test {
 	use mtma_node_null_core::config::Config as MtmaNullConfig;
 	use mtma_node_test_types::prelude::Prelude;
 
+	#[ignore] // this is just an example, so it's not expected to pass
 	#[tokio::test]
 	#[tracing_test::traced_test]
 	async fn test_global_storage_includes_null() -> Result<(), anyhow::Error> {
diff --git a/checks/migrator/util/types/src/check.rs b/checks/migrator/util/types/src/check.rs
@@ -5,13 +5,13 @@ use mtma_node_test_types::prelude::Prelude;
 /// Errors thrown when working with the [Config].
 #[derive(Debug, thiserror::Error)]
 pub enum CheckError {
-	#[error("failed to run prelude: {0}")]
+	#[error("checked migration encountered an error while running prelude: {0}")]
 	Prelude(#[source] Box<dyn std::error::Error + Send + Sync>),
-	#[error("failed to run migration: {0}")]
+	#[error("checked migration encountered an error while running migration: {0}")]
 	Migration(#[source] Box<dyn std::error::Error + Send + Sync>),
-	#[error("failed to satisfy criteria: {0}")]
+	#[error("checked migration encountered an error while satisfying criteria: {0}")]
 	Criteria(#[source] Box<dyn std::error::Error + Send + Sync>),
-	#[error("internal error: {0}")]
+	#[error("checked migration encountered an internal error: {0}")]
 	Internal(#[source] Box<dyn std::error::Error + Send + Sync>),
 }
 
diff --git a/checks/migrator/util/types/src/criterion.rs b/checks/migrator/util/types/src/criterion.rs
@@ -42,7 +42,7 @@ impl MovementMigratorClient {
 /// Errors thrown when working with the [Config].
 #[derive(Debug, thiserror::Error)]
 pub enum CriterionError {
-	#[error("failed to build from config: {0}")]
+	#[error("criterion was not satisfied: {0}")]
 	Unsatisfied(#[source] Box<dyn std::error::Error + Send + Sync>),
 }
 
diff --git a/checks/node/util/types/src/check.rs b/checks/node/util/types/src/check.rs
@@ -5,13 +5,13 @@ use mtma_node_types::migration::Migrationish;
 /// Errors thrown when working with the [Config].
 #[derive(Debug, thiserror::Error)]
 pub enum CheckError {
-	#[error("failed to run prelude: {0}")]
+	#[error("check encountered an error while running prelude: {0}")]
 	Prelude(#[source] Box<dyn std::error::Error + Send + Sync>),
-	#[error("failed to run migration: {0}")]
+	#[error("check encountered an error while running migration: {0}")]
 	Migration(#[source] Box<dyn std::error::Error + Send + Sync>),
-	#[error("failed to satisfy criteria: {0}")]
+	#[error("check encountered an error while satisfying criteria: {0}")]
 	Criteria(#[source] Box<dyn std::error::Error + Send + Sync>),
-	#[error("internal error: {0}")]
+	#[error("check encountered an internal error: {0}")]
 	Internal(#[source] Box<dyn std::error::Error + Send + Sync>),
 }
 
diff --git a/checks/node/util/types/src/criterion.rs b/checks/node/util/types/src/criterion.rs
@@ -7,9 +7,9 @@ pub use mtma_node_types::executor::movement_executor::MovementNode;
 /// Errors thrown when working with the [Config].
 #[derive(Debug, thiserror::Error)]
 pub enum CriterionError {
-	#[error("the criterion is unsatisfied: {0}")]
+	#[error("the criterion was not satisfied: {0}")]
 	Unsatisfied(#[source] Box<dyn std::error::Error + Send + Sync>),
-	#[error("internal error: {0}")]
+	#[error("criterion encountered an internal error: {0}")]
 	Internal(#[source] Box<dyn std::error::Error + Send + Sync>),
 }
 
diff --git a/checks/node/util/types/src/prelude.rs b/checks/node/util/types/src/prelude.rs
@@ -17,7 +17,7 @@ const DEFAULT_ROUND: u64 = 0;
 /// Errors thrown when working with the [Config].
 #[derive(Debug, thiserror::Error)]
 pub enum PreludeError {
-	#[error("internal error: {0}")]
+	#[error("prelude encountered an internal error: {0}")]
 	Internal(#[source] Box<dyn std::error::Error + Send + Sync>),
 }
 
diff --git a/clap-markdown-list b/clap-markdown-list
@@ -1,4 +1,5 @@
 movement
+movement-aptos
 mtma-check
 mtma-check-dev
 mtma-migrate
diff --git a/docs/cli/README.md b/docs/cli/README.md
@@ -1,5 +1,6 @@
 # CLIs
 - [movement](../../util/movement/movement/docs/cli/README.md)
+- [movement-aptos](../../util/movement-aptos/movement-aptos/docs/cli/README.md)
 - [mtma-check](../../migration/cli/check/docs/cli/README.md)
 - [mtma-check-dev](../../migration/cli/check-dev/docs/cli/README.md)
 - [mtma-migrate](../../migration/cli/migrate/docs/cli/README.md)
diff --git a/migration/core/migrator/mtma-null/src/migrate/config.rs b/migration/core/migrator/mtma-null/src/migrate/config.rs
@@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize};
 /// Errors thrown when working with the [Config].
 #[derive(Debug, thiserror::Error)]
 pub enum MigrateConfigError {
-	#[error("failed to build from config: {0}")]
+	#[error("failed to build mtma-null migrator from config: {0}")]
 	Build(#[source] Box<dyn std::error::Error + Send + Sync>),
 }
 
diff --git a/migration/core/migrator/mtma-null/src/migrate/migrate.rs b/migration/core/migrator/mtma-null/src/migrate/migrate.rs
@@ -33,7 +33,7 @@ fn copy_dir_recursive(src: &Path, dst: &Path) -> std::io::Result<()> {
 /// Errors thrown during the migration.
 #[derive(Debug, thiserror::Error)]
 pub enum MigrateError {
-	#[error("failed to migrate: {0}")]
+	#[error("mtma-null migrator failed to migrate: {0}")]
 	Migrate(#[source] Box<dyn std::error::Error + Send + Sync>),
 }
 
diff --git a/migration/core/mtma/src/migrate/config.rs b/migration/core/mtma/src/migrate/config.rs
@@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize};
 /// Errors thrown when working with the [Config].
 #[derive(Debug, thiserror::Error)]
 pub enum MigrateConfigError {
-	#[error("failed to build from config: {0}")]
+	#[error("mtma failed to build from config: {0}")]
 	Build(#[source] Box<dyn std::error::Error + Send + Sync>),
 }
 
diff --git a/util/movement-aptos/movement-aptos-core/Cargo.toml b/util/movement-aptos/movement-aptos-core/Cargo.toml
@@ -25,10 +25,18 @@ kestrel = { workspace = true }
 reqwest = { workspace = true }
 aptos-cached-packages = { workspace = true }
 rand = { workspace = true }
+chrono = { workspace = true }
+uuid = { workspace = true }
+serde_yaml = { workspace = true }
+tracing = { workspace = true }
 
 [dev-dependencies]
 uuid = { workspace = true }
 chrono = { workspace = true }
+tracing-test = { workspace = true }
+
+[build-dependencies]
+cargo_metadata = { workspace = true }
 
 [lints]
 workspace = true
diff --git a/util/movement-aptos/movement-aptos-core/build.rs b/util/movement-aptos/movement-aptos-core/build.rs
@@ -0,0 +1,77 @@
+// Clones https://github.com/movementlabsxyz/aptos-core/commit/e7ad67cfaa13127d41be7eb3b889e62efa7a2bf9
+// Runs cargo build -p aptos in it and copies the aptos binary to a known location in target
+// Caches to avoid rebuilding unless the provided hash has changed
+// Uses hash to ensure cash collisions are properly detected.
+// There are a lot of issues with aptos-core builds in nix with crane and similar, so this is a bit of a work around.
+
+use cargo_metadata::MetadataCommand;
+use std::env;
+use std::fs;
+use std::path::PathBuf;
+use std::process::Command;
+
+const APTOS_CORE_REPO: &str = "https://github.com/movementlabsxyz/aptos-core";
+const APTOS_CORE_COMMIT: &str = "fbfde356df4b1316995d03d59542005f1b277f66";
+const CACHE_DIR: &str = "aptos_build_cache";
+
+fn main() {
+	// Get workspace metadata
+	let metadata = MetadataCommand::new().exec().expect("Failed to get cargo metadata");
+
+	// Get the workspace root
+	let workspace_root = metadata.workspace_root;
+	let workspace_release_dir = workspace_root.join("target/release");
+
+	// ensure the workspace release directory exists
+	fs::create_dir_all(&workspace_release_dir)
+		.expect("Failed to create workspace release directory");
+
+	// Get the out directory
+	let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
+	let cache_path = out_dir.join(CACHE_DIR);
+	let commit_cache = cache_path.join(format!(
+		"{}_{}",
+		APTOS_CORE_COMMIT,
+		env::var("TARGET").unwrap_or_default()
+	));
+	let aptos_bin_path = commit_cache.join("target/release/aptos");
+
+	if fs::metadata(&aptos_bin_path).is_ok() {
+		println!("cargo:warning=Using cached aptos binary");
+	} else {
+		println!("cargo:warning=No cache found, building from scratch");
+
+		// ensure the parent directory for the commit cache exists
+		let parent_dir = commit_cache.parent().unwrap();
+		fs::create_dir_all(parent_dir).expect("Failed to create parent directory for commit cache");
+
+		// clone if not exists
+		if !fs::metadata(&commit_cache).is_ok() {
+			println!("cargo:warning=Cloning repository...");
+			Command::new("git")
+				.args(&["clone", APTOS_CORE_REPO, &commit_cache.to_string_lossy()])
+				.status()
+				.expect("Failed to clone repository");
+		}
+
+		// check out the desired commit
+		println!("cargo:warning=Checking out commit {}", APTOS_CORE_COMMIT);
+		Command::new("git")
+			.args(&["checkout", APTOS_CORE_COMMIT])
+			.current_dir(&commit_cache)
+			.status()
+			.expect("Failed to checkout commit");
+
+		// build the aptos binary
+		println!("cargo:warning=Building aptos binary...");
+		Command::new("cargo")
+			.current_dir(&commit_cache)
+			.args(&["build", "-p", "aptos", "--release"])
+			.status()
+			.expect("Failed to build aptos");
+	}
+
+	// copy aptos binary to the workspace release directory
+	fs::copy(&aptos_bin_path, &workspace_release_dir.join("aptos"))
+		.expect("Failed to copy aptos binary to workspace release directory");
+}
diff --git a/util/movement-aptos/movement-aptos-core/src/config.rs b/util/movement-aptos/movement-aptos-core/src/config.rs
@@ -3,14 +3,15 @@ use clap::Parser;
 use jsonlvar::Jsonl;
 use orfile::Orfile;
 use serde::{Deserialize, Serialize};
+use std::fs::File;
 use std::path::PathBuf;
 use std::str::FromStr;
 
 use crate::movement_aptos::{runtime, MovementAptos};
 use aptos_node::create_single_node_test_config;
 
 #[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct NodeConfigWrapper(NodeConfig);
+pub struct NodeConfigWrapper(pub(crate) NodeConfig);
 
 impl NodeConfigWrapper {
 	pub fn new(node_config: NodeConfig) -> Self {
@@ -74,7 +75,7 @@ impl Config {
 		let rng = rand::thread_rng();
 
 		let node_config = create_single_node_test_config(
-			&None,
+			&Some(db_dir.join("config.json")),
 			&None,
 			db_dir.as_path(),
 			true,
@@ -90,10 +91,22 @@ impl Config {
 
 	/// Builds the config into a [MovementAptos] runner.
 	pub fn build(&self) -> Result<MovementAptos<runtime::TokioTest>, ConfigError> {
-		Ok(MovementAptos::<runtime::TokioTest>::try_new(
-			self.node_config.0.clone(),
-			self.log_file.clone(),
-		)
-		.map_err(|e| ConfigError::Internal(e.into()))?)
+		// get the workspace dir
+		let workspace_dir =
+			self.node_config.node_config().base.working_dir.clone().unwrap_or_default();
+
+		// get the config path
+		let config_path = workspace_dir.join("config.yaml");
+
+		// write the config to the config path
+		let config_file = File::create(config_path).map_err(|e| ConfigError::Internal(e.into()))?;
+		serde_json::to_writer(config_file, &self.node_config.node_config())
+			.map_err(|e| ConfigError::Internal(e.into()))?;
+
+		Ok(MovementAptos::<runtime::TokioTest>::new(
+			self.node_config.node_config().clone(),
+			false,
+			workspace_dir,
+		))
 	}
 }
diff --git a/util/movement-aptos/movement-aptos-core/src/movement_aptos.rs b/util/movement-aptos/movement-aptos-core/src/movement_aptos.rs
diff --git a/util/movement-aptos/movement-aptos/Cargo.toml b/util/movement-aptos/movement-aptos/Cargo.toml
diff --git a/util/movement-aptos/movement-aptos/docs/cli/README.md b/util/movement-aptos/movement-aptos/docs/cli/README.md
diff --git a/util/movement-aptos/movement-aptos/src/cli.rs b/util/movement-aptos/movement-aptos/src/cli.rs
diff --git a/util/movement-aptos/movement-aptos/src/cli/run.rs b/util/movement-aptos/movement-aptos/src/cli/run.rs
diff --git a/util/movement-aptos/movement-aptos/src/lib.rs b/util/movement-aptos/movement-aptos/src/lib.rs
diff --git a/util/movement-aptos/movement-aptos/src/main.rs b/util/movement-aptos/movement-aptos/src/main.rs

-Original file line number
+Diff line change
@@ @@ -1,6 +1,7 @@ @@
 target
 *.pem
 .debug
 +debug
 .env*
 .idea
 .direnv
Original file line number	Diff line number	Diff line change
`@@ -42,7 +42,7 @@ impl MovementMigratorClient {`
`42`	`42`	`/// Errors thrown when working with the [Config].`
`43`	`43`	`#[derive(Debug, thiserror::Error)]`
`44`	`44`	`pub enum CriterionError {`
`45`		`- #[error("failed to build from config: {0}")]`
	`45`	`+ #[error("criterion was not satisfied: {0}")]`
`46`	`46`	`Unsatisfied(#[source] Box<dyn std::error::Error + Send + Sync>),`
`47`	`47`	`}`
`48`	`48`
Original file line number	Diff line number	Diff line change
`@@ -17,7 +17,7 @@ const DEFAULT_ROUND: u64 = 0;`
`17`	`17`	`/// Errors thrown when working with the [Config].`
`18`	`18`	`#[derive(Debug, thiserror::Error)]`
`19`	`19`	`pub enum PreludeError {`
`20`		`- #[error("internal error: {0}")]`
	`20`	`+ #[error("prelude encountered an internal error: {0}")]`
`21`	`21`	`Internal(#[source] Box<dyn std::error::Error + Send + Sync>),`
`22`	`22`	`}`
`23`	`23`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`movement`
	`2`	`+movement-aptos`
`2`	`3`	`mtma-check`
`3`	`4`	`mtma-check-dev`
`4`	`5`	`mtma-migrate`