feat: simplistic version of box environment for demonstration.

Author: l-monninger

Cargo.lock

@@ -31,7 +31,8 @@ members = [
     "util/movement/*",
     "util/movement-aptos/*",
     "util/bcs-ext",
-    "util/types"
+    "util/types",
+    "util/util"
 
 ]
 
@@ -215,6 +216,7 @@ movement-aptos = { path = "util/movement-aptos/movement-aptos" }
 movement-aptos-core = { path = "util/movement-aptos/core" }
 movement-aptos-core-util = { path = "util/movement-aptos/core-util" }
 mtma-types = { path = "util/types" }
+mtma-util = { path = "util/util" }
 
 ## Environments
 ### util

Cargo.toml

@@ -31,7 +31,7 @@ pub mod test {
 			// Create a Movement instance with the config
 			let movement = Movement::new(
 				movement_config.clone(),
-				movement_core::MovementWorkspace::try_temp()?,
+				movement_core::MovementWorkspace::try_temp()?.into(),
 				Overlays::default(),
 				true,
 				true,

checks/migrator/checks/pre-l1-merge/src/pre_l1_merge.rs

@@ -28,6 +28,7 @@ walkdir = { workspace = true }
 uuid = { workspace = true }
 tracing = { workspace = true }
 mtma-types = { workspace = true }
+mtma-util = { workspace = true }
 
 [lints]
 workspace = true

migration/core/migrator/mtma-null/Cargo.toml

@@ -8,28 +8,9 @@ use mtma_types::movement_aptos::aptos_db::AptosDB;
 use mtma_types::movement_aptos::aptos_storage_interface::DbReaderWriter;
 
 use anyhow::Context;
-use std::fs;
+use mtma_util::file::copy_dir_recursive;
 use std::path::Path;
 use tracing::info;
-use walkdir::WalkDir;
-
-/// Copies a directory recursively.
-///
-/// todo: move this out of the migration module
-fn copy_dir_recursive(src: &Path, dst: &Path) -> std::io::Result<()> {
-	for entry in WalkDir::new(src) {
-		let entry = entry?;
-		let rel_path = entry.path().strip_prefix(src).unwrap();
-		let dest_path = dst.join(rel_path);
-
-		if entry.file_type().is_dir() {
-			fs::create_dir_all(&dest_path)?;
-		} else {
-			fs::copy(entry.path(), &dest_path)?;
-		}
-	}
-	Ok(())
-}
 
 /// Errors thrown during the migration.
 #[derive(Debug, thiserror::Error)]

migration/core/migrator/mtma-null/src/migrate/migrate.rs

@@ -28,5 +28,7 @@ walkdir = { workspace = true }
 uuid = { workspace = true }
 tracing = { workspace = true }
 mtma-types = { workspace = true }
+mtma-util = { workspace = true }
+
 [lints]
 workspace = true

migration/core/node/mtma-null/Cargo.toml

@@ -7,28 +7,9 @@ use mtma_types::movement_aptos::aptos_db::AptosDB;
 use mtma_types::movement_aptos::aptos_storage_interface::DbReaderWriter;
 
 use anyhow::Context;
-use std::fs;
+use mtma_util::file::copy_dir_recursive;
 use std::path::Path;
 use tracing::info;
-use walkdir::WalkDir;
-
-/// Copies a directory recursively.
-///
-/// todo: move this out of the migration module
-fn copy_dir_recursive(src: &Path, dst: &Path) -> std::io::Result<()> {
-	for entry in WalkDir::new(src) {
-		let entry = entry?;
-		let rel_path = entry.path().strip_prefix(src).unwrap();
-		let dest_path = dst.join(rel_path);
-
-		if entry.file_type().is_dir() {
-			fs::create_dir_all(&dest_path)?;
-		} else {
-			fs::copy(entry.path(), &dest_path)?;
-		}
-	}
-	Ok(())
-}
 
 /// Errors thrown during the migration.
 #[derive(Debug, thiserror::Error)]

migration/core/node/mtma-null/src/migrate/migrate.rs

@@ -28,6 +28,8 @@ movement-client = { workspace = true }
 aptos-rest-client = { workspace = true }
 aptos-config = { workspace = true }
 mtma-types = { workspace = true }
+reqwest = { workspace = true }
+mtma-util = { workspace = true }
 
 [lints]
 workspace = true

migration/util/migrator-types/Cargo.toml

@@ -5,6 +5,11 @@ pub use movement_core::{Overlay, Overlays};
 use mtma_node_types::executor::MovementNode;
 pub use mtma_types::movement::aptos_types::{chain_id::ChainId, state_store::TStateView};
 use mtma_types::movement::movement_client::rest_client::Client as MovementRestClient;
+pub mod live;
+pub use live::LiveMigrator;
+use mtma_util::file::copy_dir_recursive;
+use std::path::PathBuf;
+use tracing::warn;
 
 /// An enum supporting different types of runners.
 ///
@@ -14,6 +19,8 @@ use mtma_types::movement::movement_client::rest_client::Client as MovementRestCl
 pub enum Runner {
 	/// [Movement] runner.
 	Movement(Movement),
+	/// [LiveMigrator] runner.
+	Live(LiveMigrator),
 }
 
 /// The Movement migration struct as would be presented in the criterion.
@@ -35,6 +42,7 @@ impl MovementMigrator {
 	pub async fn run(&self) -> Result<(), anyhow::Error> {
 		match &self.runner {
 			Runner::Movement(movement) => Ok(movement.run().await?),
+			Runner::Live(live) => Ok(live.run().await?),
 		}
 	}
 
@@ -71,6 +79,7 @@ impl MovementMigrator {
 					.context("failed to wait for Movement rest api")?;
 				Ok(rest_api.listen_url().to_string())
 			}
+			Runner::Live(live) => live.wait_for_live_rest_api_url().await,
 		}
 	}
 
@@ -94,13 +103,41 @@ impl MovementMigrator {
 			Runner::Movement(movement) => {
 				MovementNode::try_from_dir(movement.workspace_path().to_path_buf()).await
 			}
+			Runner::Live(live) => MovementNode::try_from_dir(live.dir.clone()).await,
 		}
 	}
 
 	/// Sets the overlays for the runner.
 	pub fn set_overlays(&mut self, overlays: Overlays) {
 		match &mut self.runner {
 			Runner::Movement(movement) => movement.set_overlays(overlays),
+			Runner::Live(_live) => {
+				warn!("Setting overlays for live migrator is not supported as the runner is already live. If you want to set overlays consider snapshotting the live migrator and using that as a movement runner.");
+			}
+		}
+	}
+
+	/// Gets the dir for the runner
+	pub fn dir(&self) -> PathBuf {
+		match &self.runner {
+			Runner::Movement(movement) => movement.workspace_path().to_path_buf(),
+			Runner::Live(live) => live.dir.clone(),
 		}
 	}
+
+	/// Recursively copies the dir for the runner to the given path
+	pub async fn copy_dir(&self, path: PathBuf) -> Result<(), anyhow::Error> {
+		let dir = self.dir();
+		copy_dir_recursive(&dir, &path).context("failed to copy dir for MovementMigrator")?;
+		Ok(())
+	}
+
+	/// Snapshots the migrator
+	///
+	/// NOTE: this can be used to, for example, transition from a live migrator to a movement runner.
+	pub async fn snapshot(&self, path: PathBuf) -> Result<Self, anyhow::Error> {
+		self.copy_dir(path.clone()).await?;
+
+		Ok(Self::new(Runner::Movement(Movement::try_from_dot_movement_dir(path)?)))
+	}
 }

migration/util/migrator-types/src/migrator/movement_migrator.rs

@@ -0,0 +1,59 @@
+use std::path::PathBuf;
+use tracing::warn;
+
+#[derive(Debug, Clone)]
+pub struct LiveMigrator {
+	pub rest_api_url: String,
+	pub dir: PathBuf,
+}
+
+impl LiveMigrator {
+	pub fn new(rest_api_url: String, dir: PathBuf) -> Self {
+		Self { rest_api_url, dir }
+	}
+}
+
+impl LiveMigrator {
+	pub async fn run(&self) -> Result<(), anyhow::Error> {
+		// checks for liveness by polling the rest api url
+		// once it becomes read the firs time, it should not later return an error
+		let mut ready = false;
+		loop {
+			match reqwest::get(self.rest_api_url.clone()).await {
+				Ok(response) => {
+					if response.status().is_success() {
+						ready = true;
+						continue;
+					} else if response.status().is_server_error() && ready {
+						return Err(anyhow::anyhow!("live migrator failed health check"));
+					} else {
+						warn!("live migrator not yet ready: response: {:?}", response);
+					}
+				}
+				Err(e) => {
+					warn!("failed to poll live migrator: {}", e);
+					if ready {
+						return Err(anyhow::anyhow!("live migrator failed health check"));
+					}
+				}
+			}
+		}
+	}
+
+	pub async fn wait_for_live_rest_api_url(&self) -> Result<String, anyhow::Error> {
+		// us request to poll the rest API url until is ready
+		loop {
+			match reqwest::get(self.rest_api_url.clone()).await {
+				Ok(response) => {
+					if response.status().is_success() {
+						return Ok(self.rest_api_url.clone());
+					}
+					warn!("rest api url is not ready yet: response: {:?}", response);
+				}
+				Err(e) => {
+					warn!("failed to poll rest api url: {}", e);
+				}
+			}
+		}
+	}
+}