Merge branch 'l-monninger/account-balances-redux' of https://github.com/movementlabsxyz/movement-to-maptos into l-monninger/expanded-docs

Author: l-monninger

Cargo.lock

@@ -84,6 +84,7 @@ libp2p = { version = "0.55.0", features = ["tcp", "quic"] }
 chrono = "0.4.31"
 rand = "0.7.3"
 uuid = "1.10.0"
+glob = "0.3.2"
 
 poem = { version = "=3.1.3", features = ["anyhow", "compression", "rustls"] }
 poem-openapi = { version = "=5.1.2", features = ["swagger-ui", "url"] }

Cargo.toml

@@ -29,6 +29,7 @@ mtma-node-test-types = { workspace = true }
 [dev-dependencies]
 tracing-test = { workspace = true }
 tracing = { workspace = true }
+tempfile = { workspace = true }
 
 [lints]
 workspace = true

checks/migrator/checks/sketchpad/Cargo.toml

@@ -28,5 +28,9 @@ uuid = { workspace = true }
 walkdir = { workspace = true }
 mtma-types = { workspace = true }
 
+[dev-dependencies]
+tempfile = { workspace = true }
+tracing-test = { workspace = true }
+
 [lints]
 workspace = true

migration/util/node-types/Cargo.toml

@@ -34,20 +34,74 @@ pub struct MovementNode {
 	opt_executor: MovementOptExecutor,
 }
 
-/// Copies a directory recursively.
-fn copy_dir_recursive(src: &Path, dst: &Path) -> std::io::Result<()> {
-	for entry in WalkDir::new(src) {
-		let entry = entry?;
-		let rel_path = entry.path().strip_prefix(src).unwrap();
-		let dest_path = dst.join(rel_path);
-
-		if entry.file_type().is_dir() {
-			fs::create_dir_all(&dest_path)?;
-		} else {
-			fs::copy(entry.path(), &dest_path)?;
-		}
-	}
-	Ok(())
+/// Copies a directory recursively while ignoring specified paths
+fn copy_dir_recursive_with_ignore(src: &Path, ignore_paths: impl IntoIterator<Item = impl AsRef<Path>>, dst: &Path) -> Result<(), anyhow::Error> {
+    let ignore_paths: Vec<_> = ignore_paths.into_iter().collect();
+    let mut errors: Vec<Result<(), anyhow::Error>> = Vec::new();
+
+    // Configure WalkDir to be more resilient
+    let walker = WalkDir::new(src)
+        .follow_links(false)
+        .same_file_system(true)
+        .into_iter()
+        .filter_entry(|entry| {
+            // Early filter for ignored paths to avoid permission errors
+            let path = entry.path();
+            !ignore_paths.iter().any(|ignore| path.to_string_lossy().contains(ignore.as_ref().to_string_lossy().as_ref()))
+        });
+
+    for entry in walker {
+        let entry = match entry {
+            Ok(e) => e,
+            Err(e) => {
+                // Check if this is a permission error on an ignored path
+                if let Some(path) = e.path() {
+                    if ignore_paths.iter().any(|ignore| path.to_string_lossy().contains(ignore.as_ref().to_string_lossy().as_ref())) {
+                        debug!("Ignoring permission error on ignored path: {}", path.display());
+                        continue;
+                    }
+                }
+                // For other errors, log with info for now and fail.
+                info!("Error accessing path: {:?}", e);
+				errors.push(Err(anyhow::anyhow!("failed to get entry: {:?}", e)));
+                continue;
+            }
+        };
+
+        debug!("Processing entry: {}", entry.path().display());
+
+        let path = entry.path();
+        let dest_path = dst.join(path.strip_prefix(src).context("failed to strip prefix")?);
+
+        if entry.file_type().is_dir() {
+            match fs::create_dir_all(&dest_path) {
+                Ok(_) => (),
+                Err(e) => errors.push(Err(e.into())),
+            }
+        } else {
+            if let Some(parent) = dest_path.parent() {
+                match fs::create_dir_all(parent) {
+                    Ok(_) => (),
+                    Err(e) => errors.push(Err(e.into())),
+                }
+            }
+            match fs::copy(path, &dest_path) {
+                Ok(_) => (),
+                Err(e) => errors.push(Err(e.into())),
+            }
+        }
+    }
+
+    // Combine all results into one
+    if !errors.is_empty() {
+        let mut total_error_message = String::from("failed to copy directory with the following errors: ");
+        for error in errors {
+            total_error_message = total_error_message + &format!("{:?}\n", error);
+        }
+        return Err(anyhow::anyhow!(total_error_message));
+    }
+
+    Ok(())
 }
 
 /// Sets all permission in a directory recursively.
@@ -74,13 +128,20 @@ impl MovementNode {
 
 		// Copy the entire .movement directory recursively
 		let movement_dir = dir.join(".movement");
-		copy_dir_recursive(&movement_dir, &debug_dir)?;
+
+		// set the permissions on the movement dir to 755
+		// Note: this would mess up celestia node permissions, but we don't care about that here.
+		// We really only care about maptos db permissions.
+		fs::set_permissions(&movement_dir, Permissions::from_mode(0o755)).context(format!("failed to set permissions on the movement directory {}", movement_dir.display()))?;
+
+		// don't copy anything from the celestia directory
+		copy_dir_recursive_with_ignore(&movement_dir, [PathBuf::from("celestia")], &debug_dir).context("failed to copy movement dir")?;
 
 		// Set all permissions in the debug directory recursively
 		// Note: this would mess up celestia node permissions, but we don't care about that here.
 		// We really only care about maptos db permissions.
 		// TODO: tighten the copying accordingly.
-		set_permissions_recursive(&debug_dir, Permissions::from_mode(0o755))?;
+		set_permissions_recursive(&debug_dir, Permissions::from_mode(0o755)).context(format!("failed to set permissions on the debug directory {}", debug_dir.display()))?;
 
 		let movement_args = MovementArgs { movement_path: Some(debug_dir.display().to_string()) };
 
@@ -415,3 +476,65 @@ impl<'a> Iterator for AccountAddressIterator<'a> {
 		self.get_next_address()
 	}
 }
+
+#[cfg(test)]
+mod test {
+	use super::*;
+	use tempfile::TempDir;
+
+	#[test]
+	#[tracing_test::traced_test]
+	fn test_copy_dir_recursive_with_ignore() -> Result<(), anyhow::Error> {
+		// put some file in a temp dir
+		let temp_dir = TempDir::new()?;
+
+		// write a file that should be copied
+		let file_path = temp_dir.path().join("maptos").join("test_file.txt");
+		fs::create_dir_all(file_path.parent().context("failed to get parent directory for file that should be copied")?).context("failed to create directory")?;
+		fs::write(file_path, "test").context("failed to write file that should be copied")?;
+
+		// write a file that should not be copied
+		let file_path = temp_dir.path().join("celestia").join("test_file2.txt");
+		fs::create_dir_all(file_path.parent().context("failed to get parent directory for file that should not be copied")?).context("failed to create directory")?;
+		fs::write(file_path, "test").context("failed to write file that should not be copied")?;
+
+		// create the target temp dir
+		let dst = TempDir::new()?;
+
+		// copy the file to a new dir, ignoring celestia directory
+		copy_dir_recursive_with_ignore(&temp_dir.path(), [PathBuf::from("celestia")], &dst.path()).context("failed to copy directory")?;
+
+		// check that the file was copied
+		assert!(dst.path().join("maptos").join("test_file.txt").exists());
+
+		// check that the celestia directory was not copied
+		assert!(!dst.path().join("celestia").join("test_file2.txt").exists());
+
+		Ok(())
+	}
+
+	// Somehow the following test is failing on CI: https://github.com/movementlabsxyz/movement-migration/actions/runs/15386989846/job/43287594343
+	// This indicates that failure is not due to the inability to ignore copy, but rather some issue performing an oepration that requires permissions.
+	#[test]
+	fn test_are_you_kidding_me() -> Result<(), anyhow::Error> {
+
+		let source_dir = TempDir::new()?;
+		let target_dir = TempDir::new()?;
+
+		let path_that_must_be_ignored = source_dir.path().join(".movement/celestia/c1860ae680eb2d91927b/.celestia-app/keyring-test");
+
+		fs::create_dir_all(path_that_must_be_ignored.parent().context("failed to get parent directory for path that must be ignored")?).context("failed to create directory")?;
+		// write a file that must not be ignored
+		fs::write(path_that_must_be_ignored.clone(), "test").context("failed to write file that must not be ignored")?;
+		// set permissions to 000 on the file and then on the parent directory
+		fs::set_permissions(path_that_must_be_ignored.clone(), Permissions::from_mode(0o000)).context("failed to set permissions on file that must not be ignored")?;
+		fs::set_permissions(path_that_must_be_ignored.parent().context("failed to get parent directory for path that must be ignored")?, Permissions::from_mode(0o000)).context("failed to set permissions on parent directory that must not be ignored")?;
+
+		copy_dir_recursive_with_ignore(source_dir.path(), ["celestia"], target_dir.path()).context("failed to copy directory")?;
+
+		assert!(!target_dir.path().join("celestia").join("c1860ae680eb2d91927b").join(".celestia-app").join("keyring-test").exists());
+
+		Ok(())
+	}
+
+}