Compare balances for specific ledger versions (#210)

## Description
Add state comparison cmd.

Co-authored-by: musitdev <[email protected]>

Author: sebtomba

movement-migration/validation-tool/src/checks/node.rs

@@ -1,43 +1,31 @@
 // Copyright (c) Aptos Foundation
 // SPDX-License-Identifier: Apache-2.0
 
-use crate::{
-    checks::node::global_storage_includes::GlobalStorageIncludes,
-    types::storage::{MovementAptosStorage, MovementStorage},
-};
-use clap::Parser;
-use std::path::PathBuf;
+use crate::checks::node::global_storage_includes::CompareDbCmd;
+use crate::checks::node::state_diff::CompareStatesCmd;
+use clap::Subcommand;
 
 mod global_storage_includes;
+mod state_diff;
 
-#[derive(Parser)]
-#[clap(
-    name = "migration-node-validation",
-    about = "Validates data conformity after movement migration."
-)]
-pub struct Command {
-    #[clap(long = "movement", help = "The path to the movement database.")]
-    pub movement_db: PathBuf,
-    #[clap(
-        long = "movement-aptos",
-        help = "The path to the movement Aptos database."
-    )]
-    pub movement_aptos_db: PathBuf,
+#[derive(Subcommand, Debug)]
+#[clap(rename_all = "kebab-case", about = "Node database verification tool")]
+pub enum NodeValidation {
+    CompareDb(CompareDbCmd),
+    CompareStates(CompareStatesCmd),
 }
 
-impl Command {
+impl NodeValidation {
     pub async fn run(self) -> anyhow::Result<()> {
-        let movement_storage = MovementStorage::open(&self.movement_db)?;
-        let movement_aptos_storage = MovementAptosStorage::open(&self.movement_aptos_db)?;
-
-        GlobalStorageIncludes::satisfies(&movement_storage, &movement_aptos_storage)?;
-
-        Ok(())
+        match self {
+            NodeValidation::CompareDb(cmd) => cmd.run().await,
+            NodeValidation::CompareStates(cmd) => cmd.run().await,
+        }
     }
 }
 
 #[test]
 fn verify_tool() {
     use clap::CommandFactory;
-    Command::command().debug_assert()
+    NodeValidation::command().debug_assert()
 }

movement-migration/validation-tool/src/checks/node/global_storage_includes.rs

@@ -14,14 +14,47 @@ use aptos_types::{
     },
 };
 use bytes::Bytes;
+use clap::Parser;
 use move_core_types::{account_address::AccountAddress, language_storage::StructTag};
 use std::cmp::Ordering;
 use std::collections::BTreeSet;
-use std::fmt::Display;
-use std::fmt::Formatter;
+use std::fmt::{Display, Formatter};
+use std::path::PathBuf;
 use std::str::FromStr;
 use tracing::{debug, info};
 
+#[derive(Parser, Debug)]
+#[clap(
+    name = "compare-database",
+    about = "Validates data conformity after movement migration."
+)]
+pub struct CompareDbCmd {
+    #[clap(long = "movement", help = "The path to the movement database.")]
+    pub movement_db: PathBuf,
+    #[clap(
+        long = "movement-aptos",
+        help = "The path to the movement Aptos database."
+    )]
+    pub movement_aptos_db: PathBuf,
+}
+
+impl CompareDbCmd {
+    pub async fn run(self) -> anyhow::Result<()> {
+        let movement_storage = MovementStorage::open(&self.movement_db)?;
+        let movement_aptos_storage = MovementAptosStorage::open(&self.movement_aptos_db)?;
+
+        GlobalStorageIncludes::satisfies(&movement_storage, &movement_aptos_storage)?;
+
+        Ok(())
+    }
+}
+
+#[test]
+fn verify_tool() {
+    use clap::CommandFactory;
+    CompareDbCmd::command().debug_assert()
+}
+
 #[derive(Debug)]
 pub enum FailedComparison {
     MissingStateValue(StateKey),
@@ -132,16 +165,16 @@ impl PartialOrd for FailedComparison {
 impl Display for FailedComparison {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         match self {
-            FailedComparison::MissingStateValue(movement_state_key) =>
+            FailedComparison::MissingStateValue(state_key) =>
                 write!(f,
                     "Movement Aptos is missing a value for {:?}",
-                    movement_state_key
+                    state_key
                 )
                    ,
-            FailedComparison::NotMissingStateValue(movement_state_key) =>
+            FailedComparison::NotMissingStateValue(state_key) =>
                 write!(f,
                     "Movement Aptos is unexpectedly not missing a value for {:?}",
-                    movement_state_key
+                    state_key
                 ),
             FailedComparison::RawStateDiverge {
                 state_key,

movement-migration/validation-tool/src/checks/node/state_diff.rs

@@ -0,0 +1,131 @@
+// Copyright (c) Aptos Foundation
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::checks::node::global_storage_includes::GlobalStorageIncludes;
+use crate::types::storage::{MovementAptosStorage, MovementStorage};
+use clap::Parser;
+use std::path::PathBuf;
+use std::str::FromStr;
+use tracing::info;
+
+#[derive(Parser, Debug)]
+#[clap(
+    name = "compare-states",
+    about = "Compares balances for each transaction at specific ledger versions"
+)]
+pub struct CompareStatesCmd {
+    #[clap(long = "movement-db", help = "Path to the Movement database.")]
+    pub movement_db: PathBuf,
+    #[clap(long = "aptos-db", help = "Path to the Aptos database.")]
+    pub aptos_db: PathBuf,
+    #[arg(help = "First hash,version,version tuple")]
+    first: String,
+    #[arg(help = "Second hash,version,version tuple")]
+    second: String,
+}
+
+impl CompareStatesCmd {
+    pub async fn run(&self) -> anyhow::Result<()> {
+        let movement_storage = MovementStorage::open(&self.movement_db)?;
+        let aptos_storage = MovementAptosStorage::open(&self.aptos_db)?;
+
+        compare_states(&movement_storage, &aptos_storage, &self.first, &self.second).await?;
+
+        Ok(())
+    }
+}
+
+#[test]
+fn verify_tool() {
+    use clap::CommandFactory;
+    CompareStatesCmd::command().debug_assert()
+}
+
+async fn compare_states(
+    movement_storage: &MovementStorage,
+    aptos_storage: &MovementAptosStorage,
+    first: &str,
+    second: &str,
+) -> anyhow::Result<()> {
+    let (hash1, aptos_version1, movement_version1) = parse_line(first)?;
+    let (hash2, aptos_version2, movement_version2) = parse_line(second)?;
+
+    info!(
+        "Comparing post transaction {}: Movement version: {}, Aptos version: {}",
+        hash1, movement_version1, aptos_version1
+    );
+
+    let result1 = GlobalStorageIncludes::compare_db(
+        movement_storage,
+        movement_version1,
+        aptos_storage,
+        aptos_version1,
+    )?;
+
+    info!(
+        "Comparing post transaction {}: Movement version: {}, Aptos version: {}",
+        hash2, movement_version2, aptos_version2
+    );
+
+    let result2 = GlobalStorageIncludes::compare_db(
+        movement_storage,
+        movement_version2,
+        aptos_storage,
+        aptos_version2,
+    )?;
+
+    let diff = result2
+        .into_iter()
+        .filter(|c| result1.contains(c))
+        .collect::<Vec<_>>();
+
+    for comparison in diff {
+        info!("{}", comparison);
+    }
+
+    Ok(())
+}
+
+// async fn compare_balances(
+//     movement_storage: &MovementStorage,
+//     aptos_storage: &MovementAptosStorage,
+//     path: &PathBuf,
+// ) -> anyhow::Result<()> {
+//     use tokio::fs::File;
+//     use tokio::io::{AsyncBufReadExt, BufReader};
+//     let file = File::open(path).await?;
+//     let reader = BufReader::new(file);
+//     let mut lines = reader.lines();
+//
+//     while let Some(line) = lines.next_line().await? {
+//         let (hash, aptos_version, movement_version) = parse_line(&line)?;
+//         info!(
+//             "Processing transaction {}: Aptos version {}, Movement version {}",
+//             hash, aptos_version, movement_version
+//         );
+//         let diff = GlobalStorageIncludes::compare_db(
+//             movement_storage,
+//             movement_version,
+//             aptos_storage,
+//             aptos_version,
+//         )?;
+//
+//         for comparison in diff {}
+//     }
+//
+//     Ok(())
+// }
+
+fn parse_line(line: &str) -> anyhow::Result<(&str, u64, u64)> {
+    let parts = line.split(',').collect::<Vec<_>>();
+    let parts: [&str; 3] = parts.try_into().map_err(|v: Vec<&str>| {
+        anyhow::anyhow!(
+            "Expected 3 parts extracted from the line. Found {}",
+            v.len()
+        )
+    })?;
+    let [hash, aptos_version, movement_version] = parts;
+    let aptos_version = u64::from_str(aptos_version)?;
+    let movement_version = u64::from_str(movement_version)?;
+    Ok((hash, aptos_version, movement_version))
+}

movement-migration/validation-tool/src/lib.rs

@@ -14,7 +14,8 @@ mod types;
 )]
 pub enum ValidationTool {
     Api(checks::api::Command),
-    Node(checks::node::Command),
+    #[clap(subcommand)]
+    Node(checks::node::NodeValidation),
 }
 
 impl ValidationTool {