Compare balances for specific ledger versions

movement-migration/validation-tool/src/checks/node.rs

@@ -1,43 +1,31 @@
 // Copyright (c) Aptos Foundation
 // SPDX-License-Identifier: Apache-2.0
 
-use crate::{
-    checks::node::global_storage_includes::GlobalStorageIncludes,
-    types::storage::{MovementAptosStorage, MovementStorage},
-};
-use clap::Parser;
-use std::path::PathBuf;
+use crate::checks::node::global_storage_includes::CompareDbCmd;
+use crate::checks::node::state_diff::CompareStatesCmd;
+use clap::Subcommand;
 
 mod global_storage_includes;
+mod state_diff;
 
-#[derive(Parser)]
-#[clap(
-    name = "migration-node-validation",
-    about = "Validates data conformity after movement migration."
-)]
-pub struct Command {
-    #[clap(long = "movement", help = "The path to the movement database.")]
-    pub movement_db: PathBuf,
-    #[clap(
-        long = "movement-aptos",
-        help = "The path to the movement Aptos database."
-    )]
-    pub movement_aptos_db: PathBuf,
+#[derive(Subcommand, Debug)]
+#[clap(rename_all = "kebab-case", about = "Node database verification tool")]
+pub enum NodeValidation {
+    CompareDb(CompareDbCmd),
+    CompareStates(CompareStatesCmd),
 }
 
-impl Command {
+impl NodeValidation {
     pub async fn run(self) -> anyhow::Result<()> {
-        let movement_storage = MovementStorage::open(&self.movement_db)?;
-        let movement_aptos_storage = MovementAptosStorage::open(&self.movement_aptos_db)?;
-
-        GlobalStorageIncludes::satisfies(&movement_storage, &movement_aptos_storage)?;
-
-        Ok(())
+        match self {
+            NodeValidation::CompareDb(cmd) => cmd.run().await,
+            NodeValidation::CompareStates(cmd) => cmd.run().await,
+        }
     }
 }
 
 #[test]
 fn verify_tool() {
     use clap::CommandFactory;
-    Command::command().debug_assert()
+    NodeValidation::command().debug_assert()
 }

movement-migration/validation-tool/src/checks/node/global_storage_includes.rs

@@ -14,10 +14,46 @@ use aptos_types::{
     },
 };
 use bytes::Bytes;
+use clap::Parser;
 use move_core_types::{account_address::AccountAddress, language_storage::StructTag};
+use std::fmt::{Display, Formatter};
+use std::path::PathBuf;
 use std::str::FromStr;
 use tracing::{debug, info};
 
+#[derive(Parser, Debug)]
+#[clap(
+    name = "compare-database",
+    about = "Validates data conformity after movement migration."
+)]
+pub struct CompareDbCmd {
+    #[clap(long = "movement", help = "The path to the movement database.")]
+    pub movement_db: PathBuf,
+    #[clap(
+        long = "movement-aptos",
+        help = "The path to the movement Aptos database."
+    )]
+    pub movement_aptos_db: PathBuf,
+}
+
+impl CompareDbCmd {
+    pub async fn run(self) -> anyhow::Result<()> {
+        let movement_storage = MovementStorage::open(&self.movement_db)?;
+        let movement_aptos_storage = MovementAptosStorage::open(&self.movement_aptos_db)?;
+
+        GlobalStorageIncludes::satisfies(&movement_storage, &movement_aptos_storage)?;
+
+        Ok(())
+    }
+}
+
+#[test]
+fn verify_tool() {
+    use clap::CommandFactory;
+    CompareDbCmd::command().debug_assert()
+}
+
+#[derive(Debug, PartialEq)]
 pub enum FailedComparison {
     MissingStateValue(StateKey),
     NotMissingStateValue(StateKey),
@@ -38,66 +74,63 @@ pub enum FailedComparison {
     },
 }
 
-impl From<FailedComparison> for ValidationError {
-    fn from(fail: FailedComparison) -> Self {
-        match fail {
-           FailedComparison::MissingStateValue(movement_state_key) => ValidationError::Unsatisfied(
-                        format!(
-                            "Movement Aptos is missing a value for {:?}",
-                            movement_state_key
-                        )
-                        .into(),
-                    ),
-           FailedComparison::NotMissingStateValue(movement_state_key) => ValidationError::Unsatisfied(
-                                format!(
-                                    "Movement Aptos is unexpectedly not missing a value for {:?}",
-                                    movement_state_key
-                                )
-                                .into(),
-                            ),
-    FailedComparison::RawStateDiverge {
-        movement_state_key,
-        movement_value,
-        maptos_state_value,
-    } => ValidationError::Unsatisfied(
-                format!(
+impl Display for FailedComparison {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        match self {
+            FailedComparison::MissingStateValue(movement_state_key) =>
+                write!(f,
+                    "Movement Aptos is missing a value for {:?}",
+                    movement_state_key
+                )
+                   ,
+            FailedComparison::NotMissingStateValue(movement_state_key) =>
+                write!(f,
+                    "Movement Aptos is unexpectedly not missing a value for {:?}",
+                    movement_state_key
+                ),
+            FailedComparison::RawStateDiverge {
+                movement_state_key,
+                movement_value,
+                maptos_state_value,
+            } =>
+                write!(f,
                     "Movement state value for {:?} is {:?}, while Movement Aptos state value is {:?}",
                     movement_state_key,
                     movement_value,
                     maptos_state_value
-                )
-                    .into(),
-            ),
-    FailedComparison::AccountDiverge {
-        address,
-        movement_account,
-        movement_aptos_account,
-    } => ValidationError::Unsatisfied(
-                format!(
+                ),
+            FailedComparison::AccountDiverge {
+                address,
+                movement_account,
+                movement_aptos_account,
+            } =>
+                write!(f,
                     "Movement account for {:?} is {:?}, while Movement Aptos account is {:?}",
                     address.to_standard_string(),
                     movement_account,
                     movement_aptos_account
-                )
-                .into(),
-            ),
-    FailedComparison::BalanceDiverge {
-        address,
-        movement_balance,
-        movement_aptos_balance,
-    } => ValidationError::Unsatisfied(
-                format!(
+                ),
+            FailedComparison::BalanceDiverge {
+                address,
+                movement_balance,
+                movement_aptos_balance,
+            } =>
+                write!(f,
                     "Movement balance for 0x{} is {} coin(s), while Movement Aptos balance is {} coin(s)",
                     address.short_str_lossless(),
                     movement_balance,
                     movement_aptos_balance
-                )
-                    .into(),
-            ),
+                ),
         }
     }
 }
 
+impl From<FailedComparison> for ValidationError {
+    fn from(fail: FailedComparison) -> Self {
+        ValidationError::Unsatisfied(fail.to_string().into())
+    }
+}
+
 /// This check iterates over all global state keys starting at ledger version 0.
 /// For each state key it fetches the state view for the latest ledger version,
 /// from the old Movment database and the new Aptos database. The state view bytes

movement-migration/validation-tool/src/checks/node/state_diff.rs

@@ -0,0 +1,131 @@
+// Copyright (c) Aptos Foundation
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::checks::node::global_storage_includes::GlobalStorageIncludes;
+use crate::types::storage::{MovementAptosStorage, MovementStorage};
+use clap::Parser;
+use std::path::PathBuf;
+use std::str::FromStr;
+use tracing::info;
+
+#[derive(Parser, Debug)]
+#[clap(
+    name = "compare-states",
+    about = "Compares balances for each transaction at specific ledger versions"
+)]
+pub struct CompareStatesCmd {
+    #[clap(long = "movement-db", help = "Path to the Movement database.")]
+    pub movement_db: PathBuf,
+    #[clap(long = "aptos-db", help = "Path to the Aptos database.")]
+    pub aptos_db: PathBuf,
+    #[arg(help = "First hash,version,version tuple")]
+    first: String,
+    #[arg(help = "Second hash,version,version tuple")]
+    second: String,
+}
+
+impl CompareStatesCmd {
+    pub async fn run(&self) -> anyhow::Result<()> {
+        let movement_storage = MovementStorage::open(&self.movement_db)?;
+        let aptos_storage = MovementAptosStorage::open(&self.aptos_db)?;
+
+        compare_states(&movement_storage, &aptos_storage, &self.first, &self.second).await?;
+
+        Ok(())
+    }
+}
+
+#[test]
+fn verify_tool() {
+    use clap::CommandFactory;
+    CompareStatesCmd::command().debug_assert()
+}
+
+async fn compare_states(
+    movement_storage: &MovementStorage,
+    aptos_storage: &MovementAptosStorage,
+    first: &str,
+    second: &str,
+) -> anyhow::Result<()> {
+    let (hash1, aptos_version1, movement_version1) = parse_line(first)?;
+    let (hash2, aptos_version2, movement_version2) = parse_line(second)?;
+
+    info!(
+        "Comparing post transaction {}: Movement version: {}, Aptos version: {}",
+        hash1, movement_version1, aptos_version1
+    );
+
+    let result1 = GlobalStorageIncludes::compare_db(
+        movement_storage,
+        movement_version1,
+        aptos_storage,
+        aptos_version1,
+    )?;
+
+    info!(
+        "Comparing post transaction {}: Movement version: {}, Aptos version: {}",
+        hash2, movement_version2, aptos_version2
+    );
+
+    let result2 = GlobalStorageIncludes::compare_db(
+        movement_storage,
+        movement_version2,
+        aptos_storage,
+        aptos_version2,
+    )?;
+
+    let diff = result2
+        .into_iter()
+        .filter(|c| result1.contains(c))
+        .collect::<Vec<_>>();
+
+    for comparison in diff {
+        info!("{}", comparison);
+    }
+
+    Ok(())
+}
+
+// async fn compare_balances(
+//     movement_storage: &MovementStorage,
+//     aptos_storage: &MovementAptosStorage,
+//     path: &PathBuf,
+// ) -> anyhow::Result<()> {
+//     use tokio::fs::File;
+//     use tokio::io::{AsyncBufReadExt, BufReader};
+//     let file = File::open(path).await?;
+//     let reader = BufReader::new(file);
+//     let mut lines = reader.lines();
+//
+//     while let Some(line) = lines.next_line().await? {
+//         let (hash, aptos_version, movement_version) = parse_line(&line)?;
+//         info!(
+//             "Processing transaction {}: Aptos version {}, Movement version {}",
+//             hash, aptos_version, movement_version
+//         );
+//         let diff = GlobalStorageIncludes::compare_db(
+//             movement_storage,
+//             movement_version,
+//             aptos_storage,
+//             aptos_version,
+//         )?;
+//
+//         for comparison in diff {}
+//     }
+//
+//     Ok(())
+// }
+
+fn parse_line(line: &str) -> anyhow::Result<(&str, u64, u64)> {
+    let parts = line.split(',').collect::<Vec<_>>();
+    let parts: [&str; 3] = parts.try_into().map_err(|v: Vec<&str>| {
+        anyhow::anyhow!(
+            "Expected 3 parts extracted from the line. Found {}",
+            v.len()
+        )
+    })?;
+    let [hash, aptos_version, movement_version] = parts;
+    let aptos_version = u64::from_str(aptos_version)?;
+    let movement_version = u64::from_str(movement_version)?;
+    Ok((hash, aptos_version, movement_version))
+}

movement-migration/validation-tool/src/lib.rs

@@ -14,7 +14,8 @@ mod types;
 )]
 pub enum ValidationTool {
     Api(checks::api::Command),
-    Node(checks::node::Command),
+    #[clap(subcommand)]
+    Node(checks::node::NodeValidation),
 }
 
 impl ValidationTool {