Post-migration integrity validation tool (#186)

* feat: Add initial structure of the movement migration validation tool.

* feat: Add global storage includes check to validation tool

* feat: Add matching feature flags api check to validation tool

* fix: validation tool default tracing log level

* fix: AptosDB pruner config for readonly access

* feature: remove api global feature flag check

* feature: add integration test to validation tool

* feature: compare deserialized account resources and balances

Author: sebtomba

Cargo.lock

@@ -153,6 +153,7 @@ members = [
     "keyless/pepper/example-client-rust",
     "keyless/pepper/service",
     "mempool",
+    "movement-migration/validation-tool",
     "network/benchmark",
     "network/builder",
     "network/discovery",
@@ -828,6 +829,7 @@ which = "4.2.5"
 whoami = "1.5.0"
 # This allows for zeroize 1.6 to be used. Version 1.2.0 of x25519-dalek locks zeroize to 1.3.
 x25519-dalek = { git = "https://github.com/aptos-labs/x25519-dalek", rev = "b9cdbaf36bf2a83438d9f660e5a708c82ed60d8e" }
+xz2 = "0.1"
 z3tracer = "0.8.0"
 
 # MOVE DEPENDENCIES

Cargo.toml

@@ -0,0 +1,33 @@
+[package]
+name = "validation-tool"
+version = "0.1.0"
+authors.workspace = true
+edition.workspace = true
+homepage.workspace = true
+license.workspace = true
+publish.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+
+[dependencies]
+anyhow = { workspace = true }
+aptos-config = { workspace = true }
+aptos-db = { workspace = true }
+aptos-rest-client = { workspace = true }
+aptos-storage-interface = { workspace = true }
+aptos-types = { workspace = true }
+bcs = { workspace = true }
+bytes = { workspace = true }
+clap = { workspace = true }
+either = { workspace = true }
+serde_json = { workspace = true }
+move-core-types = { workspace = true }
+thiserror = { workspace = true }
+tokio = { workspace = true }
+tracing = { workspace = true }
+tracing-subscriber = { workspace = true }
+
+[dev-dependencies]
+tar = { workspace = true }
+tempfile = { workspace = true }
+xz2 = { workspace = true }

movement-migration/validation-tool/Cargo.toml

@@ -0,0 +1,6 @@
+// Copyright (c) Aptos Foundation
+// SPDX-License-Identifier: Apache-2.0
+
+pub mod api;
+pub mod error;
+pub mod node;

movement-migration/validation-tool/assets/maptos.tar.xz

@@ -0,0 +1,37 @@
+// Copyright (c) Aptos Foundation
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::types::api::{MovementAptosRestClient, MovementRestClient};
+use clap::Parser;
+
+#[derive(Parser)]
+#[clap(
+    name = "migration-api-validation",
+    about = "Validates api conformity after movement migration."
+)]
+pub struct Command {
+    #[clap(long = "movement", help = "The url of the Movement REST endpoint.")]
+    pub movement_rest_api_url: String,
+    #[clap(value_parser)]
+    #[clap(
+        long = "movement-aptos",
+        help = "The url of the Movement Aptos REST endpoint."
+    )]
+    pub movement_aptos_rest_api_url: String,
+}
+
+impl Command {
+    pub async fn run(self) -> anyhow::Result<()> {
+        let _movement_rest_client = MovementRestClient::new(&self.movement_rest_api_url)?;
+        let _movement_aptos_rest_client =
+            MovementAptosRestClient::new(&self.movement_aptos_rest_api_url)?;
+
+        Ok(())
+    }
+}
+
+#[test]
+fn verify_tool() {
+    use clap::CommandFactory;
+    Command::command().debug_assert()
+}

movement-migration/validation-tool/src/checks.rs

@@ -0,0 +1,134 @@
+// Copyright (c) Aptos Foundation
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::checks::error::ValidationError;
+use crate::types::api::{MovementAptosRestClient, MovementRestClient};
+use aptos_rest_client::aptos_api_types::{EntryFunctionId, MoveType, ViewRequest};
+use serde_json::json;
+use std::collections::HashSet;
+use std::str::FromStr;
+
+pub struct GlobalFeatureCheck;
+
+impl GlobalFeatureCheck {
+    pub async fn satisfies(
+        movement_rest_client: &MovementRestClient,
+        movement_aptos_rest_client: &MovementAptosRestClient,
+    ) -> Result<(), ValidationError> {
+        let mut errors = vec![];
+        let expected_active = HashSet::from([73]);
+        let expected_inactive = HashSet::<u64>::new();
+
+        let mut aptos_request = ViewRequest {
+            function: EntryFunctionId::from_str("0x1::features::is_enabled")
+                .map_err(|e| ValidationError::Internal(e.into()))?,
+            type_arguments: vec![MoveType::U64],
+            arguments: vec![],
+        };
+
+        let mut maptos_request = ViewRequest {
+            function: EntryFunctionId::from_str("0x1::features::is_enabled")
+                .map_err(|e| ValidationError::Internal(e.into()))?,
+            type_arguments: vec![MoveType::U64],
+            arguments: vec![],
+        };
+
+        for feature_id in 1u64..=100 {
+            aptos_request.arguments = vec![json!(feature_id)];
+            maptos_request.arguments = vec![json!(feature_id)];
+
+            // Check feature for Aptos executor
+            let aptos_active = movement_aptos_rest_client
+                .view(&aptos_request, None)
+                .await
+                .map_err(|e| {
+                    ValidationError::Internal(
+                        format!("failed to get Aptos feature flag {}: {:?}", feature_id, e).into(),
+                    )
+                })?
+                .into_inner();
+
+            let aptos_active = aptos_active.get(0).ok_or_else(|| {
+                ValidationError::Internal(
+                    format!(
+                        "failed to get Aptos feature flag {}: response is empty",
+                        feature_id
+                    )
+                    .into(),
+                )
+            })?;
+
+            let aptos_active = aptos_active.as_bool().ok_or_else(|| {
+                ValidationError::Internal(
+                    format!(
+                        "failed to get Aptos feature flag {}: can't convert {:?} into a bool",
+                        feature_id, aptos_active
+                    )
+                    .into(),
+                )
+            })?;
+
+            // Check feature for Maptos executor
+            let maptos_active = movement_rest_client
+                .view(&maptos_request, None)
+                .await
+                .map_err(|e| {
+                    ValidationError::Internal(
+                        format!(
+                            "failed to get Movement feature flag {}: {:?}",
+                            feature_id, e
+                        )
+                        .into(),
+                    )
+                })?
+                .into_inner();
+
+            let maptos_active = maptos_active.get(0).ok_or_else(|| {
+                ValidationError::Internal(
+                    format!(
+                        "failed to get Movement feature flag {}: response is empty",
+                        feature_id
+                    )
+                    .into(),
+                )
+            })?;
+
+            let maptos_active = maptos_active.as_bool().ok_or_else(|| {
+                ValidationError::Internal(
+                    format!(
+                        "failed to get Movement feature flag {}: can't convert {:?} into a bool",
+                        feature_id, aptos_active
+                    )
+                    .into(),
+                )
+            })?;
+
+            if !expected_active.contains(&feature_id) {
+                if !maptos_active {
+                    errors.push(format!(
+                        "Feature {}: Aptos={}, Maptos={} — expected to be active",
+                        feature_id, aptos_active, maptos_active
+                    ));
+                }
+            } else if !expected_inactive.contains(&feature_id) {
+                if maptos_active {
+                    errors.push(format!(
+                        "Feature {}: Aptos={}, Maptos={} — expected to be inactive",
+                        feature_id, aptos_active, maptos_active
+                    ));
+                }
+            } else if aptos_active != maptos_active {
+                errors.push(format!(
+                    "Feature {}: Aptos={}, Maptos={} — expected to match",
+                    feature_id, aptos_active, maptos_active
+                ));
+            }
+        }
+
+        if !errors.is_empty() {
+            return Err(ValidationError::Unsatisfied(errors.join("\n").into()));
+        }
+
+        Ok(())
+    }
+}

movement-migration/validation-tool/src/checks/api.rs

@@ -0,0 +1,10 @@
+// Copyright (c) Aptos Foundation
+// SPDX-License-Identifier: Apache-2.0
+
+#[derive(Debug, thiserror::Error)]
+pub enum ValidationError {
+    #[error("the criterion was not satisfied: {0}")]
+    Unsatisfied(#[source] Box<dyn std::error::Error + Send + Sync>),
+    #[error("criterion encountered an internal error: {0}")]
+    Internal(#[source] Box<dyn std::error::Error + Send + Sync>),
+}

movement-migration/validation-tool/src/checks/api/matching_feature_flags.rs

@@ -0,0 +1,41 @@
+// Copyright (c) Aptos Foundation
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::checks::node::global_storage_includes::GlobalStorageIncludes;
+use crate::types::storage::{MovementAptosStorage, MovementStorage};
+use clap::Parser;
+use std::path::PathBuf;
+
+mod global_storage_includes;
+
+#[derive(Parser)]
+#[clap(
+    name = "migration-node-validation",
+    about = "Validates data conformity after movement migration."
+)]
+pub struct Command {
+    #[clap(long = "movement", help = "The path to the movement database.")]
+    pub movement_db: PathBuf,
+    #[clap(
+        long = "movement-aptos",
+        help = "The path to the movement Aptos database."
+    )]
+    pub movement_aptos_db: PathBuf,
+}
+
+impl Command {
+    pub async fn run(self) -> anyhow::Result<()> {
+        let movement_storage = MovementStorage::open(&self.movement_db)?;
+        let movement_aptos_storage = MovementAptosStorage::open(&self.movement_aptos_db)?;
+
+        GlobalStorageIncludes::satisfies(&movement_storage, &movement_aptos_storage)?;
+
+        Ok(())
+    }
+}
+
+#[test]
+fn verify_tool() {
+    use clap::CommandFactory;
+    Command::command().debug_assert()
+}