feat(ethexe): improve database migrations (#5188)

Co-authored-by: Gleb Smirnov <gleb.s@gear-tech.io>

Author: liferooter

Cargo.lock

@@ -111,7 +111,6 @@ members = [
     "ethexe/*",
     "ethexe/runtime/common",
     "ethexe/service/utils",
-    "ethexe/db/init",
 ]
 
 [workspace.dependencies]

Cargo.toml

@@ -16,19 +16,21 @@
 // You should have received a copy of the GNU General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
+use crate::params::{MergeParams, Params};
 use anyhow::{Context, Result, anyhow, ensure};
 use clap::Parser;
 use ethexe_common::{
     Announce, HashOf, SimpleBlockData,
     db::{AnnounceStorageRO, DBGlobals, GlobalsStorageRO, OnChainStorageRO},
+    gear::CANONICAL_QUARANTINE,
 };
 use ethexe_db::{
-    Database, RawDatabase, RocksDatabase,
+    Database, InitConfig, RawDatabase, RocksDatabase,
     iterator::{BlockNode, DatabaseIterator},
     verifier::IntegrityVerifier,
     visitor::{self},
 };
-use ethexe_processor::{Processor, ProcessorConfig};
+use ethexe_processor::{DEFAULT_CHUNK_SIZE, Processor, ProcessorConfig};
 use indicatif::{ProgressBar, ProgressStyle};
 use std::{collections::HashSet, path::PathBuf};
 
@@ -38,15 +40,13 @@ const PROGRESS_BAR_TEMPLATE: &str = "{spinner:.green} [{elapsed_precise}] [{wide
 /// Run checks on ethexe database, see more in [`super::Command::Check`].
 #[derive(Debug, Parser)]
 pub struct CheckCommand {
-    /// Path to database directory (including router addr subdirectory).
-    #[arg(long)]
-    pub db: PathBuf,
-
-    #[arg(long, default_value = "2")]
-    pub chunk_size: usize,
+    /// CLI parameters to be merged with file ones before execution.
+    #[clap(flatten)]
+    pub params: Params,
 
-    #[arg(long, default_value = "4")]
-    pub canonical_quarantine: u8,
+    /// Override database location.
+    #[arg(long)]
+    pub db: Option<PathBuf>,
 
     /// Perform computations of announces, by default from start announce to latest computed announce.
     #[arg(long, alias = "compute")]
@@ -56,12 +56,21 @@ pub struct CheckCommand {
     #[arg(long, alias = "integrity")]
     pub integrity_check: bool,
 
+    /// Perform migrations before checking the database.
+    #[arg(long)]
+    pub migrate: bool,
+
     /// Enable logging verbosity (debug level by default), disables progress bar.
     #[arg(short, long)]
     pub verbose: bool,
 }
 
 impl CheckCommand {
+    pub fn with_params(mut self, params: Params) -> Self {
+        self.params = self.params.merge(params);
+        self
+    }
+
     /// Execute the command.
     pub fn exec(self) -> Result<()> {
         tokio::runtime::Builder::new_multi_thread()
@@ -80,17 +89,48 @@ impl CheckCommand {
             self.integrity_check = true;
         }
 
-        let rocks_db = RocksDatabase::open(self.db).context("failed to open rocks database")?;
-        let db = Database::try_from_raw(RawDatabase::from_one(&rocks_db))?;
+        let rocks_db = RocksDatabase::open(
+            self.db
+                .or_else(|| self.params.node.as_ref().map(|node| node.db_dir()))
+                .context("missing database path")?,
+        )
+        .context("failed to open rocks database")?;
+        let raw_db = RawDatabase::from_one(&rocks_db);
+
+        let db = if self.migrate {
+            let ethereum_config = self
+                .params
+                .ethereum
+                .context("missing Ethereum-related configuration")?
+                .into_config()?;
+
+            ethexe_db::initialize_db(
+                InitConfig {
+                    ethereum_rpc: ethereum_config.rpc.clone(),
+                    router_address: ethereum_config.router_address,
+                    slot_duration_secs: ethereum_config.block_time.as_secs(),
+                },
+                raw_db.overlaid(),
+            )
+            .await?
+        } else {
+            Database::try_from_raw(raw_db)?
+        };
 
         let globals = db.globals().clone();
 
+        let node_params = self.params.node.unwrap_or_default();
         let checker = Checker {
             db,
             globals,
             progress_bar: !self.verbose,
-            chunk_size: self.chunk_size,
-            canonical_quarantine: self.canonical_quarantine,
+            chunk_size: node_params
+                .chunk_processing_threads
+                .unwrap_or(DEFAULT_CHUNK_SIZE)
+                .get(),
+            canonical_quarantine: node_params
+                .canonical_quarantine
+                .unwrap_or(CANONICAL_QUARANTINE),
         };
 
         if self.integrity_check {

ethexe/cli/src/commands/check.rs

@@ -36,7 +36,7 @@ pub enum Command {
     /// Keystore manipulations.
     Key(KeyCommand),
     /// Run the node.
-    Run(Box<RunCommand>),
+    Run(RunCommand),
     /// Submit a transaction.
     Tx(TxCommand),
     /// Check ethexe database for integrity and/or computation correctness.
@@ -50,9 +50,9 @@ impl Command {
     fn with_file_params(self, file_params: Params) -> Self {
         match self {
             Self::Key(key_cmd) => Self::Key(key_cmd.with_params(file_params)),
-            Self::Run(run_cmd) => Self::Run(Box::new(run_cmd.with_params(file_params))),
+            Self::Run(run_cmd) => Self::Run(run_cmd.with_params(file_params)),
             Self::Tx(tx_cmd) => Self::Tx(tx_cmd.with_params(file_params)),
-            Self::Check(check_cmd) => Self::Check(check_cmd),
+            Self::Check(check_cmd) => Self::Check(check_cmd.with_params(file_params)),
         }
     }
 

ethexe/cli/src/commands/mod.rs

@@ -38,7 +38,7 @@ gear-workspace-hack.workspace = true
 gprimitives = { workspace = true, features = ["ethexe"] }
 serde_json.workspace = true
 indoc.workspace = true
-scale-info = { workspace = true, features = ["docs"] }
+hex.workspace = true
 
 [features]
 default = ["std"]

ethexe/common/Cargo.toml

@@ -9,16 +9,21 @@ repository.workspace = true
 
 [dependencies]
 ethexe-common = { workspace = true, features = ["std"] }
+ethexe-ethereum.workspace = true
 ethexe-runtime-common = { workspace = true, features = ["std"] }
 gear-core = { workspace = true, features = ["std"] }
 gprimitives = { workspace = true, features = ["std"] }
+gsigner.workspace = true
 
+alloy.workspace = true
 anyhow.workspace = true
 dashmap.workspace = true
+log.workspace = true
 parity-scale-codec.workspace = true
 tracing.workspace = true
 paste.workspace = true
 derive_more.workspace = true
+scale-info = { workspace = true, features = ["derive"] }
 auto_impl.workspace = true
 gear-workspace-hack.workspace = true
 delegate.workspace = true
@@ -38,7 +43,11 @@ version = "0.21"
 scopeguard.workspace = true
 tempfile.workspace = true
 ethexe-common = { workspace = true, features = ["mock"] }
+indoc.workspace = true
+scale-info = { workspace = true, features = ["docs"] }
+sha3.workspace = true
+hex.workspace = true
 
 [features]
 default = ["mock"]
-mock = ["ethexe-common/mock"]
+mock = ["ethexe-common/mock"]