feat(utils): cache migrated TestNode databases as snapshots

Add Db::open_no_sync() for fast test DB loading without durability.
Add TestNode::new_from_db() and convenience methods to load
pre-migrated database snapshots instead of running the slow
clone+build+migrate flow per test. Add generate_migration_db binary
to produce the .tar.gz snapshots, and Makefile targets to extract them.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: kariy

Cargo.lock

@@ -17,6 +17,12 @@ SNOS_DB_DIR := $(DB_FIXTURES_DIR)/snos
 COMPATIBILITY_DB_TAR ?= $(DB_FIXTURES_DIR)/1_6_0.tar.gz
 COMPATIBILITY_DB_DIR ?= $(DB_FIXTURES_DIR)/1_6_0
 
+SPAWN_AND_MOVE_DB_TAR ?= $(DB_FIXTURES_DIR)/spawn_and_move.tar.gz
+SPAWN_AND_MOVE_DB_DIR := $(DB_FIXTURES_DIR)/spawn_and_move
+
+SIMPLE_DB_TAR ?= $(DB_FIXTURES_DIR)/simple.tar.gz
+SIMPLE_DB_DIR := $(DB_FIXTURES_DIR)/simple
+
 CONTRACTS_CRATE := crates/contracts
 CONTRACTS_DIR := $(CONTRACTS_CRATE)/contracts
 CONTRACTS_BUILD_DIR := $(CONTRACTS_CRATE)/build
@@ -62,7 +68,7 @@ snos-artifacts: $(SNOS_OUTPUT)
 db-compat-artifacts: $(COMPATIBILITY_DB_DIR)
 	@echo "Database compatibility test artifacts prepared successfully."
 
-test-artifacts: $(SNOS_DB_DIR) $(SNOS_OUTPUT) $(COMPATIBILITY_DB_DIR) contracts
+test-artifacts: $(SNOS_DB_DIR) $(SNOS_OUTPUT) $(COMPATIBILITY_DB_DIR) $(SPAWN_AND_MOVE_DB_DIR) $(SIMPLE_DB_DIR) contracts
 	@echo "All test artifacts prepared successfully."
 
 build-explorer:
@@ -110,6 +116,18 @@ $(COMPATIBILITY_DB_DIR): $(COMPATIBILITY_DB_TAR)
 		mv katana_db $(notdir $(COMPATIBILITY_DB_DIR)) || { echo "Failed to extract backward compatibility test database\!"; exit 1; }
 	@echo "Backward compatibility database extracted successfully."
 
+$(SPAWN_AND_MOVE_DB_DIR): $(SPAWN_AND_MOVE_DB_TAR)
+	@echo "Extracting spawn-and-move test database..."
+	@cd $(DB_FIXTURES_DIR) && \
+		tar -xzf $(notdir $(SPAWN_AND_MOVE_DB_TAR)) || { echo "Failed to extract spawn-and-move test database\!"; exit 1; }
+	@echo "Spawn-and-move test database extracted successfully."
+
+$(SIMPLE_DB_DIR): $(SIMPLE_DB_TAR)
+	@echo "Extracting simple test database..."
+	@cd $(DB_FIXTURES_DIR) && \
+		tar -xzf $(notdir $(SIMPLE_DB_TAR)) || { echo "Failed to extract simple test database\!"; exit 1; }
+	@echo "Simple test database extracted successfully."
+
 check-llvm:
 ifndef MLIR_SYS_190_PREFIX
 	$(error Could not find a suitable LLVM 19 toolchain (mlir), please set MLIR_SYS_190_PREFIX env pointing to the LLVM 19 dir)
@@ -180,5 +198,5 @@ snos-deps-macos: install-pyenv
 
 clean:
 	echo "Cleaning up generated files..."
-	-rm -rf $(SNOS_DB_DIR) $(COMPATIBILITY_DB_DIR) $(SNOS_OUTPUT) $(EXPLORER_UI_DIST) $(CONTRACTS_BUILD_DIR)
+	-rm -rf $(SNOS_DB_DIR) $(COMPATIBILITY_DB_DIR) $(SPAWN_AND_MOVE_DB_DIR) $(SIMPLE_DB_DIR) $(SNOS_OUTPUT) $(EXPLORER_UI_DIST) $(CONTRACTS_BUILD_DIR)
 	echo "Clean complete."

Makefile

@@ -111,6 +111,31 @@ impl Db {
         Ok(Self { env, version: CURRENT_DB_VERSION })
     }
 
+    /// Opens an existing database at the given `path` with [`SyncMode::UtterlyNoSync`] for
+    /// write performance, similar to [`Db::in_memory`] but on an existing path.
+    ///
+    /// This is intended for test scenarios where a pre-populated database snapshot needs to be
+    /// loaded quickly without durability guarantees.
+    pub fn open_no_sync<P: AsRef<Path>>(path: P) -> anyhow::Result<Self> {
+        let version = get_db_version(&path)?;
+        if version != CURRENT_DB_VERSION && !is_block_compatible_version(&version) {
+            return Err(anyhow!(DatabaseVersionError::MismatchVersion {
+                expected: CURRENT_DB_VERSION,
+                found: version,
+            }));
+        }
+
+        let env = mdbx::DbEnvBuilder::new()
+            .max_size(GIGABYTE * 10)
+            .growth_step((GIGABYTE / 2) as isize)
+            .sync(SyncMode::UtterlyNoSync)
+            .build(path)?;
+
+        env.create_default_tables()?;
+
+        Ok(Self { env, version })
+    }
+
     // Open the database at the given `path` in read-write mode.
     pub fn open<P: AsRef<Path>>(path: P) -> anyhow::Result<Self> {
         Self::open_inner(path, false)

crates/storage/db/src/lib.rs

@@ -19,22 +19,26 @@ async-trait.workspace = true
 futures.workspace = true
 rand.workspace = true
 thiserror.workspace = true
-tokio = { workspace = true, features = [ "macros", "signal", "time" ], default-features = false }
+tokio = { workspace = true, features = [ "macros", "rt-multi-thread", "signal", "time" ], default-features = false }
 
 # node-only dependencies
 katana-chain-spec = { workspace = true, optional = true }
 katana-core = { workspace = true, optional = true }
+katana-db = { workspace = true, optional = true }
 katana-executor = { workspace = true, optional = true }
 katana-node = { workspace = true, optional = true }
 katana-provider = { workspace = true, optional = true }
 katana-rpc-server = { workspace = true, optional = true }
+clap = { workspace = true, optional = true }
 starknet = { workspace = true, optional = true }
 tempfile = { workspace = true, optional = true }
 
 [features]
 node = [
+    "clap",
     "katana-chain-spec",
     "katana-core",
+    "katana-db",
     "katana-executor",
     "katana-node",
     "katana-provider",
@@ -43,3 +47,8 @@ node = [
     "tempfile",
 ]
 explorer = [ "node", "katana-node/explorer" ]
+
+[[bin]]
+name = "generate_migration_db"
+path = "src/bin/generate_migration_db.rs"
+required-features = [ "node" ]

crates/utils/Cargo.toml

@@ -0,0 +1,100 @@
+/// Binary to generate pre-migrated database snapshots for testing.
+///
+/// These snapshots are loaded by `TestNode::new_with_spawn_and_move_db()` and
+/// `TestNode::new_with_simple_db()` to avoid the slow migration process (git clone + scarb build +
+/// sozo migrate) in each test run.
+///
+/// Usage:
+///   cargo run --bin generate_migration_db --features node -- --example spawn-and-move --output tests/fixtures/db/spawn_and_move.tar.gz
+///   cargo run --bin generate_migration_db --features node -- --example simple --output tests/fixtures/db/simple.tar.gz
+use std::path::{Path, PathBuf};
+use std::process::Command;
+
+use clap::Parser;
+use katana_utils::node::{test_config, TestNode};
+
+#[derive(Parser)]
+#[command(about = "Generate pre-migrated database snapshots for testing")]
+struct Args {
+    /// The dojo example to migrate (e.g. "spawn-and-move" or "simple").
+    #[arg(long)]
+    example: String,
+
+    /// Output path for the .tar.gz archive.
+    #[arg(long)]
+    output: PathBuf,
+}
+
+fn create_tar_gz(db_dir: &Path, output: &Path) -> std::io::Result<()> {
+    // Derive the directory name from the output file stem (e.g., "spawn_and_move" from
+    // "spawn_and_move.tar.gz")
+    let stem = output
+        .file_stem()
+        .and_then(|s| s.to_str())
+        .and_then(|s| s.strip_suffix(".tar"))
+        .unwrap_or_else(|| {
+            output.file_stem().and_then(|s| s.to_str()).expect("invalid output file name")
+        });
+
+    let parent = output.parent().expect("output path must have a parent directory");
+    std::fs::create_dir_all(parent)?;
+
+    // Create the archive directory structure expected by the Makefile extraction targets.
+    // The tar should contain `<stem>/mdbx.dat`, `<stem>/db.version`, etc.
+    let staging_dir = tempfile::tempdir()?;
+    let inner_dir = staging_dir.path().join(stem);
+    std::fs::create_dir_all(&inner_dir)?;
+
+    // Copy db files into the staging directory
+    for entry in std::fs::read_dir(db_dir)? {
+        let entry = entry?;
+        if entry.file_type()?.is_file() {
+            std::fs::copy(entry.path(), inner_dir.join(entry.file_name()))?;
+        }
+    }
+
+    let status = Command::new("tar")
+        .args(["-czf"])
+        .arg(output)
+        .arg("-C")
+        .arg(staging_dir.path())
+        .arg(stem)
+        .status()?;
+
+    if !status.success() {
+        return Err(std::io::Error::other("tar command failed"));
+    }
+
+    Ok(())
+}
+
+#[tokio::main]
+async fn main() {
+    let args = Args::parse();
+
+    println!("Starting node with test_config()...");
+    let node = TestNode::new_with_config(test_config()).await;
+
+    println!("Migrating example '{}'...", args.example);
+    match args.example.as_str() {
+        "spawn-and-move" => {
+            node.migrate_spawn_and_move().await.expect("failed to migrate spawn-and-move");
+        }
+        "simple" => {
+            node.migrate_simple().await.expect("failed to migrate simple");
+        }
+        other => {
+            eprintln!("Unknown example: {other}. Supported: spawn-and-move, simple");
+            std::process::exit(1);
+        }
+    }
+
+    // Get the database path from the running node
+    let db_path = node.handle().node().db().path().to_path_buf();
+
+    println!("Creating archive at {}...", args.output.display());
+    let output_abs = std::env::current_dir().unwrap().join(&args.output);
+    create_tar_gz(&db_path, &output_abs).expect("failed to create tar.gz archive");
+
+    println!("Done! Snapshot saved to {}", args.output.display());
+}

crates/utils/src/bin/generate_migration_db.rs

@@ -1,5 +1,5 @@
 use std::net::SocketAddr;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::process::Command;
 use std::sync::Arc;
 
@@ -74,6 +74,52 @@ impl TestNode {
                 .expect("failed to launch node"),
         }
     }
+
+    /// Creates a [`TestNode`] from a pre-existing database directory.
+    ///
+    /// Copies the database to a temp directory so each test gets its own mutable copy.
+    /// The database is opened with [`SyncMode::UtterlyNoSync`] for test performance.
+    pub async fn new_from_db(db_path: &Path) -> Self {
+        Self::new_from_db_with_config(db_path, test_config()).await
+    }
+
+    /// Creates a [`TestNode`] from a pre-existing database directory with a custom config.
+    ///
+    /// Copies the database to a temp directory so each test gets its own mutable copy.
+    /// The database is opened with [`SyncMode::UtterlyNoSync`] for test performance.
+    pub async fn new_from_db_with_config(db_path: &Path, config: Config) -> Self {
+        let temp_dir = tempfile::Builder::new()
+            .disable_cleanup(true)
+            .tempdir()
+            .expect("failed to create temp dir");
+
+        copy_db_dir(db_path, temp_dir.path()).expect("failed to copy database");
+
+        let db = katana_db::Db::open_no_sync(temp_dir.path()).expect("failed to open database");
+        let provider = DbProviderFactory::new(db.clone());
+
+        Self {
+            node: Node::build_with_provider(db, provider, config)
+                .expect("failed to build node")
+                .launch()
+                .await
+                .expect("failed to launch node"),
+        }
+    }
+
+    /// Creates a [`TestNode`] with a pre-migrated `spawn-and-move` database snapshot.
+    pub async fn new_with_spawn_and_move_db() -> Self {
+        let db_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+            .join("../../tests/fixtures/db/spawn_and_move");
+        Self::new_from_db(&db_path).await
+    }
+
+    /// Creates a [`TestNode`] with a pre-migrated `simple` database snapshot.
+    pub async fn new_with_simple_db() -> Self {
+        let db_path =
+            PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/fixtures/db/simple");
+        Self::new_from_db(&db_path).await
+    }
 }
 
 impl ForkTestNode {
@@ -277,6 +323,17 @@ fn run_sozo_migrate(
     Ok(())
 }
 
+/// Copies all files from `src` to `dst` (flat copy, no subdirectories).
+fn copy_db_dir(src: &Path, dst: &Path) -> std::io::Result<()> {
+    for entry in std::fs::read_dir(src)? {
+        let entry = entry?;
+        if entry.file_type()?.is_file() {
+            std::fs::copy(entry.path(), dst.join(entry.file_name()))?;
+        }
+    }
+    Ok(())
+}
+
 pub fn test_config() -> Config {
     let sequencing = SequencingConfig::default();
     let dev = DevConfig { fee: false, account_validation: true, fixed_gas_prices: None };