test(multiversion-tests): add E2E, upgrade/rollback tests and CI integration

Author: glottologist

.github/workflows/multiversion.yml

@@ -0,0 +1,45 @@
+name: Multiversion Tests
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 4 * * 1" # Monday 04:00 UTC
+
+concurrency:
+  group: multiversion-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: 1
+
+jobs:
+  multiversion-tests:
+    name: Cross-version integration tests
+    runs-on: ubuntu-latest-16-cores
+    timeout-minutes: 60
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: "1.93.0"
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y clang libgmp-dev pkg-config libssl-dev
+
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: |
+            . -> target
+            multiversion-tests -> multiversion-tests/target
+
+      - name: Run multiversion tests
+        working-directory: multiversion-tests
+        run: cargo test --test '*' -- --ignored --test-threads=1 --nocapture

.gitignore

@@ -1,5 +1,5 @@
 /target
-**/target
+multiversion-tests/target
 .reth
 **/.DS_Store
 ./database

multiversion-tests/Cargo.lock

@@ -8,16 +8,15 @@ rust-version = "1.93.0"
 publish = false
 
 [dependencies]
-tokio = { version = "1", features = ["rt-multi-thread", "macros", "process", "signal", "time", "fs", "io-util"] }
+tokio = { version = "1", features = ["rt-multi-thread", "macros", "process", "signal", "time", "fs", "io-util", "sync"] }
 reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 toml = "0.8"
 tempfile = "3"
 tracing = "0.1"
-tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
 thiserror = "2"
-nix = { version = "0.29", features = ["signal", "process", "mman"] }
+nix = { version = "0.29", features = ["signal", "process"] }
 
 [dev-dependencies]
 test-log = { version = "0.2", features = ["trace"] }

multiversion-tests/Cargo.toml

@@ -1,9 +1,104 @@
-# Baseline config template for multiversion tests
-# Fields are patched by config::generate_config()
-api_port = 0
-gossip_port = 0
-base_directory = ""
-mining_key = ""
-mining_address = ""
-peers = []
-is_genesis = false
+node_mode = "Peer"
+sync_mode = "Full"
+mining_key = "0000000000000000000000000000000000000000000000000000000000000001"
+reward_address = "0x0000000000000000000000000000000000000001"
+base_directory = "/tmp/irys-test"
+consensus = "Testing"
+trusted_peers = []
+
+[network_defaults]
+public_ip = "127.0.0.1"
+bind_ip = "127.0.0.1"
+
+[gossip]
+public_port = 0
+bind_port = 0
+public_ip = "127.0.0.1"
+bind_ip = "127.0.0.1"
+
+[http]
+public_port = 0
+bind_port = 0
+public_ip = "127.0.0.1"
+bind_ip = "127.0.0.1"
+
+[reth.network]
+public_port = 0
+bind_port = 0
+use_random_ports = true
+public_ip = "127.0.0.1"
+bind_ip = "127.0.0.1"
+
+[storage]
+num_writes_before_sync = 100
+
+[data_sync]
+max_pending_chunk_requests = 1000
+max_storage_throughput_bps = 209715200
+bandwidth_adjustment_interval = "5s"
+chunk_request_timeout = "10s"
+
+[packing.local]
+cpu_packing_concurrency = 2
+gpu_packing_batch_size = 0
+
+[cache]
+cache_clean_lag = 0
+max_cache_size_bytes = 10737418240
+prune_at_capacity_percent = 80.0
+
+[vdf]
+parallel_verification_thread_limit = 4
+
+[mempool]
+max_pending_pledge_items = 1000
+max_pledges_per_item = 10
+max_pending_chunk_items = 500
+max_chunks_per_item = 100
+max_preheader_chunks_per_item = 50
+max_preheader_data_path_bytes = 4096
+max_valid_items = 10000
+max_invalid_items = 5000
+max_valid_chunks = 5000
+max_valid_submit_txs = 3000
+max_valid_commitment_addresses = 1000
+max_commitments_per_address = 5
+max_concurrent_mempool_tasks = 30
+max_concurrent_chunk_ingress_tasks = 30
+chunk_writer_buffer_size = 4096
+
+[[oracles]]
+type = "mock"
+initial_price = 1.0
+incremental_change = 0.00000000000001
+smoothing_interval = 15
+initial_direction_up = true
+poll_interval_ms = 10000
+
+[p2p_handshake]
+max_concurrent_handshakes = 32
+max_peers_per_response = 25
+max_retries = 8
+backoff_base_secs = 1
+backoff_cap_secs = 60
+blocklist_ttl_secs = 600
+server_peer_list_cap = 25
+
+[p2p_gossip]
+broadcast_batch_size = 50
+broadcast_batch_throttle_interval = 100
+enable_scoring = true
+max_concurrent_gossip_chunks = 50
+
+[p2p_pull]
+top_active_window = 10
+sample_size = 5
+max_attempts = 5
+
+[sync]
+block_batch_size = 50
+periodic_sync_check_interval_secs = 30
+retry_block_request_timeout_secs = 30
+enable_periodic_sync_check = true
+wait_queue_slot_timeout_secs = 30
+wait_queue_slot_max_attempts = 3

multiversion-tests/claude/prompts.md

@@ -1,3 +1,4 @@
+use std::fs::File;
 use std::path::{Path, PathBuf};
 use std::process::Stdio;
 
@@ -72,6 +73,11 @@ impl BinaryResolver {
             return Ok(resolved("new", rev, cached));
         }
 
+        let _lock = self.acquire_build_lock(&rev).await?;
+        if cached.exists() {
+            return Ok(resolved("new", rev, cached));
+        }
+
         self.build_at(&self.repo_root, &cached).await?;
         Ok(resolved("new", rev, cached))
     }
@@ -89,11 +95,19 @@ impl BinaryResolver {
             return Ok(resolved(git_ref, rev, cached));
         }
 
+        let _lock = self.acquire_build_lock(&rev).await?;
+        if cached.exists() {
+            return Ok(resolved(git_ref, rev, cached));
+        }
+
         let worktree_path = self.cache_dir.join(format!("worktree-{rev}"));
+        self.prune_stale_worktrees().await;
         self.create_worktree(git_ref, &worktree_path).await?;
-        let result = self.build_at(&worktree_path, &cached).await;
-        self.remove_worktree(&worktree_path).await?;
-        result?;
+        let build_result = self.build_at(&worktree_path, &cached).await;
+        if let Err(e) = self.remove_worktree(&worktree_path).await {
+            tracing::warn!(error = %e, path = %worktree_path.display(), "failed to remove worktree");
+        }
+        build_result?;
 
         Ok(resolved(git_ref, rev, cached))
     }
@@ -110,14 +124,29 @@ impl BinaryResolver {
         if !path.exists() {
             return Err(BinaryError::NotFound { path });
         }
-        let rev = self.git_rev("HEAD").await?;
-        Ok(Some(resolved(label, rev, path)))
+        Ok(Some(resolved(label, "env-override".into(), path)))
     }
 
     fn cached_binary_path(&self, rev: &str) -> PathBuf {
         self.cache_dir.join(format!("irys-{rev}"))
     }
 
+    /// Inter-process exclusive lock for a specific revision build.
+    /// Prevents parallel test processes from creating the same worktree simultaneously.
+    /// The lock is released when the returned File is dropped.
+    async fn acquire_build_lock(&self, rev: &str) -> Result<File, BinaryError> {
+        tokio::fs::create_dir_all(&self.cache_dir).await?;
+        let lock_path = self.cache_dir.join(format!("build-{rev}.lock"));
+        tokio::task::spawn_blocking(move || -> Result<File, std::io::Error> {
+            let file = File::create(lock_path)?;
+            file.lock()?;
+            Ok(file)
+        })
+        .await
+        .map_err(|e| BinaryError::Io(std::io::Error::other(e)))?
+        .map_err(BinaryError::Io)
+    }
+
     async fn git_rev(&self, git_ref: &str) -> Result<String, BinaryError> {
         validate_git_ref(git_ref)?;
         self.git_rev_at(&self.repo_root, git_ref).await
@@ -127,7 +156,7 @@ impl BinaryResolver {
         validate_git_ref(git_ref)?;
 
         let output = Command::new("git")
-            .args(["rev-parse", "--short=12", "--"])
+            .args(["rev-parse", "--short=12"])
             .arg(git_ref)
             .current_dir(dir)
             .stdout(Stdio::piped())
@@ -172,16 +201,37 @@ impl BinaryResolver {
             tokio::fs::create_dir_all(parent).await?;
         }
 
+        let built = match self
+            .try_cargo_build(work_dir, &["--profile", "debug-release"], "debug-release")
+            .await
+        {
+            Ok(path) => path,
+            Err(BinaryError::BuildFailed { ref stderr, .. })
+                if stderr.contains("is not defined") =>
+            {
+                tracing::info!("debug-release profile unavailable, falling back to --release");
+                self.try_cargo_build(work_dir, &["--release"], "release")
+                    .await?
+            }
+            Err(e) => return Err(e),
+        };
+
+        tokio::fs::copy(&built, output_path).await?;
+        Ok(())
+    }
+
+    async fn try_cargo_build(
+        &self,
+        work_dir: &Path,
+        profile_args: &[&str],
+        target_dir_name: &str,
+    ) -> Result<PathBuf, BinaryError> {
+        let mut args = vec!["build"];
+        args.extend(profile_args);
+        args.extend(["--bin", "irys", "-p", "irys-chain"]);
+
         let output = Command::new("cargo")
-            .args([
-                "build",
-                "--profile",
-                "debug-release",
-                "--bin",
-                "irys",
-                "-p",
-                "irys-chain",
-            ])
+            .args(&args)
             .current_dir(work_dir)
             .stdout(Stdio::piped())
             .stderr(Stdio::piped())
@@ -196,13 +246,24 @@ impl BinaryResolver {
             });
         }
 
-        let built = work_dir.join("target/debug-release/irys");
+        let built = work_dir.join(format!("target/{target_dir_name}/irys"));
         if !built.exists() {
             return Err(BinaryError::NotFound { path: built });
         }
-        tokio::fs::copy(&built, output_path).await?;
+        Ok(built)
+    }
 
-        Ok(())
+    async fn prune_stale_worktrees(&self) {
+        let result = Command::new("git")
+            .args(["worktree", "prune"])
+            .current_dir(&self.repo_root)
+            .stdout(Stdio::null())
+            .stderr(Stdio::null())
+            .status()
+            .await;
+        if let Err(e) = result {
+            tracing::warn!(error = %e, "git worktree prune failed");
+        }
     }
 
     async fn remove_worktree(&self, path: &Path) -> Result<(), BinaryError> {
@@ -211,12 +272,19 @@ impl BinaryResolver {
             .arg(path)
             .current_dir(&self.repo_root)
             .stdout(Stdio::null())
-            .stderr(Stdio::null())
+            .stderr(Stdio::piped())
             .output()
             .await;
 
-        if let Err(e) = output {
-            tracing::warn!(path = %path.display(), error = %e, "failed to remove worktree");
+        match output {
+            Ok(o) if !o.status.success() => {
+                let stderr = String::from_utf8_lossy(&o.stderr);
+                tracing::warn!(path = %path.display(), stderr = %stderr, "worktree removal failed");
+            }
+            Err(e) => {
+                tracing::warn!(path = %path.display(), error = %e, "failed to remove worktree");
+            }
+            _ => {}
         }
 
         Ok(())