Improve runtime of tests (#1854)

Improves CI runtime by working around slow tests. The last commit on main, tests took 36min. Tests are currently completing in this PR in 12m.

   *  switch to nextest
    * separate build and test steps
    * use --retries (retries and marks tests that succeed upon retry as FLAKY
    * split slow dev-node tests to separate workflow. That new workflow is *not* required for merge
 

---------

Co-authored-by: tbro <[email protected]>
Co-authored-by: sveitser <[email protected]>

Author: tbro

.github/workflows/slowtest.yaml

@@ -0,0 +1,64 @@
+name: SlowEst
+
+on:
+  push:
+    branches:
+      - main
+      - release-*
+    tags:
+      # YYYYMMDD
+      - "20[0-9][0-9][0-1][0-9][0-3][0-9]*"
+  schedule:
+    - cron: "0 0 * * 1"
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ !contains(github.ref, 'main')}}
+
+env:
+  RUSTFLAGS: '--cfg async_executor_impl="async-std" --cfg async_channel_impl="async-std"'
+  RUST_LOG: info,libp2p=off,node=error
+
+jobs:
+  slowest:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Fix submodule permissions check
+        run: |
+          git config --global --add safe.directory '*'
+
+      - name: Install Foundry
+        uses: foundry-rs/foundry-toolchain@v1
+        with:
+          version: nightly
+
+      - uses: taiki-e/install-action@nextest
+
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Enable Rust Caching
+        uses: Swatinem/rust-cache@v2
+
+      - name: Build
+        # Build test binary with `testing` feature, which requires `hotshot_example` config
+        run: |
+          export RUSTFLAGS="$RUSTFLAGS --cfg hotshot_example"
+          export PATH="$PWD/target/release:$PATH"
+          cargo build --locked --bin diff-test --release
+          cargo nextest run --locked --release --workspace --all-features --no-run
+        timeout-minutes: 90
+
+      - name: SlowEst
+        env:
+          CARGO_TERM_COLOR: always
+        # Build test binary with `testing` feature, which requires `hotshot_example` config
+        run: |
+          export RUSTFLAGS="$RUSTFLAGS --cfg hotshot_example"
+          export PATH="$PWD/target/release:$PATH"
+          cargo nextest run --locked --release --workspace --all-features --verbose -E 'test(slow_)'
+        timeout-minutes: 25

.github/workflows/test.yml

@@ -34,10 +34,7 @@ jobs:
         with:
           version: nightly
 
-      - name: Install just command runner
-        run: |
-          sudo snap install --edge --classic just
-          just --version
+      - uses: taiki-e/install-action@nextest
 
       - name: Checkout Repository
         uses: actions/checkout@v4
@@ -47,12 +44,21 @@ jobs:
       - name: Enable Rust Caching
         uses: Swatinem/rust-cache@v2
 
-      - name: Test
+      - name: Build
         # Build test binary with `testing` feature, which requires `hotshot_example` config
         run: |
           export RUSTFLAGS="$RUSTFLAGS --cfg hotshot_example"
           export PATH="$PWD/target/release:$PATH"
           cargo build --locked --bin diff-test --release
-          cargo test --locked --release --workspace --all-features --no-run
-          cargo test --locked --release --workspace --all-features --verbose -- --test-threads 1 --nocapture
+          cargo nextest run --locked --release --workspace --all-features --no-run
         timeout-minutes: 90
+
+      - name: Test
+        env:
+          CARGO_TERM_COLOR: always
+        # Build test binary with `testing` feature, which requires `hotshot_example` config
+        run: |
+          export RUSTFLAGS="$RUSTFLAGS --cfg hotshot_example"
+          export PATH="$PWD/target/release:$PATH"
+          cargo nextest run --locked --release --workspace --all-features --retries 2 --verbose -E '!test(slow_)'
+        timeout-minutes: 5

flake.nix

@@ -197,6 +197,7 @@
             cargo-audit
             cargo-edit
             cargo-sort
+            cargo-nextest
             typos
             just
             fenix.packages.${system}.rust-analyzer

justfile

@@ -34,8 +34,15 @@ anvil *args:
     docker run -p 127.0.0.1:8545:8545 ghcr.io/foundry-rs/foundry:latest "anvil {{args}}"
 
 test:
-    cargo build --bin diff-test --release
-    cargo test --release --all-features
+	@echo 'Omitting slow tests. Use `test-slow` for those. Or `test-all` for all tests.'
+	cargo nextest run --locked --release --workspace --all-features --retries 2 --verbose -E '!test(slow_)'
+
+test-slow:
+	@echo 'Only slow tests are included. Use `test` for those deemed not slow. Or `test-all` for all tests.'
+	cargo nextest run --locked --release --workspace --all-features --verbose -E 'test(slow_)'
+
+test-all:
+	cargo nextest run --locked --release --workspace --all-features --verbose
 
 clippy:
     cargo clippy --workspace --all-features --all-targets -- -D warnings

sequencer/src/bin/espresso-dev-node.rs

@@ -541,7 +541,7 @@ mod tests {
     // - APIs update
     // - Types (like `Header`) update
     #[async_std::test]
-    async fn dev_node_test() {
+    async fn slow_dev_node_test() {
         setup_test();
 
         let builder_port = pick_unused_port().unwrap();
@@ -849,7 +849,7 @@ mod tests {
     }
 
     #[async_std::test]
-    async fn dev_node_multiple_lc_providers_test() {
+    async fn slow_dev_node_multiple_lc_providers_test() {
         setup_test();
 
         let builder_port = pick_unused_port().unwrap();