feat: e2e test in GitHub action (#210)

colinlyguo · web-flow · commit c030533e9878 · 2025-07-29T00:03:11.000+08:00
* test: add e2e tests in github action * add linux/arm64 in Build docker image * fix block production test * address comments * restructuring * resolve no_std CI failure * fix no_std CI failure * fix no_std CI failure * fix no_std CI failure * replace random sleep with wait_for_services * error propagation * tweaks * tweak .dockerignore * revert unused scroll-tech/revm changes * fix no_std CI failure * remove common.rs * fix block propagation * add docker buildx in CI * change docker compose cmd * remove linux/arm64 * remove unused files * add --test-threads=1 * seperate tests and docker-compose tests * tweaks * tweaks * tweaks * add --test-threads=1 back * tweak * rename test names * refactor: move alloy-provider test features to tests/Cargo.toml * tweak configs * chore: remove unnecessary test declarations from Cargo.toml * fix CI script * address comments * use scroll provider * refactor docker-compose.rs * change to ad-hoc image build * address comments * temp remove integration-docker-compose * Revert "temp remove integration-docker-compose" This reverts commit 3aaeb32. * add cleanups * exclude test(docker) * Revert "add cleanups" This reverts commit e18f760. * use stable toolchain * add cleanups * remove Free up disk space step * add cleanups * tweak cleanups * tweak test.yaml * fix * increase retry times * run make pr * try fix CI * fix * add back Free up disk space * address comments
diff --git a/.dockerignore b/.dockerignore
@@ -5,6 +5,7 @@
 !/bin
 !/crates
 !/testing
+!/tests
 !book.toml
 !Cargo.lock
 !Cargo.toml
diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
@@ -128,11 +128,11 @@ jobs:
           - type: wasm
             target: wasm32-unknown-unknown
             exclude: |
-              scroll-engine,scroll-wire,rollup-node,scroll-network,rollup-node-manager,rollup-node-watcher,scroll-db,scroll-migration,rollup-node-indexer,scroll-codec,scroll-derivation-pipeline,rollup-node-providers,rollup-node-sequencer,rollup-node-signer
+              scroll-engine,scroll-wire,rollup-node,scroll-network,rollup-node-manager,rollup-node-watcher,scroll-db,scroll-migration,rollup-node-indexer,scroll-codec,scroll-derivation-pipeline,rollup-node-providers,rollup-node-sequencer,rollup-node-signer,tests
           - type: riscv
             target: riscv32imac-unknown-none-elf
             exclude: |
-              scroll-engine,scroll-wire,rollup-node,scroll-network,rollup-node-manager,rollup-node-watcher,scroll-db,scroll-migration,rollup-node-indexer,scroll-codec,scroll-derivation-pipeline,rollup-node-providers,rollup-node-sequencer,rollup-node-signer
+              scroll-engine,scroll-wire,rollup-node,scroll-network,rollup-node-manager,rollup-node-watcher,scroll-db,scroll-migration,rollup-node-indexer,scroll-codec,scroll-derivation-pipeline,rollup-node-providers,rollup-node-sequencer,rollup-node-signer,tests
     steps:
       - uses: actions/checkout@v4
       - uses: rui314/setup-mold@v1
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -53,5 +53,47 @@ jobs:
       - name: Run integration tests
         run: |
           cargo nextest run --all-features --workspace --locked \
-            --no-tests=pass -E 'kind(test)' \
+            --no-tests=pass -E 'kind(test) and not test(docker)' \
             -- --skip test_should_consolidate_to_block_15k
+
+  integration-docker-compose:
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    env:
+      RUST_BACKTRACE: 1
+    steps:
+      - uses: actions/checkout@v4
+      - uses: rui314/setup-mold@v1
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: true
+      - uses: taiki-e/install-action@v2
+        with:
+          tool: nextest@0.9.98
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Free up disk space
+        run: |
+          echo "Initial disk usage:"
+          df -h
+          sudo rm -rf /usr/share/dotnet || true
+          sudo rm -rf /opt/ghc || true
+          sudo rm -rf /usr/local/share/boost || true
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY" || true
+          sudo rm -rf /usr/local/lib/android || true
+          sudo rm -rf /usr/local/share/powershell || true
+          sudo rm -rf /usr/share/swift || true
+          sudo rm -rf /opt/hostedtoolcache || true
+          echo "Disk usage after cleanup:"
+          df -h
+      - name: Run Docker Compose integration tests
+        run: |
+          cargo nextest run --all-features --workspace --locked \
+            --no-tests=pass -E 'test(docker)' --test-threads=1 \
+            --failure-output immediate \
+            --success-output never \
+            --verbose
+        env:
+          RUST_LOG: debug
+          RUST_BACKTRACE: full
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -23,6 +23,7 @@ members = [
     "crates/sequencer",
     "crates/signer",
     "crates/watcher",
+    "tests",
 ]
 
 resolver = "2"
@@ -203,6 +204,7 @@ thiserror = "2.0"
 tokio = { version = "1.39", default-features = false }
 tokio-stream = { version = "0.1", default-features = false }
 tracing = "0.1.0"
+getrandom = { version = "0.2", features = ["js"] }
 
 [patch.crates-io]
 revm = { git = "https://github.com/scroll-tech/revm", branch = "feat/reth-v78" }
diff --git a/crates/node/src/args.rs b/crates/node/src/args.rs
@@ -358,7 +358,8 @@ pub struct BeaconProviderArgs {
     #[arg(
         long = "beacon.blob-source",
         id = "beacon_blob_source",
-        value_name = "BEACON_BLOB_SOURCE"
+        value_name = "BEACON_BLOB_SOURCE",
+        default_value = "mock"
     )]
     pub blob_source: BlobSource,
     /// The compute units per second for the provider.
diff --git a/tests/Cargo.toml b/tests/Cargo.toml
@@ -0,0 +1,19 @@
+[package]
+name = "tests"
+version = "0.0.1"
+edition = "2021"
+
+[lib]
+name = "tests"
+path = "src/lib.rs"
+
+[dependencies]
+alloy-primitives = { workspace = true }
+alloy-network = { workspace = true }
+alloy-provider = { workspace = true, features = ["reqwest", "ws"] }
+alloy-rpc-types-eth = { workspace = true }
+scroll-alloy-network = { workspace = true }
+tokio = { workspace = true, features = ["rt", "time"] }
+eyre = { workspace = true }
+getrandom = { workspace = true }
+tracing = { workspace = true }
diff --git a/tests/discovery-secret b/tests/discovery-secret
@@ -0,0 +1 @@
+6381271558bd1ae75c7b092553e3ecc04b104c38f34e126399ce4da4fd5e8730
diff --git a/tests/docker-compose.test.yml b/tests/docker-compose.test.yml
@@ -0,0 +1,51 @@
+version: '3.8'
+
+services:
+  rollup-node-sequencer:
+    build:
+      context: ../
+      dockerfile: Dockerfile
+    container_name: rollup-node-sequencer
+    entrypoint: ["bash", "/launch_rollup_node_sequencer.bash"]
+    environment:
+      - ENV=dev
+      - RUST_LOG=info
+    ports:
+      - "8545:8545"   # JSON-RPC
+      - "8546:8546"   # WebSocket
+      - "6060:6060"   # Metrics
+    volumes:
+      - ./launch_rollup_node_sequencer.bash:/launch_rollup_node_sequencer.bash:ro
+      - ./discovery-secret:/l2reth/discovery-secret:ro
+      - l2reth-sequencer:/l2reth
+    networks:
+      - test-scroll-network
+
+  rollup-node-follower:
+    build:
+      context: ../
+      dockerfile: Dockerfile
+    container_name: rollup-node-follower
+    entrypoint: ["bash", "/launch_rollup_node_follower.bash"]
+    environment:
+      - ENV=dev
+      - RUST_LOG=info
+    ports:
+      - "8547:8545"   # JSON-RPC
+      - "8548:8546"   # WebSocket
+      - "6061:6060"   # Metrics
+    volumes:
+      - ./launch_rollup_node_follower.bash:/launch_rollup_node_follower.bash:ro
+      - l2reth-follower:/l2reth
+    networks:
+      - test-scroll-network
+    depends_on:
+      - rollup-node-sequencer
+
+networks:
+  test-scroll-network:
+    driver: bridge
+
+volumes:
+  l2reth-sequencer:
+  l2reth-follower:
diff --git a/tests/launch_rollup_node_follower.bash b/tests/launch_rollup_node_follower.bash
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -e
+
+exec rollup-node node --chain dev --datadir=/l2reth --metrics=0.0.0.0:6060 --network.scroll-wire --network.bridge \
+  --http --http.addr=0.0.0.0 --http.port=8545 --http.corsdomain "*" --http.api admin,debug,eth,net,trace,txpool,web3,rpc,reth,ots,flashbots,miner,mev \
+  --ws --ws.addr=0.0.0.0 --ws.port=8546 --ws.api admin,debug,eth,net,trace,txpool,web3,rpc,reth,ots,flashbots,miner,mev \
+  --log.stdout.format log-fmt -vvv \
+  --test \
+  --txpool.pending-max-count=1000 \
+  --builder.gaslimit=20000000 \
+  --rpc.max-connections=5000 \
+  --trusted-peers enode://3983278a7cab48862d9ab3187278edf376a0736a7deb55472a5650592f6922ce626a1ea7d74b77b9a679694b343f5e93ea97d5d60a9db4e4b51bb0c23a36d01b@rollup-node-sequencer:30303
diff --git a/tests/launch_rollup_node_sequencer.bash b/tests/launch_rollup_node_sequencer.bash
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -e
+
+exec rollup-node node --chain dev --datadir=/l2reth --metrics=0.0.0.0:6060 --network.scroll-wire --network.bridge \
+  --http --http.addr=0.0.0.0 --http.port=8545 --http.corsdomain "*" --http.api admin,debug,eth,net,trace,txpool,web3,rpc,reth,ots,flashbots,miner,mev \
+  --ws --ws.addr=0.0.0.0 --ws.port=8546 --ws.api admin,debug,eth,net,trace,txpool,web3,rpc,reth,ots,flashbots,miner,mev \
+  --log.stdout.format log-fmt -vvv \
+  --test \
+  --sequencer.enabled \
+  --sequencer.block-time 250 \
+  --sequencer.payload-building-duration 230 \
+  --txpool.pending-max-count=1000 \
+  --builder.gaslimit=20000000 \
+  --rpc.max-connections=5000
diff --git a/tests/src/docker_compose.rs b/tests/src/docker_compose.rs
diff --git a/tests/src/lib.rs b/tests/src/lib.rs
diff --git a/tests/tests/block_production.rs b/tests/tests/block_production.rs
diff --git a/tests/tests/block_propagation.rs b/tests/tests/block_propagation.rs

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+6381271558bd1ae75c7b092553e3ecc04b104c38f34e126399ce4da4fd5e8730`