TA1: Add devnet test for batcher restart (#204)

* Add devnet test for batcher restart

* Check error returns

* Separate op-geth instances for each L2 node

* Build devnet dockers in CI

* Build op-deployer in CI

* Try larger runner

* Increase test outage and recovery time

* Try to speed up transaction verification

* Do not drop batches before we have seen a finalized L1 block

* Remove unnecessary sleep

* Build containers in dependency order

* Don't copy config file into Docker image at build time

* Checkout submodules in CI

* Don't copy config file into Docker image at build time

* Run devnet test in separate workflow

Author: jbearer

.github/workflows/espresso-devnet-tests.yaml

@@ -0,0 +1,60 @@
+name: Run Espresso Devnet tests
+on:
+  pull_request:
+    branches:
+      - "celo-integration*"
+  push:
+    branches:
+      - "master"
+      - "celo-integration*"
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-24.04-8core
+    env:
+        ESPRESSO_DEVNET_TESTS_LIVENESS_PERIOD: '1m'
+        ESPRESSO_DEVNET_TESTS_OUTAGE_PERIOD: '1m'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+
+      - name: Install Nix
+        uses: nixbuild/nix-quick-install-action@v30
+        with:
+          nix_conf: |
+            keep-env-derivations = true
+            keep-outputs = true
+      - name: Restore Nix cache
+        id: cache-nix-restore
+        uses: nix-community/cache-nix-action/restore@v6
+        with:
+          primary-key: nix-${{ runner.os }}-${{ hashFiles('**/*.nix', '**/flake.lock') }}
+      - name: Set up Nix environment
+        uses: nicknovitski/nix-develop@v1
+
+      - name: Cache Go modules
+        uses: actions/setup-go@v5
+
+      - name: Compile contracts
+        run: just compile-contracts
+
+      - name: Build Devnet
+        run: |
+          cd op-deployer
+          just
+          export PATH=$PATH:$PWD/bin
+          cd ../espresso
+          ./scripts/prepare-allocs.sh
+          docker compose build
+
+      - name: Run Devnet tests
+        run: go test -timeout 30m -p 1 -count 1 -v  ./espresso/devnet-tests/...
+
+      - name: Save Nix cache
+        uses: nix-community/cache-nix-action/save@v6
+        if: always() && steps.cache-nix-restore.outputs.hit-primary-key != 'true'
+        with:
+          primary-key: ${{ steps.cache-nix-restore.outputs.primary-key }}

.github/workflows/espresso-integration.yaml

@@ -48,7 +48,8 @@ jobs:
           total: 4
           packages: "./espresso/..."
       - name: Run Go tests for group ${{ matrix.group }}
-        run: go test -timeout 30m -p 1 -count 1 -v -run "^(${{ steps.test_split.outputs.run}})$" ./espresso/...
+        run: |
+          go test -short -timeout 30m -p 1 -count 1 -v -run "^(${{ steps.test_split.outputs.run}})$" ./espresso/...
 
       - name: Save Nix cache
         uses: nix-community/cache-nix-action/save@v6

.vscode/settings.json

@@ -1,4 +1,5 @@
 {
   "editorconfig.generateAuto": false,
-  "files.trimTrailingWhitespace": true
+  "files.trimTrailingWhitespace": true,
+  "makefile.configureOnOpen": false
 }

espresso/devnet-tests/README.md

@@ -0,0 +1,14 @@
+# Espresso Devnet Tests
+
+Test various end-to-end functionalities in a locally running devnet.
+
+## Running
+
+`go test ./espresso/devnet-tests/...`
+
+Configure how long it takes to run the tests vs how stringent the tests are by setting
+`ESPRESSO_DEVNET_TESTS_LIVENESS_PERIOD` and `ESPRESSO_DEVNET_TESTS_OUTAGE_PERIOD`. These determine
+how long we need the devnet to run in a healthy state before considering a run successful, and how
+long to let an unhealthy state persist before attempting recovery, respectively. For the fullest
+test these are set to `1m` and `10m`, but for quick testing, more reasonable values would be around
+`10s`.

espresso/devnet-tests/batcher_restart_test.go

@@ -0,0 +1,44 @@
+package devnet_tests
+
+import (
+	"context"
+	"testing"
+
+	"github.com/ethereum/go-ethereum"
+	"github.com/stretchr/testify/require"
+)
+
+func TestBatcherRestart(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	d := NewDevnet(ctx, t)
+	require.NoError(t, d.Up())
+	defer func() {
+		require.NoError(t, d.Down())
+	}()
+
+	// Send a transaction just to check that everything has started up ok.
+	require.NoError(t, d.RunSimpleL2Burn())
+
+	// Shut down the batcher and have another transaction submitted while it is down.
+	require.NoError(t, d.ServiceDown("op-batcher"))
+	d.SleepOutageDuration()
+
+	receipt, err := d.SubmitSimpleL2Burn()
+	require.NoError(t, err)
+
+	// Check that while the batcher is down, the verifier does NOT process submitted transactions.
+	d.SleepOutageDuration()
+	_, err = d.L2Verif.TransactionReceipt(ctx, receipt.Receipt.TxHash)
+	require.ErrorIs(t, err, ethereum.NotFound)
+
+	// Bring the batcher back up and check that it processes the transaction which was submitted
+	// while it was down.
+	require.NoError(t, d.ServiceUp("op-batcher"))
+	require.NoError(t, d.VerifySimpleL2Burn(receipt))
+
+	// Submit another transaction at the end just to check that things stay working.
+	d.SleepRecoveryDuration()
+	require.NoError(t, d.RunSimpleL2Burn())
+}