merge main

Author: dailinsubjam

.github/workflows/espresso-devnet-tests.yaml

@@ -0,0 +1,60 @@
+name: Run Espresso Devnet tests
+on:
+  pull_request:
+    branches:
+      - "celo-integration*"
+  push:
+    branches:
+      - "master"
+      - "celo-integration*"
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-24.04-8core
+    env:
+        ESPRESSO_DEVNET_TESTS_LIVENESS_PERIOD: '1m'
+        ESPRESSO_DEVNET_TESTS_OUTAGE_PERIOD: '1m'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+
+      - name: Install Nix
+        uses: nixbuild/nix-quick-install-action@v30
+        with:
+          nix_conf: |
+            keep-env-derivations = true
+            keep-outputs = true
+      - name: Restore Nix cache
+        id: cache-nix-restore
+        uses: nix-community/cache-nix-action/restore@v6
+        with:
+          primary-key: nix-${{ runner.os }}-${{ hashFiles('**/*.nix', '**/flake.lock') }}
+      - name: Set up Nix environment
+        uses: nicknovitski/nix-develop@v1
+
+      - name: Cache Go modules
+        uses: actions/setup-go@v5
+
+      - name: Compile contracts
+        run: just compile-contracts
+
+      - name: Build Devnet
+        run: |
+          cd op-deployer
+          just
+          export PATH=$PATH:$PWD/bin
+          cd ../espresso
+          ./scripts/prepare-allocs.sh
+          docker compose build
+
+      - name: Run Devnet tests
+        run: go test -timeout 30m -p 1 -count 1 -v  ./espresso/devnet-tests/...
+
+      - name: Save Nix cache
+        uses: nix-community/cache-nix-action/save@v6
+        if: always() && steps.cache-nix-restore.outputs.hit-primary-key != 'true'
+        with:
+          primary-key: ${{ steps.cache-nix-restore.outputs.primary-key }}

.github/workflows/espresso-integration.yaml

@@ -48,7 +48,8 @@ jobs:
           total: 4
           packages: "./espresso/..."
       - name: Run Go tests for group ${{ matrix.group }}
-        run: go test -timeout 30m -p 1 -count 1 -v -run "^(${{ steps.test_split.outputs.run}})$" ./espresso/...
+        run: |
+          go test -short -timeout 30m -p 1 -count 1 -v -run "^(${{ steps.test_split.outputs.run}})$" ./espresso/...
 
       - name: Save Nix cache
         uses: nix-community/cache-nix-action/save@v6

.vscode/settings.json

@@ -1,4 +1,5 @@
 {
   "editorconfig.generateAuto": false,
-  "files.trimTrailingWhitespace": true
+  "files.trimTrailingWhitespace": true,
+  "makefile.configureOnOpen": false
 }

README_ESPRESSO.md

@@ -389,3 +389,87 @@ In order to refresh this AMI one needs to:
 1. Create an AWS EC2 instance with the characteristics described in (see `.github/workflows/enclave.yaml` *Launch EC2 Instance* job).
 2. Copy the script `espresso/scrips/enclave-prepare-ami.sh` in the EC2 instance (e.g. using scp) and run it.
 3. [Export the AMI instance](https://docs.aws.amazon.com/toolkit-for-visual-studio/latest/user-guide/tkv-create-ami-from-instance.html).
+
+## Demo to Celo
+For convenience some scripts have been added to make it easier to showcase the
+results, and monitor the progress of the docker compose file.  The primary
+script concerns evaluating `optimism_syncStatus` and displaying the results.
+
+This script requires the commands `tmux`, and `watch` to be installed and
+in the `PATH`. Check to see if you have them, and if you don't, be sure to
+install them using whatever method you deem necessary in order to run the
+script.
+
+After that has been done you should be able to spin up the simple script
+using the following command:
+```console
+./espresso/scripts/demo_tmux_get_sync_status.sh
+```
+
+This will launch a `tmux` session setup with a script to automatically
+query and display the `optimism_syncStatus` result for the `sequencer`,
+`verifier`, and `caff-node`.
+
+It assumes that the `docker-file.yml` is being run with the default values
+and will attempt to connect to them as needed.
+
+If you're not used to `tmux` you should be able to disconnect from the session
+using `<C-b> d`. This only detaches from the session, the session will still
+exist and be running in the background.  You can kill the session using the
+following command:
+```console
+tmux kill-session
+```
+
+Or you can reattach to it using this command instead:
+```console
+tmux attach
+```
+
+If you want to target different RPC endpoints for optimism, if you're not
+running the local demo, and want to target the remote, you can always
+specify environment variables before running the script:
+```console
+OP_RPC_SEQUENCER=http://sequencer.example.com:4545 \
+OP_RPC_VERIFIER=http://verifier.example.com:4545 \
+OP_RPC_CAFF=http://caff.example.com:4545 \
+./espresso/scripts/demo_tmux_get_sync_status.sh
+```
+
+### Prepare for the Demo
+* Go to the scripts directory.
+```console
+cd espresso/scripts
+```
+* Allow access to scripts.
+```console
+chmod +x startup.sh
+chmod +x logs.sh
+chmod +x shutdown.sh
+```
+
+### Prebuild Everything and Start All Services
+Note that `l2-genesis` is expected to take around 2 minutes.
+```console
+./startup.sh
+```
+
+### View Logs
+There are 15 services in total, as listed in `logs.sh`. It is supported to run logs for any
+service, but we may want to show logs selectively, e.g., by running the following commands one by
+one. Note that some service names are replaced by more convenient alias, but it is also suported to
+use their full names.
+```console
+./logs.sh l1-geth
+./logs.sh dev-node
+./logs.sh op-geth-sequencer
+./logs.sh sequencer
+./logs.sh verifier
+./logs.sh caff-node
+./logs.sh batcher
+```
+
+### Shut Down All Services
+```console
+./shutdown.sh
+```

espresso/.env

@@ -24,6 +24,8 @@ L1_HTTP_PORT=8545
 L1_BEACON_PORT=5052
 
 ROLLUP_PORT=9545
+VERIFIER_PORT=9546
+CAFF_PORT=9547
 
 OP_ENGINE_PORT=8552
 OP_HTTP_PORT=8546

espresso/devnet-tests/README.md

@@ -0,0 +1,14 @@
+# Espresso Devnet Tests
+
+Test various end-to-end functionalities in a locally running devnet.
+
+## Running
+
+`go test ./espresso/devnet-tests/...`
+
+Configure how long it takes to run the tests vs how stringent the tests are by setting
+`ESPRESSO_DEVNET_TESTS_LIVENESS_PERIOD` and `ESPRESSO_DEVNET_TESTS_OUTAGE_PERIOD`. These determine
+how long we need the devnet to run in a healthy state before considering a run successful, and how
+long to let an unhealthy state persist before attempting recovery, respectively. For the fullest
+test these are set to `1m` and `10m`, but for quick testing, more reasonable values would be around
+`10s`.

espresso/devnet-tests/batcher_restart_test.go

@@ -0,0 +1,44 @@
+package devnet_tests
+
+import (
+	"context"
+	"testing"
+
+	"github.com/ethereum/go-ethereum"
+	"github.com/stretchr/testify/require"
+)
+
+func TestBatcherRestart(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	d := NewDevnet(ctx, t)
+	require.NoError(t, d.Up())
+	defer func() {
+		require.NoError(t, d.Down())
+	}()
+
+	// Send a transaction just to check that everything has started up ok.
+	require.NoError(t, d.RunSimpleL2Burn())
+
+	// Shut down the batcher and have another transaction submitted while it is down.
+	require.NoError(t, d.ServiceDown("op-batcher"))
+	d.SleepOutageDuration()
+
+	receipt, err := d.SubmitSimpleL2Burn()
+	require.NoError(t, err)
+
+	// Check that while the batcher is down, the verifier does NOT process submitted transactions.
+	d.SleepOutageDuration()
+	_, err = d.L2Verif.TransactionReceipt(ctx, receipt.Receipt.TxHash)
+	require.ErrorIs(t, err, ethereum.NotFound)
+
+	// Bring the batcher back up and check that it processes the transaction which was submitted
+	// while it was down.
+	require.NoError(t, d.ServiceUp("op-batcher"))
+	require.NoError(t, d.VerifySimpleL2Burn(receipt))
+
+	// Submit another transaction at the end just to check that things stay working.
+	d.SleepRecoveryDuration()
+	require.NoError(t, d.RunSimpleL2Burn())
+}