Cherry-pick and add regression tests for celestia integration

.github/workflows/regression-test.yml

@@ -0,0 +1,79 @@
+name: Regression Test CI
+run-name: CI triggered from @${{ github.actor }} of ${{ github.head_ref }}
+
+on:
+  workflow_dispatch:
+
+# Cancel in-progress jobs except for integration branch
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ !contains(github.ref, 'integration/')}}
+
+jobs:
+  regression_test:
+    runs-on: ubuntu-24.04
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      # Install wscat for finality node smoke test to listen to sequencer feed
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: "16"
+
+      - name: Install wscat
+        run: npm install -g wscat
+
+      - name: Install Foundry
+        uses: foundry-rs/foundry-toolchain@v1
+        with:
+          version: nightly
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver-opts: network=host
+
+      # Explicitly building the docker images because it's not clear how we can
+      # use the github action (gha) cache if we let docker compose build the images.
+      - uses: docker/build-push-action@v5
+        with:
+          context: ./rollupcreator
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          # Use default branch from test-node.bash
+          build-args: |
+            NITRO_CONTRACTS_BRANCH=develop
+
+      - uses: docker/build-push-action@v5
+        with:
+          context: ./scripts
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - uses: docker/build-push-action@v5
+        with:
+          context: ./mock-sequencer
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - uses: docker/build-push-action@v5
+        with:
+          context: ./tokenbridge
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          # Use default branch from test-node.bash
+          build-args: |
+            TOKEN_BRIDGE_BRANCH=v1.2.2
+
+      - name: Regression Test with Latest Espresso Image
+        run: ./regression-test.bash
+
+      - name: Dump docker logs on failure
+        if: failure()
+        run: |
+          docker compose logs

docker-compose.yaml

@@ -244,6 +244,19 @@ services:
       retries: 5
       start_period: 40s
 
+  mock-sequencer:
+    pid: host
+    build: mock-sequencer/
+    environment:
+      - REMOTE_WS_URL=ws://sequencer:9642
+      - HTTP_PORT=10000
+    ports:
+      - "127.0.0.1:10000:10000"
+      - "127.0.0.1:9643:9642"
+    depends_on:
+      - sequencer
+      - espresso-dev-node
+
   sequencer-on-espresso:
     pid: host # allow debugging
     image: espresso-integration-testnode
@@ -498,7 +511,7 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock
 
   espresso-dev-node:
-    image: ghcr.io/espressosystems/espresso-sequencer/espresso-dev-node:20250428-dev-node-decaf-pos
+    image: ghcr.io/espressosystems/espresso-sequencer/espresso-dev-node:20250710
     ports:
       - "$ESPRESSO_SEQUENCER_API_PORT:$ESPRESSO_SEQUENCER_API_PORT"
       - "$ESPRESSO_BUILDER_PORT:$ESPRESSO_BUILDER_PORT"

espresso-tests/.env

@@ -7,14 +7,8 @@ export PARENT_CHAIN_RPC_URL="http://localhost:8545"
 export CHILD_CHAIN_RPC_URL="http://localhost:8547"
 # Environment variables for new OSP deployment
 # These are essential for the upgrade
-export PARENT_CHAIN_UPGRADE_EXECUTOR="0x513D9F96d4D0563DEbae8a0DC307ea0E46b10ed7"
-export PARENT_UPGRADE_EXECUTOR_ADDRESS="0x513D9F96d4D0563DEbae8a0DC307ea0E46b10ed7"
-export CHILD_CHAIN_UPGRADE_EXUCTOR_ADDRESS="0xD59870177729b1Fa7CCdA1d2E245C57C6ad5F9F6"
-# Environment variables for osp migration action contract
-export ROLLUP_ADDRESS="0x1b836843Ef0B1731fea7C69d7d3847327DD137c2"
-export PROXY_ADMIN_ADDRESS="0x2A1f38c9097e7883570e0b02BFBE6869Cc25d8a3"
-# Environment variables for ArbOS upgrade action.
-export UPGRADE_TIMESTAMP="1723664126"
+CHILD_CHAIN_UPGRADE_EXUCTOR_ADDRESS="0xD59870177729b1Fa7CCdA1d2E245C57C6ad5F9F6"
+UPGRADE_TIMESTAMP="1723664126"
 
 # The reader addr is only important if the parent chain is not an arbitrum chain, this is important for the batch poster.
 export READER_ADDRESS="0x7DD3F2a3fAeF3B9F2364c335163244D3388Feb83"
@@ -25,6 +19,4 @@ export OLD_BATCH_POSTER_ADDRESS="0xe2148eE53c0755215Df69b2616E552154EdC584f"
 export NEW_BATCH_POSTER_ADDRESS="0xe2148eE53c0755215Df69b2616E552154EdC584f"
 export BATCH_POSTER_MANAGER_ADDRESS="0xe2148eE53c0755215Df69b2616E552154EdC584f"
 export PARENT_CHAIN_IS_ARBITRUM="false"
-export ESPRESSO_TEE_VERIFIER_ADDRESS="0x165155D6aBB370Cb10ad1bF835e723F662d51C86"
-export INBOX_ADDRESS="0x9f8c1c641336A371031499e3c362e40d58d0f254"
-export TARGET_WASM_MODULE_ROOT="0xe81f986823a85105c5fd91bb53b4493d38c0c26652d23f76a7405ac889908287"
+DELAY_BUFFERABLE=false

espresso-tests/create-espresso-integrated-nitro-node.bash

@@ -8,7 +8,7 @@ espresso=true
 simpleWithValidator=false
 
 # docker pull and tag the espresso integration nitro node.
-docker pull $ESPRESSO_VERSION
+docker pull $ESPRESSO_VERSION --platform linux/amd64
 
 docker tag $ESPRESSO_VERSION espresso-integration-testnode
 

espresso-tests/migration-test.bash

@@ -10,28 +10,87 @@ set -x # print each command before executing it, for debugging
 
 
 
+function forge3 {
+  forge $@ --root espresso-migration-3.1.0
+}
+
+
 # Find directory of this script, the project, and the orbit-actions submodule
 TEST_DIR="$(dirname $(readlink -f $0))"
+TEST_SCRIPT_DIR="v2.1.3-migration"
+TESTNODE_LOG_FILE=$(mktemp -t nitro-test-node-logs-XXXXXXXX)
+ESPRESSO_DEVNODE_LOG_FILE=$(mktemp -t espresso-dev-node-logs-XXXXXXXX)
 TESTNODE_DIR="$(dirname "$TEST_DIR")"
 ORBIT_ACTIONS_DIR="$TESTNODE_DIR/orbit-actions"
+ORBIT_MIGRATION_ACTION_DIR="contracts/parent-chain/espresso-migration/"
+ENV_FILE="$TEST_DIR/.env"
+# Hide docker compose warnings about orphaned containers.
+export COMPOSE_IGNORE_ORPHANS=true
+
+v3=false
+forge=forge
+ESPRESSO_NITRO_CONTRACTS_BRANCH=v2.1.3-8e58a9a
+# This commit matches v2.1.0 release of nitro-contracts, with additional support to set arb owner through upgrade executor
+NITRO_CONTRACTS_BRANCH="99c07a7db2fcce75b751c5a2bd4936e898cda065"
+BROADCAST_DIR="broadcast"
+PROXY_ADMIN_ADDRESS="0x2A1f38c9097e7883570e0b02BFBE6869Cc25d8a3"
+if [[ ${VERSION:-2} == "3" ]]; then
+    info "Using v3 migration scripts"
+    v3=true
+    TEST_SCRIPT_DIR="v3.1.0-migration"
+    ESPRESSO_NITRO_CONTRACTS_BRANCH=develop
+    ORBIT_MIGRATION_ACTION_DIR="espresso-migration-3.1.0/"
+    BROADCAST_DIR="espresso-migration-3.1.0/broadcast"
+    NITRO_CONTRACTS_BRANCH="v3.1.0"
+    PROXY_ADMIN_ADDRESS="0x275FC51309e5928Cb085b463ADEF5cbD45c76b62"
+
+    forge="forge3"
+else
+    info "Using v2 migration scripts"
+fi
+
+info Ensuring docker compose project is stopped
+run docker compose down -v --remove-orphans
 
 # Change to orbit actions directory, update the submodule, and install any dependencies for the purposes of the test.
 cd "$ORBIT_ACTIONS_DIR"
 
-git submodule update --init
-forge update
-yarn
+info "Ensuring submodules are checked out"
+run git submodule update --init --recursive
+
+info "Ensuring nodejs dependencies are installed"
+run yarn
+
+info "Ensuring we can compile the migration smart contracts"
+run $forge build
 
 # Change to the top level directory for the purposes of the test.
 cd "$TESTNODE_DIR"
 
-# Initialize a standard network not compatible with espresso to simulate a pre-upgrade orbit network e.g. not needed for the real migration
-./test-node.bash --simple --init-force --tokenbridge --detach --no-build-utils
+# NOTE: the test-node.bash script (or potentially docker compose) does not work
+# well with the `fmt` utility function and hangs at the end. I don't know why.
+# Furthermore the long warning lines don't work with the `fmt` function but I
+# can't work out a way to be able to filter the lines (e. g. grep -v WARN) and
+# still have the output show up.
+
+info Deploying a vanilla Nitro stack locally, to be migrated to Espresso later.
+emph ./test-node.bash --simple --init-force --tokenbridge --detach
+if [ "$DEBUG" = "true" ]; then
+  ./test-node.bash --simple --init-force --tokenbridge --detach
+else
+  info "This command starts up an entire Nitro stack. It takes a long time."
+  info "Run \`tail -f $TESTNODE_LOG_FILE\` to see logs, if necessary."
+  echo
+  ./test-node.bash --simple --init-force --tokenbridge --detach > "$TESTNODE_LOG_FILE" 2>&1
+fi
 
 # Start espresso sequencer node for the purposes of the test e.g. not needed for the real migration.
 docker compose up espresso-dev-node --detach
 
-# Export environment variables in .env file
+info "Waiting for espresso dev node to start"
+sleep 200
+
+info "Load environment variables in $ENV_FILE"
 # A similar env file should be supplied for whatever
 . "$TEST_DIR/.env"
 
@@ -41,8 +100,15 @@ ROLLUP_ADDRESS=$(docker compose run --entrypoint cat scripts /config/deployed_ch
 
 # A convoluted way to get the address of the child chain upgrade executor, maybe there's a better way?
 # These steps below are just for the purposes of the test. In a real deployment operators will likely already know their child-chain's upgrade executor address, and it should be included in a .env file for the migration run.
-INBOX_ADDRESS=$(docker compose run --entrypoint cat scripts /config/deployed_chain_info.json | jq -r '.[0].rollup.inbox' | tail -n 1 | tr -d '\r\n')
-L1_TOKEN_BRIDGE_CREATOR_ADDRESS=$(docker compose run --entrypoint cat scripts /tokenbridge-data/network.json | jq -r '.l1TokenBridgeCreator' | tail -n 1 | tr -d '\r\n')
+INBOX_ADDRESS=$(get-addr /config/deployed_chain_info.json '.[0].rollup.inbox')
+declare -p INBOX_ADDRESS
+
+PARENT_CHAIN_UPGRADE_EXECUTOR=$(get-addr /config/deployed_chain_info.json '.[0].rollup["upgrade-executor"]')
+declare -p PARENT_CHAIN_UPGRADE_EXECUTOR
+
+L1_TOKEN_BRIDGE_CREATOR_ADDRESS=$(get-addr /tokenbridge-data/network.json '.l1TokenBridgeCreator')
+declare -p L1_TOKEN_BRIDGE_CREATOR_ADDRESS
+
 CHILD_CHAIN_UPGRADE_EXECUTOR_ADDRESS=$(cast call $L1_TOKEN_BRIDGE_CREATOR_ADDRESS 'inboxToL2Deployment(address)(address,address,address,address,address,address,address,address,address)' $INBOX_ADDRESS | tail -n 2 | head -n 1 | tr -d '\r\n')
 
 # Export l2 owner private key and address
@@ -51,35 +117,39 @@ CHILD_CHAIN_UPGRADE_EXECUTOR_ADDRESS=$(cast call $L1_TOKEN_BRIDGE_CREATOR_ADDRES
 PRIVATE_KEY="$(docker compose run scripts print-private-key --account l2owner | tail -n 1 | tr -d '\r\n')"
 OWNER_ADDRESS="$(docker compose run scripts print-address --account l2owner | tail -n 1 | tr -d '\r\n')"
 
+info "Disabling validator whitelist"
+run cast send $PARENT_CHAIN_UPGRADE_EXECUTOR "executeCall(address,bytes)" $ROLLUP_ADDRESS "$(cast calldata 'setValidatorWhitelistDisabled(bool)' true)" --rpc-url $PARENT_CHAIN_RPC_URL --private-key $PRIVATE_KEY
+
 cd $ORBIT_ACTIONS_DIR
-forge update
-echo "Deploying mock espresso tee verifier"
-forge script --chain $PARENT_CHAIN_CHAIN_ID ../espresso-tests/DeployMockVerifier.s.sol:DeployMockVerifier --rpc-url $PARENT_CHAIN_RPC_URL --broadcast -vvvv
+info "Deploying mock espresso TEE verifier"
+run $forge script --chain $PARENT_CHAIN_CHAIN_ID ../espresso-tests/$TEST_SCRIPT_DIR/DeployMockVerifier.s.sol:DeployMockVerifier --rpc-url $PARENT_CHAIN_RPC_URL --broadcast -vvvv
 
-ESPRESSO_TEE_VERIFIER_ADDRESS=$(cat broadcast/DeployMockVerifier.s.sol/1337/run-latest.json | jq -r '.transactions[0].contractAddress' | cast to-checksum)
-echo "Mock TEE Address:"
-echo $ESPRESSO_TEE_VERIFIER_ADDRESS
+ESPRESSO_TEE_VERIFIER_ADDRESS=$(cat $BROADCAST_DIR/DeployMockVerifier.s.sol/1337/run-latest.json | jq -r '.transactions[0].contractAddress' | cast to-checksum)
+declare -p ESPRESSO_TEE_VERIFIER_ADDRESS
 
 # Echo for debug
 echo "Deploying and initializing Espresso SequencerInbox"
 # ** Essential migration step ** Forge script to deploy the new SequencerInbox. We do this to later point the rollups challenge manager to the espresso integrated OSP.
-forge script --chain $PARENT_CHAIN_CHAIN_ID contracts/parent-chain/espresso-migration/DeployAndInitEspressoSequencerInbox.s.sol:DeployAndInitEspressoSequencerInbox --rpc-url $PARENT_CHAIN_RPC_URL --broadcast -vvvv
+run $forge script --chain $PARENT_CHAIN_CHAIN_ID ../espresso-tests/$TEST_SCRIPT_DIR/DeployAndInitEspressoSequencerInboxForTest.s.sol:DeployAndInitEspressoSequencerInbox --rpc-url $PARENT_CHAIN_RPC_URL --broadcast -vvvv --skip-simulation --private-key $PRIVATE_KEY
 
 # Extract new_osp_entry address from run-latest.json
 #  * Essential migration sub step * These addresses are likely known addresses to operators in the event of a real migration after they have deployed the new OSP contracts, however, if operators create a script for the migration, this command is useful.
-NEW_SEQUENCER_INBOX_IMPL_ADDRESS=$(cat broadcast/DeployAndInitEspressoSequencerInbox.s.sol/1337/run-latest.json | jq -r '.transactions[0].contractAddress'| cast to-checksum)
+NEW_SEQUENCER_INBOX_IMPL_ADDRESS=$(cat $BROADCAST_DIR/DeployAndInitEspressoSequencerInboxForTest.s.sol/1337/run-latest.json | jq -r '.receipts[0].contractAddress'| cast to-checksum)
+declare -p NEW_SEQUENCER_INBOX_IMPL_ADDRESS
+
 # Echo for debugging.
 echo "Deployed new SequencerInbox at $NEW_SEQUENCER_INBOX_IMPL_ADDRESS"
 
 # Echo for debug
 echo "Deploying Espresso SequencerInbox migration action"
 
 # ** Essential migration step ** Forge script to deploy Espresso OSP migration action
-forge script --chain $PARENT_CHAIN_CHAIN_ID contracts/parent-chain/espresso-migration/DeployEspressoSequencerMigrationAction.s.sol:DeployEspressoSequencerMigrationAction --rpc-url $PARENT_CHAIN_RPC_URL --broadcast -vvvv
+run $forge script --chain $PARENT_CHAIN_CHAIN_ID $ORBIT_MIGRATION_ACTION_DIR/DeployEspressoSequencerMigrationAction.s.sol:DeployEspressoSequencerMigrationAction --rpc-url $PARENT_CHAIN_RPC_URL --broadcast -vvvv
 
 # Capture new OSP address
 # * Essential migration sub step ** Essential migration sub step * operators will be able to manually determine this address while running the upgrade, but this can be useful if they wish to make a script.
-SEQUENCER_MIGRATION_ACTION=$(cat broadcast/DeployEspressoSequencerMigrationAction.s.sol/1337/run-latest.json | jq -r '.transactions[0].contractAddress' | cast to-checksum)
+SEQUENCER_MIGRATION_ACTION=$(cat $BROADCAST_DIR/DeployEspressoSequencerMigrationAction.s.sol/1337/run-latest.json | jq -r '.transactions[0].contractAddress' | cast to-checksum)
+declare -p SEQUENCER_MIGRATION_ACTION
 
 echo "Deployed new EspressoSequencerMigrationAction at $SEQUENCER_MIGRATION_ACTION"
 
@@ -106,8 +176,27 @@ cast send $PARENT_CHAIN_UPGRADE_EXECUTOR "execute(address, bytes)" $SEQUENCER_MI
 
 echo "Executed SequencerMigrationAction via UpgradeExecutor"
 
+function get_latest_confirmed_v2() {
+  result=$(cast call --rpc-url $PARENT_CHAIN_RPC_URL $ROLLUP_ADDRESS 'latestConfirmed()(uint256)')
+  echo $result
+}
+
+function get_latest_confirmed_v3() {
+  result=$(cast call --rpc-url $PARENT_CHAIN_RPC_URL $ROLLUP_ADDRESS 'latestConfirmed()(bytes32)')
+  echo $result
+}
+
+function get_latest_confirmed() {
+  if $v3; then
+    get_latest_confirmed_v3
+  else
+    get_latest_confirmed_v2
+  fi
+}
+
 # Get the number of confirmed nodes before the upgrade to ensure the staker is still working.
-NUM_CONFIRMED_NODES_BEFORE_UPGRADE=$(cast call --rpc-url $PARENT_CHAIN_RPC_URL $ROLLUP_ADDRESS 'latestConfirmed()(uint256)')
+NUM_CONFIRMED_NODES_BEFORE_UPGRADE=$(get_latest_confirmed)
+info "Before upgrade: $NUM_CONFIRMED_NODES_BEFORE_UPGRADE"
 
 
 # Wait for CHILD_CHAIN_RPC_URL to be available
@@ -172,10 +261,14 @@ fi
 # Echo successful balance update
 echo "Balance of $RECIPIENT_ADDRESS changed from $BALANCE_ORIG to $BALANCE_NEW"
 
-# Check that the staker is making progress after the upgrade
-while [ "$NUM_CONFIRMED_NODES_BEFORE_UPGRADE" == "$(cast call --rpc-url $PARENT_CHAIN_RPC_URL $ROLLUP_ADDRESS 'latestConfirmed()(uint256)')" ]; do
-  echo "Waiting for confirmed nodes ..."
+info Check that the staker is making progress after the upgrade
+echo
+
+START=$SECONDS
+echo "Waiting for confirmed nodes."
+while [ "$NUM_CONFIRMED_NODES_BEFORE_UPGRADE" == "$(get_latest_confirmed)" ]; do
   sleep 5
+  echo "Waited $(( SECONDS - START )) seconds for confirmed nodes. $NUM_CONFIRMED_NODES_BEFORE_UPGRADE"
 done
 # Echo to confirm that stakers are behaving normally.
 echo "Confirmed nodes have progressed"