Merge pull request #526 from lidofinance/feat/next-vote

Omnibuses (#194 Lido V3 + #195 Operations + #196 LDO recover + #197 LDO recover)

Author: katamarinaki

.github/workflows/core_tests.yml

@@ -15,15 +15,14 @@ on:
 jobs:
   run-tests:
     name: Core repo tests in docker
-    runs-on: "ubuntu-latest"
+    runs-on: [voting-private-runners]
     timeout-minutes: 120
 
     services:
-
       tests-runner:
-        image: ghcr.io/lidofinance/scripts:v19
+        image: ghcr.io/lidofinance/scripts:v21
         ports:
-          - 8546:8546
+          - 8545:8545
         volumes:
           - ${{ github.workspace }}:/root/scripts
         options: >-
@@ -34,25 +33,38 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Run init script
-        run: docker exec -e CORE_BRANCH tests-runner bash -c 'make init'
+        run: docker exec -e CORE_BRANCH tests-runner bash -c 'PYTHONPATH=$PWD make init'
         env:
           CORE_BRANCH: master
 
       - name: Run node
-        run: docker exec -e ETH_RPC_URL --detach tests-runner bash -c 'NODE_PORT=8546 make node'
+        run: docker exec -e ETH_RPC_URL --detach tests-runner bash -c 'NODE_PORT=8545 make node'
         env:
           ETH_RPC_URL: ${{ secrets.ETH_RPC_URL }}
 
       - name: Check that the fork is ready
         shell: bash
-        run: timeout 30 bash -c 'until printf "" 2>>/dev/null >>/dev/tcp/$0/$1; do sleep 1; done' 127.0.0.1 8546
+        run: |
+          echo "Waiting for fork node on 127.0.0.1:8545..."
+          sleep 10
+          for i in {1..30}; do
+            if (echo > /dev/tcp/127.0.0.1/8545) >/dev/null 2>&1; then
+              echo "Fork is ready ✅"
+              exit 0
+            fi
+            echo "Not ready yet... ($i/30)"
+            sleep 1
+          done
+          echo "❌ Fork was not ready after 30 seconds" >&2
+          exit 1
 
       - name: Prepare test environment with Brownie
-        run: docker exec -e GITHUB_TOKEN -e ETH_RPC_URL tests-runner bash -c 'make ci-prepare-environment'
+        run: docker exec -e GITHUB_TOKEN -e ETH_RPC_URL -e ETHERSCAN_TOKEN tests-runner bash -c 'make ci-prepare-environment'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          ETHERSCAN_TOKEN: ${{ secrets.ETHERSCAN_TOKEN }}
 
       - name: Run core tests
         run: docker exec -e CORE_TESTS_TARGET_RPC_URL tests-runner bash -c 'make test-core'
         env:
-          CORE_TESTS_TARGET_RPC_URL: http://127.0.0.1:8546
+          CORE_TESTS_TARGET_RPC_URL: http://127.0.0.1:8545

.github/workflows/dual_governance_regression.yml

@@ -26,15 +26,14 @@ jobs:
       cancel-in-progress: true
 
     env:
-      NODE_VERSION: '20'
-      PYTHON_VERSION: '3.10'
-      POETRY_VERSION: '1.8.2'
-      HARDHAT_NODE_URL: 'http://127.0.0.1:8545'
-      ANVIL_NODE_URL: 'http://127.0.0.1:8555'
+      NODE_VERSION: "20"
+      PYTHON_VERSION: "3.10"
+      POETRY_VERSION: "1.8.2"
+      HARDHAT_NODE_URL: "http://127.0.0.1:8545"
+      ANVIL_NODE_URL: "http://127.0.0.1:8555"
       CURL_PARAMS: '-X POST -H "Content-Type: application/json" -d "{\"jsonrpc\":\"2.0\",\"method\":\"eth_blockNumber\",\"params\":[],\"id\":1}"'
 
     steps:
-
       ################################################################
       # Clone repositories`
       ################################################################
@@ -50,7 +49,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: lidofinance/dual-governance
-          ref: feature/escrow-mainnet-deploy
+          ref: develop
           path: dual-governance
           persist-credentials: false
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -63,7 +62,7 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: ${{ env.NODE_VERSION }}
-          cache: 'yarn'
+          cache: "yarn"
           cache-dependency-path: scripts/yarn.lock
 
       - name: Install Poetry
@@ -141,11 +140,17 @@ jobs:
           done
         working-directory: scripts
 
+      - name: Import network config for brownie
+        shell: bash
+        run: poetry run brownie networks import network-config.yaml True
+        working-directory: scripts
+
       - name: Prepare test environment with Brownie
-        run: poetry run brownie run scripts/ci/prepare_environment --network mainnet-fork
+        run: PYTHONPATH=$PWD poetry run brownie run scripts/ci/prepare_environment --network mfh-1
         working-directory: scripts
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          ETHERSCAN_TOKEN: ${{ secrets.ETHERSCAN_TOKEN }}
 
       - name: Run regression tests
         run: npm run test:regressions -- --load-accounts

.github/workflows/large_vote_ci.yml

@@ -36,7 +36,7 @@ jobs:
           vote: "large"
           rpc_url: ${{ secrets.ETH_RPC_URL }}
           etherscan: ${{ secrets.ETHERSCAN_TOKEN }}
-          command: "make test-1/2"
+          command: "PYTHONPATH=$PWD make test-1/2"
 
   run-tests-large-2:
     name: Brownie fork LARGE tests 2
@@ -62,4 +62,4 @@ jobs:
           vote: "large"
           rpc_url: ${{ secrets.ETH_RPC_URL }}
           etherscan: ${{ secrets.ETHERSCAN_TOKEN }}
-          command: "make test-2/2"
+          command: "PYTHONPATH=$PWD make test-2/2"

.github/workflows/normal_vote_ci.yml

@@ -13,7 +13,16 @@ jobs:
   run-tests-normal-1:
     name: Brownie fork NORMAL tests 1
     runs-on: [voting-private-runners]
-    timeout-minutes: 100
+    timeout-minutes: 150
+
+    services:
+      hardhat-node:
+        image: ghcr.io/lidofinance/hardhat-node:2.26.0
+        ports:
+          - 8545:8545
+        env:
+          ETH_RPC_URL: ${{ secrets.ETH_RPC_URL }}
+          HARDFORK: "cancun"
 
     steps:
       - uses: actions/checkout@v4
@@ -23,12 +32,21 @@ jobs:
           vote: "normal"
           rpc_url: ${{ secrets.ETH_RPC_URL }}
           etherscan: ${{ secrets.ETHERSCAN_TOKEN }}
-          command: "make test-1/2"
+          command: "PYTHONPATH=$PWD make test-1/2"
 
   run-tests-normal-2:
     name: Brownie fork NORMAL tests 2
     runs-on: [voting-private-runners]
-    timeout-minutes: 100
+    timeout-minutes: 150
+
+    services:
+      hardhat-node:
+        image: ghcr.io/lidofinance/hardhat-node:2.26.0
+        ports:
+          - 8546:8545
+        env:
+          ETH_RPC_URL: ${{ secrets.ETH_RPC_URL }}
+          HARDFORK: "cancun"
 
     steps:
       - uses: actions/checkout@v4
@@ -38,4 +56,4 @@ jobs:
           vote: "normal"
           rpc_url: ${{ secrets.ETH_RPC_URL }}
           etherscan: ${{ secrets.ETHERSCAN_TOKEN }}
-          command: "make test-2/2"
+          command: "PYTHONPATH=$PWD make test-2/2"

Dockerfile

@@ -233,6 +233,40 @@ RUN if [ "$TARGETARCH" = "arm64" ]; then \
       git clean -d -x -f; \
     fi
 
+RUN if [ "$TARGETARCH" = "arm64" ]; then \
+      # build solc-v0.8.25
+      git checkout v0.8.25; \
+      # the compiler throws warnings when compiling this version, and the warnings are treated as errors.
+      # we disable treating the warnings as errors, unless the build doesn't succeed
+      grep -rl '\-Werror' ./cmake/EthCompilerSettings.cmake | xargs sed -i 's/\-Werror/\-Wno\-error/g'; \
+      # there is no sudo in the container, but we are under root so we do not need it
+      grep -rl 'sudo make install' ./scripts/build.sh | xargs sed -i 's/sudo make install/make install/g'; \
+      # build solc faster
+      grep -rl 'make -j2' ./scripts/build.sh | xargs sed -i 's/make -j2/make -j4/g'; \
+      ./scripts/build.sh; \
+      mv /usr/local/bin/solc /root/.solcx/solc-v0.8.25; \
+      git checkout .; \
+      git checkout develop; \
+      git clean -d -x -f; \
+    fi
+
+RUN if [ "$TARGETARCH" = "arm64" ]; then \
+      # build solc-v0.8.21
+      git checkout v0.8.21; \
+      # the compiler throws warnings when compiling this version, and the warnings are treated as errors.
+      # we disable treating the warnings as errors, unless the build doesn't succeed
+      grep -rl '\-Werror' ./cmake/EthCompilerSettings.cmake | xargs sed -i 's/\-Werror/\-Wno\-error/g'; \
+      # there is no sudo in the container, but we are under root so we do not need it
+      grep -rl 'sudo make install' ./scripts/build.sh | xargs sed -i 's/sudo make install/make install/g'; \
+      # build solc faster
+      grep -rl 'make -j2' ./scripts/build.sh | xargs sed -i 's/make -j2/make -j4/g'; \
+      ./scripts/build.sh; \
+      mv /usr/local/bin/solc /root/.solcx/solc-v0.8.21; \
+      git checkout .; \
+      git checkout develop; \
+      git clean -d -x -f; \
+    fi
+
 RUN if [ "$TARGETARCH" = "arm64" ]; then \
      # build solc-v0.6.11
      git checkout v0.6.11; \
@@ -303,6 +337,8 @@ RUN if [ "$TARGETARCH" = "arm64" ]; then /root/.solcx/solc-v0.7.6 --version | gr
 RUN if [ "$TARGETARCH" = "arm64" ]; then /root/.solcx/solc-v0.8.15 --version | grep 'Version: 0.8.15+commit.e14f2714' || (echo "Incorrect solc-v0.8.15 version" && exit 1) fi
 RUN if [ "$TARGETARCH" = "arm64" ]; then /root/.solcx/solc-v0.8.19 --version | grep 'Version: 0.8.19+commit.7dd6d404' || (echo "Incorrect solc-v0.8.19 version" && exit 1) fi
 RUN if [ "$TARGETARCH" = "arm64" ]; then /root/.solcx/solc-v0.8.24 --version | grep 'Version: 0.8.24+commit.e11b9ed9' || (echo "Incorrect solc-v0.8.24 version" && exit 1) fi
+RUN if [ "$TARGETARCH" = "arm64" ]; then /root/.solcx/solc-v0.8.25 --version | grep 'Version: 0.8.25+commit.b61c2a91' || (echo "Incorrect solc-v0.8.25 version" && exit 1) fi
+RUN if [ "$TARGETARCH" = "arm64" ]; then /root/.solcx/solc-v0.8.21 --version | grep 'Version: 0.8.21+commit.d9974bed' || (echo "Incorrect solc-v0.8.21 version" && exit 1) fi
 RUN if [ "$TARGETARCH" = "arm64" ]; then /root/.solcx/solc-v0.6.11 --version | grep 'Version: 0.6.11+commit.5ef660b1' || (echo "Incorrect solc-v0.6.11 version" && exit 1) fi
 RUN if [ "$TARGETARCH" = "arm64" ]; then /root/.vvm/vyper-0.3.7 --version | grep '0.3.7+' || (echo "Incorrect vyper-0.3.7 version" && exit 1) fi
 

Makefile

@@ -4,6 +4,13 @@ define run_2nd_test
 	poetry run $(1)
 endef
 
+# Get the latest block number from the target RPC node to use as FORKING_BLOCK_NUMBER for core tests
+__get_rpc_latest_block_number:
+	@curl -s -X POST $(CORE_TESTS_TARGET_RPC_URL) \
+	  -H "Content-Type: application/json" \
+	  --data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \
+	  | sed -E 's/.*"result":"([^"]+)".*/\1/' \
+	  | xargs printf "%d"
 
 test:
 ifdef vote
@@ -22,6 +29,7 @@ CORE_DIR ?= lido-core
 CORE_BRANCH ?= master
 NODE_PORT ?= 8545
 SECONDARY_NETWORK ?= mfh-2
+NETWORK_STATE_FILE ?= deployed-mainnet.json
 
 test-1/2:
 	poetry run brownie test tests/[tc]*.py tests/regression/test_staking_router_stake_distribution.py --durations=20 --network mfh-1
@@ -36,8 +44,8 @@ init-scripts:
 # because current brownie version does not support pm install from the config file
 	poetry install && \
 	yarn && \
-	poetry run brownie pm install OpenZeppelin/openzeppelin-contracts@4.0.0 && \
-	poetry run brownie compile && \
+	PYTHONPATH=$$PWD poetry run brownie pm install OpenZeppelin/openzeppelin-contracts@4.0.0 && \
+	PYTHONPATH=$$PWD poetry run brownie compile && \
 	poetry run brownie networks import network-config.yaml True
 
 debug:
@@ -78,21 +86,36 @@ node3:
 	npx hardhat node --fork $(ETH_RPC_URL3) --port $(NODE_PORT)
 
 test-core:
+	LATEST_BLOCK_NUMBER=$$($(MAKE) --no-print-directory __get_rpc_latest_block_number) && \
+	echo "LATEST_BLOCK_NUMBER: $$LATEST_BLOCK_NUMBER" && \
 	cd $(CORE_DIR) && \
-	FORK_RPC_URL=$(CORE_TESTS_TARGET_RPC_URL) \
 	RPC_URL=$(CORE_TESTS_TARGET_RPC_URL) \
+	NETWORK_STATE_FILE=$(NETWORK_STATE_FILE) \
+	FORKING_BLOCK_NUMBER=$$LATEST_BLOCK_NUMBER \
 	yarn test:integration
 
 slots:
-	@echo "Input https://github.com/lidofinance/protocol-onchain-mon-bots/blob/main/bots/ethereum-steth-v2/src/utils/constants.ts file content (end with Enter and Ctrl+D):"
-	@cat | grep -v "import { StorageSlot } from '../entity/storage_slot'" | sed 's/StorageSlot/any/g' > slots.ts
+	@echo "Input https://github.com/lidofinance/protocol-onchain-mon-bots/blob/main/bots/ethereum-steth-v2/src/utils/constants.mainnet.ts file content (end with Enter and Ctrl+D):"
+	@cat | grep -v "import { ERC20 } from './constants'" | sed 's/ERC20/any/g' > slots2.ts
+	@cat \
+	| grep -v "import { ERC20 } from './constants'" \
+	| sed 's/ERC20/any/g' \
+	> slots.ts
+
+	@echo "Input https://github.com/lidofinance/protocol-onchain-mon-bots/blob/main/bots/ethereum-steth-v2/src/services/storage-watcher/constants.ts file content (end with Enter and Ctrl+D):"
+	@cat \
+	| grep -v "import { StorageSlot } from '../../entity/storage_slot'" \
+	| sed "s|import { Address } from '../../utils/constants.mainnet'|import { Address } from './slots2.ts'|g" \
+	| sed 's/StorageSlot/any/g' \
+	> slots.ts
 	@cat check_storage_slots.ts >> slots.ts
 	@echo "Checking storage slots against 127.0.0.1:8545..."
 	@npx tsx slots.ts
 	@rm -f slots.ts
+	@rm -f slots2.ts
 
 ci-prepare-environment:
-	poetry run brownie run scripts/ci/prepare_environment --network $(SECONDARY_NETWORK)
+	poetry run brownie run scripts/ci/prepare_environment --network mfh-1
 
 enact-fork:
 	poetry run brownie run $(vote) start_and_execute_vote_on_fork_manual --network=mfh-1

README.md

@@ -49,7 +49,7 @@ _*may be optionally set when running tests asynchronously to reduce the risk of
 Run the container in the `scripts` directory and specify the ENV VARs:
 
 ```shell
-docker run --name scripts -v "$(pwd)":/root/scripts -e ETH_RPC_URL -e ETH_RPC_URL2 -e ETH_RPC_URL3 -e PINATA_CLOUD_TOKEN -e DEPLOYER -e ETHERSCAN_TOKEN -e ETHERSCAN_TOKEN2 -e ETHERSCAN_TOKEN3 -d ghcr.io/lidofinance/scripts:v19
+docker run --name scripts -v "$(pwd)":/root/scripts -e ETH_RPC_URL -e ETH_RPC_URL2 -e PINATA_CLOUD_TOKEN -e DEPLOYER -e ETHERSCAN_TOKEN -e ETHERSCAN_TOKEN2 -p 8545:8545 -d ghcr.io/lidofinance/scripts:v21
 ```
 
 #### Step 4. Initialize container