diff --git a/.github/workflows/artifact_benchmark_fast.yaml b/.github/workflows/artifact_benchmark_fast.yaml deleted file mode 100644 index fa610289a8..0000000000 --- a/.github/workflows/artifact_benchmark_fast.yaml +++ /dev/null @@ -1,43 +0,0 @@ -name: Build Benchmark Fixtures - -on: - push: - branches: - - "forks/osaka" - - "forks/amsterdam" - paths: - # Benchmark test files - - "tests/benchmark/**" - # Precompile specs used by benchmark tests - - "tests/prague/eip2537_bls_12_381_precompiles/spec.py" - - "tests/osaka/eip7951_p256verify_precompiles/spec.py" - # Benchmark specific framework files - - "packages/testing/src/execution_testing/benchmark/**" - # CI benchmark configuration - - ".github/configs/evm.yaml" - - ".github/configs/feature.yaml" - - ".github/actions/build-fixtures/**" - - ".github/actions/build-evm-base/**" - - ".github/actions/build-evm-client/evmone/**" - # This workflow itself - - ".github/workflows/artifact_benchmark_fast.yaml" - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }} - cancel-in-progress: true - -jobs: - build: - runs-on: [self-hosted-ghr, size-gigachungus-x64] - timeout-minutes: 720 - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - with: - submodules: true - - - uses: ./.github/actions/build-fixtures - with: - release_name: benchmark_fast - uv_version: ${{ vars.UV_VERSION }} - python_version: ${{ vars.DEFAULT_PYTHON_VERSION }} diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml new file mode 100644 index 0000000000..9d11edce54 --- /dev/null +++ b/.github/workflows/benchmark.yaml @@ -0,0 +1,138 @@ +name: Benchmark Tests + +on: + push: + branches: + - "forks/osaka" + - "forks/amsterdam" + paths: + # Benchmark test files + - "tests/benchmark/**" + # Precompile specs used by benchmark tests + - "tests/prague/eip2537_bls_12_381_precompiles/spec.py" + - "tests/osaka/eip7951_p256verify_precompiles/spec.py" + # Benchmark specific framework files + - "packages/testing/src/execution_testing/benchmark/**" + # CI benchmark configuration + - ".github/configs/evm.yaml" + - ".github/configs/feature.yaml" + - ".github/actions/build-fixtures/**" + - ".github/actions/build-evm-base/**" + - ".github/actions/build-evm-client/evmone/**" + # This workflow itself + - ".github/workflows/benchmark.yaml" + pull_request: + paths: + - "tests/benchmark/**" + - "packages/testing/src/execution_testing/benchmark/**" + - ".github/workflows/benchmark.yaml" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + fixed-opcode-count: + name: Fixed Opcode Count Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + submodules: true + + - name: Install uv and Python + uses: astral-sh/setup-uv@0c5e2b8115b80b4c7c5ddf6ffdd634974642d182 + with: + enable-cache: false + version: ${{ vars.UV_VERSION }} + python-version: ${{ vars.DEFAULT_PYTHON_VERSION }} + + - name: Install EEST + run: uv sync --no-progress + + - uses: ./.github/actions/build-evm-base + id: evm-builder + with: + type: benchmark + + - name: Run fixed opcode count tests + run: | + uv run fill --evm-bin=${{ steps.evm-builder.outputs.evm-bin }} --fixed-opcode-count 1 --fork Prague -m repricing tests/benchmark + + fixed-opcode-count-config: + name: Fixed Opcode Count Config Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + submodules: true + + - name: Install uv and Python + uses: astral-sh/setup-uv@0c5e2b8115b80b4c7c5ddf6ffdd634974642d182 + with: + enable-cache: false + version: ${{ vars.UV_VERSION }} + python-version: ${{ vars.DEFAULT_PYTHON_VERSION }} + + - name: Install EEST + run: uv sync --no-progress + + - uses: ./.github/actions/build-evm-base + id: evm-builder + with: + type: benchmark + + - name: Generate opcode counts config + run: uv run benchmark_parser + + - name: Run fixed opcode count tests with config + run: | + uv run fill --evm-bin=${{ steps.evm-builder.outputs.evm-bin }} --fixed-opcode-count --fork Prague -m repricing tests/benchmark + + gas-benchmark: + name: Gas Benchmark Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + submodules: true + + - name: Install uv and Python + uses: astral-sh/setup-uv@0c5e2b8115b80b4c7c5ddf6ffdd634974642d182 + with: + enable-cache: false + version: ${{ vars.UV_VERSION }} + python-version: ${{ vars.DEFAULT_PYTHON_VERSION }} + + - name: Install EEST + run: uv sync --no-progress + + - uses: ./.github/actions/build-evm-base + id: evm-builder + with: + type: benchmark + + - name: Run gas benchmark tests + run: | + uv run fill --evm-bin=${{ steps.evm-builder.outputs.evm-bin }} --gas-benchmark-values 1 --generate-pre-alloc-groups --fork Prague -m "benchmark and not state_test" tests/benchmark + + # TODO: Add execute remote tests with --gas-benchmark-values + # TODO: Add execute remote tests with --fixed-opcode-count + + build-artifacts: + name: Build Benchmark Fixtures + needs: [fixed-opcode-count, fixed-opcode-count-config, gas-benchmark] + if: github.event_name == 'push' + runs-on: [self-hosted-ghr, size-gigachungus-x64] + timeout-minutes: 720 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + submodules: true + + - uses: ./.github/actions/build-fixtures + with: + release_name: benchmark_fast + uv_version: ${{ vars.UV_VERSION }} + python_version: ${{ vars.DEFAULT_PYTHON_VERSION }}