openvm/.github/workflows/benchmarks-execute.yml at 612baeb3c4e1526abfd2b4cbe951d8037441ceef · openvm-org/openvm · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
name: "Execution benchmarks"

on:
  workflow_call:
  push:
    branches: ["main"]
  # pull_request:
  #   types: [opened, synchronize, reopened, labeled]
  #   branches: ["**"]
  #   paths:
  #     - "benchmarks/execute/**"
  #     - "crates/circuits/**"
  #     - "crates/toolchain/**"
  #     - "crates/prof/**"
  #     - "crates/sdk/**"
  #     - "crates/vm/**"
  #     - "extensions/**"
  #     - "Cargo.toml"
  #     - ".github/workflows/benchmarks-execute.yml"
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow_ref }}-benchmarks-execute-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

env:
  CARGO_TERM_COLOR: always
  S3_FIXTURES_PATH: s3://openvm-public-data-sandbox-us-east-1/benchmark/fixtures
  JEMALLOC_SYS_WITH_MALLOC_CONF: "retain:true,background_thread:true,metadata_thp:always,thp:always,dirty_decay_ms:10000,muzzy_decay_ms:10000,abort_conf:true"
  TOOLCHAIN: "+nightly-2025-08-19"

jobs: {}
  # codspeed-walltime-benchmarks:
  #   name: Run codspeed walltime benchmarks
  #   runs-on:
  #     - runs-on=${{ github.run_id }}
  #     - family=m5a.16xlarge # 2.5Ghz clock speed
  #     - image=ubuntu24-full-x64
  #     - extras=s3-cache
  #
  #   env:
  #     CODSPEED_RUNNER_MODE: walltime
  #     CODSPEED_PROFILING: "false"
  #
  #   steps:
  #     - uses: runs-on/action@v2
  #     - uses: actions/checkout@v5
  #     - uses: dtolnay/rust-toolchain@stable
  #     - uses: Swatinem/rust-cache@v2
  #       with:
  #         cache-on-failure: true
  #
  #     - name: Install architecture specific tools
  #       run: |
  #         source ci/scripts/utils.sh
  #         install_s5cmd
  #
  #     - name: Pull fixtures from S3
  #       run: |
  #         mkdir -p benchmarks/fixtures
  #         s5cmd cp "${{ env.S3_FIXTURES_PATH }}/*" benchmarks/fixtures/ || echo "No fixtures found in S3"
  #
  #     - name: Install cargo-binstall
  #       uses: cargo-bins/cargo-binstall@main
  #     - name: Install codspeed
  #       run: cargo binstall --no-confirm --force cargo-codspeed
  #
  #     - name: Build benchmarks
  #       working-directory: benchmarks/execute
  #       run: cargo $TOOLCHAIN codspeed build --profile maxperf --features aot --bench execute
  #     - name: Run benchmarks
  #       uses: CodSpeedHQ/action@v3
  #       with:
  #         working-directory: benchmarks/execute
  #         run: cargo $TOOLCHAIN codspeed run
  #         token: ${{ secrets.CODSPEED_TOKEN }}
  #
  # codspeed-instrumentation-benchmarks:
  #   name: Run codspeed instrumentation benchmarks
  #   runs-on:
  #     - runs-on=${{ github.run_id }}/family=m5a.xlarge/image=ubuntu24-full-x64/extras=s3-cache
  #   if: github.event_name != 'pull_request'
  #
  #   env:
  #     CODSPEED_RUNNER_MODE: instrumentation
  #
  #   steps:
  #     - uses: runs-on/action@v2
  #     - uses: actions/checkout@v5
  #     - uses: dtolnay/rust-toolchain@stable
  #     - uses: Swatinem/rust-cache@v2
  #       with:
  #         cache-on-failure: true
  #
  #     - name: Install architecture specific tools
  #       run: |
  #         source ci/scripts/utils.sh
  #         install_s5cmd
  #
  #     - name: Pull fixtures from S3
  #       run: |
  #         mkdir -p benchmarks/fixtures
  #         s5cmd cp "${{ env.S3_FIXTURES_PATH }}/*" benchmarks/fixtures/ || echo "No fixtures found in S3"
  #
  #     - name: Install cargo-binstall
  #       uses: cargo-bins/cargo-binstall@main
  #     - name: Install codspeed
  #       run: cargo binstall --no-confirm --force cargo-codspeed
  #
  #     - name: Create codspeed-post-bench symlink
  #       run: |
  #         # codspeed-post-bench is just a sym-link
  #         which codspeed-post-bench || echo "codspeed-post-bench not found"
  #         sudo ln -sf ~/.cargo/bin/cargo-codspeed /usr/local/bin/codspeed-post-bench
  #         ls -la /usr/local/bin/codspeed-post-bench
  #
  #     - name: Build benchmarks
  #       working-directory: benchmarks/execute
  #       run: cargo $TOOLCHAIN codspeed build --features aot --bench execute
  #     - name: Run benchmarks
  #       uses: CodSpeedHQ/action@v3
  #       with:
  #         working-directory: benchmarks/execute
  #         run: cargo $TOOLCHAIN codspeed run
  #         token: ${{ secrets.CODSPEED_TOKEN }}