openvm/.github/workflows/benchmarks-execute.yml at b3288ababbd63c503d4793a46078ebdcf2c43c39 · openvm-org/openvm · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
name: "Execution benchmarks"

on:
  push:
    # TODO(ayush): remove after feat/new-execution is merged
    branches: ["main", "feat/new-execution"]
  pull_request:
    types: [opened, synchronize, reopened, labeled]
    branches: ["**"]
    paths:
      - "benchmarks/execute/**"
      - "crates/circuits/**"
      - "crates/toolchain/**"
      - "crates/prof/**"
      - "crates/sdk/**"
      - "crates/vm/**"
      - "extensions/**"
      - "Cargo.toml"
      - ".github/workflows/benchmarks-execute.yml"
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

env:
  CARGO_TERM_COLOR: always

jobs:
  codspeed-walltime-benchmarks:
    name: Run codspeed walltime benchmarks
    runs-on:
      - runs-on=${{ github.run_id }}
      - family=m5a.xlarge # 2.5Ghz clock speed
      - image=ubuntu24-full-x64
      - extras=s3-cache

    env:
      CODSPEED_RUNNER_MODE: walltime

    steps:
      - uses: runs-on/action@v1
      - uses: actions/checkout@v4
      - uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2
        with:
          cache-on-failure: true

      - name: Install cargo-binstall
        uses: cargo-bins/cargo-binstall@main
      - name: Install codspeed
        run: cargo binstall --no-confirm --force cargo-codspeed

      - name: Build benchmarks
        working-directory: benchmarks/execute
        run: cargo codspeed build --profile maxperf
      - name: Run benchmarks
        uses: CodSpeedHQ/action@v3
        with:
          working-directory: benchmarks/execute
          run: cargo codspeed run
          token: ${{ secrets.CODSPEED_TOKEN }}
        env:
          CODSPEED_RUNNER_MODE: walltime

  codspeed-instrumentation-benchmarks:
    name: Run codspeed instrumentation benchmarks
    runs-on:
      - runs-on=${{ github.run_id }}
      - family=m5a.xlarge
      - image=ubuntu24-full-x64
      - extras=s3-cache
    if: github.event_name != 'pull_request'

    env:
      CODSPEED_RUNNER_MODE: instrumentation

    steps:
      - uses: runs-on/action@v1
      - uses: actions/checkout@v4
      - uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2
        with:
          cache-on-failure: true

      - name: Install cargo-binstall
        uses: cargo-bins/cargo-binstall@main
      - name: Install codspeed
        run: cargo binstall --no-confirm --force cargo-codspeed

      - name: Build benchmarks
        working-directory: benchmarks/execute
        run: cargo codspeed build
      - name: Run benchmarks
        uses: CodSpeedHQ/action@v3
        with:
          working-directory: benchmarks/execute
          run: cargo codspeed run
          token: ${{ secrets.CODSPEED_TOKEN }}
        env:
          CODSPEED_RUNNER_MODE: instrumentation