|
1 | | -name: "benchmarks-execute" |
| 1 | +name: "Execution benchmarks" |
2 | 2 |
|
3 | 3 | on: |
4 | 4 | push: |
5 | | - branches: ["main"] |
| 5 | + # TODO(ayush): remove after feat/new-execution is merged |
| 6 | + branches: ["main", "feat/new-execution"] |
6 | 7 | pull_request: |
7 | 8 | types: [opened, synchronize, reopened, labeled] |
8 | 9 | branches: ["**"] |
|
18 | 19 | - ".github/workflows/benchmarks-execute.yml" |
19 | 20 | workflow_dispatch: |
20 | 21 |
|
| 22 | +concurrency: |
| 23 | + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} |
| 24 | + cancel-in-progress: true |
| 25 | + |
21 | 26 | env: |
22 | 27 | CARGO_TERM_COLOR: always |
| 28 | + S3_FIXTURES_PATH: s3://openvm-public-data-sandbox-us-east-1/benchmark/fixtures |
| 29 | + JEMALLOC_SYS_WITH_MALLOC_CONF: "retain:true,background_thread:true,metadata_thp:always,thp:always,dirty_decay_ms:10000,muzzy_decay_ms:10000,abort_conf:true" |
23 | 30 |
|
24 | 31 | jobs: |
25 | | - execute-benchmarks: |
| 32 | + codspeed-walltime-benchmarks: |
| 33 | + name: Run codspeed walltime benchmarks |
26 | 34 | runs-on: |
27 | 35 | - runs-on=${{ github.run_id }} |
28 | | - - runner=8cpu-linux-x64 |
| 36 | + - family=m5a.xlarge # 2.5Ghz clock speed |
| 37 | + - image=ubuntu24-full-x64 |
| 38 | + - extras=s3-cache |
| 39 | + |
| 40 | + env: |
| 41 | + CODSPEED_RUNNER_MODE: walltime |
| 42 | + |
29 | 43 | steps: |
| 44 | + - uses: runs-on/action@v1 |
30 | 45 | - uses: actions/checkout@v4 |
31 | | - |
32 | | - - name: Set up Rust |
33 | | - uses: actions-rs/toolchain@v1 |
| 46 | + - uses: dtolnay/rust-toolchain@stable |
| 47 | + - uses: Swatinem/rust-cache@v2 |
34 | 48 | with: |
35 | | - profile: minimal |
36 | | - toolchain: stable |
37 | | - override: true |
| 49 | + cache-on-failure: true |
38 | 50 |
|
39 | | - - name: Run execution benchmarks |
40 | | - working-directory: benchmarks/execute |
41 | | - run: cargo run | tee benchmark_output.log |
| 51 | + - name: Install architecture specific tools |
| 52 | + run: | |
| 53 | + source ci/scripts/utils.sh |
| 54 | + install_s5cmd |
42 | 55 |
|
43 | | - - name: Parse benchmark results |
| 56 | + - name: Pull fixtures from S3 |
| 57 | + run: | |
| 58 | + mkdir -p benchmarks/fixtures |
| 59 | + s5cmd cp "${{ env.S3_FIXTURES_PATH }}/*" benchmarks/fixtures/ || echo "No fixtures found in S3" |
| 60 | +
|
| 61 | + - name: Install cargo-binstall |
| 62 | + uses: cargo-bins/cargo-binstall@main |
| 63 | + - name: Install codspeed |
| 64 | + run: cargo binstall --no-confirm --force cargo-codspeed |
| 65 | + |
| 66 | + - name: Build benchmarks |
44 | 67 | working-directory: benchmarks/execute |
| 68 | + run: cargo codspeed build --profile maxperf |
| 69 | + - name: Run benchmarks |
| 70 | + uses: CodSpeedHQ/action@v3 |
| 71 | + with: |
| 72 | + working-directory: benchmarks/execute |
| 73 | + run: cargo codspeed run |
| 74 | + token: ${{ secrets.CODSPEED_TOKEN }} |
| 75 | + |
| 76 | + codspeed-instrumentation-benchmarks: |
| 77 | + name: Run codspeed instrumentation benchmarks |
| 78 | + runs-on: |
| 79 | + - runs-on=${{ github.run_id }} |
| 80 | + - family=m5a.xlarge |
| 81 | + - image=ubuntu24-full-x64 |
| 82 | + - extras=s3-cache |
| 83 | + if: github.event_name != 'pull_request' |
| 84 | + |
| 85 | + env: |
| 86 | + CODSPEED_RUNNER_MODE: instrumentation |
| 87 | + |
| 88 | + steps: |
| 89 | + - uses: runs-on/action@v1 |
| 90 | + - uses: actions/checkout@v4 |
| 91 | + - uses: dtolnay/rust-toolchain@stable |
| 92 | + - uses: Swatinem/rust-cache@v2 |
| 93 | + with: |
| 94 | + cache-on-failure: true |
| 95 | + |
| 96 | + - name: Install architecture specific tools |
45 | 97 | run: | |
46 | | - # Determine if running in GitHub Actions environment |
47 | | - if [ -n "$GITHUB_STEP_SUMMARY" ]; then |
48 | | - SUMMARY_FILE="$GITHUB_STEP_SUMMARY" |
49 | | - echo "### Benchmark Results Summary" >> "$SUMMARY_FILE" |
50 | | - else |
51 | | - SUMMARY_FILE="benchmark_summary.md" |
52 | | - echo "### Benchmark Results Summary" > "$SUMMARY_FILE" |
53 | | - echo "Saving summary to $SUMMARY_FILE" |
54 | | - fi |
55 | | -
|
56 | | - # Set up summary table header |
57 | | - echo "| Program | Total Time (ms) |" >> "$SUMMARY_FILE" |
58 | | - echo "| ------- | --------------- |" >> "$SUMMARY_FILE" |
59 | | -
|
60 | | - # Variables to track current program and total time |
61 | | - current_program="" |
62 | | - total_time=0 |
63 | | -
|
64 | | - # Process the output file line by line |
65 | | - while IFS= read -r line; do |
66 | | - # Check if line contains "Running program" message |
67 | | - if [[ $line =~ ｉ\ \[info\]:\ Running\ program:\ ([a-zA-Z0-9_-]+) ]]; then |
68 | | - # If we were processing a program, output its results |
69 | | - if [[ -n "$current_program" ]]; then |
70 | | - echo "| $current_program | $total_time |" >> "$SUMMARY_FILE" |
71 | | - fi |
72 | | -
|
73 | | - # Start tracking new program |
74 | | - current_program="${BASH_REMATCH[1]}" |
75 | | - total_time=0 |
76 | | - fi |
77 | | -
|
78 | | - # Check for program completion to catch programs that might have no execution segments |
79 | | - if [[ $line =~ ｉ\ \[info\]:\ Completed\ program:\ ([a-zA-Z0-9_-]+) ]]; then |
80 | | - completed_program="${BASH_REMATCH[1]}" |
81 | | - # If no segments were found for this program, ensure it's still in the output |
82 | | - if [[ "$current_program" == "$completed_program" && $total_time == 0 ]]; then |
83 | | - echo "| $current_program | 0 |" >> "$SUMMARY_FILE" |
84 | | - current_program="" |
85 | | - fi |
86 | | - fi |
87 | | -
|
88 | | - # Check if line contains execution time (looking for the format with ms or s) |
89 | | - if [[ $line =~ execute_segment\ \[\ ([0-9.]+)(ms|s)\ \|\ [0-9.]+%\ \]\ segment ]]; then |
90 | | - segment_time="${BASH_REMATCH[1]}" |
91 | | - unit="${BASH_REMATCH[2]}" |
92 | | -
|
93 | | - # Convert to milliseconds if in seconds |
94 | | - if [[ "$unit" == "s" ]]; then |
95 | | - segment_time=$(echo "scale=6; $segment_time * 1000" | bc) |
96 | | - fi |
97 | | -
|
98 | | - # Add segment time to total |
99 | | - total_time=$(echo "scale=6; $total_time + $segment_time" | bc) |
100 | | - fi |
101 | | - done < benchmark_output.log |
102 | | -
|
103 | | - # Output the last program result if there was one |
104 | | - if [[ -n "$current_program" ]]; then |
105 | | - echo "| $current_program | $total_time |" >> "$SUMMARY_FILE" |
106 | | - fi |
107 | | -
|
108 | | - # If not in GitHub Actions, print the summary to the terminal |
109 | | - if [ -z "$GITHUB_STEP_SUMMARY" ]; then |
110 | | - echo -e "\nBenchmark Summary:" |
111 | | - cat "$SUMMARY_FILE" |
112 | | - fi |
| 98 | + source ci/scripts/utils.sh |
| 99 | + install_s5cmd |
| 100 | +
|
| 101 | + - name: Pull fixtures from S3 |
| 102 | + run: | |
| 103 | + mkdir -p benchmarks/fixtures |
| 104 | + s5cmd cp "${{ env.S3_FIXTURES_PATH }}/*" benchmarks/fixtures/ || echo "No fixtures found in S3" |
| 105 | +
|
| 106 | + - name: Install cargo-binstall |
| 107 | + uses: cargo-bins/cargo-binstall@main |
| 108 | + - name: Install codspeed |
| 109 | + run: cargo binstall --no-confirm --force cargo-codspeed |
| 110 | + |
| 111 | + - name: Build benchmarks |
| 112 | + working-directory: benchmarks/execute |
| 113 | + run: cargo codspeed build |
| 114 | + - name: Run benchmarks |
| 115 | + uses: CodSpeedHQ/action@v3 |
| 116 | + with: |
| 117 | + working-directory: benchmarks/execute |
| 118 | + run: cargo codspeed run |
| 119 | + token: ${{ secrets.CODSPEED_TOKEN }} |
0 commit comments