CCF/.github/workflows/bencher-ab.yml at b889d2b3772f6d5ac1509f0d7fc6a027db3b5ec9 · microsoft/CCF · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the Apache 2.0 License.

name: "Benchmark A/B Testing"

on:
  pull_request:
    types:
      - labeled
      - synchronize
      - opened
      - reopened

permissions: read-all

jobs:
  benchmark_pr:
    name: Benchmark PR
    runs-on:
      [
        self-hosted,
        1ES.Pool=gha-vmss-d16av5-ci,
        "JobId=bab_benchmark_pr-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}",
      ]
    if: &check_trigger_conditions ${{ (github.event.action == 'labeled' && github.event.label.name == 'bench-ab') || (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'bench-ab')) || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'}}
    container:
      image: mcr.microsoft.com/azurelinux/base/core:3.0
      options: --user root
    steps:
      - name: Setup container dependencies
        run: |
          gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY
          tdnf -y update && tdnf -y install ca-certificates git

      - uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Install dependencies
        run: ./scripts/setup-ci.sh

      - name: Confirm platform
        run: python3 tests/infra/platform_detection.py virtual

      - name: Build and run benchmarks
        run: |
          git config --global --add safe.directory /__w/CCF/CCF
          mkdir build
          cd build
          cmake -GNinja -DWORKER_THREADS=2 ..
          ninja
          # Microbenchmarks
          ./tests.sh -VV -L benchmark
          # End to end performance tests
          ./tests.sh -VV -L perf -C perf
          # Convert microbenchmark output to bencher json
          source env/bin/activate
          PYTHONPATH=../tests python convert_pico_to_bencher.py

      - name: Upload PR results
        uses: actions/upload-artifact@v7
        with:
          name: benchmark-pr-${{ github.run_id }}
          path: build/bencher.json
          retention-days: 7

  benchmark_main:
    name: Benchmark Main
    runs-on:
      [
        self-hosted,
        1ES.Pool=gha-vmss-d16av5-ci,
        "JobId=bab_benchmark_main-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}",
      ]
    if: *check_trigger_conditions
    container:
      image: mcr.microsoft.com/azurelinux/base/core:3.0
      options: --user root
    steps:
      - name: Setup container dependencies
        run: |
          gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY
          tdnf -y update && tdnf -y install ca-certificates git

      - uses: actions/checkout@v6
        with:
          ref: main
          fetch-depth: 0

      - name: Install dependencies
        run: ./scripts/setup-ci.sh

      - name: Confirm platform
        run: python3 tests/infra/platform_detection.py virtual

      - name: Build and run benchmarks
        run: |
          git config --global --add safe.directory /__w/CCF/CCF
          mkdir build
          cd build
          cmake -GNinja -DWORKER_THREADS=2 ..
          ninja
          # Microbenchmarks
          ./tests.sh -VV -L benchmark
          # End to end performance tests
          ./tests.sh -VV -L perf -C perf
          # Convert microbenchmark output to bencher json
          source env/bin/activate
          PYTHONPATH=../tests python convert_pico_to_bencher.py

      - name: Upload main results
        uses: actions/upload-artifact@v7
        with:
          name: benchmark-main-${{ github.run_id }}
          path: build/bencher.json
          retention-days: 7

  compare:
    name: Compare Results
    runs-on:
      [
        self-hosted,
        1ES.Pool=gha-vmss-d16av5-ci,
        "JobId=bab_compare-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}",
      ]
    container:
      image: mcr.microsoft.com/azurelinux/base/core:3.0
      options: --user root
    needs: [benchmark_pr, benchmark_main]
    if: *check_trigger_conditions
    steps:
      - name: Setup container dependencies
        run: |
          gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY
          tdnf -y update && tdnf -y install ca-certificates git

      - uses: actions/checkout@v6

      - name: Download artifacts
        uses: actions/download-artifact@v8
        with:
          pattern: benchmark-*-${{ github.run_id }}
          merge-multiple: false

      - name: Generate comparison
        run: |
          echo "# Benchmark Comparison: main vs PR" > report.md
          echo "" >> report.md
          echo "**PR Commit:** \`${{ github.event.pull_request.head.sha }}\`" >> report.md
          echo "**Base Commit:** \`${{ github.event.pull_request.base.sha }}\`" >> report.md
          echo "**Run ID:** [${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})" >> report.md
          echo "" >> report.md
          echo '```' >> report.md
          python3 scripts/compare_bencher_ab.py \
            benchmark-main-${{ github.run_id }}/bencher.json \
            benchmark-pr-${{ github.run_id }}/bencher.json \
            --label1 "main" --label2 "PR" >> report.md
          echo '```' >> report.md

          cat report.md >> $GITHUB_STEP_SUMMARY