casbin-pycasbin/.github/workflows/performance-pr.yml at 1df15d5d55921d0b32c4c618c95256d6492c474f · apache/casbin-pycasbin · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
name: Performance Comparison for Pull Requests

on:
  pull_request:
    branches: [master]

jobs:
  benchmark-pr:
    name: Run Benchmark
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: benchmark_adapter
            target: benchmark_adapter.py
            extra_args: ""
          - name: benchmark_management_api
            target: benchmark_management_api.py
            extra_args: ""
          - name: benchmark_model_large
            target: benchmark_model.py
            extra_args: "-k large"
          - name: benchmark_model_others
            target: benchmark_model.py
            extra_args: "-k 'not large'"
          - name: benchmark_role_manager
            target: benchmark_role_manager.py
            extra_args: ""
    steps:
      - name: Checkout PR branch
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt -r requirements_dev.txt

      # Run benchmark on PR branch
      - name: Run benchmark on PR branch
        run: |
          pytest -o "python_files=${{ matrix.target }}" ${{ matrix.extra_args }} tests/benchmarks --benchmark-json=pr-${{ matrix.name }}.json

      # Checkout base branch and run benchmark
      - name: Checkout base branch
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.base.sha }}
          clean: false
          path: base

      - name: Run benchmark on base branch
        working-directory: base
        run: |
          pip install -r requirements.txt -r requirements_dev.txt
          pytest -o "python_files=${{ matrix.target }}" ${{ matrix.extra_args }} tests/benchmarks --benchmark-json=../base-${{ matrix.name }}.json

      - name: Upload benchmark shard
        uses: actions/upload-artifact@v4
        with:
          name: benchmark-shard-${{ matrix.name }}
          path: |
            pr-${{ matrix.name }}.json
            base-${{ matrix.name }}.json

  report:
    name: Generate Report
    needs: benchmark-pr
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: Download all benchmark shards
        uses: actions/download-artifact@v4
        with:
          pattern: benchmark-shard-*
          merge-multiple: true
          path: benchmark_data

      - name: Merge Benchmark Results
        run: |
          python .github/scripts/merge_benchmarks.py base.json benchmark_data/base-*.json
          python .github/scripts/merge_benchmarks.py pr.json benchmark_data/pr-*.json

      # Save commit SHAs for display
      - name: Save commit info
        id: commits
        run: |
          BASE_SHA="${{ github.event.pull_request.base.sha }}"
          HEAD_SHA="${{ github.event.pull_request.head.sha }}"
          echo "base_short=${BASE_SHA:0:7}" >> $GITHUB_OUTPUT
          echo "head_short=${HEAD_SHA:0:7}" >> $GITHUB_OUTPUT

      # Compare benchmarks using script
      - name: Compare benchmarks
        id: benchstat
        run: |
          cat > comparison.md << 'EOF'
          ## Benchmark Comparison

          Comparing base branch (`${{ steps.commits.outputs.base_short }}`)
          vs PR branch (`${{ steps.commits.outputs.head_short }}`)

          ```
          EOF
          python3 .github/scripts/pytest_benchstat.py base.json pr.json >> comparison.md || true
          echo '```' >> comparison.md

          # Post-process to append percentage + emoji column (🚀 faster < -10%, 🐌 slower > +10%, otherwise ➡️)
          if [ ! -f comparison.md ]; then
            echo "comparison.md not found after benchstat." >&2
            exit 1
          fi
          python3 .github/scripts/benchmark_formatter.py

      # Save PR number
      - name: Save PR number
        run: |
          PR_NUMBER="${{ github.event.pull_request.number }}"
          if [ -z "$PR_NUMBER" ]; then
            echo "Error: Pull request number is not available in event payload." >&2
            exit 1
          fi
          echo "$PR_NUMBER" > pr_number.txt

      # Upload benchmark results
      - name: Upload benchmark results
        uses: actions/upload-artifact@v4
        with:
          name: benchmark-results
          path: |
            comparison.md
            pr_number.txt