firewall-python/.github/workflows/benchmark.yml at main · AikidoSec/firewall-python · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
name: 📊 Benchmarks
on:
  push: {}
  workflow_call: {}
permissions:
  contents: read

jobs:
  benchmark_sql_algorithm:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Start databases
        working-directory:  ./sample-apps/databases
        run: docker compose up --build -d

      - name: Set up Python 3.9
        uses: actions/setup-python@v5
        with:
          python-version: 3.9
      - name: Setup safe-chain
        run: curl -fsSL https://github.com/AikidoSec/safe-chain/releases/latest/download/install-safe-chain.sh | sh -s -- --ci

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          make dev_install
      - name: Install aikido_zen
        run: |
          make build
          mv ./dist/aikido_zen-*.tar.gz ./dist/aikido_zen.tar.gz
          pip install ./dist/aikido_zen.tar.gz
      - name: run sql benchmarks
        run: |
          poetry run python ./benchmarks/sql_benchmark/sql_benchmark_fw.py
          poetry run python ./benchmarks/sql_benchmark/sql_benchmark_no_fw.py

  benchmark_with_flask_mysql:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Start databases
        working-directory:  ./sample-apps/databases
        run: docker compose up --build -d
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies & build
        run: |
          python -m pip install --upgrade pip
          make install && make build
      - name: Start flask-mysql
        working-directory: ./sample-apps/flask-mysql
        run: nohup make runBenchmark & nohup make runZenDisabled &

      - name: Check app health (max 3 attempts)
        uses: nick-fields/retry@14672906e672a08bd6eeb15720e9ed3ce869cdd4 # v2.9.0
        with:
          timeout_minutes: 5
          max_attempts: 3
          command: |
            chmod +x ./sample-apps/scripts/health_check.sh
            cd ./sample-apps/flask-mysql && make health-check

      - name: Install K6
        uses: grafana/setup-k6-action@ffe7d7290dfa715e48c2ccc924d068444c94bde2 # v1
      - name: Run flask-mysql k6 Benchmark (max 3 attempts)
        uses: nick-fields/retry@14672906e672a08bd6eeb15720e9ed3ce869cdd4 # v2.9.0
        with:
          timeout_minutes: 20
          max_attempts: 3
          command: |
            k6 run -q ./benchmarks/flask-mysql-benchmarks.js

  wrk_benchmarks:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    strategy:
      fail-fast: false
      matrix:
        app: [starlette-postgres-uvicorn, flask-mysql-uwsgi, flask-mysql]
        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Start databases
        working-directory:  ./sample-apps/databases
        run: docker compose up --build -d
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies & build
        run: |
          python -m pip install --upgrade pip
          make install && make build
      - name: Start app
        working-directory: ./sample-apps/${{ matrix.app }}
        run: nohup make runBenchmark & nohup make runZenDisabled &

      - name: Check app health (max 3 attempts)
        uses: nick-fields/retry@14672906e672a08bd6eeb15720e9ed3ce869cdd4 # v2.9.0
        with:
          timeout_minutes: 5
          max_attempts: 3
          command: |
            chmod +x ./sample-apps/scripts/health_check.sh
            cd ./sample-apps/${{ matrix.app }}/ && make health-check

      - name: Install wrk
        run: |
          sudo apt-get update
          sudo apt-get install -y wrk
      - name: Run benchmarks (max 3 attempts)
        uses: nick-fields/retry@14672906e672a08bd6eeb15720e9ed3ce869cdd4 # v2.9.0
        with:
          timeout_minutes: 20
          max_attempts: 3
          command: |
            cd ./benchmarks/wrk_benchmark && make benchmark_${{ matrix.app }}