Add support for manually runnung the real-time benchmark on PRs

kocsismate · kocsismate · commit ae11f49f7027 · 2025-07-29T09:03:05.000+02:00
diff --git a/.github/workflows/real-time-benchmark.yml b/.github/workflows/real-time-benchmark.yml
@@ -2,13 +2,49 @@ name: Real-time Benchmark
 on:
   schedule:
     - cron: "30 0 * * *"
+  workflow_dispatch:
+    inputs:
+      repository:
+        description: 'Repository name that is going to be benchmarked (e.g. "johndoe/php-src")'
+        required: true
+        type: string
+      branch:
+        description: 'Branch name that is going to be benchmarked (e.g. "my-branch")'
+        required: true
+        type: string
+      commit:
+        description: 'Full commit SHA that is going to be benchmarked (e.g. "123456789a...")'
+        required: true
+        type: string
+      baseline_commit:
+        description: 'Full commit SHA that is compared against the one provided by the "commit" input (e.g. "abcdef123456...")'
+        required: true
+        type: string
+      id:
+        description: 'ID of the benchmarked code. It should be a valid identifier (e.g. "master", "php_85").'
+        required: true
+        type: string
+      jit:
+        description: 'Whether JIT is benchmarked'
+        required: false
+        type: choice
+        options:
+          - "0"
+          - "1"
 permissions:
   contents: read
 jobs:
   REAL_TIME_BENCHMARK:
     name: REAL_TIME_BENCHMARK
-    if: github.repository == 'php/php-src'
+    if: github.repository == 'kocsismate/php-src' || github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-22.04
+    env:
+      REPOSITORY: ${{ inputs.repository || 'php/php-src' }}
+      BRANCH: ${{ inputs.branch || 'master' }}
+      COMMIT: ${{ inputs.commit || github.sha }}
+      BASELINE_COMMIT: ${{ inputs.baseline_commit || 'd5f6e56610c729710073350af318c4ea1b292cfe' }}
+      ID: ${{ inputs.id || 'master' }}
+      JIT: ${{ inputs.jit || '1' }}
     steps:
       - name: Install dependencies
         run: |
@@ -29,73 +65,143 @@ jobs:
           ref: 'main'
           fetch-depth: 1
           path: 'php-version-benchmarks'
-      - name: Checkout php-src
+      - name: Checkout php-src (benchmarked version)
         uses: actions/checkout@v4
         with:
-          repository: 'php/php-src'
-          ref: '${{ github.sha }}'
+          repository: '${{ env.REPOSITORY }}'
+          ref: '${{ env.COMMIT }}'
           fetch-depth: 100
-          path: 'php-version-benchmarks/tmp/php_master'
+          path: 'php-version-benchmarks/tmp/php_${{ env.ID }}'
+      - name: Checkout php-src (baseline version)
+        uses: actions/checkout@v4
+        with:
+          repository: '${{ env.REPOSITORY }}'
+          ref: '${{ env.BASELINE_COMMIT }}'
+          fetch-depth: 100
+          path: 'php-version-benchmarks/tmp/php_baseline'
       - name: Setup benchmark results
         run: |
           git config --global user.name "Benchmark"
           git config --global user.email "benchmark@php.net"
-          
+
           rm -rf ./php-version-benchmarks/docs/results
       - name: Checkout benchmark data
+        if: github.repository == 'kocsismate/php-src' && github.event_name != 'workflow_dispatch'
         uses: actions/checkout@v4
         with:
           repository: php/real-time-benchmark-data
           ssh-key: ${{ secrets.PHP_VERSION_BENCHMARK_RESULTS_DEPLOY_KEY }}
           path: 'php-version-benchmarks/docs/results'
-      - name: Set benchmark config
+      - name: Setup infra config
         run: |
           set -e
 
-          # Set infrastructure config
           cp ./php-version-benchmarks/config/infra/aws/x86_64-metal.ini.dist ./php-version-benchmarks/config/infra/aws/x86_64-metal.ini
           ESCAPED_DOCKER_REGISTRY=$(printf '%s\n' "${{ secrets.PHP_VERSION_BENCHMARK_DOCKER_REGISTRY }}" | sed -e 's/[\/&]/\\&/g')
           sed -i "s/INFRA_DOCKER_REGISTRY=public.ecr.aws\/abcdefgh/INFRA_DOCKER_REGISTRY=$ESCAPED_DOCKER_REGISTRY/g" ./php-version-benchmarks/config/infra/aws/x86_64-metal.ini
           cp ./php-version-benchmarks/build/infrastructure/config/aws.tfvars.dist ./php-version-benchmarks/build/infrastructure/config/aws.tfvars
           sed -i 's/access_key = ""/access_key = "${{ secrets.PHP_VERSION_BENCHMARK_AWS_ACCESS_KEY }}"/g' ./php-version-benchmarks/build/infrastructure/config/aws.tfvars
           sed -i 's/secret_key = ""/secret_key = "${{ secrets.PHP_VERSION_BENCHMARK_AWS_SECRET_KEY }}"/g' ./php-version-benchmarks/build/infrastructure/config/aws.tfvars
+      - name: Setup PHP config - baseline PHP version
+        if: github.repository == 'kocsismate/php-src' && github.event_name != 'workflow_dispatch'
+        run: |
+          set -e
+
+          BASELINE_SHORT_SHA="$(echo "${{ env.BASELINE_COMMIT }}" | cut -c1-4)"
+
+          cat << EOF > ./php-version-benchmarks/config/php/baseline.ini
+            PHP_NAME="PHP - baseline@$BASELINE_SHORT_SHA"
+            PHP_ID=php_baseline
+
+            PHP_REPO=https://github.com/${{ env.REPOSITORY }}.git
+            PHP_BRANCH=${{ env.BRANCH }}
+            PHP_COMMIT=${{ env.BASELINE_COMMIT }}
+
+            PHP_OPCACHE=2
+            PHP_JIT=0
+          EOF
+      - name: Setup PHP config - baseline PHP version with JIT (manual only)
+        if: github.repository == 'kocsismate/php-src' && github.event_name == 'workflow_dispatch' && inputs.jit == '1'
+        run: |
+          set -e
+
+          BASELINE_SHORT_SHA="$(echo "${{ env.BASELINE_COMMIT }}" | cut -c1-4)"
+          
+          cat << EOF > ./php-version-benchmarks/config/php/baseline.ini
+            PHP_NAME="PHP - baseline@$BASELINE_SHORT_SHA (JIT)"
+            PHP_ID=php_baseline_jit
+
+            PHP_REPO=https://github.com/${{ env.REPOSITORY }}.git
+            PHP_BRANCH=${{ env.BRANCH }}
+            PHP_COMMIT=${{ env.BASELINE_COMMIT }}
+
+            PHP_OPCACHE=2
+            PHP_JIT=${{ env.JIT }}
+          EOF
+
+          git clone ./php-version-benchmarks/tmp/php_baseline/ ./php-version-benchmarks/tmp/php_baseline_jit
+      - name: Setup PHP config - previous PHP version (scheduled only)
+        if: github.repository == 'kocsismate/php-src' && github.event_name != 'workflow_dispatch'
+        run: |
+          set -e
 
           YEAR="$(date '+%Y')"
           DATABASE="./php-version-benchmarks/docs/results/$YEAR/database.tsv"
           if [ -f "$DATABASE" ]; then
             LAST_RESULT_SHA="$(tail -n 2 "$DATABASE" | head -n 1 | cut -f 6)"
           else
             YESTERDAY="$(date -d "-2 day 23:59:59" '+%Y-%m-%d %H:%M:%S')"
-            LAST_RESULT_SHA="$(cd ./php-version-benchmarks/tmp/php_master/ && git --no-pager log --until="$YESTERDAY" -n 1 --pretty='%H')"
+            LAST_RESULT_SHA="$(cd ./php-version-benchmarks/tmp/php_${{ env.ID }}/ && git --no-pager log --until="$YESTERDAY" -n 1 --pretty='%H')"
           fi
 
-          BASELINE_SHA="d5f6e56610c729710073350af318c4ea1b292cfe"
-          BASELINE_SHORT_SHA="$(echo "$BASELINE_SHA" | cut -c1-4)"
-
-          # Set config for the baseline PHP version
-          cp ./php-version-benchmarks/config/php/master.ini.dist ./php-version-benchmarks/config/php/master_baseline.ini
-          sed -i 's/PHP_NAME="PHP - master"/PHP_NAME="PHP - baseline@'"$BASELINE_SHORT_SHA"'"/g' ./php-version-benchmarks/config/php/master_baseline.ini
-          sed -i "s/PHP_ID=php_master/PHP_ID=php_master_baseline/g" ./php-version-benchmarks/config/php/master_baseline.ini
-          sed -i "s/PHP_COMMIT=/PHP_COMMIT=$BASELINE_SHA/g" ./php-version-benchmarks/config/php/master_baseline.ini
-          sed -i "s/PHP_OPCACHE=1/PHP_OPCACHE=2/g" ./php-version-benchmarks/config/php/master_baseline.ini
-
-          # Set config for the previous PHP version
-          cp ./php-version-benchmarks/config/php/master.ini.dist ./php-version-benchmarks/config/php/master_last.ini
-          sed -i 's/PHP_NAME="PHP - master"/PHP_NAME="PHP - previous master"/g' ./php-version-benchmarks/config/php/master_last.ini
-          sed -i "s/PHP_ID=php_master/PHP_ID=php_master_previous/g" ./php-version-benchmarks/config/php/master_last.ini
-          sed -i "s/PHP_COMMIT=/PHP_COMMIT=$LAST_RESULT_SHA/g" ./php-version-benchmarks/config/php/master_last.ini
-          sed -i "s/PHP_OPCACHE=1/PHP_OPCACHE=2/g" ./php-version-benchmarks/config/php/master_last.ini
-
-          # Set config for the current PHP version
-          cp ./php-version-benchmarks/config/php/master.ini.dist ./php-version-benchmarks/config/php/master_now.ini
-          sed -i "s/PHP_COMMIT=/PHP_COMMIT=${{ github.sha }}/g" ./php-version-benchmarks/config/php/master_now.ini
-
-          # Set config for current PHP version with JIT
-          git clone ./php-version-benchmarks/tmp/php_master/ ./php-version-benchmarks/tmp/php_master_jit
-          cp ./php-version-benchmarks/config/php/master_jit.ini.dist ./php-version-benchmarks/config/php/master_now_jit.ini
-          sed -i "s/PHP_COMMIT=/PHP_COMMIT=${{ github.sha }}/g" ./php-version-benchmarks/config/php/master_now_jit.ini
-
-          # Set test configs
+          cat << EOF > ./php-version-benchmarks/config/php/previous.ini
+            PHP_NAME="PHP - previous ${{ env.BRANCH }}"
+            PHP_ID=php_previous
+
+            PHP_REPO=https://github.com/${{ env.REPOSITORY }}.git
+            PHP_BRANCH=${{ env.BRANCH }}
+            PHP_COMMIT=$LAST_RESULT_SHA
+
+            PHP_OPCACHE=1
+            PHP_JIT=0
+          EOF
+      - name: Setup PHP config - benchmarked PHP version
+        run: |
+          set -e
+
+          cat << EOF > ./php-version-benchmarks/config/php/this.ini
+            PHP_NAME="PHP - ${{ env.BRANCH }}"
+            PHP_ID=php_${{ env.ID }}
+
+            PHP_REPO=https://github.com/${{ env.REPOSITORY }}.git
+            PHP_BRANCH=${{ env.BRANCH }}
+            PHP_COMMIT=${{ env.COMMIT }}
+
+            PHP_OPCACHE=1
+            PHP_JIT=0
+          EOF
+      - name: Setup PHP config - benchmarked PHP version with JIT
+        if: env.JIT == '1'
+        run: |
+          set -e
+
+          cat << EOF > ./php-version-benchmarks/config/php/this_jit.ini
+            PHP_NAME="PHP - ${{ env.BRANCH }} (JIT)"
+            PHP_ID=php_${{ env.ID }}_jit
+
+            PHP_REPO=https://github.com/${{ env.REPOSITORY }}.git
+            PHP_BRANCH=${{ env.BRANCH }}
+            PHP_COMMIT=${{ env.COMMIT }}
+
+            PHP_OPCACHE=1
+            PHP_JIT=${{ env.JIT }}
+          EOF
+
+          git clone ./php-version-benchmarks/tmp/php_${{ env.ID }}/ ./php-version-benchmarks/tmp/php_${{ env.ID }}_jit
+      - name: Setup test config
+        run: |
+          set -e
+
           cp ./php-version-benchmarks/config/test/1_laravel.ini.dist ./php-version-benchmarks/config/test/1_laravel.ini
           cp ./php-version-benchmarks/config/test/2_symfony_main.ini.dist ./php-version-benchmarks/config/test/2_symfony_main.ini
           cp ./php-version-benchmarks/config/test/4_wordpress.ini.dist ./php-version-benchmarks/config/test/4_wordpress.ini
@@ -104,6 +210,7 @@ jobs:
       - name: Run benchmark
         run: ./php-version-benchmarks/benchmark.sh run aws
       - name: Store results
+        if: github.repository == 'kocsismate/php-src' && github.event_name != 'workflow_dispatch'
         run: |
           set -ex
 
@@ -119,6 +226,17 @@ jobs:
           fi
           git commit -m "Add result for ${{ github.repository }}@${{ github.sha }}"
           git push
+      - name: Comment results
+        if: github.repository != 'php/php-src' && github.event_name == 'workflow_dispatch'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          PR_NUMBER=$(gh pr list --head "${{ inputs.branch }}" --state open --json number --jq '.[0].number')
+          if [ ! -z "$PR_NUMBER" ]; then
+            YEAR="$(date '+%Y')"
+            NEWEST_RESULT_DIRECTORY=$(ls -td ./php-version-benchmarks/docs/results/$YEAR/*/ | head -1)
+            gh pr comment $PR_NUMBER --body-file "$NEWEST_RESULT_DIRECTORY/result.md"
+          fi
       - name: Cleanup
         if: always()
         run: |
