real-time-benchmark.yml

name: Real-time Benchmark
on:
  schedule:
    - cron: "30 0 * * *"
  workflow_dispatch:
    inputs:
      pull_request:
        description: 'PR number that is going to be benchmarked (e.g. "1234")'
        required: true
        type: number
      jit:
        description: 'Whether JIT is benchmarked'
        required: false
        default: "0"
        type: choice
        options:
          - "0"
          - "1"
      collect_extended_perf_stats:
        description: 'Whether to collect extended perf stats as artifacts'
        required: true
        default: "0"
        type: choice
        options:
          - "0"
          - "1"
      debug_environment:
        description: 'Whether to collect environment debug logs as artifacts'
        required: true
        default: "0"
        type: choice
        options:
          - "0"
          - "1"
      run_micro_bench:
        description: 'Whether to run the micro_bench.php test'
        required: true
        default: "0"
        type: choice
        options:
          - "0"
          - "1"
permissions:
  contents: read
  pull-requests: write
concurrency:
  group: ${{ github.workflow }}-${{ github.event_name }}
  cancel-in-progress: false
jobs:
  REAL_TIME_BENCHMARK:
    name: REAL_TIME_BENCHMARK
    if: github.repository == 'php/php-src' || github.event_name == 'workflow_dispatch'
    runs-on: ubuntu-24.04
    env:
      REPOSITORY: ${{ github.repository }}
      BRANCH: "master"
      COMMIT: ${{ github.sha }}
      BASELINE_COMMIT: "d5f6e56610c729710073350af318c4ea1b292cfe"
      ID: "master"
      JIT: "1"
      COLLECT_EXTENDED_PERF_STATS: "0"
      DEBUG_ENVIRONMENT: "0"
      RUN_MICRO_BENCH: "0"
      YEAR: ""
    steps:
      - name: Setup benchmark environment
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          YEAR="$(date '+%Y')"
          echo "YEAR=$YEAR" >> $GITHUB_ENV

          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
            PR_INFO=$(gh pr view ${{ inputs.pull_request }} --json headRepositoryOwner,headRepository,headRefName,headRefOid,baseRefName --repo ${{ github.repository }} | jq -r '.headRepositoryOwner.login, .headRepository.name, .headRefName, .headRefOid, .baseRefName')

            REPOSITORY="$(echo "$PR_INFO" | sed -n '1p')/$(echo "$PR_INFO" | sed -n '2p')"
            echo "REPOSITORY=$REPOSITORY" >> $GITHUB_ENV

            BRANCH=$(echo "$PR_INFO" | sed -n '3p')
            echo "BRANCH=$BRANCH" >> $GITHUB_ENV

            COMMIT=$(echo "$PR_INFO" | sed -n '4p')
            echo "COMMIT=$COMMIT" >> $GITHUB_ENV

            BASELINE_BRANCH=$(echo "$PR_INFO" | sed -n '5p')

            BASELINE_COMMIT=$(gh api /repos/${{ github.repository }}/compare/$BASELINE_BRANCH...$COMMIT --jq '.merge_base_commit.sha')
            echo "BASELINE_COMMIT=$BASELINE_COMMIT" >> $GITHUB_ENV

            echo "ID=benchmarked" >> $GITHUB_ENV

            echo "JIT=${{ inputs.jit }}" >> $GITHUB_ENV
            echo "COLLECT_EXTENDED_PERF_STATS=${{ inputs.collect_extended_perf_stats }}" >> $GITHUB_ENV
            echo "DEBUG_ENVIRONMENT=${{ inputs.debug_environment }}" >> $GITHUB_ENV
            echo "RUN_MICRO_BENCH=${{ inputs.run_micro_bench }}" >> $GITHUB_ENV
          fi

      - name: Install dependencies
        run: |
          set -e
          sudo apt-get update
          sudo apt-get install gpg

          wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
          gpg --no-default-keyring --keyring /usr/share/keyrings/hashicorp-archive-keyring.gpg --fingerprint
          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
          export DEBIAN_FRONTEND=noninteractive
          sudo apt-get update -y
          sudo apt-get install -y terraform=1.5.7-*
      - name: Checkout benchmark suite
        uses: actions/checkout@v6
        with:
          repository: 'kocsismate/php-version-benchmarks'
          ref: 'main'
          fetch-depth: 1
          path: 'php-version-benchmarks'
      - name: Checkout php-src (benchmarked version)
        uses: actions/checkout@v6
        with:
          repository: '${{ env.REPOSITORY }}'
          ref: '${{ env.COMMIT }}'
          fetch-depth: 100
          path: 'php-version-benchmarks/tmp/php_${{ env.ID }}'
      - name: Checkout php-src (baseline version)
        uses: actions/checkout@v6
        with:
          repository: '${{ env.REPOSITORY }}'
          ref: '${{ env.BASELINE_COMMIT }}'
          fetch-depth: 100
          path: 'php-version-benchmarks/tmp/php_baseline'
      - name: Setup benchmark results
        run: |
          git config --global user.name "Benchmark"
          git config --global user.email "benchmark@php.net"

          rm -rf ./php-version-benchmarks/docs/results
      - name: Checkout benchmark data
        if: github.event_name != 'workflow_dispatch'
        uses: actions/checkout@v6
        with:
          repository: php/real-time-benchmark-data
          ssh-key: ${{ secrets.PHP_VERSION_BENCHMARK_RESULTS_DEPLOY_KEY }}
          path: 'php-version-benchmarks/docs/results'
      - name: Setup PHP config - baseline PHP version
        run: |
          set -e

          BASELINE_SHORT_SHA="$(echo "${{ env.BASELINE_COMMIT }}" | cut -c 1-7)"

          cat << EOF > ./php-version-benchmarks/config/php/baseline.ini
            PHP_NAME="PHP - baseline@$BASELINE_SHORT_SHA"
            PHP_ID=php_baseline

            PHP_REPO=${{ github.server_url }}/${{ env.REPOSITORY }}.git
            PHP_BRANCH=${{ env.BRANCH }}
            PHP_COMMIT=${{ env.BASELINE_COMMIT }}

            PHP_JIT=0
          EOF
      - name: Setup PHP config - baseline PHP version with JIT
        if: github.event_name == 'workflow_dispatch' && inputs.jit == '1'
        run: |
          set -e

          BASELINE_SHORT_SHA="$(echo "${{ env.BASELINE_COMMIT }}" | cut -c 1-7)"

          cat << EOF > ./php-version-benchmarks/config/php/baseline_jit.ini
            PHP_NAME="PHP - baseline@$BASELINE_SHORT_SHA (JIT)"
            PHP_ID=php_baseline_jit

            PHP_REPO=${{ github.server_url }}/${{ env.REPOSITORY }}.git
            PHP_BRANCH=${{ env.BRANCH }}
            PHP_COMMIT=${{ env.BASELINE_COMMIT }}

            PHP_JIT=${{ env.JIT }}
          EOF

          git clone ./php-version-benchmarks/tmp/php_baseline/ ./php-version-benchmarks/tmp/php_baseline_jit
      - name: Setup PHP config - previous PHP version
        if: github.event_name != 'workflow_dispatch'
        run: |
          set -e

          DATABASE="./php-version-benchmarks/docs/results/${{ env.YEAR }}/database.tsv"
          if [ -f "$DATABASE" ]; then
            LAST_RESULT_SHA="$(tail -n 2 "$DATABASE" | head -n 1 | cut -f 6)"
          else
            YESTERDAY="$(date -d "-2 day 23:59:59" '+%Y-%m-%d %H:%M:%S')"
            LAST_RESULT_SHA="$(cd ./php-version-benchmarks/tmp/php_${{ env.ID }}/ && git --no-pager log --until="$YESTERDAY" -n 1 --pretty='%H')"
          fi

          echo "LAST_RESULT_SHA=$LAST_RESULT_SHA" >> $GITHUB_ENV

          cat << EOF > ./php-version-benchmarks/config/php/previous.ini
            PHP_NAME="PHP - previous ${{ env.BRANCH }}"
            PHP_ID=php_previous

            PHP_REPO=${{ github.server_url }}/${{ env.REPOSITORY }}.git
            PHP_BRANCH=${{ env.BRANCH }}
            PHP_COMMIT=$LAST_RESULT_SHA

            PHP_JIT=0
          EOF
      - name: Setup PHP config - benchmarked PHP version
        run: |
          set -e

          cat << EOF > ./php-version-benchmarks/config/php/this.ini
            PHP_NAME="PHP - ${{ env.BRANCH }}"
            PHP_ID=php_${{ env.ID }}

            PHP_REPO=${{ github.server_url }}/${{ env.REPOSITORY }}.git
            PHP_BRANCH=${{ env.BRANCH }}
            PHP_COMMIT=${{ env.COMMIT }}

            PHP_JIT=0
          EOF
      - name: Setup PHP config - benchmarked PHP version with JIT
        if: env.JIT == '1'
        run: |
          set -e

          cat << EOF > ./php-version-benchmarks/config/php/this_jit.ini
            PHP_NAME="PHP - ${{ env.BRANCH }} (JIT)"
            PHP_ID=php_${{ env.ID }}_jit

            PHP_REPO=${{ github.server_url }}/${{ env.REPOSITORY }}.git
            PHP_BRANCH=${{ env.BRANCH }}
            PHP_COMMIT=${{ env.COMMIT }}

            PHP_JIT=${{ env.JIT }}
          EOF

          git clone ./php-version-benchmarks/tmp/php_${{ env.ID }}/ ./php-version-benchmarks/tmp/php_${{ env.ID }}_jit
      - name: Setup test config
        run: |
          set -e

          cp ./php-version-benchmarks/config/test/1_laravel.ini.dist ./php-version-benchmarks/config/test/1_laravel.ini
          cp ./php-version-benchmarks/config/test/2_symfony_main.ini.dist ./php-version-benchmarks/config/test/2_symfony_main.ini
          cp ./php-version-benchmarks/config/test/4_wordpress.ini.dist ./php-version-benchmarks/config/test/4_wordpress.ini
          cp ./php-version-benchmarks/config/test/5_bench.php.ini.dist ./php-version-benchmarks/config/test/5_bench.php.ini

          if [ "${{ env.RUN_MICRO_BENCH }}" -eq "1" ]; then
            cp ./php-version-benchmarks/config/test/6_micro_bench.php.ini.dist ./php-version-benchmarks/config/test/6_micro_bench.php.ini
          fi
      - name: Setup infra config
        run: |
          set -e
          
          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
            WORKSPACE="manual"
            BASE_COMMIT="$(echo "${{ env.BASELINE_COMMIT }}" | cut -c 1-10)"
          else
            WORKSPACE="nightly"
            BASE_COMMIT="$(echo "${{ env.LAST_RESULT_SHA }}" | cut -c 1-10)"
          fi
          COMPARE_COMMIT="$(echo "${{ env.COMMIT }}" | cut -c 1-10)"

          cp ./php-version-benchmarks/config/infra/aws/x86_64-metal.ini.dist ./php-version-benchmarks/config/infra/aws/x86_64-metal.ini
          sed -i "s|INFRA_DOCKER_REGISTRY=public.ecr.aws/abcdefgh|INFRA_DOCKER_REGISTRY=${{ secrets.PHP_VERSION_BENCHMARK_DOCKER_REGISTRY }}|g" ./php-version-benchmarks/config/infra/aws/x86_64-metal.ini
          sed -i "s|INFRA_WORKSPACE=|INFRA_WORKSPACE=$WORKSPACE|g" ./php-version-benchmarks/config/infra/aws/x86_64-metal.ini
          sed -i "s/INFRA_COLLECT_EXTENDED_PERF_STATS=0/INFRA_COLLECT_EXTENDED_PERF_STATS=${{ env.COLLECT_EXTENDED_PERF_STATS }}/g" ./php-version-benchmarks/config/infra/aws/x86_64-metal.ini
          sed -i "s/INFRA_DEBUG_ENVIRONMENT=0/INFRA_DEBUG_ENVIRONMENT=${{ env.DEBUG_ENVIRONMENT }}/g" ./php-version-benchmarks/config/infra/aws/x86_64-metal.ini

          cp ./php-version-benchmarks/build/infrastructure/config/aws.tfvars.dist ./php-version-benchmarks/build/infrastructure/config/aws.tfvars
          sed -i 's/access_key = ""/access_key = "${{ secrets.PHP_VERSION_BENCHMARK_AWS_ACCESS_KEY }}"/g' ./php-version-benchmarks/build/infrastructure/config/aws.tfvars
          sed -i 's/secret_key = ""/secret_key = "${{ secrets.PHP_VERSION_BENCHMARK_AWS_SECRET_KEY }}"/g' ./php-version-benchmarks/build/infrastructure/config/aws.tfvars
          sed -i 's/state_bucket = ""/state_bucket = "${{ secrets.PHP_VERSION_BENCHMARK_STATE_BUCKET }}"/g' ./php-version-benchmarks/build/infrastructure/config/aws.tfvars
          sed -i 's/github_token = ""/github_token = "${{ secrets.GITHUB_TOKEN }}"/g' ./php-version-benchmarks/build/infrastructure/config/aws.tfvars

          WORKFLOW_RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
          sed -i "s|log_url = \"\"|log_url = \"$WORKFLOW_RUN_URL\"|g" ./php-version-benchmarks/build/infrastructure/config/aws.tfvars
          sed -i 's|artifact_url = ""|artifact_url = "#ARTIFACT_URL#"|g' ./php-version-benchmarks/build/infrastructure/config/aws.tfvars

          CHANGESET_URL="${{ github.server_url }}/${{ github.repository }}/compare/$BASE_COMMIT..$COMPARE_COMMIT"
          sed -i 's|extra_title = ""|extra_title = "Changeset"|g' ./php-version-benchmarks/build/infrastructure/config/aws.tfvars
          sed -i "s|extra_text = \"\"|extra_text = \"$CHANGESET_URL\"|g" ./php-version-benchmarks/build/infrastructure/config/aws.tfvars
      - name: Run benchmark
        run: |
          set -e
          
          ./php-version-benchmarks/benchmark.sh run aws
          
          echo "NEWEST_RESULT_DIRECTORY=$(ls -td ${{ github.workspace }}/php-version-benchmarks/docs/results/${{ env.YEAR }}/*/ | head -1)" >> $GITHUB_ENV
      - name: Upload artifacts
        id: upload
        uses: actions/upload-artifact@v6
        with:
          name: results
          path: |
            ./php-version-benchmarks/tmp/results/${{ env.YEAR }}/**/*
          retention-days: 30
      - name: Commit results
        if: github.repository == 'php/php-src' && github.event_name != 'workflow_dispatch'
        run: |
          set -ex

          sed -i "s|#ARTIFACT_URL#|${{ steps.upload.outputs.artifact-url }}|g" "${NEWEST_RESULT_DIRECTORY}result.md"

          cd ./php-version-benchmarks/docs/results
          git pull --autostash
          if [ -e ".git/MERGE_HEAD" ]; then
            echo "Merging, can't proceed"
            exit 1
          fi
          git add .
          if git diff --cached --quiet; then
            exit 1
          fi
          git commit -m "Add result for ${{ github.repository }}@${{ github.sha }}"
          git push
      - name: Comment results
        if: github.event_name == 'workflow_dispatch'
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          cd ./php-version-benchmarks/tmp/php_${{ env.ID }}
          gh pr comment ${{ inputs.pull_request }} --body-file "${NEWEST_RESULT_DIRECTORY}result.md" --repo ${{ github.repository }}