call-perf-test.yml

name: Call perf tests

on:
  workflow_call:
    inputs:
      project:
        description: "Project(s) to launch tests (space separated)"
        required: false
        type: string
      run_id_source:
        description: "Repo source for provided workflow run ID"
        required: false
        type: string
      run_id:
        description: "Workflow ID to use for the new version of wheel"
        required: false
        type: string
      ref:
        description: "Git ref to checkout"
        required: false
        type: string
      test-filter:
        description: "Only run tests that contains"
        required: false
        type: string
      sh-runner:
        description: "Run on shared runner"
        required: false
        type: boolean
      perf_regression_check:
        description: "Enable perf metrics regression testing"
        required: false
        type: boolean
      docker-image:
        description: "Docker image used in tests"
        required: false
        type: string
        default: "ghcr.io/tenstorrent/tt-forge-slim:latest"
      project-filter:
        description: "Project filter"
        required: false
        type: string
        default: tt-forge
      update-wheel:
        description: "Update wheel for the project"
        required: false
        type: boolean
      parent_run_id:
        description: "Parent run id is used to track child workflows in automated dispatch workflow calls"
        required: false
        type: string
        default: ""
      matrix:
        description: "Matrix of all test jobs that are ran"
        required: true
        type: string
      skip-device-perf:
        description: "Skip device performance measurement"
        required: false
        type: boolean

jobs:
  run-perf-benchmarks:
    container:
      image: ${{ inputs.sh-runner && format('harbor.ci.tenstorrent.net/{0}', inputs.docker-image) || inputs.docker-image }}
      options: --device /dev/tenstorrent --user root
      volumes:
        - /dev/hugepages:/dev/hugepages
        - /dev/hugepages-1G:/dev/hugepages-1G
        - /etc/udev/rules.d:/etc/udev/rules.d
        - /lib/modules:/lib/modules
        - /opt/tt_metal_infra/provisioning/provisioning_env:/opt/tt_metal_infra/provisioning/provisioning_env
        - /usr/local/bin:/usr/local/bin
        - /mnt/dockercache:/mnt/dockercache

    timeout-minutes: 60

    # tt-torch reqs are not needed by tt-torch, but rather by benchmarking infra, should be moved elsewhere: https://github.com/tenstorrent/tt-forge/issues/177
    strategy:
      fail-fast: false
      matrix:
        build: ${{ fromJson(inputs.matrix) }}

    runs-on: ${{ inputs.sh-runner && format('tt-ubuntu-2204-{0}-stable', matrix.build.runs-on) || fromJson(format('["{0}", "in-service"]', matrix.build.runs-on)) }}

    env:
      # TODO: Revisit the addition of these env vars https://github.com/tenstorrent/tt-metal/issues/20161
      TRACY_NO_INVARIANT_CHECK: 1

    name: "${{ matrix.build.project }}-${{ matrix.build.name }} (${{ inputs.sh-runner && format('{0}-shared', matrix.build.runs-on) || (matrix.build.runs-on) }}, ${{ matrix.build.bs }}, ${{ matrix.build.lp }}) benchmark"
    steps:

    - uses: actions/checkout@v4
      with:
        repository: 'tenstorrent/tt-forge'
        fetch-depth: 1
        ref: ${{ inputs.ref || github.ref }}
        submodules: 'recursive'

    - name: Set caching env variables
      shell: bash
      run: |
        shared_runners=${{ inputs.sh-runner }}
        if [[ "$shared_runners" == "" || "$shared_runners" == "false" ]]; then
          echo "DOCKER_CACHE_ROOT=/mnt/dockercache" >> $GITHUB_ENV
        else
          echo "IRD_LF_CACHE=${{ vars.IRD_LF_CACHE }}" >> $GITHUB_ENV
        fi

    - name: Fix HOME Directory
      shell: bash
      run: |
        # Issue [HOME is overridden for containers](https://github.com/actions/runner/issues/863)
        h=$(getent passwd $(id -un) | cut -d: -f6)
        if [ "$h" = "$HOME" ]; then
          echo "HOME fine: $HOME"
          exit 0
        fi
        echo "HOME=$HOME was broken. Setting it to $h"
        ls -ld $HOME
        ls -ld $h
        echo "USER: $USER"
        echo "id: $(id)"
        echo "HOME=$h" >> $GITHUB_ENV

    - name: Install system and python dependencies
      shell: bash
      run: |
        if [ "${{ matrix.build.project }}" == "tt-forge-onnx" ]; then
          apt-get update -y -qq
          apt-get install -y -qq --no-install-recommends sqlite3
        fi
        if [ -n "${{ matrix.build.libreq }}" ]; then
          apt-get update -y -qq
          apt-get install -y -qq --no-install-recommends ${{ matrix.build.libreq }}
        fi

        if [ -n "${{ matrix.build.pyreq }}" ]; then
          pip install ${{ matrix.build.pyreq }}
        fi
        if [ "${{ matrix.build.project }}" == "tt-xla" ]; then
          pip install torchvision==0.24.0+cpu --index-url https://download.pytorch.org/whl/cpu
        fi

    - name: Install upgraded wheel
      if: (inputs.project && inputs.run_id) || inputs.update-wheel
      uses: ./.github/actions/install-wheel
      with:
        project: ${{ matrix.build.project }}
        run_id: ${{ inputs.run_id }}
        run_id_source: ${{ inputs.run_id_source }}

    - name: Fetch job id
      id: fetch-job-id
      uses: tenstorrent/tt-github-actions/.github/actions/job_id@main
      with:
        job_name: "${{ matrix.build.project }}-${{ matrix.build.name }} (${{ inputs.sh-runner && format('{0}-shared', matrix.build.runs-on) || (matrix.build.runs-on) }}, ${{ matrix.build.bs }}, ${{ matrix.build.lp }}) benchmark"

    - name: Set reusable strings
      id: strings
      shell: bash
      env:
        JOB_ID: ${{ steps.fetch-job-id.outputs.job_id }}
      run: |
        echo "work-dir=$(pwd)" >> "$GITHUB_OUTPUT"
        echo "build-output-dir=$(pwd)/build" >> "$GITHUB_OUTPUT"
        echo "perf_report_path=$(pwd)/benchmark_reports" >> "$GITHUB_OUTPUT"
        echo "perf_report_json_file=$(pwd)/benchmark_reports/benchmark_${{ matrix.build.project }}_e2e_${{ matrix.build.name }}_${{ matrix.build.bs }}_${{ matrix.build.lp }}_${{ steps.fetch-job-id.outputs.job_id }}.json" >> "$GITHUB_OUTPUT"

    - name: Run Perf Benchmark
      shell: bash
      env:
        HF_TOKEN: ${{ secrets.HF_TOKEN }}
        HF_HOME: /mnt/dockercache/huggingface
        TORCH_HOME: /mnt/dockercache/torchhub
        HF_HUB_DISABLE_PROGRESS_BARS: 1
      run: |
        echo "Create perf report directory"
        mkdir -p ${{ steps.strings.outputs.perf_report_path }}

        # Caused by tensorflow installation in tt-xla: https://github.com/tenstorrent/tt-forge/issues/477
        if [ "${{ matrix.build.project }}" == "tt-xla" ]; then
          pip uninstall tensorflow -y
        fi

        SITE_PACKAGES_DIR=$(python -c "import site; print(site.getsitepackages()[0])")
        if [ "${{ matrix.build.project }}" == "tt-forge-onnx" ]; then
          export TT_METAL_HOME="$SITE_PACKAGES_DIR/forge/tt-metal"
        elif [ "${{ matrix.build.project }}" == "tt-xla" ]; then
          export TT_METAL_HOME="$SITE_PACKAGES_DIR/pjrt_plugin_tt/tt-metal"
        fi

        echo "Run benchmark for ${{ matrix.build.project }} - ${{ matrix.build.name }}"

        if [ -z "${{ matrix.build.pytest }}" ]; then
          # Run with legacy benchmark script
          python benchmark/benchmark.py -p ${{ matrix.build.project}} -m ${{ matrix.build.name }} -bs ${{ matrix.build.bs }} -df ${{ matrix.build.df }} -lp ${{ matrix.build.lp }} ${{ matrix.build.input_sequence_length && format('-isl {0}', matrix.build.input_sequence_length) }} -ts ${{ matrix.build.ts }} -o ${{ steps.strings.outputs.perf_report_json_file }} ${{ inputs.run_id_source && format('-r {0}', inputs.run_id_source) }}
        else
          # Run with pytest
          pytest -svv "${{ matrix.build.pytest }}" --output-file=${{ steps.strings.outputs.perf_report_json_file }}
        fi

    - name: Dump stablehlo to report
      if: (success() || failure()) && !matrix.build.skip-stablehlo-dump
      shell: bash
      run: |
        echo "Dump stablehlo to report"
        mkdir -p ${{ steps.strings.outputs.perf_report_path }}/stablehlo
        if [ "${{ matrix.build.name }}" == "resnet_jax" ]; then
          cp ./modules/${{ matrix.build.name }}_stablehlo.mlir ${{ steps.strings.outputs.perf_report_path }}/stablehlo/stablehlo.mlir 2>/dev/null || true
        else
          cp ./modules/irs/shlo_*.mlir ${{ steps.strings.outputs.perf_report_path }}/stablehlo/ 2>/dev/null || true
        fi

    - name: Upload StableHLO MLIR separately
      id: upload-stablehlo-mlir
      uses: actions/upload-artifact@v4
      if: (success() || failure()) && !matrix.build.skip-stablehlo-dump
      with:
        name: stablehlo-mlir-${{ steps.fetch-job-id.outputs.job_id }}
        path: ${{ steps.strings.outputs.perf_report_path }}/stablehlo/

    - name: Dump ttir to report
      id: dump-ttir
      if: (success() || failure()) && !matrix.build.skip-ttir-dump
      shell: bash
      run: |
        echo "Dump ttir to report"
        mkdir -p ${{ steps.strings.outputs.perf_report_path }}/ttir
        ttir_artifact_name="ttir-mlir-${{ steps.fetch-job-id.outputs.job_id }}"
        if [ "${{ matrix.build.name }}" == "resnet_jax" ]; then
          cp ./modules/${{ matrix.build.name }}_ttir.mlir ${{ steps.strings.outputs.perf_report_path }}/ttir/ttir.mlir
        else
          cp ./modules/irs/ttir_*.mlir ${{ steps.strings.outputs.perf_report_path }}/ttir/
        fi
        # Group TTIR MLIR files under a folder named like the files but without the gN segment
        ttir_first_file="$(ls -1 ${{ steps.strings.outputs.perf_report_path }}/ttir/ttir_*.mlir 2>/dev/null | head -n 1 || true)"
        if [ -n "$ttir_first_file" ]; then
          ttir_base_name="$(basename "$ttir_first_file")"
          ttir_folder_name="$(echo "$ttir_base_name" | sed -E 's/_g[0-9]+_/_/' | sed -E 's/_run[0-9a-fA-F]+_/_/' | sed -E 's/\.mlir$//')"
          mkdir -p "${{ steps.strings.outputs.perf_report_path }}/ttir/${ttir_folder_name}"
          mv ${{ steps.strings.outputs.perf_report_path }}/ttir/ttir_*.mlir "${{ steps.strings.outputs.perf_report_path }}/ttir/${ttir_folder_name}/"
          ttir_artifact_name="ttir-mlir-${ttir_folder_name}"
        fi
        echo "artifact_name=${ttir_artifact_name}" >> "$GITHUB_OUTPUT"

    - name: Upload TTIR MLIR separately
      id: upload-ttir-mlir
      uses: actions/upload-artifact@v4
      if: (success() || failure()) && !matrix.build.skip-ttir-dump
      with:
        name: ${{ steps.dump-ttir.outputs.artifact_name }}
        path: ${{ steps.strings.outputs.perf_report_path }}/ttir/

    - name: Dump ttnn to report
      id: dump-ttnn
      if: (success() || failure()) && !matrix.build.skip-ttnn-dump
      shell: bash
      run: |
        echo "Dump ttnn to report"
        mkdir -p ${{ steps.strings.outputs.perf_report_path }}/ttnn
        ttnn_artifact_name="ttnn-mlir-${{ steps.fetch-job-id.outputs.job_id }}"
        if [ "${{ matrix.build.name }}" == "resnet_jax" ]; then
          cp ./modules/${{ matrix.build.name }}_ttnn.mlir ${{ steps.strings.outputs.perf_report_path }}/ttnn/ttnn.mlir
        else
          cp ./modules/irs/ttnn_*.mlir ${{ steps.strings.outputs.perf_report_path }}/ttnn/
        fi
        # Group TTNN MLIR files under a folder named like the files but without the gN segment
        ttnn_first_file="$(ls -1 ${{ steps.strings.outputs.perf_report_path }}/ttnn/ttnn_*.mlir 2>/dev/null | head -n 1 || true)"
        if [ -n "$ttnn_first_file" ]; then
          ttnn_base_name="$(basename "$ttnn_first_file")"
          ttnn_folder_name="$(echo "$ttnn_base_name" | sed -E 's/_g[0-9]+_/_/' | sed -E 's/_run[0-9a-fA-F]+_/_/' | sed -E 's/\.mlir$//')"
          mkdir -p "${{ steps.strings.outputs.perf_report_path }}/ttnn/${ttnn_folder_name}"
          mv ${{ steps.strings.outputs.perf_report_path }}/ttnn/ttnn_*.mlir "${{ steps.strings.outputs.perf_report_path }}/ttnn/${ttnn_folder_name}/"
          ttnn_artifact_name="ttnn-mlir-${ttnn_folder_name}"
        fi
        echo "artifact_name=${ttnn_artifact_name}" >> "$GITHUB_OUTPUT"

    - name: Upload TTNN MLIR separately
      id: upload-ttnn-mlir
      uses: actions/upload-artifact@v4
      if: (success() || failure()) && !matrix.build.skip-ttnn-dump
      with:
        name: ${{ steps.dump-ttnn.outputs.artifact_name }}
        path: ${{ steps.strings.outputs.perf_report_path }}/ttnn/

    - name: Find workflow run ID for ttrt wheel download
      id: find-run
      shell: bash
      run: |
        project_name="${{ matrix.build.project }}"
        if [[ "${{ matrix.build.project }}" == "tt-xla" ]]; then
          project_name="pjrt-plugin-tt"
        fi

        description_field_key="tt-mlir-commit"

        mlir_sha=$(python .github/scripts/get_description_field_from_wheel.py "$project_name" "$description_field_key")
        echo "Mlir commit sha: $mlir_sha"

        RUN_ID=$(curl -s \
          -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
          "https://api.github.com/repos/tenstorrent/tt-mlir/actions/workflows/on-push.yml/runs?head_sha=${mlir_sha}&per_page=1" \
          | jq -r '.workflow_runs[0].id')

        if [[ -z "$RUN_ID" ]]; then
          echo "The tt-mlir commit doesn't contain a ttrt wheel artifact. Run an artifact building workflow on the commit (e.g. 'On PR')."
          exit 1
        fi

        echo "run_id=$RUN_ID" >> $GITHUB_OUTPUT
        echo "mlir_sha=$mlir_sha" >> $GITHUB_OUTPUT
        echo "Workflow URL: https://github.com/tenstorrent/tt-mlir/actions/runs/$RUN_ID"
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

    - name: Download ttrt wheel
      if: ${{ !matrix.build.skip-device-perf && !inputs.skip-device-perf }}
      uses: dawidd6/action-download-artifact@v11
      with:
        run_id: ${{ steps.find-run.outputs.run_id }}
        name: "ttrt-whl-tracy"
        repo: tenstorrent/tt-mlir
        check_artifacts: true
        path: ./
        skip_unpack: true

    - name: Extract ttrt wheel
      if: ${{ !matrix.build.skip-device-perf && !inputs.skip-device-perf }}
      shell: bash
      run: |
        echo "Extracting ttrt wheel"
        mkdir ttrt-whl-tracy
        mv ttrt-whl-tracy.zip ttrt-whl-tracy/ttrt-whl-tracy.zip
        unzip -q ttrt-whl-tracy/ttrt-whl-tracy.zip -d ttrt-whl-tracy

    - name: Install ttrt wheel
      shell: bash
      if: ${{ !matrix.build.skip-device-perf && !inputs.skip-device-perf }}
      run: |
        echo "Install ttrt"
        python -m venv ttrt-venv
        source ttrt-venv/bin/activate
        apt-get update -y -qq
        python --version
        apt-get install -y -qq --no-install-recommends libtbb12 libcapstone4
        pip install ttrt-whl-tracy/ttrt*.whl --upgrade
        pip install torch==2.3.0 --index-url https://download.pytorch.org/whl/cpu
        pip install pandas==2.3.3

    - name: Run Device Perf
      if: ${{ !matrix.build.skip-device-perf && !inputs.skip-device-perf && matrix.build.runs-on != 'p150' && matrix.build.runs-on != 'llmbox' }}
      shell: bash
      run: |
        source ttrt-venv/bin/activate

        echo "Save artifacts"
        ttrt query --save-artifacts

        echo "Copy binaries"
        mkdir flatbuffers
        cp ./modules/*.ttnn flatbuffers

        echo "Run ttrt perf"
        ttrt perf flatbuffers --ignore-version

        echo "Write device perf to report"
        python ./benchmark/device_perf.py ttrt-artifacts ${{ steps.strings.outputs.perf_report_json_file }}

        echo "Copy device perf CSVs"
        mkdir -p ${{ steps.strings.outputs.perf_report_path }}/device_perf
        for dir in ttrt-artifacts/*/; do
          binary_name=$(basename "$dir")
          csv_file="$dir/perf/ops_perf_results_minus_const_eval_and_input_layout_conversions.csv"
          if [ -f "$csv_file" ]; then
            cp "$csv_file" "${{ steps.strings.outputs.perf_report_path }}/device_perf/${binary_name}.csv"
            echo "Copied $csv_file to ${binary_name}.csv"
          fi
        done

    - name: Upload Device Perf separately
      id: upload-device-perf
      uses: actions/upload-artifact@v4
      if: ${{ !matrix.build.skip-device-perf && !inputs.skip-device-perf && matrix.build.runs-on != 'p150' && matrix.build.runs-on != 'llmbox' }}
      with:
        name: device-perf-${{ steps.fetch-job-id.outputs.job_id }}
        path: ${{ steps.strings.outputs.perf_report_path }}/device_perf/
        compression-level: 0

    - name: Add config fields to perf report
      shell: bash
      run: |
        PERF_REPORT_FILE="${{ steps.strings.outputs.perf_report_json_file }}"
        STABLEHLO_URL="${{ !matrix.build.skip-stablehlo-dump && steps.upload-stablehlo-mlir.outputs.artifact-url || '' }}"
        TTIR_URL="${{ !matrix.build.skip-ttir-dump && steps.upload-ttir-mlir.outputs.artifact-url || '' }}"
        TTNN_URL="${{ !matrix.build.skip-ttnn-dump && steps.upload-ttnn-mlir.outputs.artifact-url || '' }}"
        DEVICE_PERF_URL="${{ !matrix.build.skip-device-perf && !inputs.skip-device-perf && matrix.build.runs-on != 'p150' && matrix.build.runs-on != 'llmbox' && steps.upload-device-perf.outputs.artifact-url || '' }}"
        MLIR_SHA="${{ steps.find-run.outputs.mlir_sha || '' }}"
        JOB_ID_URL="https://github.com/tenstorrent/tt-forge/actions/runs/${{ github.run_id }}/job/${{ steps.fetch-job-id.outputs.job_id }}"

        python benchmark/extend_result_config.py "$PERF_REPORT_FILE" \
          ${STABLEHLO_URL:+--stablehlo-url "$STABLEHLO_URL"} \
          ${TTIR_URL:+--ttir-url "$TTIR_URL"} \
          ${TTNN_URL:+--ttnn-url "$TTNN_URL"} \
          ${DEVICE_PERF_URL:+--device-perf-url "$DEVICE_PERF_URL"} \
          ${MLIR_SHA:+--mlir-sha "$MLIR_SHA"} \
          --job-id-url "$JOB_ID_URL"

    - name: Upload Perf Report
      uses: actions/upload-artifact@v4
      if: success() || failure()
      with:
        name: perf-reports-${{ steps.fetch-job-id.outputs.job_id }}
        path: ${{ steps.strings.outputs.perf_report_path }}

    # note: potentially improve this by looking at the last nigthly with the specific model's
    # perf data present, not only at the last nigthly
    - name: Set perf query strings
      id: set-query-strings
      if: inputs.perf_regression_check
      shell: bash
      run: |
        echo "model_name=$(cat ${{ steps.strings.outputs.perf_report_json_file }} | jq -r '.model')" >> $GITHUB_OUTPUT

        workflow_id=$(curl -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
          -H "Accept: application/vnd.github.v3+json" \
          "https://api.github.com/repos/tenstorrent/tt-forge/actions/workflows/perf-benchmark.yml/runs?status=completed&branch=main&per_page=100&actor=github-actions\[bot\]" \
          | jq -r '.workflow_runs[] | select(.name | contains("draft") | not) | .id' | head -1)
        echo "last_nightly_workflow_id=$workflow_id" >> $GITHUB_OUTPUT

        arch=""
        if [[ "${{ matrix.build.runs-on-original }}" == "n150" ]]; then
          arch="wormhole"
        elif [[ "${{ matrix.build.runs-on-original }}" == "p150" ]]; then
          arch="blackhole"
        elif [[ "${{ matrix.build.runs-on-original }}" == "llmbox" ]]; then
          arch="wormhole_llmbox"
        fi
        echo "arch=$arch" >> $GITHUB_OUTPUT


    - name: Fetch previous perf results
      id: prev-perf
      if: inputs.perf_regression_check
      uses: ./.github/actions/superset-api
      with:
        query: 'benchmarks/last_measurement'
        query_params: '{"project":"tt-forge/${{ matrix.build.project }}","ml_model_name":"${{ steps.set-query-strings.outputs.model_name }}","github_pipeline_id":"${{ steps.set-query-strings.outputs.last_nightly_workflow_id }}","arch":"${{ steps.set-query-strings.outputs.arch }}"}'

    - name: Check perf regression
      if: inputs.perf_regression_check
      shell: bash
      run: |
        if [[ "${{ steps.prev-perf.outputs.response }}" == "[]" ]]; then
          echo "Performance report not found in the last nightly"
          exit 0
        fi

        apt-get update -y -qq
        apt-get install -y -qq bc

        prev_perf_report='${{ steps.prev-perf.outputs.response }}'

        prev_total_samples=$(echo "$prev_perf_report" | jq -r '.[] | select(.name == "total_samples") | .last_value')
        prev_total_time=$(echo "$prev_perf_report" | jq -r '.[] | select(.name == "total_time") | .last_value')
        prev_samples_per_sec=$(echo "scale=6; $prev_total_samples / $prev_total_time" | bc -l)

        current_perf_report_file=${{ steps.strings.outputs.perf_report_json_file }}

        current_total_samples=$(jq -r '.measurements[] | select(.measurement_name == "total_samples") | .value' "$current_perf_report_file")
        current_total_time=$(jq -r '.measurements[] | select(.measurement_name == "total_time") | .value' "$current_perf_report_file")
        current_samples_per_sec=$(echo "scale=6; $current_total_samples / $current_total_time" | bc -l)

        perf_diff=$(echo "scale=4; (($prev_samples_per_sec - $current_samples_per_sec) / $prev_samples_per_sec) * 100" | bc -l)

        echo "Previous performance:"
        echo "  Total samples: $prev_total_samples"
        echo "  Total time: $prev_total_time"
        echo "  Samples per second: $prev_samples_per_sec"
        echo ""
        echo "Current performance:"
        echo "  Total samples: $current_total_samples"
        echo "  Total time: $current_total_time"
        echo "  Samples per second: $current_samples_per_sec"
        echo ""

        if (( $(echo "$perf_diff >= 5.0" | bc -l) )); then
          echo "Performance regression > 5% detected! Performance dropped by ${perf_diff}%"
          exit 1
        fi