[CI][benchmarks][Liger-Kernel] Fixed issues with dependencies and runtime (#3660)

Closes #3650

Currently need to solve:
- [x] When one of benchmarks fail, CI stops, so no artefacts available

Author: Egor-Krivov

.github/workflows/third-party-benchmarks.yml

@@ -1,5 +1,4 @@
 name: Third party benchmarks
-run-name: ${{ inputs.run_name }}
 
 on:
   workflow_dispatch:
@@ -12,10 +11,6 @@ on:
         description: Tag for benchmark results
         type: string
         default: "test"
-      run_name:
-        description: Run name
-        type: string
-        default: "Triton benchmarks"
       use_pyenv_python:
         description: Use Python built with pyenv
         type: boolean
@@ -24,12 +19,6 @@ on:
     # About midnight PST (UTC-8)
     - cron: "5 10 * * *"
 
-
-# Cancels in-progress PR runs when the PR is updated.  Manual runs are never cancelled.
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event_name == 'workflow_dispatch' && github.run_id || github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
 permissions: read-all
 
 env:
@@ -92,7 +81,7 @@ jobs:
       - name: Install benchmark dependencies
         id: install
         run: |
-          pip install transformers pandas
+          pip install transformers pandas pytest
 
       - name: Create reports dir
         run: |
@@ -109,10 +98,15 @@ jobs:
           git clone https://github.com/linkedin/Liger-Kernel
           pip install -e Liger-Kernel
 
-          bash ./run_benchmarks.sh
+          # To remember return code, but still copy results
+          RET_CODE=0
+          bash ./run_benchmarks.sh || RET_CODE=$?
 
           cp Liger-Kernel/benchmark/data/all_benchmark_data.csv $REPORTS/liger-raw.csv
-          python transform.py $REPORTS/liger-raw.csv $REPORTS/liger-report.csv
+          python transform.py $REPORTS/liger-raw.csv $REPORTS/liger-report.csv --tag $TAG
+
+          # Return the captured return code at the end
+          exit "$RET_CODE"
 
       - name: Upload benchmark reports
         if: ${{ steps.install.outcome == 'success' && !cancelled() }}

benchmarks/third_party/liger_kernels/README.md

@@ -6,5 +6,5 @@ https://github.com/linkedin/Liger-Kernel/tree/main
 
 We run benchmarks for Liger Kernels in CI, the process is
 1. Run individual benchmarks.
-2. Convert results from Liger-Lernels format into our format.
+2. Convert results from Liger-Kernels format into our format.
 3. Upload file with results as an artefact for further processing.

benchmarks/third_party/liger_kernels/run_benchmarks.sh

@@ -2,6 +2,28 @@
 
 set -euo pipefail
 
+
+# Array to keep track of failed benchmarks
+FAILED_BENCHMARKS=()
+
 for file in Liger-Kernel/benchmark/scripts/benchmark_*; do
-    python "$file"
+    if python "$file"; then
+        echo "Benchmark ran successfully: $file"
+    else
+        echo "Error: Benchmark failed for $file."
+        FAILED_BENCHMARKS+=("$file")
+    fi
 done
+
+# Print failed benchmarks
+if [ ${#FAILED_BENCHMARKS[@]} -ne 0 ]; then
+    echo "The following benchmarks failed:"
+    for failed_bench in "${FAILED_BENCHMARKS[@]}"; do
+        echo "$failed_bench"
+    done
+    exit 1
+else
+    echo "All benchmarks completed successfully."
+fi
+
+exit 0