[CI][Do not merge] Enable UR benchmarking dashboard in intel/llvm

.github/workflows/sycl-docs.yml

@@ -47,8 +47,10 @@ jobs:
         mkdir $GITHUB_WORKSPACE/install_docs
         cd $GITHUB_WORKSPACE/install_docs
         mkdir clang
+        mkdir benchmarks
         mv $GITHUB_WORKSPACE/build/tools/sycl/doc/html/* .
         mv $GITHUB_WORKSPACE/build/tools/clang/docs/html/* clang/
+        cp -r $GITHUB_WORKSPACE/devops/scripts/benchmarks/html benchmarks/
         touch .nojekyll
     # Upload the generated docs as an artifact and deploy to GitHub Pages.
     - name: Upload artifact

.github/workflows/sycl-linux-run-tests.yml

@@ -126,6 +126,7 @@ on:
           - '["cts-cpu"]'
           - '["Linux", "build"]'
           - '["cuda"]'
+          - '["Linux", "bmg"]'
           - '["PVC_PERF"]'
       image:
         type: choice
@@ -154,6 +155,7 @@ on:
           - e2e
           - cts
           - compute-benchmarks
+          - benchmark-v2
 
       env:
         description: |
@@ -329,3 +331,12 @@ jobs:
       env:
         RUNNER_TAG: ${{ inputs.runner }}
         GITHUB_TOKEN: ${{ secrets.LLVM_SYCL_BENCHMARK_TOKEN }}
+
+    - name: Run Benchmarks
+      if: inputs.tests_selector == 'benchmark-v2'
+      uses: ./devops/actions/run-tests/benchmark_v2
+      with:
+        target_devices: ${{ inputs.target_devices }}
+      env:
+        RUNNER_TAG: ${{ inputs.runner }}
+        GITHUB_TOKEN: ${{ secrets.LLVM_SYCL_BENCHMARK_TOKEN }}

.github/workflows/ur-build-hw.yml

@@ -151,4 +151,4 @@ jobs:
 
     - name: Get information about platform
       if: ${{ always() }}
-      run: ${{github.workspace}}/unified-runtime/.github/scripts/get_system_info.sh
+      run: ${{github.workspace}}/devops/scripts/get_system_info.sh

devops/actions/run-tests/benchmark/action.yml

@@ -46,27 +46,6 @@ runs:
           echo "# This workflow is not guaranteed to work with other backends."
           echo "#" ;;
       esac
-  - name: Compute CPU core range to run benchmarks on
-    shell: bash
-    run: |
-      # Taken from ur-benchmark-reusable.yml:
-
-      # Compute the core range for the first NUMA node; second node is used by
-      # UMF. Skip the first 4 cores as the kernel is likely to schedule more
-      # work on these.
-      CORES="$(lscpu | awk '
-        /NUMA node0 CPU|On-line CPU/ {line=$0}
-        END {
-          split(line, a, " ")
-          split(a[4], b, ",")
-          sub(/^0/, "4", b[1])
-          print b[1]
-        }')"
-      echo "CPU core range to use: $CORES"
-      echo "CORES=$CORES" >> $GITHUB_ENV
-
-      ZE_AFFINITY_MASK=0
-      echo "ZE_AFFINITY_MASK=$ZE_AFFINITY_MASK" >> $GITHUB_ENV
   - name: Run compute-benchmarks
     shell: bash
     run: |
@@ -90,7 +69,7 @@ runs:
       echo "-----"
       sycl-ls
       echo "-----"
-      taskset -c "$CORES" ./devops/scripts/benchmarking/benchmark.sh -n '${{ runner.name }}' -s || exit 1
+      ./devops/scripts/benchmarking/benchmark.sh -n '${{ runner.name }}' -s || exit 1
   - name: Push compute-benchmarks results
     if: always()
     shell: bash

devops/actions/run-tests/benchmark_v2/action.yml

@@ -0,0 +1,134 @@
+name: 'Run Benchmarks'
+
+# This action assumes the following prerequisites:
+#
+# - SYCL is placed in ./toolchain -- TODO change this
+# - /devops has been checked out in ./devops.
+# - env.GITHUB_TOKEN was properly set, because according to Github, that's
+#   apparently the recommended way to pass a secret into a github action:
+
+#   https://docs.github.com/en/actions/security-for-github-actions/security-guides/using-secrets-in-github-actions#accessing-your-secrets
+#
+# - env.RUNNER_TAG set to the runner tag used to run this workflow: Currently,
+#   only specific runners are fully supported.
+
+inputs:
+  target_devices:
+    type: string
+    required: True
+
+runs:
+  using: "composite"
+  steps:
+  - name: Check specified runner type / target backend
+    shell: bash
+    env:
+      TARGET_DEVICE: ${{ inputs.target_devices }}
+      RUNNER_NAME: ${{ runner.name }}
+    run: |
+      case "$RUNNER_TAG" in
+        '["PVC_PERF"]' ) ;;
+        *)
+          echo "#"
+          echo "# WARNING: Only specific tuned runners are fully supported."
+          echo "# This workflow is not guaranteed to work with other runners."
+          echo "#" ;;
+      esac
+
+      # Ensure runner name has nothing injected
+      # TODO: in terms of security, is this overkill?
+      if [ -z "$(printf '%s' "$RUNNER_NAME" | grep -oE '^[a-zA-Z0-9_-]+$')" ]; then
+          echo "Bad runner name, please ensure runner name is [a-zA-Z0-9_-]."
+          exit 1
+      fi
+      echo "RUNNER_NAME=$RUNNER_NAME" >> $GITHUB_ENV 
+
+      # input.target_devices is not directly used, as this allows code injection
+      case "$TARGET_DEVICE" in
+        level_zero:*) ;;
+        *)
+          echo "#"
+          echo "# WARNING: Only level_zero backend is fully supported."
+          echo "# This workflow is not guaranteed to work with other backends."
+          echo "#" ;;
+      esac
+      echo "ONEAPI_DEVICE_SELECTOR=$TARGET_DEVICE" >> $GITHUB_ENV 
+
+  - name: Compute CPU core range to run benchmarks on
+    shell: bash
+    run: |
+      # Compute the core range for the first NUMA node; second node is used by
+      # UMF. Skip the first 4 cores as the kernel is likely to schedule more
+      # work on these.
+      CORES="$(lscpu | awk '
+        /NUMA node0 CPU|On-line CPU/ {line=$0}
+        END {
+          split(line, a, " ")
+          split(a[4], b, ",")
+          sub(/^0/, "4", b[1])
+          print b[1]
+        }')"
+      echo "CPU core range to use: $CORES"
+      echo "CORES=$CORES" >> $GITHUB_ENV
+
+      ZE_AFFINITY_MASK=0
+      echo "ZE_AFFINITY_MASK=$ZE_AFFINITY_MASK" >> $GITHUB_ENV
+  - name: Checkout results repo
+    shell: bash
+    run: |
+      git clone -b unify-ci https://github.com/intel/llvm-ci-perf-results
+  - name: Run compute-benchmarks
+    shell: bash
+    run: |
+      # TODO generate summary + display helpful message here
+      export CMPLR_ROOT=./toolchain
+      echo "-----"
+      sycl-ls
+      echo "-----"
+      pip install --user --break-system-packages -r ./devops/scripts/benchmarks/requirements.txt
+      echo "-----"
+      mkdir -p "./llvm-ci-perf-results/$RUNNER_NAME"
+      taskset -c "$CORES" ./devops/scripts/benchmarks/main.py \
+        "$(realpath ./llvm_test_workdir)" \
+        --sycl "$(realpath ./toolchain)" \
+        --save baseline \
+        --output-html remote \
+        --results-dir "./llvm-ci-perf-results/$RUNNER_NAME" \
+        --output-dir "./llvm-ci-perf-results/$RUNNER_NAME" \
+        --preset Minimal
+      echo "-----"
+      ls
+  - name: Push compute-benchmarks results
+    if: always()
+    shell: bash
+    run: |
+      # TODO redo configuration
+      # $(python ./devops/scripts/benchmarking/load_config.py ./devops constants)
+
+      cd "./llvm-ci-perf-results"
+      git config user.name "SYCL Benchmarking Bot"
+      git config user.email "[email protected]"
+      git pull
+      git add .
+      # Make sure changes have been made
+      if git diff --quiet && git diff --cached --quiet; then
+        echo "No new results added, skipping push."
+      else
+        git commit -m "[GHA] Upload compute-benchmarks results from https://github.com/intel/llvm/actions/runs/${{ github.run_id }}"
+        git push "https://[email protected]/intel/llvm-ci-perf-results.git" unify-ci
+      fi
+#  - name: Find benchmark result artifact here
+#    if: always()
+#    shell: bash
+#    run: |
+#      cat << EOF
+#      #
+#      # Artifact link for benchmark results here:
+#      #
+#      EOF
+#  - name: Archive compute-benchmark results
+#    if: always()
+#    uses: actions/upload-artifact@v4
+#    with:
+#      name: Compute-benchmark run ${{ github.run_id }} (${{ runner.name }})
+#      path: ./artifact

unified-runtime/scripts/benchmarks/benches/base.py → devops/scripts/benchmarks/benches/base.py

@@ -6,7 +6,7 @@
 import os
 import shutil
 from pathlib import Path
-from .result import Result
+from utils.result import Result
 from options import options
 from utils.utils import download, run
 import urllib.request
@@ -55,16 +55,25 @@ def create_data_path(self, name, skip_data_dir=False):
             data_path = os.path.join(self.directory, name)
         else:
             data_path = os.path.join(self.directory, "data", name)
-            if options.rebuild and Path(data_path).exists():
+            if options.redownload and Path(data_path).exists():
                 shutil.rmtree(data_path)
 
         Path(data_path).mkdir(parents=True, exist_ok=True)
 
         return data_path
 
-    def download(self, name, url, file, untar=False, unzip=False, skip_data_dir=False):
+    def download(
+        self,
+        name,
+        url,
+        file,
+        untar=False,
+        unzip=False,
+        skip_data_dir=False,
+        checksum="",
+    ):
         self.data_path = self.create_data_path(name, skip_data_dir)
-        return download(self.data_path, url, file, untar, unzip)
+        return download(self.data_path, url, file, untar, unzip, checksum)
 
     def name(self):
         raise NotImplementedError()