Merge pull request #1876 from pbalcer/modularize-benchmarks

generalize benchmark script, add more benchmarks

Author: pbalcer

.github/scripts/compute_benchmarks.py

@@ -42,7 +42,7 @@ permissions:
 jobs:
   e2e-build-hw:
     # Run only on upstream; forks will not have the HW
-    if: github.repository == 'oneapi-src/unified-runtime'
+    # if: github.repository == 'oneapi-src/unified-runtime'
     name: Build SYCL, UR, run Compute Benchmarks
     strategy:
       matrix:
@@ -88,6 +88,9 @@ jobs:
       with:
         path: ur-repo
 
+    - name: Install pip packages
+      run: pip install -r ${{github.workspace}}/ur-repo/third_party/requirements.txt
+
     # We need to fetch special ref for proper PR's merge commit. Note, this ref may be absent if the PR is already merged.
     - name: Fetch PR's merge commit
       if: ${{ inputs.pr_no != 0 }}
@@ -131,44 +134,13 @@ jobs:
     - name: Build SYCL
       run: cmake --build ${{github.workspace}}/sycl_build -j
 
-    - name: Set additional env. vars
-      run: |
-        echo "${{github.workspace}}/sycl_build/bin" >> $GITHUB_PATH
-        echo "LD_LIBRARY_PATH=${{github.workspace}}/sycl_build/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV
-
-    # Running (newly built) sycl-ls sets up some extra variables
-    - name: Setup SYCL variables
-      run: |
-        which clang++ sycl-ls
-        SYCL_PI_TRACE=-1 sycl-ls
-
-    - name: Checkout Compute Benchmarks
-      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-      with:
-        repository: intel/compute-benchmarks
-        path: compute-benchmarks-repo
-        submodules: recursive
-
-    - name: Configure Compute Benchmarks
-      run: >
-        cmake
-        -B ${{github.workspace}}/compute-benchmarks-build/
-        -S ${{github.workspace}}/compute-benchmarks-repo/
-        -DCMAKE_BUILD_TYPE=Release
-        -DBUILD_SYCL=ON
-        -DSYCL_COMPILER_ROOT=${{github.workspace}}/sycl_build
-        -DALLOW_WARNINGS=ON
-
-    - name: Build Compute Benchmarks
-      run: cmake --build ${{github.workspace}}/compute-benchmarks-build/ -j
-
     - name: Set oneAPI Device Selector
       run: |
         echo "ONEAPI_DEVICE_SELECTOR=${{ matrix.adapter.str_name }}:${{ matrix.adapter.unit }}" >> $GITHUB_ENV
 
-    - name: Run SYCL API Overhead benchmark
+    - name: Run benchmarks
       id: benchmarks
-      run: ${{ github.workspace }}/ur-repo/.github/scripts/compute_benchmarks.py ${{ github.workspace }}/compute-benchmarks-build/bin/ ${{ inputs.bench_script_params }}
+      run: numactl -N 0 ${{ github.workspace }}/ur-repo/scripts/benchmarks/main.py ~/bench_workdir ${{github.workspace}}/sycl_build ${{ inputs.bench_script_params }}
 
     - name: Add comment to PR
       uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
@@ -188,7 +160,7 @@ jobs:
           const test_status = '${{ steps.benchmarks.outcome }}';
           const job_status = '${{ job.status }}';
           const params = '${{ inputs.bench_script_params }}';
-          const body = `Compute Benchmarks ${adapter} run (with params: ${params}):\n${url}\nJob status: ${job_status}. Test status: ${test_status}.\n ${markdown}`;
+          const body = `Compute Benchmarks ${adapter} run (${params}):\n${url}\nJob status: ${job_status}. Test status: ${test_status}.\n ${markdown}`;
 
           github.rest.issues.createComment({
             issue_number: pr_no,

.github/workflows/benchmarks_compute.yml

@@ -0,0 +1,26 @@
+# Unified Runtime Benchmark Runner
+
+Scripts for running performance tests on SYCL and Unified Runtime.
+
+## Benchmarks
+
+- [Velocity Bench](https://github.com/oneapi-src/Velocity-Bench)
+- [Compute Benchmarks](https://github.com/intel/compute-benchmarks/)
+
+## Running
+
+`$ ./main.py ~/benchmarks_workdir/ ~/llvm/build/`
+
+This will download and build everything in `~/benchmarks_workdir/` using the compiler in `~/llvm/build/`, and then run the benchmarks. The results will be stored in `benchmark_results.md`.
+
+The scripts will try to reuse the files stored in `~/benchmarks_workdir/`, but the benchmarks will be rebuilt every time. To avoid that, use `-no-rebuild` option.
+
+## Requirements
+
+### Python
+
+dataclasses-json==0.6.7
+
+### System
+
+libopencv-dev

scripts/benchmarks/README.md

@@ -0,0 +1,36 @@
+# Copyright (C) 2024 Intel Corporation
+# Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
+# See LICENSE.TXT
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+from .base import Benchmark
+from .result import Result
+from .velocity import VelocityBase, VelocityBench
+from utils.utils import run
+import re
+
+class SobelFilter(VelocityBase):
+    def __init__(self, vb: VelocityBench):
+        super().__init__("sobel_filter", "sobel_filter", vb)
+        self.download_untar("sobel_filter", "https://github.com/oneapi-src/Velocity-Bench/raw/main/sobel_filter/res/sobel_filter_data.tgz?download=", "sobel_filter_data.tgz")
+
+    def name(self):
+        return "Velocity-Bench Sobel Filter"
+
+    def unit(self):
+        return "ms"
+
+    def bin_args(self) -> list[str]:
+        return ["-i", f"{self.data_path}/sobel_filter_data/silverfalls_32Kx32K.png",
+                "-n", "5"]
+
+    def extra_env_vars(self) -> dict:
+        return {"OPENCV_IO_MAX_IMAGE_PIXELS" : "1677721600"}
+
+    def parse_output(self, stdout: str) -> float:
+        match = re.search(r'sobelfilter - total time for whole calculation: (\d+\.\d+) s', stdout)
+        if match:
+            return round(float(match.group(1)) * 1000, 3)
+        else:
+            raise ValueError("Failed to parse benchmark output.")
+