Merge branch 'sycl' into add_trackers_XFAIL

Author: lbushi25

.github/CODEOWNERS

@@ -170,11 +170,16 @@ sycl/test-e2e/LLVMIntrinsicLowering/ @intel/dpcpp-spirv-reviewers
 # Sanitizer
 clang/lib/Driver/SanitizerArgs.cpp @intel/dpcpp-sanitizers-review
 libdevice/include/asan_rtl.hpp @intel/dpcpp-sanitizers-review
+libdevice/include/msan_rtl.hpp @intel/dpcpp-sanitizers-review
 libdevice/include/sanitizer_defs.hpp @intel/dpcpp-sanitizers-review
 libdevice/sanitizer/ @intel/dpcpp-sanitizers-review
 llvm/include/llvm/Transforms/Instrumentation/AddressSanitizer.h @intel/dpcpp-sanitizers-review
 llvm/include/llvm/Transforms/Instrumentation/AddressSanitizerCommon.h @intel/dpcpp-sanitizers-review
 llvm/include/llvm/Transforms/Instrumentation/AddressSanitizerOptions.h @intel/dpcpp-sanitizers-review
+llvm/include/llvm/Transforms/Instrumentation/MemorySanitizer.h @intel/dpcpp-sanitizers-review
 llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp @intel/dpcpp-sanitizers-review
+llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp @intel/dpcpp-sanitizers-review
 llvm/test/Instrumentation/AddressSanitizer/ @intel/dpcpp-sanitizers-review
+llvm/test/Instrumentation/MemorySanitizer/ @intel/dpcpp-sanitizers-review
 sycl/test-e2e/AddressSanitizer/ @intel/dpcpp-sanitizers-review
+sycl/test-e2e/MemorySanitizer/ @intel/dpcpp-sanitizers-review

.github/workflows/sycl-containers.yaml

@@ -82,7 +82,6 @@ jobs:
           file: ${{ matrix.file }}
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
-          sycl_ci_passwd: ${{ secrets.DOCKER_SUDO_PASSWORD }}
           tags: |
             ghcr.io/${{ github.repository }}/${{ matrix.file }}:${{ matrix.tag }}-${{ github.sha }}
             ghcr.io/${{ github.repository }}/${{ matrix.file }}:${{ matrix.tag }}

.github/workflows/sycl-linux-precommit.yml

@@ -72,6 +72,53 @@ jobs:
           else
             echo 'arc_tests="Matrix/"' >> "$GITHUB_OUTPUT"
           fi
+
+  build_e2e_tests:
+    needs: [build]
+    if: ${{ always() && !cancelled() && needs.build.outputs.build_conclusion == 'success' }}
+    uses: ./.github/workflows/sycl-linux-run-tests.yml
+    with:
+      name: Build e2e tests
+      runner: '["Linux", "build"]'
+      image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest
+      image_options: -u 1001
+      ref: ${{ github.sha }}
+      merge_ref: ''
+      sycl_toolchain_artifact: sycl_linux_default
+      sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }}
+      sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }}
+      e2e_testing_mode: 'build-only'
+  run_prebuilt_e2e_tests:
+    needs: [build, build_e2e_tests]
+    if: ${{ always() && !cancelled() && needs.build.outputs.build_conclusion == 'success' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: Intel
+            runner: '["Linux", "gen12"]'
+            image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest
+            image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
+            target_devices: level_zero:gpu;opencl:gpu;opencl:cpu
+            reset_intel_gpu: true
+            extra_lit_opts: --param gpu-intel-gen12=True
+    uses: ./.github/workflows/sycl-linux-run-tests.yml
+    with:
+      name: ${{ matrix.name }}
+      runner: ${{ matrix.runner }}
+      image: ${{ matrix.image }}
+      image_options: ${{ matrix.image_options }}
+      target_devices: ${{ matrix.target_devices }}
+      extra_lit_opts: --param fallback-to-build-if-requires-build-and-run=True ${{ matrix.extra_lit_opts }}
+      reset_intel_gpu: ${{ matrix.reset_intel_gpu }}
+      ref: ${{ github.sha }}
+      merge_ref: ''
+      sycl_toolchain_artifact: sycl_linux_default
+      sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }}
+      sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }}
+      e2e_binaries_artifact: sycl_e2e_bin_default
+      e2e_testing_mode: 'run-only'
+
   test:
     needs: [build, detect_changes, determine_arc_tests]
     if: ${{ always() && !cancelled() && needs.build.outputs.build_conclusion == 'success' }}
@@ -90,16 +137,9 @@ jobs:
             image_options: -u 1001 --device=/dev/dri --device=/dev/kfd
             target_devices: ext_oneapi_hip:gpu
             reset_intel_gpu: false
-          - name: Intel
-            runner: '["Linux", "gen12"]'
-            image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
-            image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
-            target_devices: level_zero:gpu;opencl:gpu;opencl:cpu
-            reset_intel_gpu: true
-            extra_lit_opts: --param gpu-intel-gen12=True
           - name: E2E tests on Intel Arc A-Series Graphics
             runner: '["Linux", "arc"]'
-            image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
+            image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest
             image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
             target_devices: level_zero:gpu;opencl:gpu
             reset_intel_gpu: true
@@ -161,12 +201,12 @@ jobs:
         include:
           - name: Intel GEN12 Graphics system
             runner: '["Linux", "gen12"]'
-            image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
+            image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest
             image_extra_opts: --device=/dev/dri
             reset_intel_gpu: true
           - name: Intel Arc A-Series Graphics system
             runner: '["Linux", "arc"]'
-            image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
+            image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest
             image_extra_opts: --device=/dev/dri
             reset_intel_gpu: true
           - name: AMD system

.github/workflows/sycl-linux-run-tests.yml

@@ -19,7 +19,7 @@ on:
 
       target_devices:
         type: string
-        required: True
+        required: False
       extra_cmake_args:
         type: string
         required: False
@@ -59,6 +59,30 @@ on:
         default: ''
         required: False
 
+      e2e_binaries_artifact:
+        description: |
+          By setting this the E2E binaries folder will not be created, rather it
+          will be downloaded and extracted from the specified artifact. When
+          running tests in `run-only` mode this must be provided.
+        type: string
+        default: ''
+        required: False
+      e2e_testing_mode:
+        description: |
+          Testing mode to run E2E tests in, can be either `full`, `build-only`
+          or `run-only`. In `build-only` mode an artifact of the E2E binaries
+          will be uploaded.
+        type: string
+        default: 'full'
+      artifact_suffix:
+        description: 'Suffix for E2E binaries artifact that is output when in `build-only`.'
+        type: string
+        default: 'default'
+      retention-days:
+        description: 'E2E binaries artifact retention period.'
+        type: string
+        default: 1
+
       reset_intel_gpu:
         type: string
         required: False
@@ -87,6 +111,7 @@ on:
           - '["amdgpu"]'
           - '["Linux", "arc"]'
           - '["cts-cpu"]'
+          - '["Linux", "build"]'
       image:
         description: |
           Use option ending with ":build" for AMDGPU, ":latest" for the rest.
@@ -142,6 +167,11 @@ on:
         options:
           - false
           - true
+      e2e_testing_mode:
+        type: choice
+        options:
+          - "full"
+          - "build-only"
 
 permissions:
   contents: read
@@ -270,8 +300,19 @@ jobs:
             cat /usr/local/lib/igc/IGCTAG.txt
           fi
 
+    - name: Download E2E Binaries
+      if: inputs.e2e_binaries_artifact != ''
+      uses: actions/download-artifact@v4
+      with:
+        name: ${{ inputs.e2e_binaries_artifact }}
+    - name: Extract E2E Binaries
+      if: inputs.e2e_binaries_artifact != ''
+      run: |
+        mkdir build-e2e
+        tar -I 'zstd' -xf e2e_binaries.tar.zst -C build-e2e
+
     - name: Deduce E2E CMake options
-      if: inputs.tests_selector == 'e2e'
+      if: inputs.tests_selector == 'e2e' && inputs.e2e_binaries_artifact == ''
       id: cmake_opts
       shell: bash
       env:
@@ -281,14 +322,14 @@ jobs:
           echo "opts=$CMAKE_EXTRA_ARGS" >> $GITHUB_OUTPUT
         fi
     - name: Configure E2E tests
-      if: inputs.tests_selector == 'e2e'
+      if: inputs.tests_selector == 'e2e' && inputs.e2e_binaries_artifact == ''
       run: |
-        cmake -GNinja -B./build-e2e -S./llvm/sycl/test-e2e -DSYCL_TEST_E2E_TARGETS="${{ inputs.target_devices }}" -DCMAKE_CXX_COMPILER="$(which clang++)" -DLLVM_LIT="$PWD/llvm/llvm/utils/lit/lit.py" ${{ steps.cmake_opts.outputs.opts }}
+        cmake -GNinja -B./build-e2e -S./llvm/sycl/test-e2e -DCMAKE_CXX_COMPILER="$(which clang++)" -DLLVM_LIT="$PWD/llvm/llvm/utils/lit/lit.py" ${{ steps.cmake_opts.outputs.opts }}
     - name: SYCL End-to-end tests
       shell: bash {0}
       if: inputs.tests_selector == 'e2e'
       env:
-        LIT_OPTS: -v --no-progress-bar --show-unsupported --show-pass --show-xfail --max-time 3600 --time-tests ${{ inputs.extra_lit_opts }}
+        LIT_OPTS: -v --no-progress-bar --show-unsupported --show-pass --show-xfail --max-time 3600 --time-tests --param test-mode=${{ inputs.e2e_testing_mode }} --param sycl_devices=${{ inputs.target_devices }} ${{ inputs.extra_lit_opts }}
       run: |
         ninja -C build-e2e check-sycl-e2e > e2e.log 2>&1
         exit_code=$?
@@ -375,3 +416,13 @@ jobs:
         grep 'exit code: [^0]' -r logs >> $GITHUB_STEP_SUMMARY
 
         exit $ret
+    - name: Pack E2E binaries
+      if: ${{ always() && !cancelled() && inputs.e2e_testing_mode == 'build-only'}}
+      run: tar -I 'zstd -9' -cf e2e_binaries.tar.zst -C ./build-e2e .
+    - name: Upload E2E binaries
+      if: ${{ always() && !cancelled() && inputs.e2e_testing_mode == 'build-only'}}
+      uses: actions/upload-artifact@v4
+      with:
+        name: sycl_e2e_bin_${{ inputs.artifact_suffix }}
+        path: e2e_binaries.tar.zst
+        retention-days: ${{ inputs.retention-days }}

.github/workflows/sycl-nightly.yml

@@ -52,7 +52,7 @@ jobs:
 
           - name: Intel L0 GPU
             runner: '["Linux", "gen12"]'
-            image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
+            image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest
             image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
             target_devices: level_zero:gpu
             reset_intel_gpu: true
@@ -61,7 +61,7 @@ jobs:
 
           - name: Intel OCL GPU
             runner: '["Linux", "gen12"]'
-            image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
+            image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest
             image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
             target_devices: opencl:gpu
             reset_intel_gpu: true
@@ -70,35 +70,35 @@ jobs:
 
           - name: OCL CPU (AMD)
             runner: '["Linux", "amdgpu"]'
-            image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
+            image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest
             image_options: -u 1001
             target_devices: opencl:cpu
             tests_selector: e2e
 
           - name: OCL CPU (Intel/GEN12)
             runner: '["Linux", "gen12"]'
-            image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
+            image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest
             image_options: -u 1001 --privileged --cap-add SYS_ADMIN
             target_devices: opencl:cpu
             tests_selector: e2e
 
           - name: OCL CPU (Intel/Arc)
             runner: '["Linux", "arc"]'
-            image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
+            image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest
             image_options: -u 1001
             target_devices: opencl:cpu
             tests_selector: e2e
 
           - name: SYCL-CTS on OCL CPU
             runner: '["Linux", "gen12"]'
-            image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
+            image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest
             image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN
             target_devices: opencl:cpu
             tests_selector: cts
 
           - name: SYCL-CTS on L0 gen12
             runner: '["Linux", "gen12"]'
-            image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
+            image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest
             image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN
             target_devices: level_zero:gpu
             tests_selector: cts
@@ -229,7 +229,7 @@ jobs:
         username: ${{ github.repository_owner }}
         password: ${{ secrets.GITHUB_TOKEN }}
         build-args: |
-          base_image=ghcr.io/intel/llvm/ubuntu2204_intel_drivers
+          base_image=ghcr.io/intel/llvm/ubuntu2404_intel_drivers
           base_tag=latest
         tags: |
           ghcr.io/${{ github.repository }}/sycl_ubuntu2204_nightly:${{ github.sha }}

.github/workflows/sycl-post-commit.yml

@@ -82,7 +82,7 @@ jobs:
     with:
       name: ${{ matrix.name }}
       runner: ${{ matrix. runner }}
-      image: ${{ matrix.image || 'ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest' }}
+      image: ${{ matrix.image || 'ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest' }}
       image_options: ${{ matrix.image_options || '-u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN' }}
       target_devices: ${{ matrix.target_devices || 'level_zero:gpu' }}
       reset_intel_gpu: ${{ matrix.reset_intel_gpu }}

.github/workflows/sycl-rel-nightly.yml

@@ -54,14 +54,14 @@ jobs:
         include:
           - name: AMD/HIP
             runner: '["Linux", "amdgpu"]'
-            image: ghcr.io/intel/llvm/ubuntu2204_build:latest
+            image: ghcr.io/intel/llvm/ubuntu2404_build:latest
             image_options: -u 1001 --device=/dev/dri --device=/dev/kfd
             target_devices: ext_oneapi_hip:gpu
             tests_selector: e2e
 
           - name: Intel L0 GPU
             runner: '["Linux", "gen12"]'
-            image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
+            image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest
             image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
             target_devices: level_zero:gpu
             reset_intel_gpu: true
@@ -70,7 +70,7 @@ jobs:
 
           - name: Intel OCL GPU
             runner: '["Linux", "gen12"]'
-            image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
+            image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest
             image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
             target_devices: opencl:gpu
             reset_intel_gpu: true
@@ -79,21 +79,21 @@ jobs:
 
           - name: Intel OCL CPU
             runner: '["Linux", "gen12"]'
-            image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
+            image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest
             image_options: -u 1001 --privileged --cap-add SYS_ADMIN
             target_devices: opencl:cpu
             tests_selector: e2e
 
           - name: SYCL-CTS on OCL CPU
             runner: '["Linux", "gen12"]'
-            image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
+            image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest
             image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN
             target_devices: opencl:cpu
             tests_selector: cts
 
           - name: SYCL-CTS on L0 gen12
             runner: '["Linux", "gen12"]'
-            image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
+            image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest
             image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN
             target_devices: level_zero:gpu
             tests_selector: cts