From ad165c51af5ea93193c54b65348056643c934c28 Mon Sep 17 00:00:00 2001
From: "Garcia Orozco, David" <david.garcia.orozco@intel.com>
Date: Wed, 11 Dec 2024 10:49:26 -0800
Subject: [PATCH 01/26] Add split testing functionality to CI and enable in
 precommit

---
 .github/workflows/sycl-linux-precommit.yml | 57 ++++++++++++--
 .github/workflows/sycl-linux-run-tests.yml | 89 +++++++++++++++++++++-
 2 files changed, 136 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/sycl-linux-precommit.yml b/.github/workflows/sycl-linux-precommit.yml
index 04665c383cbb6..b67750a0c027e 100644
--- a/.github/workflows/sycl-linux-precommit.yml
+++ b/.github/workflows/sycl-linux-precommit.yml
@@ -72,6 +72,56 @@ jobs:
           else
             echo 'arc_tests="Matrix/"' >> "$GITHUB_OUTPUT"
           fi
+
+  split_test_build:
+    needs: [build]
+    if: ${{ always() && !cancelled() && needs.build.outputs.build_conclusion == 'success' }}
+    uses: ./.github/workflows/sycl-linux-run-tests.yml
+    with:
+      name: Build e2e tests
+      runner: '["Linux", "build"]'
+      image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:alldeps
+      image_options: -u 1001
+      target_devices: opencl:cpu
+      extra_lit_opts: --param test-mode=build-only
+      ref: ${{ github.sha }}
+      merge_ref: ''
+      sycl_toolchain_artifact: sycl_linux_default
+      sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }}
+      sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }}
+      upload_artifact: 'true'
+  split_test_run:
+    needs: [build, split_test_build]
+    if: ${{ always() && !cancelled() && needs.build.outputs.build_conclusion == 'success' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: Intel
+            runner: '["Linux", "gen12"]'
+            image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
+            image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
+            target_devices: level_zero:gpu;opencl:gpu;opencl:cpu
+            reset_intel_gpu: true
+            extra_lit_opts: --param gpu-intel-gen12=True
+    uses: ./.github/workflows/sycl-linux-run-tests.yml
+    with:
+      name: ${{ matrix.name }}
+      runner: ${{ matrix.runner }}
+      image: ${{ matrix.image }}
+      image_options: ${{ matrix.image_options }}
+      target_devices: ${{ matrix.target_devices }}
+      extra_lit_opts: --param test-mode=run-only ${{ matrix.extra_lit_opts }}
+      reset_intel_gpu: ${{ matrix.reset_intel_gpu }}
+      ref: ${{ github.sha }}
+      merge_ref: ''
+      sycl_toolchain_artifact: sycl_linux_default
+      sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }}
+      sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }}
+      e2e_binaries_artifact: sycl_e2e_bin_default
+      e2e_binaries_archive: ${{ needs.split_test_build.outputs.artifact_archive_name }}
+      e2e_binaries_decompress_command: ${{ needs.split_test_build.outputs.artifact_decompress_command }}
+
   test:
     needs: [build, detect_changes, determine_arc_tests]
     if: ${{ always() && !cancelled() && needs.build.outputs.build_conclusion == 'success' }}
@@ -90,13 +140,6 @@ jobs:
             image_options: -u 1001 --device=/dev/dri --device=/dev/kfd
             target_devices: ext_oneapi_hip:gpu
             reset_intel_gpu: false
-          - name: Intel
-            runner: '["Linux", "gen12"]'
-            image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
-            image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
-            target_devices: level_zero:gpu;opencl:gpu;opencl:cpu
-            reset_intel_gpu: true
-            extra_lit_opts: --param gpu-intel-gen12=True
           - name: E2E tests on Intel Arc A-Series Graphics
             runner: '["Linux", "arc"]'
             image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml
index 73b2a1f336db7..7aa53817af391 100644
--- a/.github/workflows/sycl-linux-run-tests.yml
+++ b/.github/workflows/sycl-linux-run-tests.yml
@@ -59,6 +59,29 @@ on:
         default: ''
         required: False
 
+      e2e_binaries_artifact:
+        type: string
+        default: ''
+        required: False
+      e2e_binaries_archive:
+        type: string
+        default: ''
+        required: False
+      e2e_binaries_decompress_command:
+        type: string
+        default: ''
+        required: False
+
+      upload_artifact:
+        type: string
+        default: 'false'
+      build_artifact_suffix:
+        type: string
+        default: 'default'
+      artifact_archive_name:
+        type: string
+        default: e2e_binaries.tar.zst
+
       reset_intel_gpu:
         type: string
         required: False
@@ -78,6 +101,14 @@ on:
         default: 'false'
         required: False
 
+    outputs:
+      run_conclusion:
+        value: ${{ jobs.run.outputs.build_conclusion }}
+      artifact_archive_name:
+        value: ${{ jobs.run.outputs.artifact_archive_name }}
+      artifact_decompress_command:
+        value: ${{ jobs.run.outputs.artifact_decompress_command }}
+
   workflow_dispatch:
     inputs:
       runner:
@@ -87,6 +118,7 @@ on:
           - '["amdgpu"]'
           - '["Linux", "arc"]'
           - '["cts-cpu"]'
+          - '["Linux", "build"]'
       image:
         description: |
           Use option ending with ":build" for AMDGPU, ":latest" for the rest.
@@ -142,6 +174,11 @@ on:
         options:
           - false
           - true
+      upload_artifact:
+        type: choice
+        options:
+          - "false"
+          - "true"
 
 permissions:
   contents: read
@@ -154,8 +191,31 @@ jobs:
     container:
       image: ${{ inputs.image }}
       options: ${{ inputs.image_options }}
+    outputs:
+      run_conclusion: ${{ steps.run.conclusion }}
+      artifact_archive_name: ${{ steps.artifact_info.outputs.ARCHIVE_NAME }}
+      artifact_decompress_command: ${{ steps.artifact_info.outputs.DECOMPRESS }}
     env: ${{ fromJSON(inputs.env) }}
     steps:
+    - name: Deduce artifact archive params
+      if: inputs.upload_artifact == 'true'
+      id: artifact_info
+      run: |
+        NAME="${{inputs.artifact_archive_name}}"
+        if [ -z "$NAME" ]; then
+          NAME=e2e_binaries.tar.zst
+        fi
+        echo ARCHIVE_NAME="$NAME" >> $GITHUB_OUTPUT
+        if [ "${NAME}" != "${NAME%.tar.gz}" ]; then
+          echo COMPRESS="gzip" >> $GITHUB_OUTPUT
+          echo DECOMPRESS="gunzip" >> $GITHUB_OUTPUT
+        elif [ "${NAME}" != "${NAME%.tar.zst}" ]; then
+          echo COMPRESS="zstd -9" >> $GITHUB_OUTPUT
+          echo DECOMPRESS="zstd" >> $GITHUB_OUTPUT
+        else
+          echo "Unsupported extension"
+          exit 1
+        fi
     - name: Reset Intel GPU
       if: inputs.reset_intel_gpu == 'true'
       run: |
@@ -270,8 +330,20 @@ jobs:
             cat /usr/local/lib/igc/IGCTAG.txt
           fi
 
+    - name: Download E2E Binaries
+      if: inputs.e2e_binaries_artifact != '' && github.event_name != 'workflow_run'
+      uses: actions/download-artifact@v4
+      with:
+        name: ${{ inputs.e2e_binaries_artifact }}
+    - name: Extract/Setup E2E Binaries
+      if: inputs.e2e_binaries_artifact != ''
+      run: |
+        mkdir build-e2e
+        tar -I '${{ inputs.e2e_binaries_decompress_command }}' -xf ${{ inputs.e2e_binaries_archive }} -C build-e2e
+        rm -f ${{ inputs.e2e_binaries_artifact }}
+
     - name: Deduce E2E CMake options
-      if: inputs.tests_selector == 'e2e'
+      if: inputs.tests_selector == 'e2e' && inputs.e2e_binaries_artifact == ''
       id: cmake_opts
       shell: bash
       env:
@@ -281,14 +353,15 @@ jobs:
           echo "opts=$CMAKE_EXTRA_ARGS" >> $GITHUB_OUTPUT
         fi
     - name: Configure E2E tests
-      if: inputs.tests_selector == 'e2e'
+      if: inputs.tests_selector == 'e2e' && inputs.e2e_binaries_artifact == ''
       run: |
-        cmake -GNinja -B./build-e2e -S./llvm/sycl/test-e2e -DSYCL_TEST_E2E_TARGETS="${{ inputs.target_devices }}" -DCMAKE_CXX_COMPILER="$(which clang++)" -DLLVM_LIT="$PWD/llvm/llvm/utils/lit/lit.py" ${{ steps.cmake_opts.outputs.opts }}
+        cmake -GNinja -B./build-e2e -S./llvm/sycl/test-e2e -DCMAKE_CXX_COMPILER="$(which clang++)" -DLLVM_LIT="$PWD/llvm/llvm/utils/lit/lit.py" ${{ steps.cmake_opts.outputs.opts }}
     - name: SYCL End-to-end tests
       shell: bash {0}
       if: inputs.tests_selector == 'e2e'
       env:
         LIT_OPTS: -v --no-progress-bar --show-unsupported --show-pass --show-xfail --max-time 3600 --time-tests ${{ inputs.extra_lit_opts }}
+        ONEAPI_DEVICE_SELECTOR: ${{ inputs.target_devices }}
       run: |
         ninja -C build-e2e check-sycl-e2e > e2e.log 2>&1
         exit_code=$?
@@ -375,3 +448,13 @@ jobs:
         grep 'exit code: [^0]' -r logs >> $GITHUB_STEP_SUMMARY
 
         exit $ret
+    - name: Pack e2e binaries
+      if: ${{ always() && !cancelled() && inputs.upload_artifact == 'true'}}
+      run: tar -I '${{ steps.artifact_info.outputs.COMPRESS }}' -cf ${{ steps.artifact_info.outputs.ARCHIVE_NAME }} -C ./build-e2e .
+    - name: Upload e2e binaries
+      if: ${{ always() && !cancelled() && inputs.upload_artifact == 'true'}}
+      uses: actions/upload-artifact@v4
+      with:
+        name: sycl_e2e_bin_${{ inputs.build_artifact_suffix }}
+        path: ${{ steps.artifact_info.outputs.ARCHIVE_NAME }}
+        retention-days: ${{ inputs.retention-days }}

From d604088c8dddc613a3cae1857ebd114f68cb654c Mon Sep 17 00:00:00 2001
From: "Garcia Orozco, David" <david.garcia.orozco@intel.com>
Date: Wed, 11 Dec 2024 10:49:49 -0800
Subject: [PATCH 02/26] Add workaround for lit hang inside our containers

---
 sycl/test-e2e/format.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/sycl/test-e2e/format.py b/sycl/test-e2e/format.py
index e6d0da2c39b7b..d7d96f414fdc0 100644
--- a/sycl/test-e2e/format.py
+++ b/sycl/test-e2e/format.py
@@ -240,7 +240,7 @@ def get_extra_env(sycl_devices):
             if (is_run_line and test.config.test_mode == "build-only") or (
                 not is_run_line and test.config.test_mode == "run-only"
             ):
-                directive.command = ""
+                continue
 
             if "%{run}" not in directive.command:
                 new_script.append(directive)
@@ -292,7 +292,15 @@ def get_extra_env(sycl_devices):
             conditions,
             recursion_limit=test.config.recursiveExpansionLimit,
         )
-        useExternalSh = False
+
+        # TODO: workaround for lit hanging when executing non-existent binary
+        # inside our containers
+        if len(script) == 0:
+            return lit.Test.Result(
+                    lit.Test.UNSUPPORTED, "Lit script is empty"
+                    )
+        useExternalSh = test.config.test_mode == "run-only"
+
         result = lit.TestRunner._runShTest(
             test, litConfig, useExternalSh, script, tmpBase
         )

From 0f6313b4b890215e7abbd14e72829cd725f3f34d Mon Sep 17 00:00:00 2001
From: "Garcia Orozco, David" <david.garcia.orozco@intel.com>
Date: Wed, 11 Dec 2024 10:51:39 -0800
Subject: [PATCH 03/26] Mark as split exceptions tests newly failing on build
 stage

---
 sycl/test-e2e/Adapters/sycl-ls-gpu-default-any.cpp               | 1 +
 sycl/test-e2e/Adapters/sycl-ls-gpu-default-level-zero.cpp        | 1 +
 sycl/test-e2e/Adapters/sycl-ls-gpu-level-zero.cpp                | 1 +
 sycl/test-e2e/Adapters/sycl-ls-gpu-opencl.cpp                    | 1 +
 sycl/test-e2e/Adapters/sycl-ls-uuid-subdevs.cpp                  | 1 +
 .../is_compatible/is_compatible_spir64.cpp                       | 1 +
 .../is_compatible/is_compatible_spir64_fpga.cpp                  | 1 +
 .../is_compatible/is_compatible_spir64_gen.cpp                   | 1 +
 .../is_compatible/is_compatible_spir64_x86_64.cpp                | 1 +
 .../SpecConstants/2020/native_specialization_constant.cpp        | 1 +
 sycl/test-e2e/SpecConstants/2020/non_native/SpecConstBuffer.cpp  | 1 +
 sycl/test-e2e/SpecConstants/2020/non_native/cpu.cpp              | 1 +
 12 files changed, 12 insertions(+)

diff --git a/sycl/test-e2e/Adapters/sycl-ls-gpu-default-any.cpp b/sycl/test-e2e/Adapters/sycl-ls-gpu-default-any.cpp
index 046a41494370b..1c90ea9f1e32f 100644
--- a/sycl/test-e2e/Adapters/sycl-ls-gpu-default-any.cpp
+++ b/sycl/test-e2e/Adapters/sycl-ls-gpu-default-any.cpp
@@ -1,4 +1,5 @@
 // REQUIRES: gpu
+// REQUIRES: build-and-run-mode
 
 // TODO: Remove unsetting SYCL_DEVICE_FILTER when feature is dropped
 // RUN: env --unset=SYCL_DEVICE_FILTER  --unset=ONEAPI_DEVICE_SELECTOR sycl-ls --verbose >%t.default.out
diff --git a/sycl/test-e2e/Adapters/sycl-ls-gpu-default-level-zero.cpp b/sycl/test-e2e/Adapters/sycl-ls-gpu-default-level-zero.cpp
index 6f144553fff7b..cd6ece996d9f8 100644
--- a/sycl/test-e2e/Adapters/sycl-ls-gpu-default-level-zero.cpp
+++ b/sycl/test-e2e/Adapters/sycl-ls-gpu-default-level-zero.cpp
@@ -1,4 +1,5 @@
 // REQUIRES: gpu, level-zero
+// REQUIRES: build-and-run-mode
 
 // TODO: Remove unsetting SYCL_DEVICE_FILTER when feature is dropped
 // RUN: env --unset=SYCL_DEVICE_FILTER --unset=ONEAPI_DEVICE_SELECTOR sycl-ls --verbose >%t.default.out
diff --git a/sycl/test-e2e/Adapters/sycl-ls-gpu-level-zero.cpp b/sycl/test-e2e/Adapters/sycl-ls-gpu-level-zero.cpp
index 36eb621e923a6..f20ea34182833 100644
--- a/sycl/test-e2e/Adapters/sycl-ls-gpu-level-zero.cpp
+++ b/sycl/test-e2e/Adapters/sycl-ls-gpu-level-zero.cpp
@@ -1,4 +1,5 @@
 // REQUIRES: gpu, level_zero
+// REQUIRES: build-and-run-mode
 
 // RUN: sycl-ls --verbose >%t.default.out
 // RUN: FileCheck %s --check-prefixes=CHECK-GPU-BUILTIN,CHECK-GPU-CUSTOM --input-file %t.default.out
diff --git a/sycl/test-e2e/Adapters/sycl-ls-gpu-opencl.cpp b/sycl/test-e2e/Adapters/sycl-ls-gpu-opencl.cpp
index fa764d0629a0f..fda84100fecd6 100644
--- a/sycl/test-e2e/Adapters/sycl-ls-gpu-opencl.cpp
+++ b/sycl/test-e2e/Adapters/sycl-ls-gpu-opencl.cpp
@@ -1,4 +1,5 @@
 // REQUIRES: gpu, opencl
+// REQUIRES: build-and-run-mode
 
 // RUN: env ONEAPI_DEVICE_SELECTOR="opencl:*" sycl-ls --verbose >%t.opencl.out
 // RUN: FileCheck %s --check-prefixes=CHECK-GPU-BUILTIN,CHECK-GPU-CUSTOM --input-file %t.opencl.out
diff --git a/sycl/test-e2e/Adapters/sycl-ls-uuid-subdevs.cpp b/sycl/test-e2e/Adapters/sycl-ls-uuid-subdevs.cpp
index b7f83f8f337cd..f5d155d15ee5b 100644
--- a/sycl/test-e2e/Adapters/sycl-ls-uuid-subdevs.cpp
+++ b/sycl/test-e2e/Adapters/sycl-ls-uuid-subdevs.cpp
@@ -1,4 +1,5 @@
 /* Test to check that sycl-ls is outputting UUID and number of sub and sub-sub
+// REQUIRES: build-and-run-mode
  * devices. */
 // REQUIRES:  gpu, level_zero
 
diff --git a/sycl/test-e2e/OptionalKernelFeatures/is_compatible/is_compatible_spir64.cpp b/sycl/test-e2e/OptionalKernelFeatures/is_compatible/is_compatible_spir64.cpp
index 465a79056906a..fcf6affb809fb 100644
--- a/sycl/test-e2e/OptionalKernelFeatures/is_compatible/is_compatible_spir64.cpp
+++ b/sycl/test-e2e/OptionalKernelFeatures/is_compatible/is_compatible_spir64.cpp
@@ -1,4 +1,5 @@
 // REQUIRES: cuda, opencl, gpu, cpu
+// REQUIRES: build-and-run-mode
 
 // RUN: %clangxx -fsycl -fsycl-targets=spir64 %S/Inputs/is_compatible_with_env.cpp -o %t.out
 
diff --git a/sycl/test-e2e/OptionalKernelFeatures/is_compatible/is_compatible_spir64_fpga.cpp b/sycl/test-e2e/OptionalKernelFeatures/is_compatible/is_compatible_spir64_fpga.cpp
index 57366482e7082..1372c352c09ea 100644
--- a/sycl/test-e2e/OptionalKernelFeatures/is_compatible/is_compatible_spir64_fpga.cpp
+++ b/sycl/test-e2e/OptionalKernelFeatures/is_compatible/is_compatible_spir64_fpga.cpp
@@ -1,4 +1,5 @@
 // REQUIRES: opencl-aot, accelerator, gpu, cpu
+// REQUIRES: build-and-run-mode
 
 // RUN: %clangxx -fsycl -fsycl-targets=spir64_fpga %S/Inputs/is_compatible_with_env.cpp -o %t.out
 
diff --git a/sycl/test-e2e/OptionalKernelFeatures/is_compatible/is_compatible_spir64_gen.cpp b/sycl/test-e2e/OptionalKernelFeatures/is_compatible/is_compatible_spir64_gen.cpp
index 5adb27e0ae697..c6a01b3a6dc18 100644
--- a/sycl/test-e2e/OptionalKernelFeatures/is_compatible/is_compatible_spir64_gen.cpp
+++ b/sycl/test-e2e/OptionalKernelFeatures/is_compatible/is_compatible_spir64_gen.cpp
@@ -1,4 +1,5 @@
 // REQUIRES: ocloc, gpu, level_zero, cpu
+// REQUIRES: build-and-run-mode
 
 // RUN: %clangxx -fsycl -fsycl-targets=spir64_gen -Xsycl-target-backend "-device *" %S/Inputs/is_compatible_with_env.cpp -o %t.out
 
diff --git a/sycl/test-e2e/OptionalKernelFeatures/is_compatible/is_compatible_spir64_x86_64.cpp b/sycl/test-e2e/OptionalKernelFeatures/is_compatible/is_compatible_spir64_x86_64.cpp
index 0a6f2c39df8af..5de21b8984d71 100644
--- a/sycl/test-e2e/OptionalKernelFeatures/is_compatible/is_compatible_spir64_x86_64.cpp
+++ b/sycl/test-e2e/OptionalKernelFeatures/is_compatible/is_compatible_spir64_x86_64.cpp
@@ -1,4 +1,5 @@
 // REQUIRES: opencl-aot, cpu, gpu, level_zero
+// REQUIRES: build-and-run-mode
 
 // RUN: %clangxx -fsycl -fsycl-targets=spir64_x86_64 %S/Inputs/is_compatible_with_env.cpp -o %t.out
 
diff --git a/sycl/test-e2e/SpecConstants/2020/native_specialization_constant.cpp b/sycl/test-e2e/SpecConstants/2020/native_specialization_constant.cpp
index 170e0aad6b33f..c15c4ba892ed7 100644
--- a/sycl/test-e2e/SpecConstants/2020/native_specialization_constant.cpp
+++ b/sycl/test-e2e/SpecConstants/2020/native_specialization_constant.cpp
@@ -4,6 +4,7 @@
 
 // FIXME: This set is never satisfied all at once in our infrastructure.
 // REQUIRES: opencl, level-zero, cpu, gpu, opencl-aot, ocloc
+// REQUIRES: build-and-run-mode
 
 // RUN: %clangxx -fsycl -DJIT %s -o %t.out
 // RUN: %{run} %t.out
diff --git a/sycl/test-e2e/SpecConstants/2020/non_native/SpecConstBuffer.cpp b/sycl/test-e2e/SpecConstants/2020/non_native/SpecConstBuffer.cpp
index 3ee1d4bd2490b..bc7145049ddcb 100644
--- a/sycl/test-e2e/SpecConstants/2020/non_native/SpecConstBuffer.cpp
+++ b/sycl/test-e2e/SpecConstants/2020/non_native/SpecConstBuffer.cpp
@@ -1,4 +1,5 @@
 // REQUIRES: opencl-aot, cpu
+// REQUIRES: build-and-run-mode
 
 // RUN: %clangxx -fsycl -fsycl-targets=spir64_x86_64 %S/Inputs/common.cpp -o %t.out \
 // RUN:          -fsycl-dead-args-optimization
diff --git a/sycl/test-e2e/SpecConstants/2020/non_native/cpu.cpp b/sycl/test-e2e/SpecConstants/2020/non_native/cpu.cpp
index 8453a6556ffec..f6b7b836e873c 100644
--- a/sycl/test-e2e/SpecConstants/2020/non_native/cpu.cpp
+++ b/sycl/test-e2e/SpecConstants/2020/non_native/cpu.cpp
@@ -1,4 +1,5 @@
 // REQUIRES: opencl-aot, cpu
+// REQUIRES: build-and-run-mode
 
 // RUN: %clangxx -fsycl -fsycl-targets=spir64_x86_64 %S/Inputs/common.cpp -o %t.out \
 // RUN:          -fsycl-dead-args-optimization

From 142e6617d7417dce8a12a6e9de8d771549fade6b Mon Sep 17 00:00:00 2001
From: "Garcia Orozco, David" <david.garcia.orozco@intel.com>
Date: Wed, 11 Dec 2024 10:55:29 -0800
Subject: [PATCH 04/26] Mark as `XFAIL: run-mode` test that can pass build
 stage

---
 sycl/test-e2e/AddressSanitizer/nullpointer/private_nullptr.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sycl/test-e2e/AddressSanitizer/nullpointer/private_nullptr.cpp b/sycl/test-e2e/AddressSanitizer/nullpointer/private_nullptr.cpp
index 27b4b5a6fab17..fd7cae43dd3ea 100644
--- a/sycl/test-e2e/AddressSanitizer/nullpointer/private_nullptr.cpp
+++ b/sycl/test-e2e/AddressSanitizer/nullpointer/private_nullptr.cpp
@@ -7,7 +7,7 @@
 // RUN: %{run} not %t3.out 2>&1 | FileCheck %s
 
 // FIXME: There's an issue in gfx driver, so this test pending here.
-// XFAIL: *
+// XFAIL: run-mode
 
 #include <sycl/detail/core.hpp>
 #include <sycl/ext/oneapi/experimental/address_cast.hpp>

From ba3dd1ae0d093368d20b86d1d4b18dbec86abc0a Mon Sep 17 00:00:00 2001
From: "Garcia Orozco, David" <david.garcia.orozco@intel.com>
Date: Wed, 11 Dec 2024 10:56:13 -0800
Subject: [PATCH 05/26] Add as split exceptions tests that fail on run-only
 stage

These are mostly AOT tests
---
 sycl/test-e2e/AOT/cpu.cpp                                        | 1 +
 sycl/test-e2e/AOT/double.cpp                                     | 1 +
 sycl/test-e2e/AOT/half.cpp                                       | 1 +
 sycl/test-e2e/BFloat16/bfloat16_example_aot.cpp                  | 1 +
 sycl/test-e2e/BFloat16/bfloat16_example_aot_cpu.cpp              | 1 +
 sycl/test-e2e/Compression/compression_aot.cpp                    | 1 +
 sycl/test-e2e/Compression/compression_separate_compile.cpp       | 1 +
 .../DeviceArchitecture/device_architecture_on_device_aot.cpp     | 1 +
 sycl/test-e2e/DeviceCodeSplit/aot-cpu.cpp                        | 1 +
 sycl/test-e2e/DeviceLib/assert-aot.cpp                           | 1 +
 sycl/test-e2e/DeviceLib/cmath-aot.cpp                            | 1 +
 sycl/test-e2e/NewOffloadDriver/aot-cpu.cpp                       | 1 +
 sycl/test-e2e/NewOffloadDriver/cpu.cpp                           | 1 +
 sycl/test-e2e/NonUniformGroups/ballot_group.cpp                  | 1 +
 sycl/test-e2e/NonUniformGroups/ballot_group_algorithms.cpp       | 1 +
 sycl/test-e2e/NonUniformGroups/fixed_size_group.cpp              | 1 +
 sycl/test-e2e/NonUniformGroups/fixed_size_group_algorithms.cpp   | 1 +
 sycl/test-e2e/NonUniformGroups/opportunistic_group.cpp           | 1 +
 .../test-e2e/NonUniformGroups/opportunistic_group_algorithms.cpp | 1 +
 sycl/test-e2e/NonUniformGroups/tangle_group.cpp                  | 1 +
 sycl/test-e2e/NonUniformGroups/tangle_group_algorithms.cpp       | 1 +
 .../multi_device_bundle/device_libs_and_caching.cpp              | 1 +
 22 files changed, 22 insertions(+)

diff --git a/sycl/test-e2e/AOT/cpu.cpp b/sycl/test-e2e/AOT/cpu.cpp
index f0c71be2d481e..e7bedc04b07ba 100644
--- a/sycl/test-e2e/AOT/cpu.cpp
+++ b/sycl/test-e2e/AOT/cpu.cpp
@@ -7,6 +7,7 @@
 //===---------------------------------------------------------------------===//
 
 // REQUIRES: opencl-aot, cpu
+// REQUIRES: build-and-run-mode
 
 // RUN: %clangxx -fsycl -fsycl-targets=spir64_x86_64 %S/Inputs/aot.cpp -o %t.out
 // RUN: %{run} %t.out
diff --git a/sycl/test-e2e/AOT/double.cpp b/sycl/test-e2e/AOT/double.cpp
index ead41bfc8c741..4d48f5c7b57ec 100644
--- a/sycl/test-e2e/AOT/double.cpp
+++ b/sycl/test-e2e/AOT/double.cpp
@@ -2,6 +2,7 @@
 // using fp64 can be compiled AOT.
 
 // REQUIRES: ocloc, opencl-aot, any-device-is-cpu
+// REQUIRES: build-and-run-mode
 // RUN: %clangxx -fsycl -fsycl-targets=intel_gpu_tgllp -o %t.tgllp.out %s
 // RUN: %clangxx -fsycl -fsycl-targets=spir64_x86_64 -o %t.x86.out %s
 // RUN: %if cpu %{ %{run} %t.x86.out %}
diff --git a/sycl/test-e2e/AOT/half.cpp b/sycl/test-e2e/AOT/half.cpp
index e843ee65bbeef..5711b21735f72 100644
--- a/sycl/test-e2e/AOT/half.cpp
+++ b/sycl/test-e2e/AOT/half.cpp
@@ -2,6 +2,7 @@
 // using fp16 can be compiled AOT.
 
 // REQUIRES: ocloc, opencl-aot, any-device-is-cpu
+// REQUIRES: build-and-run-mode
 // RUN: %clangxx -fsycl -fsycl-targets=intel_gpu_tgllp -o %t.tgllp.out %s
 // RUN: %clangxx -fsycl -fsycl-targets=spir64_x86_64 -o %t.x86.out %s
 // RUN: %if cpu %{ %{run} %t.x86.out %}
diff --git a/sycl/test-e2e/BFloat16/bfloat16_example_aot.cpp b/sycl/test-e2e/BFloat16/bfloat16_example_aot.cpp
index 8337716c3191e..32c6c3aae8f06 100644
--- a/sycl/test-e2e/BFloat16/bfloat16_example_aot.cpp
+++ b/sycl/test-e2e/BFloat16/bfloat16_example_aot.cpp
@@ -4,6 +4,7 @@
 ///
 
 // REQUIRES: opencl-aot, ocloc, gpu-intel-gen12, any-device-is-cpu
+// REQUIRES: build-and-run-mode
 
 // RUN: %clangxx -fsycl -fsycl-targets=spir64 %s -o %t.out
 // RUN: %{run} %t.out
diff --git a/sycl/test-e2e/BFloat16/bfloat16_example_aot_cpu.cpp b/sycl/test-e2e/BFloat16/bfloat16_example_aot_cpu.cpp
index 2f6d893768c4e..9231a62428272 100644
--- a/sycl/test-e2e/BFloat16/bfloat16_example_aot_cpu.cpp
+++ b/sycl/test-e2e/BFloat16/bfloat16_example_aot_cpu.cpp
@@ -4,6 +4,7 @@
 ///
 
 // REQUIRES: opencl-aot, ocloc, gpu-intel-gen12, any-device-is-cpu
+// REQUIRES: build-and-run-mode
 
 // RUN: %clangxx -fsycl -fsycl-targets=spir64,spir64_gen -Xsycl-target-backend=spir64_gen "-device dg1" %s -o %t.out
 // RUN: %if cpu %{ %{run} %t.out %}
diff --git a/sycl/test-e2e/Compression/compression_aot.cpp b/sycl/test-e2e/Compression/compression_aot.cpp
index b8293f1c79351..ba95974fde781 100644
--- a/sycl/test-e2e/Compression/compression_aot.cpp
+++ b/sycl/test-e2e/Compression/compression_aot.cpp
@@ -1,5 +1,6 @@
 // End-to-End test for testing device image compression in AOT.
 // REQUIRES: zstd, opencl-aot, cpu
+// REQUIRES: build-and-run-mode
 
 // RUN: %clangxx -fsycl -fsycl-targets=spir64_x86_64 %O0 --offload-compress --offload-compression-level=3 %S/Inputs/single_kernel.cpp -o %t_compress.out
 // RUN: %{run} %t_compress.out
diff --git a/sycl/test-e2e/Compression/compression_separate_compile.cpp b/sycl/test-e2e/Compression/compression_separate_compile.cpp
index dab17e3506b4e..afbe442230e29 100644
--- a/sycl/test-e2e/Compression/compression_separate_compile.cpp
+++ b/sycl/test-e2e/Compression/compression_separate_compile.cpp
@@ -2,6 +2,7 @@
 // seperatly compile and link device images.
 
 // REQUIRES: zstd, opencl-aot, cpu, linux
+// REQUIRES: build-and-run-mode
 
 //////////////////////  Compile device images
 // RUN: %clangxx -fsycl -fsycl-targets=spir64_x86_64 -fsycl-host-compiler=clang++ -fsycl-host-compiler-options='-std=c++17 -Wno-attributes -Wno-deprecated-declarations -fPIC -DENABLE_KERNEL1' -DENABLE_KERNEL1 -c %s -o %t_kernel1_aot.o
diff --git a/sycl/test-e2e/DeviceArchitecture/device_architecture_on_device_aot.cpp b/sycl/test-e2e/DeviceArchitecture/device_architecture_on_device_aot.cpp
index a62ae965683ae..1d4c41f117136 100644
--- a/sycl/test-e2e/DeviceArchitecture/device_architecture_on_device_aot.cpp
+++ b/sycl/test-e2e/DeviceArchitecture/device_architecture_on_device_aot.cpp
@@ -1,4 +1,5 @@
 // REQUIRES: opencl-aot, cpu
+// REQUIRES: build-and-run-mode
 // RUN: %clangxx -fsycl -fsycl-targets=spir64_x86_64 %s -o %t.out
 // RUN: %{run} %t.out
 
diff --git a/sycl/test-e2e/DeviceCodeSplit/aot-cpu.cpp b/sycl/test-e2e/DeviceCodeSplit/aot-cpu.cpp
index d2937b796fe63..319c69af0faef 100644
--- a/sycl/test-e2e/DeviceCodeSplit/aot-cpu.cpp
+++ b/sycl/test-e2e/DeviceCodeSplit/aot-cpu.cpp
@@ -1,4 +1,5 @@
 // REQUIRES: opencl-aot, cpu
+// REQUIRES: build-and-run-mode
 
 // RUN: %clangxx -fsycl -fsycl-device-code-split=per_source -fsycl-targets=spir64_x86_64 -I %S/Inputs -o %t.out %S/split-per-source-main.cpp %S/Inputs/split-per-source-second-file.cpp \
 // RUN: -fsycl-dead-args-optimization
diff --git a/sycl/test-e2e/DeviceLib/assert-aot.cpp b/sycl/test-e2e/DeviceLib/assert-aot.cpp
index ccff887e80adf..f5c8c61f07d78 100644
--- a/sycl/test-e2e/DeviceLib/assert-aot.cpp
+++ b/sycl/test-e2e/DeviceLib/assert-aot.cpp
@@ -1,4 +1,5 @@
 // REQUIRES: opencl-aot, cpu, linux
+// REQUIRES: build-and-run-mode
 
 // RUN: %clangxx -DSYCL_FALLBACK_ASSERT=1 -fsycl -fsycl-targets=spir64_x86_64 %S/assert.cpp -o %t.aot.out
 // RUN: env EXPECTED_SIGNAL=SIGABRT SHOULD_CRASH=1 %{run} %t.aot.out 2>&1 | FileCheck %S/assert.cpp --check-prefixes=CHECK-MESSAGE
diff --git a/sycl/test-e2e/DeviceLib/cmath-aot.cpp b/sycl/test-e2e/DeviceLib/cmath-aot.cpp
index bb9e201de9282..c9897f652f034 100644
--- a/sycl/test-e2e/DeviceLib/cmath-aot.cpp
+++ b/sycl/test-e2e/DeviceLib/cmath-aot.cpp
@@ -1,4 +1,5 @@
 // REQUIRES: opencl-aot, cpu
+// REQUIRES: build-and-run-mode
 // UNSUPPORTED: windows
 
 // DEFINE: %{mathflags} = %if cl_options %{/clang:-fno-fast-math%} %else %{-fno-fast-math%}
diff --git a/sycl/test-e2e/NewOffloadDriver/aot-cpu.cpp b/sycl/test-e2e/NewOffloadDriver/aot-cpu.cpp
index 585b368641d6e..0016e8498bd9d 100644
--- a/sycl/test-e2e/NewOffloadDriver/aot-cpu.cpp
+++ b/sycl/test-e2e/NewOffloadDriver/aot-cpu.cpp
@@ -1,4 +1,5 @@
 // REQUIRES: opencl-aot, cpu
+// REQUIRES: build-and-run-mode
 
 // Test with `--offload-new-driver`
 // RUN: %clangxx -fsycl -fsycl-device-code-split=per_source -fsycl-targets=spir64_x86_64 -I %S/Inputs -o %t.out %S/split-per-source-main.cpp %S/Inputs/split-per-source-second-file.cpp \
diff --git a/sycl/test-e2e/NewOffloadDriver/cpu.cpp b/sycl/test-e2e/NewOffloadDriver/cpu.cpp
index ef93d69f8693b..943ea71c3386e 100644
--- a/sycl/test-e2e/NewOffloadDriver/cpu.cpp
+++ b/sycl/test-e2e/NewOffloadDriver/cpu.cpp
@@ -7,6 +7,7 @@
 //===---------------------------------------------------------------------===//
 
 // REQUIRES: opencl-aot, cpu
+// REQUIRES: build-and-run-mode
 
 // Test with  `--offload-new-driver`
 // RUN: %clangxx -fsycl -fsycl-targets=spir64_x86_64 --offload-new-driver %S/Inputs/aot.cpp -o %t.out
diff --git a/sycl/test-e2e/NonUniformGroups/ballot_group.cpp b/sycl/test-e2e/NonUniformGroups/ballot_group.cpp
index f24bffb81526a..1532c54879ecc 100644
--- a/sycl/test-e2e/NonUniformGroups/ballot_group.cpp
+++ b/sycl/test-e2e/NonUniformGroups/ballot_group.cpp
@@ -4,6 +4,7 @@
 // RUN: %if any-device-is-cpu && opencl-aot %{ %clangxx -fsycl -fsycl-targets=spir64_x86_64 -o %t.x86.out %s %}
 // RUN: %if cpu %{ %{run} %t.x86.out %}
 //
+// REQUIRES: build-and-run-mode
 // REQUIRES: cpu || gpu
 // UNSUPPORTED: hip
 
diff --git a/sycl/test-e2e/NonUniformGroups/ballot_group_algorithms.cpp b/sycl/test-e2e/NonUniformGroups/ballot_group_algorithms.cpp
index 8f6b6a8f17197..14e976edc8492 100644
--- a/sycl/test-e2e/NonUniformGroups/ballot_group_algorithms.cpp
+++ b/sycl/test-e2e/NonUniformGroups/ballot_group_algorithms.cpp
@@ -4,6 +4,7 @@
 // RUN: %if any-device-is-cpu && opencl-aot %{ %clangxx -fsycl -fsycl-targets=spir64_x86_64 -o %t.x86.out %s %}
 // RUN: %if cpu %{ %{run} %t.x86.out %}
 //
+// REQUIRES: build-and-run-mode
 // REQUIRES: cpu || gpu
 // REQUIRES: sg-32
 // REQUIRES: aspect-ext_oneapi_ballot_group
diff --git a/sycl/test-e2e/NonUniformGroups/fixed_size_group.cpp b/sycl/test-e2e/NonUniformGroups/fixed_size_group.cpp
index 939be57799dd4..5d2b98b58ddc2 100644
--- a/sycl/test-e2e/NonUniformGroups/fixed_size_group.cpp
+++ b/sycl/test-e2e/NonUniformGroups/fixed_size_group.cpp
@@ -4,6 +4,7 @@
 // RUN: %if any-device-is-cpu && opencl-aot %{ %clangxx -fsycl -fsycl-targets=spir64_x86_64 -o %t.x86.out %s %}
 // RUN: %if cpu %{ %{run} %t.x86.out %}
 //
+// REQUIRES: build-and-run-mode
 // REQUIRES: cpu || gpu
 // UNSUPPORTED: hip
 // REQUIRES: sg-32
diff --git a/sycl/test-e2e/NonUniformGroups/fixed_size_group_algorithms.cpp b/sycl/test-e2e/NonUniformGroups/fixed_size_group_algorithms.cpp
index c1c172c4189c3..dfed10e9f587e 100644
--- a/sycl/test-e2e/NonUniformGroups/fixed_size_group_algorithms.cpp
+++ b/sycl/test-e2e/NonUniformGroups/fixed_size_group_algorithms.cpp
@@ -4,6 +4,7 @@
 // RUN: %if any-device-is-cpu && opencl-aot %{ %clangxx -fsycl -fsycl-targets=spir64_x86_64 -fsycl-device-code-split=per_kernel -o %t.x86.out %s %}
 // RUN: %if cpu %{ %{run} %t.x86.out %}
 //
+// REQUIRES: build-and-run-mode
 // REQUIRES: cpu || gpu
 // REQUIRES: sg-32
 // REQUIRES: aspect-ext_oneapi_fixed_size_group
diff --git a/sycl/test-e2e/NonUniformGroups/opportunistic_group.cpp b/sycl/test-e2e/NonUniformGroups/opportunistic_group.cpp
index 18d42487b7768..2069113c3f939 100644
--- a/sycl/test-e2e/NonUniformGroups/opportunistic_group.cpp
+++ b/sycl/test-e2e/NonUniformGroups/opportunistic_group.cpp
@@ -4,6 +4,7 @@
 // RUN: %if any-device-is-cpu && opencl-aot %{ %clangxx -fsycl -fsycl-targets=spir64_x86_64 -o %t.x86.out %s %}
 // RUN: %if cpu %{ %{run} %t.x86.out %}
 //
+// REQUIRES: build-and-run-mode
 // REQUIRES: cpu || gpu
 // UNSUPPORTED: hip
 
diff --git a/sycl/test-e2e/NonUniformGroups/opportunistic_group_algorithms.cpp b/sycl/test-e2e/NonUniformGroups/opportunistic_group_algorithms.cpp
index af4d45cee862f..8e7626650e01c 100644
--- a/sycl/test-e2e/NonUniformGroups/opportunistic_group_algorithms.cpp
+++ b/sycl/test-e2e/NonUniformGroups/opportunistic_group_algorithms.cpp
@@ -4,6 +4,7 @@
 // RUN: %if any-device-is-cpu && opencl-aot %{ %clangxx -fsycl -fsycl-targets=spir64_x86_64 -o %t.x86.out %s %}
 // RUN: %if cpu %{ %{run} %t.x86.out %}
 //
+// REQUIRES: build-and-run-mode
 // REQUIRES: cpu || gpu
 // REQUIRES: sg-32
 // REQUIRES: aspect-ext_oneapi_opportunistic_group
diff --git a/sycl/test-e2e/NonUniformGroups/tangle_group.cpp b/sycl/test-e2e/NonUniformGroups/tangle_group.cpp
index 44191955048f3..686d1b34c6185 100644
--- a/sycl/test-e2e/NonUniformGroups/tangle_group.cpp
+++ b/sycl/test-e2e/NonUniformGroups/tangle_group.cpp
@@ -4,6 +4,7 @@
 // RUN: %if any-device-is-cpu && opencl-aot %{ %clangxx -fsycl -fsycl-targets=spir64_x86_64 -fno-sycl-early-optimizations -o %t.x86.out %s %}
 // RUN: %if cpu %{ %{run} %t.x86.out %}
 //
+// REQUIRES: build-and-run-mode
 // REQUIRES: cpu || gpu
 // UNSUPPORTED: cuda || hip
 
diff --git a/sycl/test-e2e/NonUniformGroups/tangle_group_algorithms.cpp b/sycl/test-e2e/NonUniformGroups/tangle_group_algorithms.cpp
index 7033c4c9e4df5..51070ed1731e4 100644
--- a/sycl/test-e2e/NonUniformGroups/tangle_group_algorithms.cpp
+++ b/sycl/test-e2e/NonUniformGroups/tangle_group_algorithms.cpp
@@ -4,6 +4,7 @@
 // RUN: %if any-device-is-cpu && opencl-aot %{ %clangxx -fsycl -fsycl-targets=spir64_x86_64 -fno-sycl-early-optimizations -o %t.x86.out %s %}
 // RUN: %if cpu %{ %{run} %t.x86.out %}
 //
+// REQUIRES: build-and-run-mode
 // REQUIRES: cpu || gpu
 // REQUIRES: sg-32
 // REQUIRES: aspect-ext_oneapi_tangle_group
diff --git a/sycl/test-e2e/ProgramManager/multi_device_bundle/device_libs_and_caching.cpp b/sycl/test-e2e/ProgramManager/multi_device_bundle/device_libs_and_caching.cpp
index d596518165762..1534b2330f552 100644
--- a/sycl/test-e2e/ProgramManager/multi_device_bundle/device_libs_and_caching.cpp
+++ b/sycl/test-e2e/ProgramManager/multi_device_bundle/device_libs_and_caching.cpp
@@ -1,4 +1,5 @@
 // REQUIRES: ocloc && gpu && linux && (opencl || level_zero)
+// REQUIRES: build-and-run-mode
 
 // Test to check several use cases for multi-device kernel bundles.
 // Test covers AOT and JIT cases. Kernel is using some math functions to enforce

From 8ec39c316d05855c508a12cd05302452f3a1ed64 Mon Sep 17 00:00:00 2001
From: "Garcia Orozco, David" <david.garcia.orozco@intel.com>
Date: Wed, 11 Dec 2024 12:19:25 -0800
Subject: [PATCH 06/26] Rename new e2e jobs

---
 .github/workflows/sycl-linux-precommit.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/sycl-linux-precommit.yml b/.github/workflows/sycl-linux-precommit.yml
index b67750a0c027e..e46e58c7b8dc4 100644
--- a/.github/workflows/sycl-linux-precommit.yml
+++ b/.github/workflows/sycl-linux-precommit.yml
@@ -73,7 +73,7 @@ jobs:
             echo 'arc_tests="Matrix/"' >> "$GITHUB_OUTPUT"
           fi
 
-  split_test_build:
+  build_e2e_tests:
     needs: [build]
     if: ${{ always() && !cancelled() && needs.build.outputs.build_conclusion == 'success' }}
     uses: ./.github/workflows/sycl-linux-run-tests.yml
@@ -90,8 +90,8 @@ jobs:
       sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }}
       sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }}
       upload_artifact: 'true'
-  split_test_run:
-    needs: [build, split_test_build]
+  run_prebuilt_e2e_tests:
+    needs: [build, build_e2e_tests]
     if: ${{ always() && !cancelled() && needs.build.outputs.build_conclusion == 'success' }}
     strategy:
       fail-fast: false
@@ -119,8 +119,8 @@ jobs:
       sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }}
       sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }}
       e2e_binaries_artifact: sycl_e2e_bin_default
-      e2e_binaries_archive: ${{ needs.split_test_build.outputs.artifact_archive_name }}
-      e2e_binaries_decompress_command: ${{ needs.split_test_build.outputs.artifact_decompress_command }}
+      e2e_binaries_archive: ${{ needs.build_e2e_tests.outputs.artifact_archive_name }}
+      e2e_binaries_decompress_command: ${{ needs.build_e2e_tests.outputs.artifact_decompress_command }}
 
   test:
     needs: [build, detect_changes, determine_arc_tests]

From ca6fdcea85d0efba9f004674d0e7d4f16aaa8922 Mon Sep 17 00:00:00 2001
From: "Garcia Orozco, David" <david.garcia.orozco@intel.com>
Date: Thu, 12 Dec 2024 07:15:28 -0800
Subject: [PATCH 07/26] Remove target_devices from build only test run

Also change how its set
---
 .github/workflows/sycl-linux-precommit.yml | 1 -
 .github/workflows/sycl-linux-run-tests.yml | 5 ++---
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/sycl-linux-precommit.yml b/.github/workflows/sycl-linux-precommit.yml
index e46e58c7b8dc4..6ba4e91e380b3 100644
--- a/.github/workflows/sycl-linux-precommit.yml
+++ b/.github/workflows/sycl-linux-precommit.yml
@@ -82,7 +82,6 @@ jobs:
       runner: '["Linux", "build"]'
       image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:alldeps
       image_options: -u 1001
-      target_devices: opencl:cpu
       extra_lit_opts: --param test-mode=build-only
       ref: ${{ github.sha }}
       merge_ref: ''
diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml
index 7aa53817af391..98300dc5b3b68 100644
--- a/.github/workflows/sycl-linux-run-tests.yml
+++ b/.github/workflows/sycl-linux-run-tests.yml
@@ -19,7 +19,7 @@ on:
 
       target_devices:
         type: string
-        required: True
+        required: False
       extra_cmake_args:
         type: string
         required: False
@@ -360,8 +360,7 @@ jobs:
       shell: bash {0}
       if: inputs.tests_selector == 'e2e'
       env:
-        LIT_OPTS: -v --no-progress-bar --show-unsupported --show-pass --show-xfail --max-time 3600 --time-tests ${{ inputs.extra_lit_opts }}
-        ONEAPI_DEVICE_SELECTOR: ${{ inputs.target_devices }}
+        LIT_OPTS: -v --no-progress-bar --show-unsupported --show-pass --show-xfail --max-time 3600 --time-tests --param sycl_devices=${{ inputs.target_devices }} ${{ inputs.extra_lit_opts }}
       run: |
         ninja -C build-e2e check-sycl-e2e > e2e.log 2>&1
         exit_code=$?

From 99747cc47b6d879c68796114c411402f8f16e718 Mon Sep 17 00:00:00 2001
From: "Garcia Orozco, David" <david.garcia.orozco@intel.com>
Date: Thu, 12 Dec 2024 07:22:19 -0800
Subject: [PATCH 08/26] Remove build_conclusion output on run-tests

---
 .github/workflows/sycl-linux-run-tests.yml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml
index 98300dc5b3b68..89240b577ca81 100644
--- a/.github/workflows/sycl-linux-run-tests.yml
+++ b/.github/workflows/sycl-linux-run-tests.yml
@@ -102,8 +102,6 @@ on:
         required: False
 
     outputs:
-      run_conclusion:
-        value: ${{ jobs.run.outputs.build_conclusion }}
       artifact_archive_name:
         value: ${{ jobs.run.outputs.artifact_archive_name }}
       artifact_decompress_command:
@@ -191,8 +189,6 @@ jobs:
     container:
       image: ${{ inputs.image }}
       options: ${{ inputs.image_options }}
-    outputs:
-      run_conclusion: ${{ steps.run.conclusion }}
       artifact_archive_name: ${{ steps.artifact_info.outputs.ARCHIVE_NAME }}
       artifact_decompress_command: ${{ steps.artifact_info.outputs.DECOMPRESS }}
     env: ${{ fromJSON(inputs.env) }}

From 04676fd0d4269be607f5ce7f667f73b621f8e8c8 Mon Sep 17 00:00:00 2001
From: "Garcia Orozco, David" <david.garcia.orozco@intel.com>
Date: Thu, 12 Dec 2024 07:57:12 -0800
Subject: [PATCH 09/26] Re-add accidentally removed `outputs:` line

---
 .github/workflows/sycl-linux-run-tests.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml
index 89240b577ca81..a28e00501ef22 100644
--- a/.github/workflows/sycl-linux-run-tests.yml
+++ b/.github/workflows/sycl-linux-run-tests.yml
@@ -189,6 +189,7 @@ jobs:
     container:
       image: ${{ inputs.image }}
       options: ${{ inputs.image_options }}
+    outputs:
       artifact_archive_name: ${{ steps.artifact_info.outputs.ARCHIVE_NAME }}
       artifact_decompress_command: ${{ steps.artifact_info.outputs.DECOMPRESS }}
     env: ${{ fromJSON(inputs.env) }}

From 0470bf20e01fc09b8499d51c83024879d1073528 Mon Sep 17 00:00:00 2001
From: "Garcia Orozco, David" <david.garcia.orozco@intel.com>
Date: Thu, 12 Dec 2024 08:22:27 -0800
Subject: [PATCH 10/26] Rename extract/setup step to just extract

---
 .github/workflows/sycl-linux-run-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml
index a28e00501ef22..e02260154b830 100644
--- a/.github/workflows/sycl-linux-run-tests.yml
+++ b/.github/workflows/sycl-linux-run-tests.yml
@@ -332,7 +332,7 @@ jobs:
       uses: actions/download-artifact@v4
       with:
         name: ${{ inputs.e2e_binaries_artifact }}
-    - name: Extract/Setup E2E Binaries
+    - name: Extract E2E Binaries
       if: inputs.e2e_binaries_artifact != ''
       run: |
         mkdir build-e2e

From 0a7a77248a085e68a4c2352b7fff1d5216542544 Mon Sep 17 00:00:00 2001
From: "Garcia Orozco, David" <david.garcia.orozco@intel.com>
Date: Thu, 12 Dec 2024 08:28:48 -0800
Subject: [PATCH 11/26] Fix formatting

---
 sycl/test-e2e/format.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/sycl/test-e2e/format.py b/sycl/test-e2e/format.py
index d7d96f414fdc0..6033fcfaac429 100644
--- a/sycl/test-e2e/format.py
+++ b/sycl/test-e2e/format.py
@@ -296,9 +296,7 @@ def get_extra_env(sycl_devices):
         # TODO: workaround for lit hanging when executing non-existent binary
         # inside our containers
         if len(script) == 0:
-            return lit.Test.Result(
-                    lit.Test.UNSUPPORTED, "Lit script is empty"
-                    )
+            return lit.Test.Result(lit.Test.UNSUPPORTED, "Lit script is empty")
         useExternalSh = test.config.test_mode == "run-only"
 
         result = lit.TestRunner._runShTest(

From b0dabe2207f095df786356f1812625bc748bf3bd Mon Sep 17 00:00:00 2001
From: "Garcia Orozco, David" <david.garcia.orozco@intel.com>
Date: Thu, 12 Dec 2024 08:34:11 -0800
Subject: [PATCH 12/26] Add retention days input

---
 .github/workflows/sycl-linux-run-tests.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml
index e02260154b830..c1ed6549356e5 100644
--- a/.github/workflows/sycl-linux-run-tests.yml
+++ b/.github/workflows/sycl-linux-run-tests.yml
@@ -81,6 +81,10 @@ on:
       artifact_archive_name:
         type: string
         default: e2e_binaries.tar.zst
+      retention-days:
+        description: 'E2E binaries artifact retention period'
+        type: string
+        default: 1
 
       reset_intel_gpu:
         type: string

From 125285c8e396af0bd13042bcbbc64b68bc95bdc4 Mon Sep 17 00:00:00 2001
From: "Garcia Orozco, David" <david.garcia.orozco@intel.com>
Date: Thu, 12 Dec 2024 08:38:21 -0800
Subject: [PATCH 13/26] rename build_artifact_suffix to artifact_suffix

---
 .github/workflows/sycl-linux-run-tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml
index c1ed6549356e5..0318be4858444 100644
--- a/.github/workflows/sycl-linux-run-tests.yml
+++ b/.github/workflows/sycl-linux-run-tests.yml
@@ -75,7 +75,7 @@ on:
       upload_artifact:
         type: string
         default: 'false'
-      build_artifact_suffix:
+      artifact_suffix:
         type: string
         default: 'default'
       artifact_archive_name:
@@ -455,6 +455,6 @@ jobs:
       if: ${{ always() && !cancelled() && inputs.upload_artifact == 'true'}}
       uses: actions/upload-artifact@v4
       with:
-        name: sycl_e2e_bin_${{ inputs.build_artifact_suffix }}
+        name: sycl_e2e_bin_${{ inputs.artifact_suffix }}
         path: ${{ steps.artifact_info.outputs.ARCHIVE_NAME }}
         retention-days: ${{ inputs.retention-days }}

From ee4a9ce2db63e0c11000de258a8082c1f3221db3 Mon Sep 17 00:00:00 2001
From: "Garcia Orozco, David" <david.garcia.orozco@intel.com>
Date: Fri, 13 Dec 2024 10:17:55 -0800
Subject: [PATCH 14/26] Add --crash to `not` in 4 tests

---
 .../OneapiDeviceSelector/illegal_input.cpp    | 20 +++++++++----------
 .../usm/queue_copy_released_pointer.cpp       |  2 +-
 .../Tracing/usm/queue_single_task_nullptr.cpp |  2 +-
 .../queue_single_task_released_pointer.cpp    |  2 +-
 4 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/sycl/test-e2e/OneapiDeviceSelector/illegal_input.cpp b/sycl/test-e2e/OneapiDeviceSelector/illegal_input.cpp
index 79675bbac23cf..35430c7b12ff6 100644
--- a/sycl/test-e2e/OneapiDeviceSelector/illegal_input.cpp
+++ b/sycl/test-e2e/OneapiDeviceSelector/illegal_input.cpp
@@ -1,15 +1,15 @@
 
 // RUN: %clangxx -fsycl -fsycl-targets=%{sycl_triple} %S/Inputs/trivial.cpp -o %t.out
-// RUN: not env ONEAPI_DEVICE_SELECTOR="macaroni:*" %{run-unfiltered-devices} %t.out
-// RUN: not env ONEAPI_DEVICE_SELECTOR=":" %{run-unfiltered-devices} %t.out
-// RUN: not env ONEAPI_DEVICE_SELECTOR="level_zero:." %{run-unfiltered-devices} %t.out
-// RUN: not env ONEAPI_DEVICE_SELECTOR="macaroni_level_zero:." %{run-unfiltered-devices} %t.out
-// RUN: not env ONEAPI_DEVICE_SELECTOR="level_zero:macaroni_gpu" %{run-unfiltered-devices} %t.out
-// RUN: not env ONEAPI_DEVICE_SELECTOR="level_zero:0..0" %{run-unfiltered-devices} %t.out
-// RUN: not env ONEAPI_DEVICE_SELECTOR="level_zero:" %{run-unfiltered-devices} %t.out
-// RUN: not env ONEAPI_DEVICE_SELECTOR="level_zero:::gpu" %{run-unfiltered-devices} %t.out
-// RUN: not env ONEAPI_DEVICE_SELECTOR="level_zero:.1" %{run-unfiltered-devices} %t.out
-// RUN: not env ONEAPI_DEVICE_SELECTOR="" %{run-unfiltered-devices} %t.out
+// RUN: not --crash env ONEAPI_DEVICE_SELECTOR="macaroni:*" %{run-unfiltered-devices} %t.out
+// RUN: not --crash env ONEAPI_DEVICE_SELECTOR=":" %{run-unfiltered-devices} %t.out
+// RUN: not --crash env ONEAPI_DEVICE_SELECTOR="level_zero:." %{run-unfiltered-devices} %t.out
+// RUN: not --crash env ONEAPI_DEVICE_SELECTOR="macaroni_level_zero:." %{run-unfiltered-devices} %t.out
+// RUN: not --crash env ONEAPI_DEVICE_SELECTOR="level_zero:macaroni_gpu" %{run-unfiltered-devices} %t.out
+// RUN: not --crash env ONEAPI_DEVICE_SELECTOR="level_zero:0..0" %{run-unfiltered-devices} %t.out
+// RUN: not --crash env ONEAPI_DEVICE_SELECTOR="level_zero:" %{run-unfiltered-devices} %t.out
+// RUN: not --crash env ONEAPI_DEVICE_SELECTOR="level_zero:::gpu" %{run-unfiltered-devices} %t.out
+// RUN: not --crash env ONEAPI_DEVICE_SELECTOR="level_zero:.1" %{run-unfiltered-devices} %t.out
+// RUN: not --crash env ONEAPI_DEVICE_SELECTOR="" %{run-unfiltered-devices} %t.out
 
 // Calling ONEAPI_DEVICE_SELECTOR with an illegal input should result in an
 // error.
diff --git a/sycl/test-e2e/Tracing/usm/queue_copy_released_pointer.cpp b/sycl/test-e2e/Tracing/usm/queue_copy_released_pointer.cpp
index 9585a9dc485ed..a37e5b8f4238c 100644
--- a/sycl/test-e2e/Tracing/usm/queue_copy_released_pointer.cpp
+++ b/sycl/test-e2e/Tracing/usm/queue_copy_released_pointer.cpp
@@ -1,6 +1,6 @@
 // UNSUPPORTED: windows || hip_amd
 // RUN: %{build} -o %t.out
-// RUN: not env SYCL_TRACE_TERMINATE_ON_WARNING=1 %{run} sycl-trace --verify %t.out | FileCheck %s
+// RUN: not --crash env SYCL_TRACE_TERMINATE_ON_WARNING=1 %{run} sycl-trace --verify %t.out | FileCheck %s
 
 // Test parameter analysis of USM function
 
diff --git a/sycl/test-e2e/Tracing/usm/queue_single_task_nullptr.cpp b/sycl/test-e2e/Tracing/usm/queue_single_task_nullptr.cpp
index 6d19f88dd5487..4c4299dd93d8e 100644
--- a/sycl/test-e2e/Tracing/usm/queue_single_task_nullptr.cpp
+++ b/sycl/test-e2e/Tracing/usm/queue_single_task_nullptr.cpp
@@ -1,6 +1,6 @@
 // UNSUPPORTED: windows || hip_amd
 // RUN: %{build} -o %t.out
-// RUN: not env SYCL_TRACE_TERMINATE_ON_WARNING=1 %{run} sycl-trace --verify %t.out | FileCheck %s
+// RUN: not --crash env SYCL_TRACE_TERMINATE_ON_WARNING=1 %{run} sycl-trace --verify %t.out | FileCheck %s
 
 // Test parameter analysis of USM usage
 
diff --git a/sycl/test-e2e/Tracing/usm/queue_single_task_released_pointer.cpp b/sycl/test-e2e/Tracing/usm/queue_single_task_released_pointer.cpp
index f87717c8efc28..61e27b7927f7b 100644
--- a/sycl/test-e2e/Tracing/usm/queue_single_task_released_pointer.cpp
+++ b/sycl/test-e2e/Tracing/usm/queue_single_task_released_pointer.cpp
@@ -1,6 +1,6 @@
 // UNSUPPORTED: windows || hip_amd
 // RUN: %{build} -o %t.out
-// RUN: not env SYCL_TRACE_TERMINATE_ON_WARNING=1 %{run} sycl-trace --verify %t.out | FileCheck %s
+// RUN: not --crash env SYCL_TRACE_TERMINATE_ON_WARNING=1 %{run} sycl-trace --verify %t.out | FileCheck %s
 
 // Test parameter analysis of USM usage
 

From 3047efb62847cdd3d72b15b1fd0220e921d9106b Mon Sep 17 00:00:00 2001
From: "Garcia Orozco, David" <david.garcia.orozco@intel.com>
Date: Mon, 16 Dec 2024 10:12:02 -0800
Subject: [PATCH 15/26] Remove workflow_run check on e2e binary download

---
 .github/workflows/sycl-linux-run-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml
index 0318be4858444..a755475a09abd 100644
--- a/.github/workflows/sycl-linux-run-tests.yml
+++ b/.github/workflows/sycl-linux-run-tests.yml
@@ -332,7 +332,7 @@ jobs:
           fi
 
     - name: Download E2E Binaries
-      if: inputs.e2e_binaries_artifact != '' && github.event_name != 'workflow_run'
+      if: inputs.e2e_binaries_artifact != ''
       uses: actions/download-artifact@v4
       with:
         name: ${{ inputs.e2e_binaries_artifact }}

From d38dce217143c340b38327ba80f0ed4d0502ba51 Mon Sep 17 00:00:00 2001
From: "Garcia Orozco, David" <david.garcia.orozco@intel.com>
Date: Mon, 16 Dec 2024 12:16:06 -0800
Subject: [PATCH 16/26] Hardcode archive name and compression/decompression
 commands

---
 .github/workflows/sycl-linux-precommit.yml |  2 -
 .github/workflows/sycl-linux-run-tests.yml | 46 ++--------------------
 2 files changed, 3 insertions(+), 45 deletions(-)

diff --git a/.github/workflows/sycl-linux-precommit.yml b/.github/workflows/sycl-linux-precommit.yml
index 6ba4e91e380b3..666355533222c 100644
--- a/.github/workflows/sycl-linux-precommit.yml
+++ b/.github/workflows/sycl-linux-precommit.yml
@@ -118,8 +118,6 @@ jobs:
       sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }}
       sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }}
       e2e_binaries_artifact: sycl_e2e_bin_default
-      e2e_binaries_archive: ${{ needs.build_e2e_tests.outputs.artifact_archive_name }}
-      e2e_binaries_decompress_command: ${{ needs.build_e2e_tests.outputs.artifact_decompress_command }}
 
   test:
     needs: [build, detect_changes, determine_arc_tests]
diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml
index a755475a09abd..9dd61d4d342a9 100644
--- a/.github/workflows/sycl-linux-run-tests.yml
+++ b/.github/workflows/sycl-linux-run-tests.yml
@@ -63,14 +63,6 @@ on:
         type: string
         default: ''
         required: False
-      e2e_binaries_archive:
-        type: string
-        default: ''
-        required: False
-      e2e_binaries_decompress_command:
-        type: string
-        default: ''
-        required: False
 
       upload_artifact:
         type: string
@@ -78,9 +70,6 @@ on:
       artifact_suffix:
         type: string
         default: 'default'
-      artifact_archive_name:
-        type: string
-        default: e2e_binaries.tar.zst
       retention-days:
         description: 'E2E binaries artifact retention period'
         type: string
@@ -105,12 +94,6 @@ on:
         default: 'false'
         required: False
 
-    outputs:
-      artifact_archive_name:
-        value: ${{ jobs.run.outputs.artifact_archive_name }}
-      artifact_decompress_command:
-        value: ${{ jobs.run.outputs.artifact_decompress_command }}
-
   workflow_dispatch:
     inputs:
       runner:
@@ -193,30 +176,8 @@ jobs:
     container:
       image: ${{ inputs.image }}
       options: ${{ inputs.image_options }}
-    outputs:
-      artifact_archive_name: ${{ steps.artifact_info.outputs.ARCHIVE_NAME }}
-      artifact_decompress_command: ${{ steps.artifact_info.outputs.DECOMPRESS }}
     env: ${{ fromJSON(inputs.env) }}
     steps:
-    - name: Deduce artifact archive params
-      if: inputs.upload_artifact == 'true'
-      id: artifact_info
-      run: |
-        NAME="${{inputs.artifact_archive_name}}"
-        if [ -z "$NAME" ]; then
-          NAME=e2e_binaries.tar.zst
-        fi
-        echo ARCHIVE_NAME="$NAME" >> $GITHUB_OUTPUT
-        if [ "${NAME}" != "${NAME%.tar.gz}" ]; then
-          echo COMPRESS="gzip" >> $GITHUB_OUTPUT
-          echo DECOMPRESS="gunzip" >> $GITHUB_OUTPUT
-        elif [ "${NAME}" != "${NAME%.tar.zst}" ]; then
-          echo COMPRESS="zstd -9" >> $GITHUB_OUTPUT
-          echo DECOMPRESS="zstd" >> $GITHUB_OUTPUT
-        else
-          echo "Unsupported extension"
-          exit 1
-        fi
     - name: Reset Intel GPU
       if: inputs.reset_intel_gpu == 'true'
       run: |
@@ -340,8 +301,7 @@ jobs:
       if: inputs.e2e_binaries_artifact != ''
       run: |
         mkdir build-e2e
-        tar -I '${{ inputs.e2e_binaries_decompress_command }}' -xf ${{ inputs.e2e_binaries_archive }} -C build-e2e
-        rm -f ${{ inputs.e2e_binaries_artifact }}
+        tar -I 'zstd' -xf e2e_binaries.tar.zst -C build-e2e
 
     - name: Deduce E2E CMake options
       if: inputs.tests_selector == 'e2e' && inputs.e2e_binaries_artifact == ''
@@ -450,11 +410,11 @@ jobs:
         exit $ret
     - name: Pack e2e binaries
       if: ${{ always() && !cancelled() && inputs.upload_artifact == 'true'}}
-      run: tar -I '${{ steps.artifact_info.outputs.COMPRESS }}' -cf ${{ steps.artifact_info.outputs.ARCHIVE_NAME }} -C ./build-e2e .
+      run: tar -I 'zstd -9' -cf e2e_binaries.tar.zst -C ./build-e2e .
     - name: Upload e2e binaries
       if: ${{ always() && !cancelled() && inputs.upload_artifact == 'true'}}
       uses: actions/upload-artifact@v4
       with:
         name: sycl_e2e_bin_${{ inputs.artifact_suffix }}
-        path: ${{ steps.artifact_info.outputs.ARCHIVE_NAME }}
+        path: e2e_binaries.tar.zst
         retention-days: ${{ inputs.retention-days }}

From 06f0abb9190df73cb35aa0becd33705baa157e81 Mon Sep 17 00:00:00 2001
From: "Garcia Orozco, David" <david.garcia.orozco@intel.com>
Date: Mon, 16 Dec 2024 12:22:48 -0800
Subject: [PATCH 17/26] Replace upload_artifact input with e2e_testing_mode

---
 .github/workflows/sycl-linux-precommit.yml |  6 +++---
 .github/workflows/sycl-linux-run-tests.yml | 18 ++++++++++--------
 2 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/sycl-linux-precommit.yml b/.github/workflows/sycl-linux-precommit.yml
index 666355533222c..b177969834df1 100644
--- a/.github/workflows/sycl-linux-precommit.yml
+++ b/.github/workflows/sycl-linux-precommit.yml
@@ -82,13 +82,12 @@ jobs:
       runner: '["Linux", "build"]'
       image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:alldeps
       image_options: -u 1001
-      extra_lit_opts: --param test-mode=build-only
       ref: ${{ github.sha }}
       merge_ref: ''
       sycl_toolchain_artifact: sycl_linux_default
       sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }}
       sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }}
-      upload_artifact: 'true'
+      e2e_testing_mode: 'build-only'
   run_prebuilt_e2e_tests:
     needs: [build, build_e2e_tests]
     if: ${{ always() && !cancelled() && needs.build.outputs.build_conclusion == 'success' }}
@@ -110,7 +109,7 @@ jobs:
       image: ${{ matrix.image }}
       image_options: ${{ matrix.image_options }}
       target_devices: ${{ matrix.target_devices }}
-      extra_lit_opts: --param test-mode=run-only ${{ matrix.extra_lit_opts }}
+      extra_lit_opts: ${{ matrix.extra_lit_opts }}
       reset_intel_gpu: ${{ matrix.reset_intel_gpu }}
       ref: ${{ github.sha }}
       merge_ref: ''
@@ -118,6 +117,7 @@ jobs:
       sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }}
       sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }}
       e2e_binaries_artifact: sycl_e2e_bin_default
+      e2e_testing_mode: 'run-only'
 
   test:
     needs: [build, detect_changes, determine_arc_tests]
diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml
index 9dd61d4d342a9..d6b30ac6b57d4 100644
--- a/.github/workflows/sycl-linux-run-tests.yml
+++ b/.github/workflows/sycl-linux-run-tests.yml
@@ -64,9 +64,9 @@ on:
         default: ''
         required: False
 
-      upload_artifact:
+      e2e_testing_mode:
         type: string
-        default: 'false'
+        default: 'full'
       artifact_suffix:
         type: string
         default: 'default'
@@ -159,11 +159,11 @@ on:
         options:
           - false
           - true
-      upload_artifact:
+      e2e_testing_mode:
         type: choice
         options:
-          - "false"
-          - "true"
+          - "full"
+          - "build-only"
 
 permissions:
   contents: read
@@ -321,7 +321,9 @@ jobs:
       shell: bash {0}
       if: inputs.tests_selector == 'e2e'
       env:
-        LIT_OPTS: -v --no-progress-bar --show-unsupported --show-pass --show-xfail --max-time 3600 --time-tests --param sycl_devices=${{ inputs.target_devices }} ${{ inputs.extra_lit_opts }}
+        LIT_OPTS: |
+        -v --no-progress-bar --show-unsupported --show-pass --show-xfail --max-time 3600 --time-tests
+        --param test-mode=${{ inputs.e2e_testing_mode }} --param sycl_devices=${{ inputs.target_devices }} ${{ inputs.extra_lit_opts }}
       run: |
         ninja -C build-e2e check-sycl-e2e > e2e.log 2>&1
         exit_code=$?
@@ -409,10 +411,10 @@ jobs:
 
         exit $ret
     - name: Pack e2e binaries
-      if: ${{ always() && !cancelled() && inputs.upload_artifact == 'true'}}
+      if: ${{ always() && !cancelled() && inputs.e2e_testing_mode == 'build-only'}}
       run: tar -I 'zstd -9' -cf e2e_binaries.tar.zst -C ./build-e2e .
     - name: Upload e2e binaries
-      if: ${{ always() && !cancelled() && inputs.upload_artifact == 'true'}}
+      if: ${{ always() && !cancelled() && inputs.e2e_testing_mode == 'build-only'}}
       uses: actions/upload-artifact@v4
       with:
         name: sycl_e2e_bin_${{ inputs.artifact_suffix }}

From 3bb4d202f1f0f347e5b20bcdd70fd363c4f942ef Mon Sep 17 00:00:00 2001
From: "Garcia Orozco, David" <david.garcia.orozco@intel.com>
Date: Mon, 16 Dec 2024 12:26:25 -0800
Subject: [PATCH 18/26] Make LIT_OPTS one line

---
 .github/workflows/sycl-linux-run-tests.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml
index d6b30ac6b57d4..a899455cb0400 100644
--- a/.github/workflows/sycl-linux-run-tests.yml
+++ b/.github/workflows/sycl-linux-run-tests.yml
@@ -321,9 +321,7 @@ jobs:
       shell: bash {0}
       if: inputs.tests_selector == 'e2e'
       env:
-        LIT_OPTS: |
-        -v --no-progress-bar --show-unsupported --show-pass --show-xfail --max-time 3600 --time-tests
-        --param test-mode=${{ inputs.e2e_testing_mode }} --param sycl_devices=${{ inputs.target_devices }} ${{ inputs.extra_lit_opts }}
+        LIT_OPTS: -v --no-progress-bar --show-unsupported --show-pass --show-xfail --max-time 3600 --time-tests --param test-mode=${{ inputs.e2e_testing_mode }} --param sycl_devices=${{ inputs.target_devices }} ${{ inputs.extra_lit_opts }}
       run: |
         ninja -C build-e2e check-sycl-e2e > e2e.log 2>&1
         exit_code=$?

From 3e7d549bb1cf7e8297943f7153ab7e2773993ed1 Mon Sep 17 00:00:00 2001
From: "Garcia Orozco, David" <david.garcia.orozco@intel.com>
Date: Mon, 16 Dec 2024 13:58:47 -0800
Subject: [PATCH 19/26] Try to split LIT_OPTS

---
 .github/workflows/sycl-linux-run-tests.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml
index a899455cb0400..cf19886e95dd0 100644
--- a/.github/workflows/sycl-linux-run-tests.yml
+++ b/.github/workflows/sycl-linux-run-tests.yml
@@ -321,7 +321,9 @@ jobs:
       shell: bash {0}
       if: inputs.tests_selector == 'e2e'
       env:
-        LIT_OPTS: -v --no-progress-bar --show-unsupported --show-pass --show-xfail --max-time 3600 --time-tests --param test-mode=${{ inputs.e2e_testing_mode }} --param sycl_devices=${{ inputs.target_devices }} ${{ inputs.extra_lit_opts }}
+        LIT_OPTS: |
+          -v --no-progress-bar --show-unsupported --show-pass --show-xfail --max-time 3600 --time-tests \
+          --param test-mode=${{ inputs.e2e_testing_mode }} --param sycl_devices=${{ inputs.target_devices }} ${{ inputs.extra_lit_opts }}
       run: |
         ninja -C build-e2e check-sycl-e2e > e2e.log 2>&1
         exit_code=$?

From a49ec589724fc67dcaefdff57ed82e25a17fc846 Mon Sep 17 00:00:00 2001
From: "Garcia Orozco, David" <david.garcia.orozco@intel.com>
Date: Mon, 16 Dec 2024 14:09:33 -0800
Subject: [PATCH 20/26] Revert "Try to split LIT_OPTS"

This reverts commit 3e7d549bb1cf7e8297943f7153ab7e2773993ed1.
---
 .github/workflows/sycl-linux-run-tests.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml
index cf19886e95dd0..a899455cb0400 100644
--- a/.github/workflows/sycl-linux-run-tests.yml
+++ b/.github/workflows/sycl-linux-run-tests.yml
@@ -321,9 +321,7 @@ jobs:
       shell: bash {0}
       if: inputs.tests_selector == 'e2e'
       env:
-        LIT_OPTS: |
-          -v --no-progress-bar --show-unsupported --show-pass --show-xfail --max-time 3600 --time-tests \
-          --param test-mode=${{ inputs.e2e_testing_mode }} --param sycl_devices=${{ inputs.target_devices }} ${{ inputs.extra_lit_opts }}
+        LIT_OPTS: -v --no-progress-bar --show-unsupported --show-pass --show-xfail --max-time 3600 --time-tests --param test-mode=${{ inputs.e2e_testing_mode }} --param sycl_devices=${{ inputs.target_devices }} ${{ inputs.extra_lit_opts }}
       run: |
         ninja -C build-e2e check-sycl-e2e > e2e.log 2>&1
         exit_code=$?

From 39518843626dfbb9e27d8a3d54d3dd42aaa43c07 Mon Sep 17 00:00:00 2001
From: "Garcia Orozco, David" <david.garcia.orozco@intel.com>
Date: Wed, 18 Dec 2024 14:00:46 -0800
Subject: [PATCH 21/26] Formatting changes

---
 .github/workflows/sycl-linux-run-tests.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml
index a899455cb0400..ad571f129903c 100644
--- a/.github/workflows/sycl-linux-run-tests.yml
+++ b/.github/workflows/sycl-linux-run-tests.yml
@@ -63,7 +63,6 @@ on:
         type: string
         default: ''
         required: False
-
       e2e_testing_mode:
         type: string
         default: 'full'
@@ -408,10 +407,10 @@ jobs:
         grep 'exit code: [^0]' -r logs >> $GITHUB_STEP_SUMMARY
 
         exit $ret
-    - name: Pack e2e binaries
+    - name: Pack E2E binaries
       if: ${{ always() && !cancelled() && inputs.e2e_testing_mode == 'build-only'}}
       run: tar -I 'zstd -9' -cf e2e_binaries.tar.zst -C ./build-e2e .
-    - name: Upload e2e binaries
+    - name: Upload E2E binaries
       if: ${{ always() && !cancelled() && inputs.e2e_testing_mode == 'build-only'}}
       uses: actions/upload-artifact@v4
       with:

From cee4ea0ca436cfc9699c9395dead582d38b584be Mon Sep 17 00:00:00 2001
From: "Garcia Orozco, David" <david.garcia.orozco@intel.com>
Date: Wed, 18 Dec 2024 14:18:07 -0800
Subject: [PATCH 22/26] Add descriptions for new parameters

---
 .github/workflows/sycl-linux-run-tests.yml | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml
index ad571f129903c..4eff7a47c6c15 100644
--- a/.github/workflows/sycl-linux-run-tests.yml
+++ b/.github/workflows/sycl-linux-run-tests.yml
@@ -60,17 +60,26 @@ on:
         required: False
 
       e2e_binaries_artifact:
+        description: |
+          By setting this the E2E binaries folder will not be created, rather it
+          will be downloaded and extracted from the specified artifact. When
+          running tests in `run-only` mode this must be provided.
         type: string
         default: ''
         required: False
       e2e_testing_mode:
+        description: |
+          Testing mode to run E2E tests in, can be either `full`, `build-only`
+          or `run-only`. In `build-only` mode an artifact of the E2E binaries
+          will be uploaded.
         type: string
         default: 'full'
       artifact_suffix:
+        description: 'Suffix for E2E binaries artifact that is output when in `build-only`.'
         type: string
         default: 'default'
       retention-days:
-        description: 'E2E binaries artifact retention period'
+        description: 'E2E binaries artifact retention period.'
         type: string
         default: 1
 

From 20006935bb9473676d79bc7faeb03c9f1b94416f Mon Sep 17 00:00:00 2001
From: "Garcia Orozco, David" <david.garcia.orozco@intel.com>
Date: Thu, 19 Dec 2024 06:47:33 -0800
Subject: [PATCH 23/26] Add `XFAIL run-mode` to two tests passing on
 `build-only`

---
 sycl/test-e2e/Matrix/joint_matrix_bf16_fill_k_cache_arg_dim.cpp | 2 +-
 .../Matrix/joint_matrix_bf16_fill_k_cache_runtime_dim.cpp       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/sycl/test-e2e/Matrix/joint_matrix_bf16_fill_k_cache_arg_dim.cpp b/sycl/test-e2e/Matrix/joint_matrix_bf16_fill_k_cache_arg_dim.cpp
index febb1230d91dd..2e27f1d6ce063 100644
--- a/sycl/test-e2e/Matrix/joint_matrix_bf16_fill_k_cache_arg_dim.cpp
+++ b/sycl/test-e2e/Matrix/joint_matrix_bf16_fill_k_cache_arg_dim.cpp
@@ -14,7 +14,7 @@
 
 // Waiting for the commit in IGC to be pulled into the driver to resolve the
 // test.
-// XFAIL: !igc-dev || gpu-intel-dg2
+// XFAIL: (!igc-dev || gpu-intel-dg2) && run-mode
 // XFAIL-TRACKER: CMPLRLLVM-63710
 
 #include "common.hpp"
diff --git a/sycl/test-e2e/Matrix/joint_matrix_bf16_fill_k_cache_runtime_dim.cpp b/sycl/test-e2e/Matrix/joint_matrix_bf16_fill_k_cache_runtime_dim.cpp
index 291852eaa612d..46fd7c7190b3d 100644
--- a/sycl/test-e2e/Matrix/joint_matrix_bf16_fill_k_cache_runtime_dim.cpp
+++ b/sycl/test-e2e/Matrix/joint_matrix_bf16_fill_k_cache_runtime_dim.cpp
@@ -14,7 +14,7 @@
 
 // Waiting for the commit in IGC to be pulled into the driver to resolve the
 // test.
-// XFAIL: !igc-dev || gpu-intel-dg2
+// XFAIL: (!igc-dev || gpu-intel-dg2) && run-mode
 // XFAIL-TRACKER: CMPLRLLVM-63710
 
 #include "common.hpp"

From 457db6396c7849522729fa06c5066e1a119c4526 Mon Sep 17 00:00:00 2001
From: "Garcia Orozco, David" <david.garcia.orozco@intel.com>
Date: Thu, 19 Dec 2024 06:53:56 -0800
Subject: [PATCH 24/26] Use fallback build option in `run-only`

---
 .github/workflows/sycl-linux-precommit.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sycl-linux-precommit.yml b/.github/workflows/sycl-linux-precommit.yml
index 418320b6a76e3..42dcd5e5e0479 100644
--- a/.github/workflows/sycl-linux-precommit.yml
+++ b/.github/workflows/sycl-linux-precommit.yml
@@ -109,7 +109,7 @@ jobs:
       image: ${{ matrix.image }}
       image_options: ${{ matrix.image_options }}
       target_devices: ${{ matrix.target_devices }}
-      extra_lit_opts: ${{ matrix.extra_lit_opts }}
+      extra_lit_opts: --param fallback-to-build-if-requires-build-and-run=True ${{ matrix.extra_lit_opts }}
       reset_intel_gpu: ${{ matrix.reset_intel_gpu }}
       ref: ${{ github.sha }}
       merge_ref: ''

From dabce66c0e1c47ced6ce3ad05dd9641d70083256 Mon Sep 17 00:00:00 2001
From: "Garcia Orozco, David" <david.garcia.orozco@intel.com>
Date: Thu, 19 Dec 2024 07:32:45 -0800
Subject: [PATCH 25/26] Add `XFAIL: run-mode` to one more test that can compile

---
 sycl/test-e2e/Matrix/get_coordinate_ops.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sycl/test-e2e/Matrix/get_coordinate_ops.cpp b/sycl/test-e2e/Matrix/get_coordinate_ops.cpp
index b32145b954422..b869fbff4ffef 100644
--- a/sycl/test-e2e/Matrix/get_coordinate_ops.cpp
+++ b/sycl/test-e2e/Matrix/get_coordinate_ops.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 // REQUIRES: aspect-ext_intel_matrix
 
-// XFAIL: !igc-dev
+// XFAIL: !igc-dev && run-mode
 // XFAIL-TRACKER: GSD-6376
 // REQUIRES-INTEL-DRIVER: lin: 30049
 

From 776e5f04f404e4e97ebede029db7afab2f85691f Mon Sep 17 00:00:00 2001
From: "Garcia Orozco, David" <david.garcia.orozco@intel.com>
Date: Fri, 20 Dec 2024 07:00:09 -0800
Subject: [PATCH 26/26] Use ubuntu 24 latest drivers container for build

---
 .github/workflows/sycl-linux-precommit.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sycl-linux-precommit.yml b/.github/workflows/sycl-linux-precommit.yml
index b9d5e49755e0a..8baa70e8b878b 100644
--- a/.github/workflows/sycl-linux-precommit.yml
+++ b/.github/workflows/sycl-linux-precommit.yml
@@ -80,7 +80,7 @@ jobs:
     with:
       name: Build e2e tests
       runner: '["Linux", "build"]'
-      image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:alldeps
+      image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest
       image_options: -u 1001
       ref: ${{ github.sha }}
       merge_ref: ''