[CI] Add oneAPI run to nightly

Signed-off-by: Sarnie, Nick <[email protected]>

Author: sarnex

.github/workflows/sycl-containers.yaml

@@ -51,10 +51,6 @@ jobs:
             file: ubuntu2404_build
             tag: latest
             build_args: ""
-          - name: Build Ubuntu 24.04 oneAPI Docker image
-            file: ubuntu2404_build_oneapi
-            tag: latest
-            build_args: ""
           - name: Intel Drivers Ubuntu 22.04 Docker image
             file: ubuntu2204_intel_drivers
             tag: latest

.github/workflows/sycl-linux-build.yml

@@ -109,6 +109,9 @@ jobs:
   build:
     name: Build + LIT
     runs-on: [Linux, build]
+    defaults:
+      run:
+         shell: bash
     container:
       image: ${{ inputs.build_image }}
       options: -u 1001:1001
@@ -152,6 +155,9 @@ jobs:
         ref: ${{ inputs.build_ref || github.sha }}
         merge_ref: ${{ inputs.merge_ref }}
         cache_path: "/__w/repo_cache/"
+    - name: Setup oneAPI env
+      if: ${{ inputs.cc == 'icx' || inputs.cxx == 'icpx' }}
+      uses: ./devops/actions/setup_linux_oneapi_env
     - name: Configure
       env:
         CC: ${{ inputs.cc }}
@@ -176,6 +182,9 @@ jobs:
     - name: check-llvm
       if: always() && !cancelled() && contains(inputs.changes, 'llvm')
       run: |
+        if [[ ${{ inputs.cc }} == 'icx' ]] || [[ ${{ inputs.cxx }} == 'icpx' ]]; then
+          export LIT_FILTER="SYCL"
+        fi
         cmake --build $GITHUB_WORKSPACE/build --target check-llvm
     - name: check-clang
       if: always() && !cancelled() && contains(inputs.changes, 'clang')
@@ -188,6 +197,9 @@ jobs:
       run: |
         # TODO consider moving this to Dockerfile.
         export LD_LIBRARY_PATH=/usr/local/cuda/compat/:/usr/local/cuda/lib64:$LD_LIBRARY_PATH
+        if [[ ${{ inputs.cc }} == 'icx' ]] || [[ ${{ inputs.cxx }} == 'icpx' ]]; then
+           export LIT_XFAIL="regression/host_tanpi_double_accuracy.cpp"
+        fi
         cmake --build $GITHUB_WORKSPACE/build --target check-sycl
     - name: check-sycl-unittests
       if: always() && !cancelled() && contains(inputs.changes, 'sycl')

.github/workflows/sycl-linux-run-tests.yml

@@ -86,6 +86,9 @@ on:
       reset_intel_gpu:
         type: string
         required: False
+      is_icx:
+        type: string
+        default: 'false'
       install_igc_driver:
         type: string
         required: False
@@ -244,7 +247,11 @@ jobs:
             sudo apt-get install -yqq libllvm14 || true;
         fi
         sudo -E bash devops/scripts/install_drivers.sh llvm/devops/dependencies.json ${{ inputs.install_dev_igc_driver == 'true' && 'llvm/devops/dependencies-igc-dev.json --use-dev-igc' || '' }} --all
+    - name: Setup oneAPI env
+      if: ${{ inputs.is_icx == 'true' }}
+      uses: ./devops/actions/setup_linux_oneapi_env
     - name: Source OneAPI TBB vars.sh
+      if: ${{ inputs.is_icx == 'false' }}
       shell: bash
       run: |
         # https://github.com/actions/runner/issues/1964 prevents us from using
@@ -346,12 +353,20 @@ jobs:
       env:
         LIT_OPTS: -v --no-progress-bar --show-unsupported --show-pass --show-xfail --max-time 3600 --time-tests --param test-mode=${{ inputs.e2e_testing_mode }} --param sycl_devices=${{ inputs.target_devices }} ${{ inputs.extra_lit_opts }}
       run: |
+        # When building with oneAPI, the oneAPI libsycl is found before the newly compiled one and the search order
+        # can't be changed by environment variables, so just rename it temporarily.
+        if [ ${{ inputs.is_icx }} == 'true' ]; then
+          sudo mv /opt/intel/oneapi/compiler/2025.0/lib/libsycl.so /opt/intel/oneapi/compiler/2025.0/lib/libsycl.so.bak
+        fi
         ninja -C build-e2e check-sycl-e2e > e2e.log 2>&1
         exit_code=$?
         cat e2e.log
         if [ $exit_code -ne 0 ]; then
           awk '/^Failed Tests|Unexpectedly Passed Tests|Unresolved tests|Testing Time/{flag=1}/FAILED: CMakeFiles/{flag=0}flag' e2e.log >> $GITHUB_STEP_SUMMARY
         fi
+        if [ ${{ inputs.is_icx }} == 'true' ]; then
+          sudo mv /opt/intel/oneapi/compiler/2025.0/lib/libsycl.so.bak /opt/intel/oneapi/compiler/2025.0/lib/libsycl.so
+        fi
         exit $exit_code
     - name: Build SYCL CTS tests
       if: inputs.tests_selector == 'cts' && inputs.sycl_cts_artifact == ''

.github/workflows/sycl-nightly.yml

@@ -37,6 +37,20 @@ jobs:
 
       artifact_archive_name: sycl_linux_shared.tar.zst
 
+  ubuntu2404_oneapi_build:
+    if: github.repository == 'intel/llvm'
+    uses: ./.github/workflows/sycl-linux-build.yml
+    secrets: inherit
+    with:
+      build_cache_root: "/__w/"
+      build_cache_suffix: oneapi
+      build_artifact_suffix: oneapi
+      build_configure_extra_args: '--hip --cuda --native_cpu --cmake-opt=-DCOMPILER_RT_BUILD_SANITIZERS=Off --cmake-opt=-DCOMPILER_RT_BUILD_MEMPROF=Off'
+      cc: icx
+      cxx: icpx
+
+      artifact_archive_name: sycl_linux.tar.zst
+
   ubuntu2204_test:
     needs: [ubuntu2204_build]
     if: ${{ always() && !cancelled() && needs.ubuntu2204_build.outputs.build_conclusion == 'success' }}
@@ -103,6 +117,24 @@ jobs:
       sycl_toolchain_archive: ${{ needs.ubuntu2204_build.outputs.artifact_archive_name }}
       sycl_toolchain_decompress_command: ${{ needs.ubuntu2204_build.outputs.artifact_decompress_command }}
 
+  ubuntu2404_oneapi_test:
+    needs: [ubuntu2404_oneapi_build]
+    if: ${{ always() && !cancelled() && needs.ubuntu2404_oneapi_build.outputs.build_conclusion == 'success' }}
+    uses: ./.github/workflows/sycl-linux-run-tests.yml
+    with:
+      name: Intel PVC L0 oneAPI
+      runner: '["Linux", "pvc"]'
+      is_icx: true
+      target_devices: level_zero:gpu
+      extra_lit_opts: -j 50
+      image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest
+      image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
+      ref: ${{ github.sha }}
+      merge_ref: ''
+      sycl_toolchain_artifact: sycl_linux_oneapi
+      sycl_toolchain_archive: ${{ needs.ubuntu2404_oneapi_build.outputs.artifact_archive_name }}
+      sycl_toolchain_decompress_command: ${{ needs.ubuntu2404_oneapi_build.outputs.artifact_decompress_command }}
+
   build-win:
     uses: ./.github/workflows/sycl-windows-build.yml
     if: github.repository == 'intel/llvm'

.github/workflows/sycl-windows-build.yml

@@ -149,7 +149,7 @@ jobs:
       shell: bash
       run: |
          if [[ ${{inputs.compiler}} == 'icx' ]]; then
-           export LIT_FILTER_OUT="host_tanpi_double_accuracy"
+           export LIT_XFAIL="regression\host_tanpi_double_accuracy.cpp"
          fi
          cmake --build build --target check-sycl
     - name: check-sycl-unittests

devops/actions/setup_linux_oneapi_env/action.yml

@@ -0,0 +1,22 @@
+name: Linux setup oneAPI env
+
+runs:
+  using: "composite"
+  steps:
+    - name: Setup oneAPI env
+      shell: bash
+      run: |
+        sudo apt-get --fix-broken -y install
+        wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor \
+        | sudo tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \
+        sudo echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" \
+        | sudo tee /etc/apt/sources.list.d/oneAPI.list && \
+        sudo apt update && sudo apt-get -y install intel-oneapi-compiler-dpcpp-cpp-2025.0
+
+        env_before=$(env | sort)         
+        source /opt/intel/oneapi/setvars.sh
+        env_after=$(env | sort)
+        changed_envvars=$(comm -13 <(echo "$env_before") <(echo "$env_after"))
+        while IFS= read -r line; do
+          echo "$line" >> $GITHUB_ENV
+        done <<< "$changed_envvars"