Merge branch 'sycl' into llvmspirv_pulldown

Author: jsji

.github/workflows/coverity.yml

@@ -46,8 +46,7 @@ jobs:
         python3 $GITHUB_WORKSPACE/src/buildbot/configure.py -w $GITHUB_WORKSPACE \
           -s $GITHUB_WORKSPACE/src -o $GITHUB_WORKSPACE/build -t Release \
           --ci-defaults --hip --cuda \
-          -DNATIVECPU_USE_OCK=Off \
-          -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=SPIRV
+          -DNATIVECPU_USE_OCK=Off
 
     - name: Build with coverity
       run: $GITHUB_WORKSPACE/cov-analysis-linux64-*/bin/cov-build --dir cov-int cmake --build $GITHUB_WORKSPACE/build --target sycl-toolchain

.github/workflows/sycl-linux-build.yml

@@ -1,5 +1,9 @@
 name: Reusable SYCL Linux build workflow
 
+defaults:
+  run:
+    shell: bash
+
 on:
   workflow_call:
     inputs:
@@ -29,39 +33,54 @@ on:
         type: string
         required: false
         default: "--hip --cuda --native_cpu"
-      build_artifact_suffix:
-        type: string
-        required: true
       build_target:
         type: string
         required: false
         default: sycl-toolchain
-      artifact_archive_name:
-        type: string
-        default: llvm_sycl.tar.zst
+
       changes:
         type: string
         description: 'Filter matches for the changed files in the PR'
         default: '[llvm, clang, sycl, llvm_spirv, xptifw, libclc]'
         required: false
+
+      # Artifacts:
+
       retention-days:
         description: 'Artifacts retention period'
         type: string
         default: 3
+
+      toolchain_artifact:
+        type: string
+        required: true
+      toolchain_artifact_filename:
+        type: string
+        default: llvm_sycl.tar.zst
+
+      pack_release:
+        type: string
+        required: false
+
       e2e_binaries_artifact:
         type: string
         required: false
-      pack_release:
+      e2e_binaries_spirv_backend_artifact:
+        type: string
+        required: false
+      e2e_binaries_preview_artifact:
         type: string
         required: false
 
     outputs:
       build_conclusion:
         value: ${{ jobs.build.outputs.build_conclusion }}
-      artifact_archive_name:
-        value: ${{ jobs.build.outputs.artifact_archive_name }}
-      artifact_decompress_command:
-        value: ${{ jobs.build.outputs.artifact_decompress_command }}
+      toolchain_artifact:
+        value: ${{ inputs.toolchain_artifact }}
+      toolchain_artifact_filename:
+        value: ${{ jobs.build.outputs.toolchain_artifact_filename }}
+      toolchain_decompress_command:
+        value: ${{ jobs.build.outputs.toolchain_decompress_command }}
 
   workflow_dispatch:
     inputs:
@@ -98,10 +117,10 @@ on:
         options:
           - "default"
 
-      build_artifact_suffix:
+      toolchain_artifact:
         type: choice
         options:
-          - "default"
+          - "sycl_linux_default"
       retention-days:
         type: choice
         options:
@@ -122,8 +141,8 @@ jobs:
       options: -u 1001:1001
     outputs:
       build_conclusion: ${{ steps.build.conclusion }}
-      artifact_archive_name: ${{ steps.artifact_info.outputs.ARCHIVE_NAME }}
-      artifact_decompress_command: ${{ steps.artifact_info.outputs.DECOMPRESS }}
+      toolchain_artifact_filename: ${{ steps.artifact_info.outputs.ARCHIVE_NAME }}
+      toolchain_decompress_command: ${{ steps.artifact_info.outputs.DECOMPRESS }}
     env:
       CCACHE_DIR: ${{ inputs.build_cache_root }}/build_cache_${{ inputs.build_cache_suffix }}
       CCACHE_MAXSIZE: 8G
@@ -132,7 +151,7 @@ jobs:
       # To reduce number of inputs parameters that is limited for manual triggers.
       id: artifact_info
       run: |
-        NAME="${{inputs.artifact_archive_name}}"
+        NAME="${{inputs.toolchain_artifact_filename}}"
         if [ -z "$NAME" ]; then
           NAME=llvm_sycl.tar.zst
         fi
@@ -163,6 +182,10 @@ jobs:
       if: ${{ inputs.cc == 'icx' || inputs.cxx == 'icpx' }}
       uses: ./devops/actions/setup_linux_oneapi_env
     - name: Configure
+      # Setting `CMAKE_INSTALL_PREFIX` is important because that's the location
+      # where `sycl-linux-run-tests.yml` unpacks the toolchain to. That location
+      # *must* match between build-only and run-only E2E tests runs and we might
+      # be creating pre-built E2E tests' binaries in this workflow.
       env:
         CC: ${{ inputs.cc }}
         CXX: ${{ inputs.cxx }}
@@ -172,26 +195,42 @@ jobs:
         mkdir -p $GITHUB_WORKSPACE/build
         cd $GITHUB_WORKSPACE/build
         python3 $GITHUB_WORKSPACE/src/buildbot/configure.py -w $GITHUB_WORKSPACE \
-          -s $GITHUB_WORKSPACE/src -o $GITHUB_WORKSPACE/build -t Release \
+          -s $GITHUB_WORKSPACE/src -o $GITHUB_WORKSPACE/build \
+          -DCMAKE_INSTALL_PREFIX=$GITHUB_WORKSPACE/toolchain \
+          -t Release \
           --ci-defaults ${{ inputs.build_configure_extra_args }} \
           -DCMAKE_C_COMPILER_LAUNCHER=ccache \
           -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
           -DLLVM_INSTALL_UTILS=ON \
-          -DNATIVECPU_USE_OCK=Off \
-          -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=SPIRV
+          -DNATIVECPU_USE_OCK=Off
     - name: Compile
       id: build
       # Emulate default value for manual dispatch as we've run out of available arguments.
       run: cmake --build $GITHUB_WORKSPACE/build --target ${{ inputs.build_target || 'sycl-toolchain' }}
     - name: check-llvm
       if: always() && !cancelled() && contains(inputs.changes, 'llvm')
+      env:
+        # Can't inline to support possible quotes inside:
+        BUILD_CONFIGURE_EXTRA_ARGS: ${{ inputs.build_configure_extra_args }}
       run: |
+        if [[ "${BUILD_CONFIGURE_EXTRA_ARGS}" == *"--use-libcxx"* ]]; then
+          # https://github.com/llvm/llvm-project/issues/59429
+          export LIT_FILTER_OUT="ExecutionEngine/MCJIT"
+        fi
+
         cmake --build $GITHUB_WORKSPACE/build --target check-llvm
     - name: check-clang
       if: always() && !cancelled() && contains(inputs.changes, 'clang')
+      env:
+        # Can't inline to support possible quotes inside:
+        BUILD_CONFIGURE_EXTRA_ARGS: ${{ inputs.build_configure_extra_args }}
       run: |
         # Can we move this to Dockerfile? Hopefully, noop on Windows.
         export XDG_CACHE_HOME=$GITHUB_WORKSPACE/os_cache
+        if [[ "${BUILD_CONFIGURE_EXTRA_ARGS}" == *"--use-libcxx"* ]]; then
+          # https://github.com/llvm/llvm-project/issues/59428
+          export LIT_FILTER_OUT="(E|e)xception"
+        fi
         cmake --build $GITHUB_WORKSPACE/build --target check-clang
     - name: check-sycl
       if: always() && !cancelled() && contains(inputs.changes, 'sycl')
@@ -234,7 +273,7 @@ jobs:
 
     - name: Pack toolchain release
       if: ${{ always() && !cancelled() && steps.build.conclusion == 'success' && inputs.pack_release == 'true' }}
-      run: tar -I '${{ steps.artifact_info.outputs.COMPRESS }}' -cf ${{ steps.artifact_info.outputs.ARCHIVE_NAME }} -C $GITHUB_WORKSPACE/build/install .
+      run: tar -I '${{ steps.artifact_info.outputs.COMPRESS }}' -cf ${{ steps.artifact_info.outputs.ARCHIVE_NAME }} -C $GITHUB_WORKSPACE/toolchain .
     - name: Upload toolchain release
       if: ${{ always() && !cancelled() && steps.build.conclusion == 'success' && inputs.pack_release == 'true' }}
       uses: actions/upload-artifact@v4
@@ -264,24 +303,20 @@ jobs:
 
     - name: Pack toolchain
       if: ${{ always() && !cancelled() && steps.build.conclusion == 'success' }}
-      run: tar -I '${{ steps.artifact_info.outputs.COMPRESS }}' -cf ${{ steps.artifact_info.outputs.ARCHIVE_NAME }} -C $GITHUB_WORKSPACE/build/install .
+      run: tar -I '${{ steps.artifact_info.outputs.COMPRESS }}' -cf ${{ steps.artifact_info.outputs.ARCHIVE_NAME }} -C $GITHUB_WORKSPACE/toolchain .
     - name: Upload toolchain
       if: ${{ always() && !cancelled() && steps.build.conclusion == 'success' }}
       uses: actions/upload-artifact@v4
       with:
-        name: sycl_linux_${{ inputs.build_artifact_suffix }}
+        name: ${{ inputs.toolchain_artifact }}
         path: ${{ steps.artifact_info.outputs.ARCHIVE_NAME }}
         retention-days: ${{ inputs.retention-days }}
 
-    - name: Copy toolchain
-      if: ${{ inputs.e2e_binaries_artifact && always() && !cancelled() && steps.build.conclusion == 'success' }}
-      # We must have the compiler in the same location as it will be in the E2E
-      # run-tests job.
-      run: cp -r $GITHUB_WORKSPACE/build/install $GITHUB_WORKSPACE/toolchain
 
     - name: Source OneAPI TBB vars.sh
-      if: ${{ inputs.e2e_binaries_artifact && always() && !cancelled() && steps.build.conclusion == 'success' }}
-      shell: bash
+      # Tasks that use the just built toolchain below, need extra environment
+      # setup. No harm in it if all of those tasks would get skipped.
+      if: ${{ always() && !cancelled() && steps.build.conclusion == 'success' }}
       run: |
         # https://github.com/actions/runner/issues/1964 prevents us from using
         # the ENTRYPOINT in the image.
@@ -309,31 +344,31 @@ jobs:
         extra_lit_opts: --param sycl_build_targets="spir;nvidia;amd"
 
     - name: Remove E2E tests before spirv-backend run
-      if: ${{ inputs.e2e_binaries_artifact && always() && !cancelled() && steps.build.conclusion == 'success' }}
+      if: ${{ inputs.e2e_binaries_spirv_backend_artifact && always() && !cancelled() && steps.build.conclusion == 'success' }}
       run: rm -rf build-e2e
 
     - name: Build E2E tests with SPIR-V Backend
-      if: ${{ inputs.e2e_binaries_artifact && always() && !cancelled() && steps.build.conclusion == 'success' }}
+      if: ${{ inputs.e2e_binaries_spirv_backend_artifact && always() && !cancelled() && steps.build.conclusion == 'success' }}
       uses: ./devops/actions/run-tests/e2e
       with:
         ref: ${{ inputs.ref || github.sha }}
         testing_mode: build-only
         target_devices: all
-        binaries_artifact: ${{ inputs.e2e_binaries_artifact }}_spirv_backend
+        binaries_artifact: ${{ inputs.e2e_binaries_spirv_backend_artifact }}
         sycl_compiler: $GITHUB_WORKSPACE/toolchain/bin/clang++
         extra_lit_opts: --param spirv-backend=True
 
     - name: Remove E2E tests before preview-mode run
-      if: ${{ inputs.e2e_binaries_artifact && always() && !cancelled() && steps.build.conclusion == 'success' }}
+      if: ${{ inputs.e2e_binaries_preview_artifact && always() && !cancelled() && steps.build.conclusion == 'success' }}
       run: rm -rf build-e2e
 
     - name: Build E2E tests in Preview Mode
-      if: ${{ inputs.e2e_binaries_artifact && always() && !cancelled() && steps.build.conclusion == 'success' }}
+      if: ${{ inputs.e2e_binaries_preview_artifact && always() && !cancelled() && steps.build.conclusion == 'success' }}
       uses: ./devops/actions/run-tests/e2e
       with:
         ref: ${{ inputs.ref || github.sha }}
         testing_mode: build-only
         target_devices: all
-        binaries_artifact: ${{ inputs.e2e_binaries_artifact }}_preview
+        binaries_artifact: ${{ inputs.e2e_binaries_preview_artifact }}
         sycl_compiler: $GITHUB_WORKSPACE/toolchain/bin/clang++
         extra_lit_opts: --param test-preview-mode=True

.github/workflows/sycl-linux-precommit-aws.yml

@@ -73,9 +73,9 @@ jobs:
       # pre-commit workflow.
       repo_ref: ${{ github.event.workflow_run.referenced_workflows[0].sha }}
 
-      sycl_toolchain_artifact: sycl_linux_default
-      sycl_toolchain_archive: llvm_sycl.tar.zst
-      sycl_toolchain_decompress_command: zstd
+      toolchain_artifact: sycl_linux_default
+      toolchain_artifact_filename: llvm_sycl.tar.zst
+      toolchain_decompress_command: zstd
 
   update-check:
     needs: [create-check, e2e-cuda]

.github/workflows/sycl-linux-precommit.yml

@@ -50,14 +50,17 @@ jobs:
     with:
       build_ref: ${{ github.sha }}
       build_cache_root: "/__w/"
-      build_artifact_suffix: "default"
       build_cache_suffix: "default"
       # Docker image has last nightly pre-installed and added to the PATH
       build_image: "ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest"
       cc: clang
       cxx: clang++
       changes: ${{ needs.detect_changes.outputs.filters }}
-      e2e_binaries_artifact: sycl_e2e_bin_default
+
+      toolchain_artifact: sycl_linux_default
+      e2e_binaries_artifact: e2e_bin
+      e2e_binaries_spirv_backend_artifact: e2e_bin_spirv_backend
+      e2e_binaries_preview_artifact: e2e_bin_preview
 
   # If a PR changes CUDA adapter, run the build on Ubuntu 22.04 as well.
   # Ubuntu 22.04 container has CUDA 12.1 installed while Ubuntu 24.0 image
@@ -70,11 +73,12 @@ jobs:
     with:
       build_ref: ${{ github.sha }}
       build_cache_root: "/__w/"
-      build_artifact_suffix: "ubuntu22"
       build_cache_suffix: "ubuntu22"
       build_image: "ghcr.io/intel/llvm/ubuntu2204_build:latest"
       changes: ${{ needs.detect_changes.outputs.filters }}
 
+      toolchain_artifact: sycl_linux_ubuntu22
+
   run_prebuilt_e2e_tests:
     needs: [build, detect_changes]
     if: ${{ always() && !cancelled() && needs.build.outputs.build_conclusion == 'success' }}
@@ -129,13 +133,13 @@ jobs:
             image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
             target_devices: level_zero:gpu;opencl:gpu;opencl:cpu
             extra_lit_opts: --param spirv-backend=True
-            e2e_binaries_artifact: sycl_e2e_bin_default_spirv_backend
+            e2e_binaries_artifact: e2e_bin_spirv_backend
           - name: Preview Mode
             runner: '["Linux", "gen12"]'
             image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
             target_devices: level_zero:gpu;opencl:gpu;opencl:cpu
             extra_lit_opts: --param test-preview-mode=True
-            e2e_binaries_artifact: sycl_e2e_bin_default_preview
+            e2e_binaries_artifact: e2e_bin_preview
 
     uses: ./.github/workflows/sycl-linux-run-tests.yml
     with:
@@ -146,10 +150,10 @@ jobs:
       target_devices: ${{ matrix.target_devices }}
       extra_lit_opts: ${{ matrix.extra_lit_opts }}
       repo_ref: ${{ github.sha }}
-      sycl_toolchain_artifact: sycl_linux_default
-      sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }}
-      sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }}
-      e2e_binaries_artifact: ${{ matrix.e2e_binaries_artifact || 'sycl_e2e_bin_default' }}
+      toolchain_artifact: ${{ needs.build.outputs.toolchain_artifact }}
+      toolchain_artifact_filename: ${{ needs.build.outputs.toolchain_artifact_filename }}
+      toolchain_decompress_command: ${{ needs.build.outputs.toolchain_decompress_command }}
+      e2e_binaries_artifact: ${{ matrix.e2e_binaries_artifact || 'e2e_bin' }}
       e2e_testing_mode: 'run-only'
 
       # Do not install drivers on AMD and CUDA runners.
@@ -185,9 +189,9 @@ jobs:
       benchmark_preset: 'Minimal'
       benchmark_dry_run: true
       repo_ref: ${{ github.sha }}
-      sycl_toolchain_artifact: sycl_linux_default
-      sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }}
-      sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }}
+      toolchain_artifact: ${{ needs.build.outputs.toolchain_artifact }}
+      toolchain_artifact_filename: ${{ needs.build.outputs.toolchain_artifact_filename }}
+      toolchain_decompress_command: ${{ needs.build.outputs.toolchain_decompress_command }}
 
   test-perf:
     needs: [build, detect_changes]
@@ -224,6 +228,6 @@ jobs:
 
       repo_ref: ${{ github.sha }}
 
-      sycl_toolchain_artifact: sycl_linux_default
-      sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }}
-      sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }}
+      toolchain_artifact: ${{ needs.build.outputs.toolchain_artifact }}
+      toolchain_artifact_filename: ${{ needs.build.outputs.toolchain_artifact_filename }}
+      toolchain_decompress_command: ${{ needs.build.outputs.toolchain_decompress_command }}