From d057e2d24def7658f151567ff724dcf21a27c15e Mon Sep 17 00:00:00 2001 From: "Sarnie, Nick" Date: Mon, 13 Jan 2025 12:44:09 -0800 Subject: [PATCH 1/4] [CI] Add oneAPI run to nightly Signed-off-by: Sarnie, Nick --- .github/workflows/sycl-containers.yaml | 4 --- .github/workflows/sycl-linux-build.yml | 12 +++++++ .github/workflows/sycl-linux-run-tests.yml | 15 +++++++++ .github/workflows/sycl-nightly.yml | 32 +++++++++++++++++++ .github/workflows/sycl-windows-build.yml | 2 +- .../actions/setup_linux_oneapi_env/action.yml | 22 +++++++++++++ .../ubuntu2404_build_oneapi.Dockerfile | 22 ------------- 7 files changed, 82 insertions(+), 27 deletions(-) create mode 100644 devops/actions/setup_linux_oneapi_env/action.yml delete mode 100644 devops/containers/ubuntu2404_build_oneapi.Dockerfile diff --git a/.github/workflows/sycl-containers.yaml b/.github/workflows/sycl-containers.yaml index 82e9665486250..4c07a978e02fa 100644 --- a/.github/workflows/sycl-containers.yaml +++ b/.github/workflows/sycl-containers.yaml @@ -51,10 +51,6 @@ jobs: file: ubuntu2404_build tag: latest build_args: "" - - name: Build Ubuntu 24.04 oneAPI Docker image - file: ubuntu2404_build_oneapi - tag: latest - build_args: "" - name: Intel Drivers Ubuntu 22.04 Docker image file: ubuntu2204_intel_drivers tag: latest diff --git a/.github/workflows/sycl-linux-build.yml b/.github/workflows/sycl-linux-build.yml index 6221afd481938..cbf53c6e9a5bf 100644 --- a/.github/workflows/sycl-linux-build.yml +++ b/.github/workflows/sycl-linux-build.yml @@ -109,6 +109,9 @@ jobs: build: name: Build + LIT runs-on: [Linux, build] + defaults: + run: + shell: bash container: image: ${{ inputs.build_image }} options: -u 1001:1001 @@ -152,6 +155,9 @@ jobs: ref: ${{ inputs.build_ref || github.sha }} merge_ref: ${{ inputs.merge_ref }} cache_path: "/__w/repo_cache/" + - name: Setup oneAPI env + if: ${{ inputs.cc == 'icx' || inputs.cxx == 'icpx' }} + uses: ./devops/actions/setup_linux_oneapi_env - name: Configure env: CC: ${{ inputs.cc }} @@ -176,6 +182,9 @@ jobs: - name: check-llvm if: always() && !cancelled() && contains(inputs.changes, 'llvm') run: | + if [[ ${{ inputs.cc }} == 'icx' ]] || [[ ${{ inputs.cxx }} == 'icpx' ]]; then + export LIT_FILTER="SYCL" + fi cmake --build $GITHUB_WORKSPACE/build --target check-llvm - name: check-clang if: always() && !cancelled() && contains(inputs.changes, 'clang') @@ -188,6 +197,9 @@ jobs: run: | # TODO consider moving this to Dockerfile. export LD_LIBRARY_PATH=/usr/local/cuda/compat/:/usr/local/cuda/lib64:$LD_LIBRARY_PATH + if [[ ${{ inputs.cc }} == 'icx' ]] || [[ ${{ inputs.cxx }} == 'icpx' ]]; then + export LIT_XFAIL="regression/host_tanpi_double_accuracy.cpp" + fi cmake --build $GITHUB_WORKSPACE/build --target check-sycl - name: check-sycl-unittests if: always() && !cancelled() && contains(inputs.changes, 'sycl') diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml index 3e4066fdc1904..bd35b8e3d62b4 100644 --- a/.github/workflows/sycl-linux-run-tests.yml +++ b/.github/workflows/sycl-linux-run-tests.yml @@ -86,6 +86,9 @@ on: reset_intel_gpu: type: string required: False + is_icx: + type: string + default: 'false' install_igc_driver: type: string required: False @@ -244,7 +247,11 @@ jobs: sudo apt-get install -yqq libllvm14 || true; fi sudo -E bash devops/scripts/install_drivers.sh llvm/devops/dependencies.json ${{ inputs.install_dev_igc_driver == 'true' && 'llvm/devops/dependencies-igc-dev.json --use-dev-igc' || '' }} --all + - name: Setup oneAPI env + if: ${{ inputs.is_icx == 'true' }} + uses: ./devops/actions/setup_linux_oneapi_env - name: Source OneAPI TBB vars.sh + if: ${{ inputs.is_icx == 'false' }} shell: bash run: | # https://github.com/actions/runner/issues/1964 prevents us from using @@ -346,12 +353,20 @@ jobs: env: LIT_OPTS: -v --no-progress-bar --show-unsupported --show-pass --show-xfail --max-time 3600 --time-tests --param test-mode=${{ inputs.e2e_testing_mode }} --param sycl_devices=${{ inputs.target_devices }} ${{ inputs.extra_lit_opts }} run: | + # When building with oneAPI, the oneAPI libsycl is found before the newly compiled one and the search order + # can't be changed by environment variables, so just rename it temporarily. + if [ ${{ inputs.is_icx }} == 'true' ]; then + sudo mv /opt/intel/oneapi/compiler/2025.0/lib/libsycl.so /opt/intel/oneapi/compiler/2025.0/lib/libsycl.so.bak + fi ninja -C build-e2e check-sycl-e2e > e2e.log 2>&1 exit_code=$? cat e2e.log if [ $exit_code -ne 0 ]; then awk '/^Failed Tests|Unexpectedly Passed Tests|Unresolved tests|Testing Time/{flag=1}/FAILED: CMakeFiles/{flag=0}flag' e2e.log >> $GITHUB_STEP_SUMMARY fi + if [ ${{ inputs.is_icx }} == 'true' ]; then + sudo mv /opt/intel/oneapi/compiler/2025.0/lib/libsycl.so.bak /opt/intel/oneapi/compiler/2025.0/lib/libsycl.so + fi exit $exit_code - name: Build SYCL CTS tests if: inputs.tests_selector == 'cts' && inputs.sycl_cts_artifact == '' diff --git a/.github/workflows/sycl-nightly.yml b/.github/workflows/sycl-nightly.yml index 0c3ff68b27efd..aab26034318c1 100644 --- a/.github/workflows/sycl-nightly.yml +++ b/.github/workflows/sycl-nightly.yml @@ -37,6 +37,20 @@ jobs: artifact_archive_name: sycl_linux_shared.tar.zst + ubuntu2404_oneapi_build: + if: github.repository == 'intel/llvm' + uses: ./.github/workflows/sycl-linux-build.yml + secrets: inherit + with: + build_cache_root: "/__w/" + build_cache_suffix: oneapi + build_artifact_suffix: oneapi + build_configure_extra_args: '--hip --cuda --native_cpu --cmake-opt=-DCOMPILER_RT_BUILD_SANITIZERS=Off --cmake-opt=-DCOMPILER_RT_BUILD_MEMPROF=Off' + cc: icx + cxx: icpx + + artifact_archive_name: sycl_linux.tar.zst + ubuntu2204_test: needs: [ubuntu2204_build] if: ${{ always() && !cancelled() && needs.ubuntu2204_build.outputs.build_conclusion == 'success' }} @@ -103,6 +117,24 @@ jobs: sycl_toolchain_archive: ${{ needs.ubuntu2204_build.outputs.artifact_archive_name }} sycl_toolchain_decompress_command: ${{ needs.ubuntu2204_build.outputs.artifact_decompress_command }} + ubuntu2404_oneapi_test: + needs: [ubuntu2404_oneapi_build] + if: ${{ always() && !cancelled() && needs.ubuntu2404_oneapi_build.outputs.build_conclusion == 'success' }} + uses: ./.github/workflows/sycl-linux-run-tests.yml + with: + name: Intel PVC L0 oneAPI + runner: '["Linux", "pvc"]' + is_icx: true + target_devices: level_zero:gpu + extra_lit_opts: -j 50 + image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest + image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN + ref: ${{ github.sha }} + merge_ref: '' + sycl_toolchain_artifact: sycl_linux_oneapi + sycl_toolchain_archive: ${{ needs.ubuntu2404_oneapi_build.outputs.artifact_archive_name }} + sycl_toolchain_decompress_command: ${{ needs.ubuntu2404_oneapi_build.outputs.artifact_decompress_command }} + build-win: uses: ./.github/workflows/sycl-windows-build.yml if: github.repository == 'intel/llvm' diff --git a/.github/workflows/sycl-windows-build.yml b/.github/workflows/sycl-windows-build.yml index de2d452de7a50..77ea70c23c8d1 100644 --- a/.github/workflows/sycl-windows-build.yml +++ b/.github/workflows/sycl-windows-build.yml @@ -149,7 +149,7 @@ jobs: shell: bash run: | if [[ ${{inputs.compiler}} == 'icx' ]]; then - export LIT_FILTER_OUT="host_tanpi_double_accuracy" + export LIT_XFAIL="regression\host_tanpi_double_accuracy.cpp" fi cmake --build build --target check-sycl - name: check-sycl-unittests diff --git a/devops/actions/setup_linux_oneapi_env/action.yml b/devops/actions/setup_linux_oneapi_env/action.yml new file mode 100644 index 0000000000000..aa344ee9e19a8 --- /dev/null +++ b/devops/actions/setup_linux_oneapi_env/action.yml @@ -0,0 +1,22 @@ +name: Linux setup oneAPI env + +runs: + using: "composite" + steps: + - name: Setup oneAPI env + shell: bash + run: | + sudo apt-get --fix-broken -y install + wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor \ + | sudo tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \ + sudo echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" \ + | sudo tee /etc/apt/sources.list.d/oneAPI.list && \ + sudo apt update && sudo apt-get -y install intel-oneapi-compiler-dpcpp-cpp-2025.0 + + env_before=$(env | sort) + source /opt/intel/oneapi/setvars.sh + env_after=$(env | sort) + changed_envvars=$(comm -13 <(echo "$env_before") <(echo "$env_after")) + while IFS= read -r line; do + echo "$line" >> $GITHUB_ENV + done <<< "$changed_envvars" \ No newline at end of file diff --git a/devops/containers/ubuntu2404_build_oneapi.Dockerfile b/devops/containers/ubuntu2404_build_oneapi.Dockerfile deleted file mode 100644 index 8eb4950f6f894..0000000000000 --- a/devops/containers/ubuntu2404_build_oneapi.Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -FROM ghcr.io/intel/llvm/ubuntu2404_build - -ENV DEBIAN_FRONTEND=noninteractive - -USER root - -# Install oneAPI - -RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor \ -| tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \ -echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" \ -| tee /etc/apt/sources.list.d/oneAPI.list - -# Install the ROCM kernel driver and oneAPI -RUN apt update && apt install -yqq intel-oneapi-compiler-dpcpp-cpp-2025.0 && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -USER sycl - -ENTRYPOINT ["/docker_entrypoint.sh"] - From 3f12ef24b9f908a9f85984be82bc75be76bbdef0 Mon Sep 17 00:00:00 2001 From: "Sarnie, Nick" Date: Thu, 16 Jan 2025 11:32:28 -0800 Subject: [PATCH 2/4] attempt fix Signed-off-by: Sarnie, Nick --- .github/workflows/sycl-linux-build.yml | 12 +----------- .github/workflows/sycl-linux-run-tests.yml | 16 ++-------------- .github/workflows/sycl-nightly.yml | 4 ++-- devops/actions/setup_linux_oneapi_env/action.yml | 2 +- 4 files changed, 6 insertions(+), 28 deletions(-) diff --git a/.github/workflows/sycl-linux-build.yml b/.github/workflows/sycl-linux-build.yml index cbf53c6e9a5bf..8971079f08f30 100644 --- a/.github/workflows/sycl-linux-build.yml +++ b/.github/workflows/sycl-linux-build.yml @@ -109,9 +109,6 @@ jobs: build: name: Build + LIT runs-on: [Linux, build] - defaults: - run: - shell: bash container: image: ${{ inputs.build_image }} options: -u 1001:1001 @@ -162,7 +159,6 @@ jobs: env: CC: ${{ inputs.cc }} CXX: ${{ inputs.cxx }} - ARGS: ${{ inputs.build_configure_extra_args }} CUDA_LIB_PATH: "/usr/local/cuda/lib64/stubs" run: | mkdir -p $CCACHE_DIR @@ -170,7 +166,7 @@ jobs: cd $GITHUB_WORKSPACE/build python3 $GITHUB_WORKSPACE/src/buildbot/configure.py -w $GITHUB_WORKSPACE \ -s $GITHUB_WORKSPACE/src -o $GITHUB_WORKSPACE/build -t Release \ - --ci-defaults $ARGS \ + --ci-defaults ${{ inputs.build_configure_extra_args }} \ --cmake-opt=-DCMAKE_C_COMPILER_LAUNCHER=ccache \ --cmake-opt=-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ --cmake-opt="-DLLVM_INSTALL_UTILS=ON" \ @@ -182,9 +178,6 @@ jobs: - name: check-llvm if: always() && !cancelled() && contains(inputs.changes, 'llvm') run: | - if [[ ${{ inputs.cc }} == 'icx' ]] || [[ ${{ inputs.cxx }} == 'icpx' ]]; then - export LIT_FILTER="SYCL" - fi cmake --build $GITHUB_WORKSPACE/build --target check-llvm - name: check-clang if: always() && !cancelled() && contains(inputs.changes, 'clang') @@ -197,9 +190,6 @@ jobs: run: | # TODO consider moving this to Dockerfile. export LD_LIBRARY_PATH=/usr/local/cuda/compat/:/usr/local/cuda/lib64:$LD_LIBRARY_PATH - if [[ ${{ inputs.cc }} == 'icx' ]] || [[ ${{ inputs.cxx }} == 'icpx' ]]; then - export LIT_XFAIL="regression/host_tanpi_double_accuracy.cpp" - fi cmake --build $GITHUB_WORKSPACE/build --target check-sycl - name: check-sycl-unittests if: always() && !cancelled() && contains(inputs.changes, 'sycl') diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml index bd35b8e3d62b4..2d0611db07b2f 100644 --- a/.github/workflows/sycl-linux-run-tests.yml +++ b/.github/workflows/sycl-linux-run-tests.yml @@ -86,7 +86,7 @@ on: reset_intel_gpu: type: string required: False - is_icx: + use_oneapi: type: string default: 'false' install_igc_driver: @@ -246,12 +246,8 @@ jobs: # If libllvm14 is already installed (dev igc docker), still return true. sudo apt-get install -yqq libllvm14 || true; fi - sudo -E bash devops/scripts/install_drivers.sh llvm/devops/dependencies.json ${{ inputs.install_dev_igc_driver == 'true' && 'llvm/devops/dependencies-igc-dev.json --use-dev-igc' || '' }} --all - - name: Setup oneAPI env - if: ${{ inputs.is_icx == 'true' }} - uses: ./devops/actions/setup_linux_oneapi_env + sudo -E bash devops/scripts/install_drivers.sh llvm/devops/dependencies.json ${{ inputs.install_dev_igc_driver == 'true' && 'llvm/devops/dependencies-igc-dev.json --use-dev-igc' || '' }} --all - name: Source OneAPI TBB vars.sh - if: ${{ inputs.is_icx == 'false' }} shell: bash run: | # https://github.com/actions/runner/issues/1964 prevents us from using @@ -353,20 +349,12 @@ jobs: env: LIT_OPTS: -v --no-progress-bar --show-unsupported --show-pass --show-xfail --max-time 3600 --time-tests --param test-mode=${{ inputs.e2e_testing_mode }} --param sycl_devices=${{ inputs.target_devices }} ${{ inputs.extra_lit_opts }} run: | - # When building with oneAPI, the oneAPI libsycl is found before the newly compiled one and the search order - # can't be changed by environment variables, so just rename it temporarily. - if [ ${{ inputs.is_icx }} == 'true' ]; then - sudo mv /opt/intel/oneapi/compiler/2025.0/lib/libsycl.so /opt/intel/oneapi/compiler/2025.0/lib/libsycl.so.bak - fi ninja -C build-e2e check-sycl-e2e > e2e.log 2>&1 exit_code=$? cat e2e.log if [ $exit_code -ne 0 ]; then awk '/^Failed Tests|Unexpectedly Passed Tests|Unresolved tests|Testing Time/{flag=1}/FAILED: CMakeFiles/{flag=0}flag' e2e.log >> $GITHUB_STEP_SUMMARY fi - if [ ${{ inputs.is_icx }} == 'true' ]; then - sudo mv /opt/intel/oneapi/compiler/2025.0/lib/libsycl.so.bak /opt/intel/oneapi/compiler/2025.0/lib/libsycl.so - fi exit $exit_code - name: Build SYCL CTS tests if: inputs.tests_selector == 'cts' && inputs.sycl_cts_artifact == '' diff --git a/.github/workflows/sycl-nightly.yml b/.github/workflows/sycl-nightly.yml index aab26034318c1..c97aaac6db551 100644 --- a/.github/workflows/sycl-nightly.yml +++ b/.github/workflows/sycl-nightly.yml @@ -45,7 +45,7 @@ jobs: build_cache_root: "/__w/" build_cache_suffix: oneapi build_artifact_suffix: oneapi - build_configure_extra_args: '--hip --cuda --native_cpu --cmake-opt=-DCOMPILER_RT_BUILD_SANITIZERS=Off --cmake-opt=-DCOMPILER_RT_BUILD_MEMPROF=Off' + build_configure_extra_args: --cmake-opt=-DCMAKE_C_FLAGS="-no-intel-lib -ffp-model=precise" --cmake-opt=-DCMAKE_CXX_FLAGS="-no-intel-lib -ffp-model=precise" --cmake-opt=-DCOMPILER_RT_BUILD_SANITIZERS=Off --cmake-opt=-DCOMPILER_RT_BUILD_MEMPROF=Off cc: icx cxx: icpx @@ -124,7 +124,7 @@ jobs: with: name: Intel PVC L0 oneAPI runner: '["Linux", "pvc"]' - is_icx: true + use_oneapi: true target_devices: level_zero:gpu extra_lit_opts: -j 50 image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest diff --git a/devops/actions/setup_linux_oneapi_env/action.yml b/devops/actions/setup_linux_oneapi_env/action.yml index aa344ee9e19a8..e54bd13d99b5e 100644 --- a/devops/actions/setup_linux_oneapi_env/action.yml +++ b/devops/actions/setup_linux_oneapi_env/action.yml @@ -19,4 +19,4 @@ runs: changed_envvars=$(comm -13 <(echo "$env_before") <(echo "$env_after")) while IFS= read -r line; do echo "$line" >> $GITHUB_ENV - done <<< "$changed_envvars" \ No newline at end of file + done <<< "$changed_envvars" From ac6b279a9220443a3113aab9355cac604f2fee81 Mon Sep 17 00:00:00 2001 From: "Sarnie, Nick" Date: Tue, 21 Jan 2025 08:23:38 -0800 Subject: [PATCH 3/4] whoops Signed-off-by: Sarnie, Nick --- .github/workflows/sycl-linux-run-tests.yml | 5 +---- .github/workflows/sycl-nightly.yml | 1 - 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml index 2d0611db07b2f..3e4066fdc1904 100644 --- a/.github/workflows/sycl-linux-run-tests.yml +++ b/.github/workflows/sycl-linux-run-tests.yml @@ -86,9 +86,6 @@ on: reset_intel_gpu: type: string required: False - use_oneapi: - type: string - default: 'false' install_igc_driver: type: string required: False @@ -246,7 +243,7 @@ jobs: # If libllvm14 is already installed (dev igc docker), still return true. sudo apt-get install -yqq libllvm14 || true; fi - sudo -E bash devops/scripts/install_drivers.sh llvm/devops/dependencies.json ${{ inputs.install_dev_igc_driver == 'true' && 'llvm/devops/dependencies-igc-dev.json --use-dev-igc' || '' }} --all + sudo -E bash devops/scripts/install_drivers.sh llvm/devops/dependencies.json ${{ inputs.install_dev_igc_driver == 'true' && 'llvm/devops/dependencies-igc-dev.json --use-dev-igc' || '' }} --all - name: Source OneAPI TBB vars.sh shell: bash run: | diff --git a/.github/workflows/sycl-nightly.yml b/.github/workflows/sycl-nightly.yml index c97aaac6db551..bf85bd5a7bc16 100644 --- a/.github/workflows/sycl-nightly.yml +++ b/.github/workflows/sycl-nightly.yml @@ -124,7 +124,6 @@ jobs: with: name: Intel PVC L0 oneAPI runner: '["Linux", "pvc"]' - use_oneapi: true target_devices: level_zero:gpu extra_lit_opts: -j 50 image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest From 02d58ef71e8b137764dc914ae1ac23d204fcf13f Mon Sep 17 00:00:00 2001 From: "Sarnie, Nick" Date: Tue, 21 Jan 2025 09:27:47 -0800 Subject: [PATCH 4/4] build sanitizers and rename archive Signed-off-by: Sarnie, Nick --- .github/workflows/sycl-nightly.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sycl-nightly.yml b/.github/workflows/sycl-nightly.yml index bf85bd5a7bc16..745d7342af1df 100644 --- a/.github/workflows/sycl-nightly.yml +++ b/.github/workflows/sycl-nightly.yml @@ -45,11 +45,11 @@ jobs: build_cache_root: "/__w/" build_cache_suffix: oneapi build_artifact_suffix: oneapi - build_configure_extra_args: --cmake-opt=-DCMAKE_C_FLAGS="-no-intel-lib -ffp-model=precise" --cmake-opt=-DCMAKE_CXX_FLAGS="-no-intel-lib -ffp-model=precise" --cmake-opt=-DCOMPILER_RT_BUILD_SANITIZERS=Off --cmake-opt=-DCOMPILER_RT_BUILD_MEMPROF=Off + build_configure_extra_args: --cmake-opt=-DCMAKE_C_FLAGS="-no-intel-lib -ffp-model=precise" --cmake-opt=-DCMAKE_CXX_FLAGS="-no-intel-lib -ffp-model=precise" cc: icx cxx: icpx - artifact_archive_name: sycl_linux.tar.zst + artifact_archive_name: sycl_linux_oneapi.tar.zst ubuntu2204_test: needs: [ubuntu2204_build]