diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml new file mode 100644 index 0000000000000..83138e4a67836 --- /dev/null +++ b/.github/workflows/coverity.yml @@ -0,0 +1,83 @@ +name: Coverity +on: + workflow_dispatch: + schedule: + - cron: '0 0 * * 0' + +permissions: read-all + +jobs: + coverity: + if: github.repository == 'intel/llvm' + name: Coverity + runs-on: [Linux, build] + container: + image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:alldeps + options: -u 1001:1001 + + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: | + devops/actions + + - name: Register cleanup after job is finished + uses: ./devops/actions/cleanup + + - uses: ./devops/actions/cached_checkout + with: + path: src + ref: ${{ github.sha }} + cache_path: "/__w/repo_cache/" + + - name: Get coverity tool + run: | + wget https://scan.coverity.com/download/linux64 --post-data "token=${{ secrets.COVERITY_TOKEN }}&project=intel%2Fllvm" -O coverity_tool.tgz + tar -xf coverity_tool.tgz + + - name: Configure + env: + CC: gcc + CXX: g++ + CUDA_LIB_PATH: "/usr/local/cuda/lib64/stubs" + run: | + mkdir -p $GITHUB_WORKSPACE/build + cd $GITHUB_WORKSPACE/build + python3 $GITHUB_WORKSPACE/src/buildbot/configure.py -w $GITHUB_WORKSPACE \ + -s $GITHUB_WORKSPACE/src -o $GITHUB_WORKSPACE/build -t Release \ + --ci-defaults --hip --cuda \ + -DNATIVECPU_USE_OCK=Off \ + -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=SPIRV + + - name: Build with coverity + run: $GITHUB_WORKSPACE/cov-analysis-linux64-*/bin/cov-build --dir cov-int cmake --build $GITHUB_WORKSPACE/build --target sycl-toolchain + + - name: Compress results + run: tar -I pigz -cf intel_llvm.tgz cov-int + + - name: Submit build + run: | + # Initialize a build. Fetch a cloud upload url. + curl -X POST \ + -d version="sycl: ${{ github.sha }}" \ + -d description="Regular build" \ + -d email=${{ secrets.COVERITY_EMAIL }} \ + -d token=${{ secrets.COVERITY_TOKEN }} \ + -d file_name="intel_llvm.tgz" \ + https://scan.coverity.com/projects/31090/builds/init \ + | tee response + + # Store response data to use in later stages. + upload_url=$(jq -r '.url' response) + build_id=$(jq -r '.build_id' response) + + # Upload the tarball to the Cloud. + curl -X PUT \ + --header 'Content-Type: application/json' \ + --upload-file $PWD/intel_llvm.tgz \ + $upload_url + + # Trigger the build on Scan. + curl -X PUT \ + -d token=${{ secrets.COVERITY_TOKEN }} \ + https://scan.coverity.com/projects/31090/builds/$build_id/enqueue diff --git a/.github/workflows/email-check.yaml b/.github/workflows/email-check.yaml index 4d4c419595b5c..4b70bf2db2cbb 100644 --- a/.github/workflows/email-check.yaml +++ b/.github/workflows/email-check.yaml @@ -2,8 +2,9 @@ name: "Check for private emails used in PRs" on: pull_request: - types: - - opened + branches: + - sycl + - sycl-rel-** permissions: contents: read diff --git a/.github/workflows/pr-code-format.yml b/.github/workflows/pr-code-format.yml index 67c80441dd3a9..789a8f92b98a6 100644 --- a/.github/workflows/pr-code-format.yml +++ b/.github/workflows/pr-code-format.yml @@ -8,7 +8,6 @@ on: branches: - main - sycl - - sycl-devops-pr/** - sycl-rel-** - 'users/**' @@ -35,7 +34,7 @@ jobs: - name: Get changed files id: changed-files - uses: tj-actions/changed-files@v45 + uses: step-security/changed-files@3dbe17c78367e7d60f00d78ae6781a35be47b4a1 # v45.0.1 with: separator: "," skip_initial_fetch: true @@ -63,10 +62,10 @@ jobs: - name: Install clang-format uses: aminya/setup-cpp@v1 with: - clangformat: 18.1.7 + clangformat: 19.1.6 - name: Setup Python env - uses: actions/setup-python@v5 + uses: actions/setup-python@v5.4.0 with: python-version: '3.11' cache: 'pip' diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index b6bd85f3d556b..d8e290896bfab 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -36,7 +36,7 @@ jobs: persist-credentials: false - name: "Run analysis" - uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # v2.4.0 + uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1 with: results_file: results.sarif results_format: sarif @@ -57,6 +57,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@662472033e021d55d94146f66f6058822b0b39fd # v3.27.0 + uses: github/codeql-action/upload-sarif@48ab28a6f5dbc2a99bf1e0131198dd8f1df78169 # v3.28.0 with: sarif_file: results.sarif diff --git a/.github/workflows/sycl-aws.yml b/.github/workflows/sycl-aws.yml index 869b734df541d..47e0cc602245c 100644 --- a/.github/workflows/sycl-aws.yml +++ b/.github/workflows/sycl-aws.yml @@ -15,15 +15,19 @@ on: description: "JSON string with array of objects with aws-type, runs-on, aws-ami, aws-spot, aws-disk, aws-timebomb, one-job properties" type: string default: '[{"runs-on":"aws_cuda-${{ github.run_id }}-${{ github.run_attempt }}","aws-ami":"ami-01cb0573cb039ab24","aws-type":["g5.2xlarge","g5.4xlarge"],"aws-disk":"/dev/sda1:64","aws-spot":"false"}]' + ref: + type: string + required: false jobs: aws: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest environment: aws steps: - uses: actions/checkout@v4 with: sparse-checkout: devops/actions/aws-ec2 + ref: ${{ inputs.ref || github.sha }} - run: npm install ./devops/actions/aws-ec2 - uses: ./devops/actions/aws-ec2 with: diff --git a/.github/workflows/sycl-containers-igc-dev.yaml b/.github/workflows/sycl-containers-igc-dev.yaml index 8bf8def9d8148..76a9b4e544f4a 100644 --- a/.github/workflows/sycl-containers-igc-dev.yaml +++ b/.github/workflows/sycl-containers-igc-dev.yaml @@ -6,11 +6,13 @@ on: - sycl paths: - 'devops/actions/build_container/**' + - 'devops/scripts/**' - 'devops/dependencies-igc-dev.json' - '.github/workflows/sycl-containers-igc-dev.yaml' pull_request: paths: - 'devops/actions/build_container/**' + - 'devops/scripts/**' - 'devops/dependencies-igc-dev.json' - '.github/workflows/sycl-containers-igc-dev.yaml' @@ -20,18 +22,18 @@ jobs: build_and_push_images: if: github.repository == 'intel/llvm' name: Build and Push IGC Dev Docker Images - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest permissions: packages: write strategy: matrix: include: - - name: Intel Drivers Ubuntu 22.04 Docker image with dev IGC - dockerfile: ubuntu2204_intel_drivers_igc_dev - imagefile: ubuntu2204_intel_drivers + - name: Intel Drivers Ubuntu 24.04 Docker image with dev IGC + dockerfile: ubuntu2404_intel_drivers_igc_dev + imagefile: ubuntu2404_intel_drivers tag: devigc build_args: | - "use_latest=false" + "use_unstable_driver=false" "use_igc_dev=true" steps: - name: Checkout diff --git a/.github/workflows/sycl-containers.yaml b/.github/workflows/sycl-containers.yaml index 7abeee17df01b..7ba768a79701c 100644 --- a/.github/workflows/sycl-containers.yaml +++ b/.github/workflows/sycl-containers.yaml @@ -29,7 +29,7 @@ jobs: build_and_push_images: if: github.repository == 'intel/llvm' name: Build and Push Docker Images - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest permissions: packages: write strategy: @@ -39,25 +39,44 @@ jobs: file: ubuntu2204_base tag: latest build_args: "" - - name: Build Ubuntu Docker image + - name: Base Ubuntu 24.04 Docker image + file: ubuntu2404_base + tag: latest + build_args: "" + - name: Build Ubuntu 22.04 Docker image file: ubuntu2204_build tag: latest build_args: "" + - name: Build Ubuntu 24.04 Docker image + file: ubuntu2404_build + tag: latest + build_args: "" - name: Intel Drivers Ubuntu 22.04 Docker image file: ubuntu2204_intel_drivers tag: latest - build_args: "use_latest=false" - - name: Intel Drivers (unstable) Ubuntu 22.04 Docker image - file: ubuntu2204_intel_drivers + build_args: "use_unstable_driver=false" + - name: Intel Drivers Ubuntu 24.04 Docker image + file: ubuntu2404_intel_drivers + tag: latest + build_args: "use_unstable_driver=false" + - name: Intel Drivers (unstable) Ubuntu 24.04 Docker image + file: ubuntu2404_intel_drivers tag: unstable - build_args: "use_latest=true" + build_args: "use_unstable_driver=true" - name: Build + Intel Drivers Ubuntu 22.04 Docker image file: ubuntu2204_intel_drivers tag: alldeps build_args: | base_image=ghcr.io/intel/llvm/ubuntu2204_build base_tag=latest - use_latest=false + use_unstable_driver=false + - name: Build + Intel Drivers Ubuntu 24.04 Docker image + file: ubuntu2404_intel_drivers + tag: alldeps + build_args: | + base_image=ghcr.io/intel/llvm/ubuntu2404_build + base_tag=latest + use_unstable_driver=false steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/sycl-detect-changes.yml b/.github/workflows/sycl-detect-changes.yml index 396cd424ff4ac..33152fb3f55e1 100644 --- a/.github/workflows/sycl-detect-changes.yml +++ b/.github/workflows/sycl-detect-changes.yml @@ -72,6 +72,9 @@ jobs: - 'sycl/include/sycl/ext/oneapi/experimental/invoke_simd.hpp' - 'sycl/include/sycl/ext/oneapi/experimental/detail/invoke_simd_types.hpp' - 'sycl/test-e2e/(ESIMD|InvokeSimd)/**' + ur: + - 'unified-runtime/**' + - .github/workflows/ur-* - name: Set output id: result @@ -84,7 +87,7 @@ jobs: return '${{ steps.changes.outputs.changes }}'; } // Treat everything as changed for huge PRs. - return ["llvm", "llvm_spirv", "clang", "sycl_jit", "xptifw", "libclc", "sycl", "ci", "esimd"]; + return ["llvm", "llvm_spirv", "clang", "sycl_jit", "xptifw", "libclc", "sycl", "ci", "esimd", "ur"]; - run: echo '${{ steps.result.outputs.result }}' diff --git a/.github/workflows/sycl-issues-ping-assignee.yml b/.github/workflows/sycl-issues-ping-assignee.yml index c809d67586e71..adb4c2e5b658d 100644 --- a/.github/workflows/sycl-issues-ping-assignee.yml +++ b/.github/workflows/sycl-issues-ping-assignee.yml @@ -20,7 +20,7 @@ jobs: run: permissions: issues: write - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} REPO: ${{ github.repository }} @@ -39,7 +39,7 @@ jobs: - name: Filter issues and ping run: | - days_to_stale=60 + days_to_stale=90 current_time=$(date +%s) cat issues.json | jq -c '.[]' | while read -r issue; do diff --git a/.github/workflows/sycl-linux-build.yml b/.github/workflows/sycl-linux-build.yml index 0dc956dfea752..b76609f9d66bd 100644 --- a/.github/workflows/sycl-linux-build.yml +++ b/.github/workflows/sycl-linux-build.yml @@ -14,7 +14,7 @@ on: build_image: type: string required: false - default: "ghcr.io/intel/llvm/ubuntu2204_build:latest" + default: "ghcr.io/intel/llvm/ubuntu2404_intel_drivers:alldeps" build_ref: type: string required: false @@ -40,16 +40,13 @@ on: description: 'Filter matches for the changed files in the PR' default: '[llvm, clang, sycl, llvm_spirv, xptifw, libclc, libdevice]' required: false - merge_ref: - description: | - Commit-ish to merge post-checkout if non-empty. Must be reachable from - the default_branch input paramter. - type: string - default: 'FETCH_HEAD' retention-days: description: 'Artifacts retention period' type: string default: 3 + e2e_binaries_artifact: + type: string + required: False outputs: build_conclusion: @@ -70,7 +67,7 @@ on: build_image: type: choice options: - - "ghcr.io/intel/llvm/sycl_ubuntu2204_nightly:build" + - 'ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest' cc: type: choice options: @@ -150,13 +147,14 @@ jobs: with: path: src ref: ${{ inputs.build_ref || github.sha }} - merge_ref: ${{ inputs.merge_ref }} cache_path: "/__w/repo_cache/" + - name: Setup oneAPI env + if: ${{ inputs.cc == 'icx' || inputs.cxx == 'icpx' }} + uses: ./devops/actions/setup_linux_oneapi_env - name: Configure env: CC: ${{ inputs.cc }} CXX: ${{ inputs.cxx }} - ARGS: ${{ inputs.build_configure_extra_args }} CUDA_LIB_PATH: "/usr/local/cuda/lib64/stubs" run: | mkdir -p $CCACHE_DIR @@ -164,15 +162,15 @@ jobs: cd $GITHUB_WORKSPACE/build python3 $GITHUB_WORKSPACE/src/buildbot/configure.py -w $GITHUB_WORKSPACE \ -s $GITHUB_WORKSPACE/src -o $GITHUB_WORKSPACE/build -t Release \ - --ci-defaults $ARGS \ - --cmake-opt=-DCMAKE_C_COMPILER_LAUNCHER=ccache \ - --cmake-opt=-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - --cmake-opt="-DLLVM_INSTALL_UTILS=ON" \ - --cmake-opt="-DNATIVECPU_USE_OCK=Off" \ - --cmake-opt="-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=SPIRV" + --ci-defaults ${{ inputs.build_configure_extra_args }} \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DLLVM_INSTALL_UTILS=ON \ + -DNATIVECPU_USE_OCK=Off \ + -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=SPIRV - name: Compile id: build - run: cmake --build $GITHUB_WORKSPACE/build + run: cmake --build $GITHUB_WORKSPACE/build --target sycl-toolchain - name: check-llvm if: always() && !cancelled() && contains(inputs.changes, 'llvm') run: | @@ -247,3 +245,53 @@ jobs: name: sycl_linux_${{ inputs.build_artifact_suffix }} path: ${{ steps.artifact_info.outputs.ARCHIVE_NAME }} retention-days: ${{ inputs.retention-days }} + + - name: Copy toolchain + if: ${{ inputs.e2e_binaries_artifact && always() && !cancelled() && steps.build.conclusion == 'success' }} + # We must have the compiler in the same location as it will be in the E2E + # run-tests job. + run: cp -r $GITHUB_WORKSPACE/build/install $GITHUB_WORKSPACE/toolchain + + - name: Source OneAPI TBB vars.sh + if: ${{ inputs.e2e_binaries_artifact && always() && !cancelled() && steps.build.conclusion == 'success' }} + shell: bash + run: | + # https://github.com/actions/runner/issues/1964 prevents us from using + # the ENTRYPOINT in the image. + env | sort > env_before + if [ -e /runtimes/oneapi-tbb/env/vars.sh ]; then + source /runtimes/oneapi-tbb/env/vars.sh; + elif [ -e /opt/runtimes/oneapi-tbb/env/vars.sh ]; then + source /opt/runtimes/oneapi-tbb/env/vars.sh; + else + echo "no TBB vars in /opt/runtimes or /runtimes"; + fi + env | sort > env_after + comm -13 env_before env_after >> $GITHUB_ENV + rm env_before env_after + + - name: Build E2E tests + if: ${{ inputs.e2e_binaries_artifact && always() && !cancelled() && steps.build.conclusion == 'success' }} + uses: ./devops/actions/run-tests/e2e + with: + ref: ${{ inputs.ref || github.sha }} + testing_mode: build-only + target_devices: all + binaries_artifact: ${{ inputs.e2e_binaries_artifact }} + cxx_compiler: $GITHUB_WORKSPACE/toolchain/bin/clang++ + extra_lit_opts: --param sycl_build_targets="spir;nvidia;amd" + + - name: Remove E2E tests before spirv-backend run + if: ${{ inputs.e2e_binaries_artifact && always() && !cancelled() && steps.build.conclusion == 'success' }} + run: rm -rf build-e2e + + - name: Build E2E tests with SPIR-V Backend + if: ${{ inputs.e2e_binaries_artifact && always() && !cancelled() && steps.build.conclusion == 'success' }} + uses: ./devops/actions/run-tests/e2e + with: + ref: ${{ inputs.ref || github.sha }} + testing_mode: build-only + target_devices: all + binaries_artifact: ${{ inputs.e2e_binaries_artifact }}_spirv_backend + cxx_compiler: $GITHUB_WORKSPACE/toolchain/bin/clang++ + extra_lit_opts: --param spirv-backend=True diff --git a/.github/workflows/sycl-linux-precommit-aws.yml b/.github/workflows/sycl-linux-precommit-aws.yml index 07d309900d764..e8bd368556aa9 100644 --- a/.github/workflows/sycl-linux-precommit-aws.yml +++ b/.github/workflows/sycl-linux-precommit-aws.yml @@ -45,7 +45,7 @@ jobs: }) aws-start: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest environment: aws steps: - uses: actions/checkout@v4 @@ -68,11 +68,10 @@ jobs: runner: '["aws_cuda-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}"]' image: ghcr.io/intel/llvm/ubuntu2204_build:latest image_options: -u 1001 --gpus all --cap-add SYS_ADMIN --env NVIDIA_DISABLE_REQUIRE=1 - target_devices: ext_oneapi_cuda:gpu + target_devices: cuda:gpu # No idea why but that seems to work and be in sync with the main # pre-commit workflow. - ref: ${{ github.event.workflow_run.referenced_workflows[0].sha }} - merge_ref: '' + repo_ref: ${{ github.event.workflow_run.referenced_workflows[0].sha }} sycl_toolchain_artifact: sycl_linux_default sycl_toolchain_archive: llvm_sycl.tar.zst @@ -107,7 +106,7 @@ jobs: aws-stop: needs: [aws-start, e2e-cuda] if: always() - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest environment: aws steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/sycl-linux-precommit.yml b/.github/workflows/sycl-linux-precommit.yml index 231d49b0500c5..9b8220f48ef57 100644 --- a/.github/workflows/sycl-linux-precommit.yml +++ b/.github/workflows/sycl-linux-precommit.yml @@ -7,12 +7,12 @@ on: pull_request: branches: - sycl - - sycl-devops-pr/** - sycl-rel-** # Do not run builds if changes are only in the following locations paths-ignore: - '.github/ISSUE_TEMPLATE/**' - '.github/CODEOWNERS' + - 'sycl/cts_exclude_filter/**' - 'sycl/doc/**' - 'sycl/gdb/**' - 'clang/docs/**' @@ -21,6 +21,7 @@ on: - '.github/workflows/sycl-windows-*.yml' - '.github/workflows/sycl-macos-*.yml' - '.github/workflows/sycl-nightly.yml' + - '.github/workflows/sycl-rel-nightly.yml' - 'devops/containers/**' - 'devops/actions/build_container/**' @@ -41,108 +42,93 @@ jobs: uses: ./.github/workflows/sycl-linux-build.yml with: build_ref: ${{ github.sha }} - merge_ref: '' build_cache_root: "/__w/" build_artifact_suffix: "default" build_cache_suffix: "default" + # Docker image has last nightly pre-installed and added to the PATH + build_image: "ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest" + cc: clang + cxx: clang++ changes: ${{ needs.detect_changes.outputs.filters }} + e2e_binaries_artifact: sycl_e2e_bin_default - determine_arc_tests: - name: Decide which Arc tests to run + run_prebuilt_e2e_tests: needs: [build, detect_changes] if: ${{ always() && !cancelled() && needs.build.outputs.build_conclusion == 'success' }} - runs-on: [Linux, aux-tasks] - timeout-minutes: 3 - outputs: - arc_tests: ${{ steps.arc_tests.outputs.arc_tests }} - steps: - - name: Determine Arc tests - id: arc_tests - run: | - if [ "${{ contains(needs.detect_changes.outputs.filters, 'devigccfg') }}" == "true" ]; then - echo 'arc_tests="(ESIMD|InvokeSimd|Matrix)/"' >> "$GITHUB_OUTPUT" - elif [ "${{ contains(needs.detect_changes.outputs.filters, 'drivers') }}" == "true" ]; then - echo 'arc_tests=""' >> "$GITHUB_OUTPUT" - elif [ "${{ contains(needs.detect_changes.outputs.filters, 'esimd') }}" == "true" ]; then - echo 'arc_tests="(ESIMD|InvokeSimd|Matrix)/"' >> "$GITHUB_OUTPUT" - else - echo 'arc_tests="Matrix/"' >> "$GITHUB_OUTPUT" - fi - test: - needs: [build, detect_changes, determine_arc_tests] - if: ${{ always() && !cancelled() && needs.build.outputs.build_conclusion == 'success' }} strategy: fail-fast: false matrix: include: + - name: GEN 12 Integrated + runner: '["Linux", "gen12"]' + image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN + target_devices: level_zero:gpu;opencl:gpu;opencl:cpu + reset_intel_gpu: true + extra_lit_opts: --param gpu-intel-gen12=True - name: NVIDIA/CUDA runner: '["Linux", "cuda"]' - image: ghcr.io/intel/llvm/ubuntu2204_build:latest image_options: -u 1001 --gpus all --cap-add SYS_ADMIN - target_devices: ext_oneapi_cuda:gpu + target_devices: cuda:gpu - name: AMD/HIP runner: '["Linux", "amdgpu"]' - image: ghcr.io/intel/llvm/ubuntu2204_build:latest image_options: -u 1001 --device=/dev/dri --device=/dev/kfd - target_devices: ext_oneapi_hip:gpu + target_devices: hip:gpu reset_intel_gpu: false - - name: Intel - runner: '["Linux", "gen12"]' - image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest - image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN - target_devices: level_zero:gpu;opencl:gpu;opencl:cpu - reset_intel_gpu: true - extra_lit_opts: --param gpu-intel-gen12=True - - name: E2E tests on Intel Arc A-Series Graphics + extra_lit_opts: -j 1 + - name: Intel Arc A-Series Graphics runner: '["Linux", "arc"]' - image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN target_devices: level_zero:gpu;opencl:gpu reset_intel_gpu: true - extra_lit_opts: --param matrix-xmx8=True --param gpu-intel-dg2=True - env: '{"LIT_FILTER":${{ needs.determine_arc_tests.outputs.arc_tests }} }' - - name: E2E tests with dev igc on Intel Arc A-Series Graphics - runner: '["Linux", "arc"]' - image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:devigc + extra_lit_opts: --param matrix-xmx8=True + - name: E2E tests on Intel Ponte Vecchio GPU + runner: '["Linux", "pvc"]' image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN target_devices: level_zero:gpu;opencl:gpu reset_intel_gpu: true - extra_lit_opts: --param matrix-xmx8=True --param gpu-intel-dg2=True - env: '{"LIT_FILTER":${{ needs.determine_arc_tests.outputs.arc_tests }} }' - use_igc_dev: true - + extra_lit_opts: -j 50 + - name: Intel Battlemage Graphics + runner: '["Linux", "bmg"]' + image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN + target_devices: level_zero:gpu + reset_intel_gpu: true + - name: SPIR-V Backend / Intel Battlemage Graphics + runner: '["Linux", "bmg"]' + image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN + target_devices: level_zero:gpu;opencl:gpu;opencl:cpu + reset_intel_gpu: true + extra_lit_opts: --param spirv-backend=True + e2e_binaries_artifact: sycl_e2e_bin_default_spirv_backend uses: ./.github/workflows/sycl-linux-run-tests.yml with: name: ${{ matrix.name }} - runner: ${{ matrix. runner }} + runner: ${{ matrix.runner }} image: ${{ matrix.image }} image_options: ${{ matrix.image_options }} target_devices: ${{ matrix.target_devices }} - reset_intel_gpu: ${{ matrix.reset_intel_gpu }} extra_lit_opts: ${{ matrix.extra_lit_opts }} - env: ${{ matrix.env || '{}' }} + reset_intel_gpu: ${{ matrix.reset_intel_gpu }} + repo_ref: ${{ github.sha }} + sycl_toolchain_artifact: sycl_linux_default + sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }} + sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }} + e2e_binaries_artifact: ${{ matrix.e2e_binaries_artifact || 'sycl_e2e_bin_default' }} + e2e_testing_mode: 'run-only' # Do not install drivers on AMD and CUDA runners. install_igc_driver: >- - ${{ !contains(matrix.target_devices, 'ext_oneapi_cuda') && - !contains(matrix.target_devices, 'ext_oneapi_hip') && + ${{ !contains(matrix.target_devices, 'cuda') && + !contains(matrix.target_devices, 'hip') && contains(needs.detect_changes.outputs.filters, 'drivers') }} install_dev_igc_driver: >- - ${{ !contains(matrix.target_devices, 'ext_oneapi_cuda') && - !contains(matrix.target_devices, 'ext_oneapi_hip') && - matrix.use_igc_dev && contains(needs.detect_changes.outputs.filters, 'devigccfg') || + ${{ !contains(matrix.target_devices, 'cuda') && + !contains(matrix.target_devices, 'hip') && + matrix.use_igc_dev && + (contains(needs.detect_changes.outputs.filters, 'devigccfg') || contains(needs.detect_changes.outputs.filters, 'drivers')) || 'false' }} # Run only if the PR does not have the 'ci-no-devigc' label. skip_run: ${{matrix.use_igc_dev && contains(github.event.pull_request.labels.*.name, 'ci-no-devigc') || 'false'}} - ref: ${{ github.sha }} - merge_ref: '' - - sycl_toolchain_artifact: sycl_linux_default - sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }} - sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }} - - test-perf: needs: [build, detect_changes] if: | @@ -156,37 +142,30 @@ jobs: include: - name: Intel GEN12 Graphics system runner: '["Linux", "gen12"]' - image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest image_extra_opts: --device=/dev/dri reset_intel_gpu: true - name: Intel Arc A-Series Graphics system runner: '["Linux", "arc"]' - image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest image_extra_opts: --device=/dev/dri reset_intel_gpu: true - name: AMD system runner: '["Linux", "amdgpu"]' - image: ghcr.io/intel/llvm/ubuntu2204_build:latest image_extra_opts: --device=/dev/dri --device=/dev/kfd - name: CUDA system runner: '["Linux", "cuda"]' - image: ghcr.io/intel/llvm/ubuntu2204_build:latest image_extra_opts: --gpus all uses: ./.github/workflows/sycl-linux-run-tests.yml with: name: Perf tests on ${{ matrix.name }} runner: ${{ matrix. runner }} - image: ${{ matrix.image }} image_options: -u 1001 --privileged --cap-add SYS_ADMIN ${{ matrix.image_extra_opts }} target_devices: all reset_intel_gpu: ${{ matrix.reset_intel_gpu }} env: '{"LIT_FILTER":"PerformanceTests/"}' extra_lit_opts: -a -j 1 --param enable-perf-tests=True - extra_cmake_args: ${{ matrix.extra_cmake_args }} - ref: ${{ github.sha }} - merge_ref: '' + repo_ref: ${{ github.sha }} sycl_toolchain_artifact: sycl_linux_default sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }} diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml index 73b2a1f336db7..da3b92623b4aa 100644 --- a/.github/workflows/sycl-linux-run-tests.yml +++ b/.github/workflows/sycl-linux-run-tests.yml @@ -12,20 +12,20 @@ on: required: True image: type: string - required: True + required: False image_options: type: string required: True target_devices: type: string - required: True + required: False extra_cmake_args: type: string required: False tests_selector: description: | - Two possible options: "e2e" and "cts". + Two possible options: "e2e" and "cts" type: string default: "e2e" @@ -35,16 +35,20 @@ on: type: string default: '' - ref: + repo_ref: type: string required: True - merge_ref: description: | - Commit-ish to merge post-checkout if non-empty. Must be reachable from - the default_branch input paramter. + Commit SHA or branch to checkout the intel/llvm repo. + devops_ref: type: string - default: 'FETCH_HEAD' required: False + description: | + Commit SHA or branch to checkout the devops directory. + tests_ref: + type: string + required: False + description: Commit SHA or branch to checkout e2e/cts tests. sycl_toolchain_artifact: type: string @@ -59,6 +63,25 @@ on: default: '' required: False + e2e_binaries_artifact: + description: | + Must be set if `e2e_testing_mode` is equal to `run-only` and the + artifact must exist. Can be set in other modes resulting in artifact + upload. + type: string + default: '' + required: False + e2e_testing_mode: + description: | + Testing mode to run E2E tests in, can be either `full`, `build-only` + or `run-only`. + type: string + default: 'full' + retention-days: + description: 'E2E/SYCL-CTS binaries artifact retention period.' + type: string + default: 1 + reset_intel_gpu: type: string required: False @@ -78,6 +101,19 @@ on: default: 'false' required: False + cts_testing_mode: + description: | + Testing mode to run SYCL-CTS in, can be either `full`, `build-only` + or `run-only`. In `build-only` mode an artifact of the CTS binaries + will be uploaded. + type: string + default: 'full' + + sycl_cts_artifact: + type: string + default: '' + required: False + workflow_dispatch: inputs: runner: @@ -86,29 +122,32 @@ on: - '["Linux", "gen12"]' - '["amdgpu"]' - '["Linux", "arc"]' + - '["Linux", "pvc"]' - '["cts-cpu"]' + - '["Linux", "build"]' + - '["cuda"]' + - '["PVC_PERF"]' image: - description: | - Use option ending with ":build" for AMDGPU, ":latest" for the rest. type: choice options: - - 'ghcr.io/intel/llvm/sycl_ubuntu2204_nightly:latest' - - 'ghcr.io/intel/llvm/sycl_ubuntu2204_nightly:build' + - 'ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest' + - 'ghcr.io/intel/llvm/ubuntu2404_intel_drivers:alldeps' image_options: description: | Use option with "--device=/dev/kfd" for AMDGPU, without it for the rest. type: choice options: - - '-u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN' - '-u 1001 --device=/dev/dri --device=/dev/kfd --privileged --cap-add SYS_ADMIN' + - '-u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN' + - '-u 1001 --gpus all --cap-add SYS_ADMIN' target_devices: type: choice options: + - 'level_zero:gpu' - 'opencl:cpu' - 'opencl:gpu' - - 'opencl:fpga' - - 'level_zero:gpu' - - 'ext_oneapi_hip:gpu' + - 'hip:gpu' + - 'cuda:gpu' tests_selector: type: choice options: @@ -121,7 +160,7 @@ on: LIT_OPTS won't work as we redefine it as part of this workflow. For SYCL CTS - CTS_TESTS_TO_BUILD to specify which categories to - build. + build, e.g. {"CTS_TESTS_TO_BUILD":"test_category1 test_category2..."}. Format: '{"VAR1":"VAL1","VAR2":"VAL2",...}' default: '{}' @@ -131,20 +170,23 @@ on: Extra options to be added to LIT_OPTS. default: '' - install_igc_driver: + reset_intel_gpu: + description: | + Reset Intel GPUs type: choice options: - false - true - install_dev_igc_driver: + e2e_testing_mode: type: choice options: - - false - - true + - "full" + - "build-only" permissions: contents: read + packages: read jobs: run: @@ -152,43 +194,35 @@ jobs: name: ${{ inputs.name }} runs-on: ${{ fromJSON(inputs.runner) }} container: - image: ${{ inputs.image }} + image: ${{ inputs.image || 'ghcr.io/intel/llvm/ubuntu2404_intel_drivers:alldeps'}} options: ${{ inputs.image_options }} env: ${{ fromJSON(inputs.env) }} steps: - name: Reset Intel GPU if: inputs.reset_intel_gpu == 'true' + shell: bash run: | - sudo mount -t debugfs none /sys/kernel/debug - sudo bash -c 'echo 1 > /sys/kernel/debug/dri/0/i915_wedged' + if [[ '${{ inputs.runner }}' == '["Linux", "bmg"]' ]]; then + sudo bash -c 'echo 0000:05:00.0 > /sys/bus/pci/drivers/xe/unbind' + sudo bash -c 'echo 1 > /sys/bus/pci/devices/0000:05:00.0/reset' + sudo bash -c 'echo 0000:05:00.0 > /sys/bus/pci/drivers/xe/bind' + else + sudo mount -t debugfs none /sys/kernel/debug + base_dir="/sys/kernel/debug/dri" + + for dir in "$base_dir"/*; do + if [ -f "$dir/i915_wedged" ]; then + sudo bash -c 'echo 1 > $0/i915_wedged' $dir + fi + done + fi - uses: actions/checkout@v4 with: - ref: ${{ inputs.ref }} + ref: ${{ inputs.devops_ref || inputs.repo_ref }} sparse-checkout: | devops - name: Register cleanup after job is finished uses: ./devops/actions/cleanup - - name: Checkout E2E tests - if: inputs.tests_selector == 'e2e' - uses: ./devops/actions/cached_checkout - with: - path: llvm - ref: ${{ inputs.ref || github.sha }} - merge_ref: ${{ inputs.merge_ref }} - cache_path: "/__w/repo_cache/" - - name: Checkout SYCL CTS tests - if: inputs.tests_selector == 'cts' - uses: ./devops/actions/cached_checkout - with: - path: khronos_sycl_cts - repository: 'KhronosGroup/SYCL-CTS' - ref: 'main' - default_branch: 'main' - cache_path: "/__w/repo_cache/" - - name: SYCL CTS GIT submodules init - if: inputs.tests_selector == 'cts' - run: | - git -C khronos_sycl_cts submodule update --init - name: Install drivers if: inputs.install_igc_driver == 'true' || inputs.install_dev_igc_driver == 'true' env: @@ -198,7 +232,7 @@ jobs: # If libllvm14 is already installed (dev igc docker), still return true. sudo apt-get install -yqq libllvm14 || true; fi - sudo -E bash devops/scripts/install_drivers.sh llvm/devops/dependencies.json ${{ inputs.install_dev_igc_driver == 'true' && 'llvm/devops/dependencies-igc-dev.json --use-dev-igc' || '' }} --all + sudo -E bash devops/scripts/install_drivers.sh devops/dependencies.json ${{ inputs.install_dev_igc_driver == 'true' && 'devops/dependencies-igc-dev.json --use-dev-igc' || '' }} --all - name: Source OneAPI TBB vars.sh shell: bash run: | @@ -270,108 +304,26 @@ jobs: cat /usr/local/lib/igc/IGCTAG.txt fi - - name: Deduce E2E CMake options - if: inputs.tests_selector == 'e2e' - id: cmake_opts - shell: bash - env: - CMAKE_EXTRA_ARGS: ${{ inputs.extra_cmake_args }} - run: | - if [ -n "$CMAKE_EXTRA_ARGS" ]; then - echo "opts=$CMAKE_EXTRA_ARGS" >> $GITHUB_OUTPUT - fi - - name: Configure E2E tests + - name: Run E2E Tests if: inputs.tests_selector == 'e2e' - run: | - cmake -GNinja -B./build-e2e -S./llvm/sycl/test-e2e -DSYCL_TEST_E2E_TARGETS="${{ inputs.target_devices }}" -DCMAKE_CXX_COMPILER="$(which clang++)" -DLLVM_LIT="$PWD/llvm/llvm/utils/lit/lit.py" ${{ steps.cmake_opts.outputs.opts }} - - name: SYCL End-to-end tests - shell: bash {0} - if: inputs.tests_selector == 'e2e' - env: - LIT_OPTS: -v --no-progress-bar --show-unsupported --show-pass --show-xfail --max-time 3600 --time-tests ${{ inputs.extra_lit_opts }} - run: | - ninja -C build-e2e check-sycl-e2e > e2e.log 2>&1 - exit_code=$? - cat e2e.log - if [ $exit_code -ne 0 ]; then - awk '/^Failed Tests|Unexpectedly Passed Tests|Unresolved tests|Testing Time/{flag=1}/FAILED: CMakeFiles/{flag=0}flag' e2e.log >> $GITHUB_STEP_SUMMARY - fi - exit $exit_code - - name: Build SYCL CTS tests - if: inputs.tests_selector == 'cts' - env: - CMAKE_EXTRA_ARGS: ${{ inputs.extra_cmake_args }} - run: | - cts_exclude_filter="" - # If CTS_TESTS_TO_BUILD is null - use filter - if [ -z "$CTS_TESTS_TO_BUILD" ]; then - if [ "${{ contains(inputs.target_devices, 'opencl:cpu') }}" = "true" ]; then - cts_exclude_filter=$PWD/devops/cts_exclude_filter_OCL_CPU - elif [ "${{ contains(inputs.target_devices, 'level_zero:gpu') }}" = "true" ]; then - cts_exclude_filter=$PWD/devops/cts_exclude_filter_L0_GPU - fi - - # List excluded SYCL CTS categories: - # SYCL_CTS_EXCLUDE_TEST_CATEGORIES - Optional file specifying a list - # of test categories to be excluded from the build. - echo "::group::Excluded test categories" - cat $cts_exclude_filter - echo "::endgroup::" - fi - - cmake -GNinja -B./build-cts -S./khronos_sycl_cts -DCMAKE_CXX_COMPILER=$(which clang++) \ - -DSYCL_IMPLEMENTATION=DPCPP \ - -DSYCL_CTS_EXCLUDE_TEST_CATEGORIES="$cts_exclude_filter" \ - -DSYCL_CTS_ENABLE_OPENCL_INTEROP_TESTS=OFF \ - -DDPCPP_INSTALL_DIR="$(dirname $(which clang++))/.." \ - $CMAKE_EXTRA_ARGS - # Ignore errors so that if one category build fails others still have a - # chance to finish and be executed at the run stage. Note that - # "test_conformance" target skips building "test_all" executable. - ninja -C build-cts -k0 $( [ -n "$CTS_TESTS_TO_BUILD" ] && echo "$CTS_TESTS_TO_BUILD" || echo "test_conformance") - - - name: SYCL CTS List devices - # Proceed with execution even if the 'build' step did not succeed. - if: inputs.tests_selector == 'cts' && (success() || failure()) - env: - ONEAPI_DEVICE_SELECTOR: ${{ inputs.target_devices }} - run: | - ./build-cts/bin/* --list-devices - - - name: Run SYCL CTS tests - # Proceed with execution even if the previous two steps did not succeed. - if: inputs.tests_selector == 'cts' && (success() || failure()) - env: - ONEAPI_DEVICE_SELECTOR: ${{ inputs.target_devices }} - # This job takes ~100min usually. But sometimes some test isn't - # responding, so the job reaches the 360min limit. Setting a lower one. - timeout-minutes: 150 - # By-default GitHub actions execute the "run" shell script with -e option, - # so the execution terminates if any command returns a non-zero status. - # Since we're using a loop to run all test-binaries separately, some test - # may fail and terminate the execution. Setting "shell" value to override - # the default behavior. - # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#custom-shell - shell: bash {0} - run: | - # Run each test category separately so that - # - crash on one would not affect others - # - multiple tests could be run in parallel - mkdir logs - find build-cts/bin/ -type f -print | \ - xargs -t -I % -P 8 sh -c 'log=logs/$(basename %).log ; echo % >$log ; date >>$log ; timeout 60m % >>$log 2>&1 ; ret=$? ; echo "exit code: $ret" >>$log ; date >>$log ; exit $ret' - ret=$? - - for f in logs/* ; do - echo "::group::$f" - cat $f - echo "::endgroup::" - done - - echo "::group::Fails:" - grep 'exit code: [^0]' -r logs - echo "::endgroup::" - - grep 'exit code: [^0]' -r logs >> $GITHUB_STEP_SUMMARY + uses: ./devops/actions/run-tests/e2e + with: + ref: ${{ inputs.tests_ref || inputs.repo_ref || github.sha }} + binaries_artifact: ${{ inputs.e2e_binaries_artifact }} + testing_mode: ${{ inputs.e2e_testing_mode }} + extra_cmake_args: ${{ inputs.extra_cmake_args }} + target_devices: ${{ inputs.target_devices }} + extra_lit_opts: ${{ inputs.extra_lit_opts }} + retention-days: ${{ inputs.retention-days }} - exit $ret + - name: Run SYCL CTS Tests + if: inputs.tests_selector == 'cts' + uses: ./devops/actions/run-tests/cts + with: + ref: ${{ inputs.tests_ref || 'main' }} + cts_exclude_ref: ${{ inputs.repo_ref }} + extra_cmake_args: ${{ inputs.extra_cmake_args }} + cts_testing_mode: ${{ inputs.cts_testing_mode }} + sycl_cts_artifact: ${{ inputs.sycl_cts_artifact }} + target_devices: ${{ inputs.target_devices }} + retention-days: ${{ inputs.retention-days }} diff --git a/.github/workflows/sycl-macos-build-and-test.yml b/.github/workflows/sycl-macos-build-and-test.yml index f25e847d8a341..9831368b413b5 100644 --- a/.github/workflows/sycl-macos-build-and-test.yml +++ b/.github/workflows/sycl-macos-build-and-test.yml @@ -50,8 +50,8 @@ jobs: python3 $GITHUB_WORKSPACE/src/buildbot/configure.py -w $GITHUB_WORKSPACE \ -s $GITHUB_WORKSPACE/src -o $GITHUB_WORKSPACE/build -t Release \ --ci-defaults $ARGS \ - --cmake-opt=-DCMAKE_C_COMPILER_LAUNCHER=ccache \ - --cmake-opt=-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - --cmake-opt="-DLLVM_INSTALL_UTILS=ON" + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DLLVM_INSTALL_UTILS=ON - name: Compile run: cmake --build $GITHUB_WORKSPACE/build --target deploy-sycl-toolchain diff --git a/.github/workflows/sycl-nightly.yml b/.github/workflows/sycl-nightly.yml index 4becda0c89f82..331911824ef3c 100644 --- a/.github/workflows/sycl-nightly.yml +++ b/.github/workflows/sycl-nightly.yml @@ -16,7 +16,7 @@ jobs: build_cache_root: "/__w/" build_artifact_suffix: default build_configure_extra_args: '--hip --cuda' - merge_ref: '' + build_image: ghcr.io/intel/llvm/ubuntu2204_build:latest retention-days: 90 # We upload the build for people to download/use, override its name and @@ -32,10 +32,23 @@ jobs: build_cache_suffix: sprod_shared build_artifact_suffix: sprod_shared build_configure_extra_args: '--shared-libs --hip --cuda --native_cpu' - merge_ref: '' artifact_archive_name: sycl_linux_shared.tar.zst + ubuntu2404_oneapi_build: + if: github.repository == 'intel/llvm' + uses: ./.github/workflows/sycl-linux-build.yml + secrets: inherit + with: + build_cache_root: "/__w/" + build_cache_suffix: oneapi + build_artifact_suffix: oneapi + build_configure_extra_args: -DCMAKE_C_FLAGS="-no-intel-lib -ffp-model=precise" -DCMAKE_CXX_FLAGS="-no-intel-lib -ffp-model=precise" + cc: icx + cxx: icpx + + artifact_archive_name: sycl_linux_oneapi.tar.zst + ubuntu2204_test: needs: [ubuntu2204_build] if: ${{ always() && !cancelled() && needs.ubuntu2204_build.outputs.build_conclusion == 'success' }} @@ -45,79 +58,96 @@ jobs: include: - name: AMD/HIP runner: '["Linux", "amdgpu"]' - image: ghcr.io/intel/llvm/ubuntu2204_build:latest image_options: -u 1001 --device=/dev/dri --device=/dev/kfd - target_devices: ext_oneapi_hip:gpu - tests_selector: e2e + target_devices: hip:gpu + + - name: NVIDIA/CUDA + runner: '["Linux", "cuda"]' + image_options: -u 1001 --gpus all --cap-add SYS_ADMIN + target_devices: cuda:gpu - - name: Intel L0 GPU + - name: Intel L0 Gen12 GPU runner: '["Linux", "gen12"]' - image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN target_devices: level_zero:gpu reset_intel_gpu: true - tests_selector: e2e - extra_lit_opts: --param gpu-intel-gen12=True - - name: Intel OCL GPU + - name: Intel L0 Ponte Vecchio GPU + runner: '["Linux", "pvc"]' + image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN + target_devices: level_zero:gpu + reset_intel_gpu: true + + - name: Intel L0 Battlemage GPU + runner: '["Linux", "bmg"]' + image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN + target_devices: level_zero:gpu + reset_intel_gpu: true + + - name: Intel L0 Arc A-Series GPU + runner: '["Linux", "arc"]' + image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN + target_devices: level_zero:gpu + reset_intel_gpu: true + + - name: Intel OCL Gen12 GPU runner: '["Linux", "gen12"]' - image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN target_devices: opencl:gpu reset_intel_gpu: true - tests_selector: e2e - extra_lit_opts: --param gpu-intel-gen12=True - name: OCL CPU (AMD) - runner: '["Linux", "amdgpu"]' - image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest + runner: '["Linux", "amdcpu"]' image_options: -u 1001 target_devices: opencl:cpu - tests_selector: e2e - name: OCL CPU (Intel/GEN12) runner: '["Linux", "gen12"]' - image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest image_options: -u 1001 --privileged --cap-add SYS_ADMIN target_devices: opencl:cpu - tests_selector: e2e - name: OCL CPU (Intel/Arc) runner: '["Linux", "arc"]' - image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest image_options: -u 1001 target_devices: opencl:cpu - tests_selector: e2e - - name: SYCL-CTS on OCL CPU - runner: '["Linux", "gen12"]' - image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest - image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN - target_devices: opencl:cpu - tests_selector: cts - - - name: SYCL-CTS on L0 gen12 - runner: '["Linux", "gen12"]' - image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest - image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN + - name: Preview mode on SPR/PVC + runner: '["Linux", "pvc"]' + image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN target_devices: level_zero:gpu - tests_selector: cts + reset_intel_gpu: true + extra_lit_opts: --param test-preview-mode=True + uses: ./.github/workflows/sycl-linux-run-tests.yml with: name: ${{ matrix.name }} runner: ${{ matrix.runner }} - image: ${{ matrix.image }} image_options: ${{ matrix.image_options }} target_devices: ${{ matrix.target_devices }} - tests_selector: ${{ matrix.tests_selector }} - extra_lit_opts: ${{ matrix.extra_lit_opts }} + tests_selector: e2e + extra_lit_opts: "--param 'cxx_flags=-D_GLIBCXX_USE_CXX11_ABI=0' ${{ matrix.extra_lit_opts }}" reset_intel_gpu: ${{ matrix.reset_intel_gpu }} - ref: ${{ github.sha }} - merge_ref: '' + repo_ref: ${{ github.sha }} sycl_toolchain_artifact: sycl_linux_default sycl_toolchain_archive: ${{ needs.ubuntu2204_build.outputs.artifact_archive_name }} sycl_toolchain_decompress_command: ${{ needs.ubuntu2204_build.outputs.artifact_decompress_command }} + ubuntu2404_oneapi_test: + needs: [ubuntu2404_oneapi_build] + if: ${{ always() && !cancelled() && needs.ubuntu2404_oneapi_build.outputs.build_conclusion == 'success' }} + uses: ./.github/workflows/sycl-linux-run-tests.yml + with: + name: Intel PVC L0 oneAPI + runner: '["Linux", "pvc"]' + target_devices: level_zero:gpu + reset_intel_gpu: true + extra_lit_opts: -j 50 + image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN + repo_ref: ${{ github.sha }} + sycl_toolchain_artifact: sycl_linux_oneapi + sycl_toolchain_archive: ${{ needs.ubuntu2404_oneapi_build.outputs.artifact_archive_name }} + sycl_toolchain_decompress_command: ${{ needs.ubuntu2404_oneapi_build.outputs.artifact_decompress_command }} + build-win: uses: ./.github/workflows/sycl-windows-build.yml if: github.repository == 'intel/llvm' @@ -139,7 +169,6 @@ jobs: name: Intel GEN12 Graphics with Level Zero runner: '["Windows","gen12"]' sycl_toolchain_archive: ${{ needs.build-win.outputs.artifact_archive_name }} - extra_lit_opts: --param gpu-intel-gen12=True cuda-aws-start: needs: [ubuntu2204_build] @@ -156,11 +185,10 @@ jobs: with: name: CUDA E2E runner: '["aws_cuda-${{ github.run_id }}-${{ github.run_attempt }}"]' - image: ghcr.io/intel/llvm/ubuntu2204_build:latest-0300ac924620a51f76c4929794637b82790f12ab + image: ghcr.io/intel/llvm/ubuntu2204_build:latest image_options: -u 1001 --gpus all --cap-add SYS_ADMIN --env NVIDIA_DISABLE_REQUIRE=1 - target_devices: ext_oneapi_cuda:gpu - ref: ${{ github.sha }} - merge_ref: '' + target_devices: cuda:gpu + repo_ref: ${{ github.sha }} sycl_toolchain_artifact: sycl_linux_default sycl_toolchain_archive: ${{ needs.ubuntu2204_build.outputs.artifact_archive_name }} @@ -168,12 +196,127 @@ jobs: cuda-aws-stop: needs: [cuda-aws-start, cuda-run-tests] - if: always() + if: always() && ${{ needs.cuda-aws-start.result != 'skipped' }} uses: ./.github/workflows/sycl-aws.yml secrets: inherit with: mode: stop + build-sycl-cts-linux: + needs: ubuntu2204_build + if: ${{ always() && !cancelled() && needs.ubuntu2204_build.outputs.build_conclusion == 'success' }} + uses: ./.github/workflows/sycl-linux-run-tests.yml + with: + name: Build SYCL-CTS for Linux + runner: '["Linux", "build"]' + cts_testing_mode: 'build-only' + image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN + tests_selector: cts + repo_ref: ${{ github.sha }} + sycl_toolchain_artifact: sycl_linux_default + sycl_toolchain_archive: ${{ needs.ubuntu2204_build.outputs.artifact_archive_name }} + sycl_toolchain_decompress_command: ${{ needs.ubuntu2204_build.outputs.artifact_decompress_command }} + sycl_cts_artifact: sycl_cts_bin_linux + + run-sycl-cts-linux: + needs: [ubuntu2204_build, build-sycl-cts-linux] + if: ${{ always() && !cancelled() && needs.ubuntu2204_build.outputs.build_conclusion == 'success' }} + strategy: + fail-fast: false + matrix: + include: + - name: SYCL-CTS on OCL CPU + runner: '["Linux", "gen12"]' + image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN + target_devices: opencl:cpu + + - name: SYCL-CTS on L0 gen12 + runner: '["Linux", "gen12"]' + image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN + target_devices: level_zero:gpu + uses: ./.github/workflows/sycl-linux-run-tests.yml + with: + name: ${{ matrix.name }} + runner: ${{ matrix.runner }} + cts_testing_mode: 'run-only' + image_options: ${{ matrix.image_options }} + target_devices: ${{ matrix.target_devices }} + tests_selector: cts + repo_ref: ${{ github.sha }} + sycl_toolchain_artifact: sycl_linux_default + sycl_toolchain_archive: ${{ needs.ubuntu2204_build.outputs.artifact_archive_name }} + sycl_toolchain_decompress_command: ${{ needs.ubuntu2204_build.outputs.artifact_decompress_command }} + sycl_cts_artifact: sycl_cts_bin_linux + + build-sycl-cts-win: + needs: build-win + if: ${{ always() && !cancelled() && needs.build-win.outputs.build_conclusion == 'success' }} + uses: ./.github/workflows/sycl-windows-run-tests.yml + with: + name: Build SYCL-CTS for Windows + runner: '["Windows", "build-e2e"]' + cts_testing_mode: 'build-only' + tests_selector: cts + repo_ref: ${{ github.sha }} + sycl_toolchain_archive: ${{ needs.build-win.outputs.artifact_archive_name }} + sycl_cts_artifact: sycl_cts_bin_win + + run-sycl-cts-win: + needs: [build-win, build-sycl-cts-win] + if: ${{ always() && !cancelled() && needs.build-win.outputs.build_conclusion == 'success' }} + strategy: + fail-fast: false + matrix: + include: + - name: SYCL-CTS on L0 gen12 + runner: '["Windows", "gen12"]' + target_devices: level_zero:gpu + uses: ./.github/workflows/sycl-windows-run-tests.yml + with: + name: ${{ matrix.name }} + runner: ${{ matrix.runner }} + cts_testing_mode: 'run-only' + target_devices: ${{ matrix.target_devices }} + tests_selector: cts + repo_ref: ${{ github.sha }} + sycl_toolchain_archive: ${{ needs.build-win.outputs.artifact_archive_name }} + sycl_cts_artifact: sycl_cts_bin_win + + aggregate_benchmark_results: + if: github.repository == 'intel/llvm' && !cancelled() + name: Aggregate benchmark results and produce historical averages + uses: ./.github/workflows/sycl-benchmark-aggregate.yml + secrets: + LLVM_SYCL_BENCHMARK_TOKEN: ${{ secrets.LLVM_SYCL_BENCHMARK_TOKEN }} + with: + lookback_days: 100 + + run-sycl-benchmarks: + needs: [ubuntu2204_build, aggregate_benchmark_results] + if: ${{ always() && !cancelled() && needs.ubuntu2204_build.outputs.build_conclusion == 'success' }} + strategy: + fail-fast: false + matrix: + include: + - name: Run compute-benchmarks on L0 PVC + runner: '["PVC_PERF"]' + image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN + target_devices: level_zero:gpu + reset_intel_gpu: true + uses: ./.github/workflows/sycl-linux-run-tests.yml + secrets: inherit + with: + name: ${{ matrix.name }} + runner: ${{ matrix.runner }} + image_options: ${{ matrix.image_options }} + target_devices: ${{ matrix.target_devices }} + tests_selector: compute-benchmarks + reset_intel_gpu: ${{ matrix.reset_intel_gpu }} + repo_ref: ${{ github.sha }} + sycl_toolchain_artifact: sycl_linux_default + sycl_toolchain_archive: ${{ needs.ubuntu2204_build.outputs.artifact_archive_name }} + sycl_toolchain_decompress_command: ${{ needs.ubuntu2204_build.outputs.artifact_decompress_command }} + nightly_build_upload: name: Nightly Build Upload if: ${{ github.ref_name == 'sycl' }} @@ -198,7 +341,7 @@ jobs: echo "TAG=$(date +'%Y-%m-%d')-${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT" fi - name: Upload binaries - uses: softprops/action-gh-release@e7a8f85e1c67a31e6ed99a94b41bd0b71bbee6b8 + uses: softprops/action-gh-release@v2.2.1 with: files: | sycl_linux.tar.gz @@ -209,7 +352,7 @@ jobs: body: "Daily build ${{ steps.tag.outputs.TAG }}" target_commitish: ${{ github.sha }} - ubuntu2204_docker_build_push: + docker_build_push: if: github.repository == 'intel/llvm' runs-on: [Linux, build] permissions: @@ -221,42 +364,16 @@ jobs: with: name: sycl_linux_default path: devops/ - - name: Build and Push Container (with drivers) - uses: ./devops/actions/build_container - with: - push: ${{ github.ref_name == 'sycl' }} - file: ubuntu2204_preinstalled - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - build-args: | - base_image=ghcr.io/intel/llvm/ubuntu2204_intel_drivers - base_tag=latest - tags: | - ghcr.io/${{ github.repository }}/sycl_ubuntu2204_nightly:${{ github.sha }} - ghcr.io/${{ github.repository }}/sycl_ubuntu2204_nightly:latest - - name: Build and Push Container (no drivers) - uses: ./devops/actions/build_container - with: - push: ${{ github.ref_name == 'sycl' }} - file: ubuntu2204_preinstalled - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - build-args: | - base_image=ghcr.io/intel/llvm/ubuntu2204_base - base_tag=latest - tags: | - ghcr.io/${{ github.repository }}/sycl_ubuntu2204_nightly:no-drivers-${{ github.sha }} - ghcr.io/${{ github.repository }}/sycl_ubuntu2204_nightly:no-drivers - - name: Build and Push Container (Build image) + - name: Build and Push Container uses: ./devops/actions/build_container with: push: ${{ github.ref_name == 'sycl' }} - file: ubuntu2204_preinstalled + file: nightly username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} build-args: | - base_image=ghcr.io/intel/llvm/ubuntu2204_build - base_tag=latest + base_image=ghcr.io/intel/llvm/ubuntu2404_intel_drivers + base_tag=alldeps tags: | - ghcr.io/${{ github.repository }}/sycl_ubuntu2204_nightly:build-${{ github.sha }} - ghcr.io/${{ github.repository }}/sycl_ubuntu2204_nightly:build + ghcr.io/${{ github.repository }}/sycl_ubuntu2404_nightly:${{ github.sha }} + ghcr.io/${{ github.repository }}/sycl_ubuntu2404_nightly:latest diff --git a/.github/workflows/sycl-post-commit.yml b/.github/workflows/sycl-post-commit.yml index 1abd497a9c97c..6c97dedf21fc2 100644 --- a/.github/workflows/sycl-post-commit.yml +++ b/.github/workflows/sycl-post-commit.yml @@ -1,16 +1,16 @@ name: SYCL Post Commit on: + workflow_dispatch: + push: branches: - sycl - - sycl-devops-pr/** - sycl-rel-** pull_request: branches: - sycl - - sycl-devops-pr/** paths: - .github/workflows/sycl-post-commit.yml - .github/workflows/sycl-linux-build.yml @@ -18,6 +18,8 @@ on: - .github/workflows/sycl-macos-build-and-test.yml - ./devops/actions/cleanup - ./devops/actions/cached_checkout + - ./devops/dependencies.json + - ./devops/dependencies-igc-dev.json concurrency: # Cancel a currently running workflow from the same PR or commit hash. @@ -27,6 +29,10 @@ concurrency: permissions: read-all jobs: + detect_changes: + if: ${{ github.event_name == 'pull_request' }} + uses: ./.github/workflows/sycl-detect-changes.yml + build-lin: name: Linux (Self build + no-assertions) if: github.repository == 'intel/llvm' @@ -35,15 +41,10 @@ jobs: build_cache_root: "/__w/llvm" build_cache_suffix: default build_artifact_suffix: default - build_configure_extra_args: --no-assertions --hip --cuda --native_cpu --cmake-opt="-DSYCL_ENABLE_STACK_PRINTING=ON" --cmake-opt="-DSYCL_LIB_WITH_DEBUG_SYMBOL=ON" - # Docker image has last nightly pre-installed and added to the PATH - build_image: "ghcr.io/intel/llvm/sycl_ubuntu2204_nightly:build" - cc: clang - cxx: clang++ - merge_ref: '' + build_configure_extra_args: --no-assertions --hip --cuda --native_cpu -DSYCL_ENABLE_STACK_PRINTING=ON -DSYCL_LIB_WITH_DEBUG_SYMBOL=ON e2e-lin: - needs: [build-lin] + needs: [detect_changes, build-lin] if: ${{ always() && !cancelled() && needs.build-lin.outputs.build_conclusion == 'success' }} strategy: fail-fast: false @@ -51,18 +52,16 @@ jobs: include: - name: Intel GEN12 Graphics with Level Zero runner: '["Linux", "gen12"]' - extra_lit_opts: --param gpu-intel-gen12=True - target_devices: level_zero:gpu;opencl:fpga + target_devices: level_zero:gpu reset_intel_gpu: true - name: Intel Arc A-Series Graphics with Level Zero runner: '["Linux", "arc"]' - extra_lit_opts: --param matrix-xmx8=True --param gpu-intel-dg2=True + extra_lit_opts: --param matrix-xmx8=True reset_intel_gpu: true - name: AMD/HIP runner: '["Linux", "amdgpu"]' - image: ghcr.io/intel/llvm/ubuntu2204_build:latest image_options: -u 1001 --device=/dev/dri --device=/dev/kfd - target_devices: ext_oneapi_hip:gpu + target_devices: hip:gpu reset_intel_gpu: false # Performance tests below. Specifics: # - only run performance tests (use LIT_FILTER env) @@ -73,7 +72,7 @@ jobs: - name: Perf tests on Intel GEN12 Graphics system runner: '["Linux", "gen12"]' env: '{"LIT_FILTER":"PerformanceTests/"}' - extra_lit_opts: -a -j 1 --param enable-perf-tests=True --param gpu-intel-gen12=True + extra_lit_opts: -a -j 1 --param enable-perf-tests=True target_devices: all reset_intel_gpu: true - name: Perf tests on Intel Arc A-Series Graphics system @@ -86,7 +85,6 @@ jobs: with: name: ${{ matrix.name }} runner: ${{ matrix. runner }} - image: ${{ matrix.image || 'ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest' }} image_options: ${{ matrix.image_options || '-u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN' }} target_devices: ${{ matrix.target_devices || 'level_zero:gpu' }} reset_intel_gpu: ${{ matrix.reset_intel_gpu }} @@ -94,19 +92,29 @@ jobs: extra_lit_opts: ${{ matrix.extra_lit_opts }} env: ${{ matrix.env || '{}' }} - ref: ${{ github.sha }} - merge_ref: '' + repo_ref: ${{ github.sha }} sycl_toolchain_artifact: sycl_linux_default sycl_toolchain_archive: ${{ needs.build-lin.outputs.artifact_archive_name }} sycl_toolchain_decompress_command: ${{ needs.build-lin.outputs.artifact_decompress_command }} + # Do not install drivers on AMD and CUDA runners. + install_igc_driver: >- + ${{ github.event_name == 'pull_request' }} && + ${{ !contains(matrix.target_devices, 'cuda') && + !contains(matrix.target_devices, 'hip') && + contains(needs.detect_changes.outputs.filters, 'drivers') }} + build-win: if: | always() && success() && github.repository == 'intel/llvm' uses: ./.github/workflows/sycl-windows-build.yml + with: + compiler: icx + build_configure_extra_args: -DCMAKE_C_FLAGS="/fp:precise /clang:-Wno-nonportable-include-path /clang:-Wno-cast-function-type-mismatch" -DCMAKE_CXX_FLAGS="/fp:precise /clang:-Wno-nonportable-include-path /clang:-Wno-cast-function-type-mismatch" -DCMAKE_EXE_LINKER_FLAGS=/manifest:no -DCMAKE_MODULE_LINKER_FLAGS=/manifest:no -DCMAKE_SHARED_LINKER_FLAGS=/manifest:no + build_cache_suffix: icx e2e-win: needs: build-win @@ -120,7 +128,7 @@ jobs: name: Intel GEN12 Graphics with Level Zero runner: '["Windows","gen12"]' sycl_toolchain_archive: ${{ needs.build-win.outputs.artifact_archive_name }} - extra_lit_opts: --param gpu-intel-gen12=True + compiler: icx macos_default: name: macOS diff --git a/.github/workflows/sycl-rel-nightly.yml b/.github/workflows/sycl-rel-nightly.yml new file mode 100644 index 0000000000000..b0487f6d435bc --- /dev/null +++ b/.github/workflows/sycl-rel-nightly.yml @@ -0,0 +1,207 @@ +name: SYCL Release Branch Nightly + +on: + workflow_dispatch: + schedule: + - cron: '0 3 * * *' + +permissions: read-all + +jobs: + # To avoid unnecessary scheduled runs this job checks if there are new commits + # since the last run. More precisely, it checks if the last commit is older + # than 24h. That means the previous Nightly already tested this commit. + check_for_new_commits: + if: github.repository == 'intel/llvm' + runs-on: ubuntu-latest + name: Check for new commits + outputs: + is_new_commit: ${{ steps.is_new_commit.outputs.is_new_commit }} + steps: + - uses: actions/checkout@v4 + with: + ref: sycl-rel-6_1_0 + - run: git show --quiet | tee -a $GITHUB_STEP_SUMMARY + + - id: is_new_commit + if: ${{ github.event_name == 'schedule' }} + run: | + if [ -z "$(git rev-list --after="24 hours" HEAD)" ]; then + echo "is_new_commit=false" >> $GITHUB_OUTPUT + fi + + ubuntu2204_build: + needs: [check_for_new_commits] + if: ${{ github.repository == 'intel/llvm' && needs.check_for_new_commits.outputs.is_new_commit != 'false' }} + uses: ./.github/workflows/sycl-linux-build.yml + secrets: inherit + with: + build_cache_root: "/__w/" + build_artifact_suffix: default + build_configure_extra_args: '--hip --cuda' + build_image: ghcr.io/intel/llvm/ubuntu2204_build:latest + build_ref: sycl-rel-6_1_0 + + # We upload the build for people to download/use, override its name and + # prefer widespread gzip compression. + artifact_archive_name: sycl_linux.tar.gz + + ubuntu2204_test: + needs: [ubuntu2204_build] + if: ${{ always() && !cancelled() && needs.ubuntu2204_build.outputs.build_conclusion == 'success' }} + strategy: + fail-fast: false + matrix: + include: + - name: AMD/HIP + runner: '["Linux", "amdgpu"]' + image_options: -u 1001 --device=/dev/dri --device=/dev/kfd + target_devices: hip:gpu + tests_selector: e2e + + - name: Intel L0 GPU + runner: '["Linux", "gen12"]' + image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN + target_devices: level_zero:gpu + reset_intel_gpu: true + tests_selector: e2e + extra_lit_opts: --param gpu-intel-gen12=True + + - name: Intel OCL GPU + runner: '["Linux", "gen12"]' + image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN + target_devices: opencl:gpu + reset_intel_gpu: true + tests_selector: e2e + extra_lit_opts: --param gpu-intel-gen12=True + + - name: Intel OCL CPU + runner: '["Linux", "gen12"]' + image_options: -u 1001 --privileged --cap-add SYS_ADMIN + target_devices: opencl:cpu + tests_selector: e2e + uses: ./.github/workflows/sycl-linux-run-tests.yml + with: + name: ${{ matrix.name }} + runner: ${{ matrix.runner }} + image_options: ${{ matrix.image_options }} + target_devices: ${{ matrix.target_devices }} + tests_selector: ${{ matrix.tests_selector }} + extra_lit_opts: ${{ matrix.extra_lit_opts }} + reset_intel_gpu: ${{ matrix.reset_intel_gpu }} + repo_ref: sycl-rel-6_1_0 + devops_ref: sycl + sycl_toolchain_artifact: sycl_linux_default + sycl_toolchain_archive: ${{ needs.ubuntu2204_build.outputs.artifact_archive_name }} + sycl_toolchain_decompress_command: ${{ needs.ubuntu2204_build.outputs.artifact_decompress_command }} + + build-win: + needs: [check_for_new_commits] + if: ${{ github.repository == 'intel/llvm' && needs.check_for_new_commits.outputs.is_new_commit != 'false' }} + uses: ./.github/workflows/sycl-windows-build.yml + with: + ref: sycl-rel-6_1_0 + + # We upload both Linux/Windows build via Github's "Releases" + # functionality, make sure Linux/Windows names follow the same pattern. + artifact_archive_name: sycl_windows.tar.gz + + e2e-win: + needs: build-win + # Continue if build was successful. + if: | + always() + && !cancelled() + && needs.build-win.outputs.build_conclusion == 'success' + uses: ./.github/workflows/sycl-windows-run-tests.yml + with: + name: Intel GEN12 Graphics with Level Zero + runner: '["Windows","gen12"]' + sycl_toolchain_archive: ${{ needs.build-win.outputs.artifact_archive_name }} + extra_lit_opts: --param gpu-intel-gen12=True + repo_ref: sycl-rel-6_1_0 + devops_ref: sycl + + cuda-aws-start: + needs: [ubuntu2204_build] + if: ${{ always() && !cancelled() && needs.ubuntu2204_build.outputs.build_conclusion == 'success' }} + uses: ./.github/workflows/sycl-aws.yml + secrets: inherit + with: + mode: start + ref: sycl-rel-6_1_0 + + cuda-run-tests: + needs: [ubuntu2204_build, cuda-aws-start] + if: ${{ always() && !cancelled() && needs.ubuntu2204_build.outputs.build_conclusion == 'success' }} + uses: ./.github/workflows/sycl-linux-run-tests.yml + with: + name: CUDA E2E + runner: '["aws_cuda-${{ github.run_id }}-${{ github.run_attempt }}"]' + image: ghcr.io/intel/llvm/ubuntu2204_build:latest + image_options: -u 1001 --gpus all --cap-add SYS_ADMIN --env NVIDIA_DISABLE_REQUIRE=1 + target_devices: cuda:gpu + repo_ref: sycl-rel-6_1_0 + devops_ref: sycl + + sycl_toolchain_artifact: sycl_linux_default + sycl_toolchain_archive: ${{ needs.ubuntu2204_build.outputs.artifact_archive_name }} + sycl_toolchain_decompress_command: ${{ needs.ubuntu2204_build.outputs.artifact_decompress_command }} + + cuda-aws-stop: + needs: [cuda-aws-start, cuda-run-tests] + if: always() + uses: ./.github/workflows/sycl-aws.yml + secrets: inherit + with: + mode: stop + ref: sycl-rel-6_1_0 + + build-sycl-cts: + needs: ubuntu2204_build + if: ${{ always() && !cancelled() && needs.ubuntu2204_build.outputs.build_conclusion == 'success' }} + uses: ./.github/workflows/sycl-linux-run-tests.yml + with: + name: Build SYCL-CTS + runner: '["Linux", "build"]' + cts_testing_mode: 'build-only' + image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN + tests_selector: cts + repo_ref: sycl-rel-6_1_0 + devops_ref: sycl + tests_ref: ead7474b9cb2189ce48025550912ccad5a72bd30 + sycl_toolchain_artifact: sycl_linux_default + sycl_toolchain_archive: ${{ needs.ubuntu2204_build.outputs.artifact_archive_name }} + sycl_toolchain_decompress_command: ${{ needs.ubuntu2204_build.outputs.artifact_decompress_command }} + sycl_cts_artifact: sycl_cts_bin_linux + + run-sycl-cts: + needs: [ubuntu2204_build, build-sycl-cts] + if: ${{ always() && !cancelled() && needs.ubuntu2204_build.outputs.build_conclusion == 'success' }} + strategy: + fail-fast: false + matrix: + include: + - name: SYCL-CTS on OCL CPU + runner: '["Linux", "gen12"]' + image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN + target_devices: opencl:cpu + + - name: SYCL-CTS on L0 gen12 + runner: '["Linux", "gen12"]' + image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN + target_devices: level_zero:gpu + uses: ./.github/workflows/sycl-linux-run-tests.yml + with: + name: ${{ matrix.name }} + runner: ${{ matrix.runner }} + cts_testing_mode: 'run-only' + image_options: ${{ matrix.image_options }} + target_devices: ${{ matrix.target_devices }} + tests_selector: cts + repo_ref: ${{ github.sha }} + devops_ref: sycl + sycl_toolchain_artifact: sycl_linux_default + sycl_toolchain_archive: ${{ needs.ubuntu2204_build.outputs.artifact_archive_name }} + sycl_toolchain_decompress_command: ${{ needs.ubuntu2204_build.outputs.artifact_decompress_command }} + sycl_cts_artifact: sycl_cts_bin_linux diff --git a/.github/workflows/sycl-stale-issues.yml b/.github/workflows/sycl-stale-issues.yml index 29afd18d22471..420dcd27a1d1c 100644 --- a/.github/workflows/sycl-stale-issues.yml +++ b/.github/workflows/sycl-stale-issues.yml @@ -11,7 +11,7 @@ jobs: permissions: issues: write # for actions/stale to close stale issues pull-requests: write # for actions/stale to close stale PRs - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - uses: actions/stale@v9 with: diff --git a/.github/workflows/sycl-sync-main.yml b/.github/workflows/sycl-sync-main.yml index 68f8578a15c3a..805ee74b1d3fd 100644 --- a/.github/workflows/sycl-sync-main.yml +++ b/.github/workflows/sycl-sync-main.yml @@ -9,7 +9,7 @@ jobs: sync: permissions: contents: write # for Git to git push - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest if: github.repository == 'intel/llvm' steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/sycl-update-gpu-driver.yml b/.github/workflows/sycl-update-gpu-driver.yml index 6315f27641dce..cd998ded1b10d 100644 --- a/.github/workflows/sycl-update-gpu-driver.yml +++ b/.github/workflows/sycl-update-gpu-driver.yml @@ -11,7 +11,7 @@ jobs: update_driver_linux: permissions: contents: write # for Git to git push - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest if: github.repository == 'intel/llvm' steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/sycl-update-igc-dev-driver.yml b/.github/workflows/sycl-update-igc-dev-driver.yml index 8a323a1e05221..3bc2407b1f7df 100644 --- a/.github/workflows/sycl-update-igc-dev-driver.yml +++ b/.github/workflows/sycl-update-igc-dev-driver.yml @@ -11,7 +11,7 @@ jobs: update_driver_linux: permissions: contents: write # for Git to git push - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest if: github.repository == 'intel/llvm' steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/sycl-weekly.yml b/.github/workflows/sycl-weekly.yml new file mode 100644 index 0000000000000..35da18c76ce9e --- /dev/null +++ b/.github/workflows/sycl-weekly.yml @@ -0,0 +1,67 @@ +name: SYCL Weekly + +on: + workflow_dispatch: + schedule: + # At 00:00 on Sunday. + - cron: '0 0 * * 0' + +permissions: read-all + +jobs: + ubuntu2204_build: + if: github.repository == 'intel/llvm' + uses: ./.github/workflows/sycl-linux-build.yml + secrets: inherit + with: + build_cache_root: "/__w/" + build_artifact_suffix: default + build_configure_extra_args: '' + + # This job builds SYCL-CTS with -fsycl-use-spirv-backend-for-spirv-gen. + build-sycl-cts: + needs: ubuntu2204_build + if: ${{ always() && !cancelled() && needs.ubuntu2204_build.outputs.build_conclusion == 'success' }} + uses: ./.github/workflows/sycl-linux-run-tests.yml + with: + name: Build SYCL-CTS + runner: '["Linux", "build"]' + cts_testing_mode: 'build-only' + image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN + tests_selector: cts + repo_ref: ${{ github.sha }} + sycl_toolchain_artifact: sycl_linux_default + sycl_toolchain_archive: ${{ needs.ubuntu2204_build.outputs.artifact_archive_name }} + sycl_toolchain_decompress_command: ${{ needs.ubuntu2204_build.outputs.artifact_decompress_command }} + extra_cmake_args: -DDPCPP_FLAGS=-fsycl-use-spirv-backend-for-spirv-gen + sycl_cts_artifact: sycl_cts_bin + + run-sycl-cts: + needs: [ubuntu2204_build, build-sycl-cts] + if: ${{ always() && !cancelled() && needs.ubuntu2204_build.outputs.build_conclusion == 'success' }} + strategy: + fail-fast: false + matrix: + include: + - name: SYCL-CTS on OCL CPU PVC w/ LLVM SPIR-V Backend + runner: '["Linux", "pvc"]' + image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN + target_devices: opencl:cpu + + - name: SYCL-CTS on L0 GPU PVC w/ LLVM SPIR-V Backend + runner: '["Linux", "pvc"]' + image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN + target_devices: level_zero:gpu + uses: ./.github/workflows/sycl-linux-run-tests.yml + with: + name: ${{ matrix.name }} + runner: ${{ matrix.runner }} + cts_testing_mode: 'run-only' + image_options: ${{ matrix.image_options }} + target_devices: ${{ matrix.target_devices }} + tests_selector: cts + repo_ref: ${{ github.sha }} + sycl_toolchain_artifact: sycl_linux_default + sycl_toolchain_archive: ${{ needs.ubuntu2204_build.outputs.artifact_archive_name }} + sycl_toolchain_decompress_command: ${{ needs.ubuntu2204_build.outputs.artifact_decompress_command }} + sycl_cts_artifact: sycl_cts_bin diff --git a/.github/workflows/sycl-windows-build.yml b/.github/workflows/sycl-windows-build.yml index 4bd537146bf31..dbf4bfd88922c 100644 --- a/.github/workflows/sycl-windows-build.yml +++ b/.github/workflows/sycl-windows-build.yml @@ -7,7 +7,7 @@ on: type: string required: false default: "default" - build_ref: + build_configure_extra_args: type: string required: false changes: @@ -15,6 +15,9 @@ on: description: 'Filter matches for the changed files in the PR' default: '[llvm, clang, sycl, llvm_spirv, xptifw, libclc, libdevice]' required: false + ref: + type: string + required: False artifact_archive_name: type: string default: llvm_sycl.tar.gz @@ -22,6 +25,10 @@ on: description: 'Artifacts retention period' type: string default: 3 + compiler: + type: string + required: false + default: "cl" outputs: build_conclusion: @@ -36,11 +43,15 @@ on: type: choice options: - "[]" + - '[sycl]' - '[llvm, clang, sycl, llvm_spirv, xptifw, libclc, libdevice]' build_cache_suffix: type: choice options: - "default" + build_configure_extra_args: + type: string + required: false artifact_archive_name: type: choice options: @@ -50,6 +61,12 @@ on: type: choice options: - 3 + compiler: + type: choice + options: + - cl + - icx + permissions: read-all jobs: @@ -57,28 +74,48 @@ jobs: name: Build + LIT runs-on: [Windows, build] environment: WindowsCILock - # TODO use cached checkout outputs: build_conclusion: ${{ steps.build.conclusion }} steps: + - name: Detect hung tests + if: always() + shell: powershell + continue-on-error: true + run: | + Invoke-WebRequest -Uri "https://raw.githubusercontent.com/intel/llvm/refs/heads/sycl/devops/scripts/windows_detect_hung_tests.ps1" -OutFile "windows_detect_hung_tests.ps1" + powershell.exe -File windows_detect_hung_tests.ps1 + $exitCode = $LASTEXITCODE + Remove-Item -Path "windows_detect_hung_tests.ps1" + exit $exitCode + - uses: actions/checkout@v4 + with: + sparse-checkout: | + devops/actions - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 with: arch: amd64 + - name: Setup oneAPI env + uses: ./devops/actions/setup_windows_oneapi_env + if: ${{ always() && !cancelled() && inputs.compiler == 'icx' }} - name: Set env run: | git config --system core.longpaths true git config --global core.autocrlf false echo "C:\Program Files\Git\usr\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - echo "SCCACHE_DIR=D:\github\_work\cache\${{ inputs.build_cache_suffix }}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - - uses: actions/checkout@v4 + echo "CCACHE_DIR=D:\github\_work\cache\${{ inputs.build_cache_suffix }}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + echo "CCACHE_MAXSIZE=10G" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + echo "LIT_OPTS='-j$env:NUMBER_OF_PROCESSORS $LIT_OPTS'" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + - name: Register cleanup after job is finished + uses: ./devops/actions/cleanup + - uses: ./devops/actions/cached_checkout with: path: src - ref: ${{ inputs.build_ref || github.sha }} - fetch-depth: 1 - - name: Register cleanup after job is finished - uses: ./src/devops/actions/cleanup + ref: ${{ inputs.ref || github.sha }} + cache_path: "D:\\\\github\\\\_work\\\\repo_cache\\\\" - name: Configure shell: cmd + env: + ARGS: ${{ inputs.build_configure_extra_args }} # TODO switch to clang-cl and lld when this is fixed https://github.com/oneapi-src/level-zero/issues/83 run: | mkdir build @@ -86,14 +123,14 @@ jobs: IF NOT EXIST D:\github\_work\cache MKDIR D:\github\_work\cache IF NOT EXIST D:\github\_work\cache\${{inputs.build_cache_suffix}} MKDIR D:\github\_work\cache\${{inputs.build_cache_suffix}} python.exe src/buildbot/configure.py -o build ^ - --ci-defaults ^ - --cmake-opt="-DCMAKE_C_COMPILER=cl" ^ - --cmake-opt="-DCMAKE_CXX_COMPILER=cl" ^ - --cmake-opt="-DCMAKE_INSTALL_PREFIX=%GITHUB_WORKSPACE%\install" ^ - --cmake-opt="-DCMAKE_CXX_COMPILER_LAUNCHER=sccache" ^ - --cmake-opt="-DCMAKE_C_COMPILER_LAUNCHER=sccache" ^ - --cmake-opt="-DLLVM_INSTALL_UTILS=ON" ^ - --cmake-opt="-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=SPIRV" + --ci-defaults %ARGS% ^ + "-DCMAKE_C_COMPILER=${{inputs.compiler}}" ^ + "-DCMAKE_CXX_COMPILER=${{inputs.compiler}}" ^ + "-DCMAKE_INSTALL_PREFIX=%GITHUB_WORKSPACE%\install" ^ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache ^ + -DCMAKE_C_COMPILER_LAUNCHER=ccache ^ + -DLLVM_INSTALL_UTILS=ON ^ + -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=SPIRV - name: Build id: build shell: bash @@ -101,7 +138,8 @@ jobs: cmake --build build --target sycl-toolchain - name: check-llvm if: always() && !cancelled() && contains(inputs.changes, 'llvm') - run: | + shell: bash + run: | cmake --build build --target check-llvm - name: check-clang if: always() && !cancelled() && contains(inputs.changes, 'clang') @@ -109,8 +147,9 @@ jobs: cmake --build build --target check-clang - name: check-sycl if: always() && !cancelled() && contains(inputs.changes, 'sycl') - run: | - cmake --build build --target check-sycl + shell: bash + run: | + cmake --build build --target check-sycl - name: check-sycl-unittests if: always() && !cancelled() && contains(inputs.changes, 'sycl') run: | @@ -127,6 +166,19 @@ jobs: if: always() && !cancelled() && contains(inputs.changes, 'libdevice') run: | cmake --build build --target check-libdevice + - name: Generate/diff new ABI symbols + if: always() && !cancelled() && contains(inputs.changes, 'sycl') + shell: bash + run: | + LLVM_BIN_PATH="build/bin" python.exe src/sycl/tools/abi_check.py --mode dump_symbols --output build/new_sycl_symbols_windows.dump build/bin/sycl?.dll + diff -Naur src/sycl/test/abi/sycl_symbols_windows.dump build/new_sycl_symbols_windows.dump || true + - name: Upload new ABI symbols + if: always() && !cancelled() && contains(inputs.changes, 'sycl') + uses: actions/upload-artifact@v4 + with: + name: sycl_windows_abi_symbols + path: build/new_sycl_symbols_windows.dump + retention-days: ${{ inputs.retention-days }} - name: Install if: ${{ always() && !cancelled() && steps.build.conclusion == 'success' }} shell: bash @@ -154,3 +206,12 @@ jobs: name: sycl_windows_default path: ${{ inputs.artifact_archive_name }} retention-days: ${{ inputs.retention-days }} + - name: Detect hung tests + if: always() + shell: powershell + run: | + Invoke-WebRequest -Uri "https://raw.githubusercontent.com/intel/llvm/refs/heads/sycl/devops/scripts/windows_detect_hung_tests.ps1" -OutFile "windows_detect_hung_tests.ps1" + powershell.exe -File windows_detect_hung_tests.ps1 + $exitCode = $LASTEXITCODE + Remove-Item -Path "windows_detect_hung_tests.ps1" + exit $exitCode diff --git a/.github/workflows/sycl-windows-precommit.yml b/.github/workflows/sycl-windows-precommit.yml index a82c2b7814d75..a9cb32731d49b 100644 --- a/.github/workflows/sycl-windows-precommit.yml +++ b/.github/workflows/sycl-windows-precommit.yml @@ -4,13 +4,13 @@ on: pull_request: branches: - sycl - - sycl-devops-pr/** - llvmspirv_pulldown - sycl-rel-** # Do not run builds if changes are only in the following locations paths-ignore: - '.github/ISSUE_TEMPLATE/**' - '.github/CODEOWNERS' + - 'sycl/cts_exclude_filter/**' - 'sycl/doc/**' - 'sycl/gdb/**' - 'clang/docs/**' @@ -20,6 +20,7 @@ on: - '.github/workflows/sycl-precommit-aws.yml' - '.github/workflows/sycl-macos-*.yml' - '.github/workflows/sycl-nightly.yml' + - '.github/workflows/sycl-rel-nightly.yml' - 'devops/containers/**' - 'devops/actions/build_container/**' @@ -50,9 +51,16 @@ jobs: always() && !cancelled() && needs.build.outputs.build_conclusion == 'success' + strategy: + fail-fast: false + matrix: + include: + - name: Intel GEN12 Graphics with Level Zero + runner: '["Windows","gen12"]' + - name: Intel Battlemage Graphics with Level Zero + runner: '["Windows","bmg"]' uses: ./.github/workflows/sycl-windows-run-tests.yml with: - name: Intel GEN12 Graphics with Level Zero - runner: '["Windows","gen12"]' + name: ${{ matrix.name }} + runner: ${{ matrix.runner }} sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }} - extra_lit_opts: --param gpu-intel-gen12=True diff --git a/.github/workflows/sycl-windows-run-tests.yml b/.github/workflows/sycl-windows-run-tests.yml index 9186392f8fa46..67d1a60f23c81 100644 --- a/.github/workflows/sycl-windows-run-tests.yml +++ b/.github/workflows/sycl-windows-run-tests.yml @@ -6,18 +6,43 @@ on: name: type: string required: True + runner: type: string required: True + target_devices: + type: string + required: False + extra_cmake_args: + type: string + required: False + tests_selector: + description: | + Two possible options: "e2e" and "cts". + type: string + default: "e2e" + extra_lit_opts: description: | Extra options to be added to LIT_OPTS. type: string default: '' - ref: + + repo_ref: + type: string + required: False + description: | + Commit SHA or branch to checkout the intel/llvm repo. + devops_ref: + type: string + required: False + description: | + Commit SHA or branch to checkout the devops directory. + tests_ref: type: string required: False + description: Commit SHA or branch to checkout e2e/cts tests. sycl_toolchain_artifact: type: string @@ -33,6 +58,28 @@ on: default: '{}' required: False + compiler: + type: string + required: false + default: "cl" + + cts_testing_mode: + description: | + Testing mode to run SYCL-CTS in, can be either `full`, `build-only` + or `run-only`. In `build-only` mode an artifact of the CTS binaries + will be uploaded. + type: string + default: 'full' + + sycl_cts_artifact: + type: string + default: '' + required: False + artifact_retention_days: + description: 'E2E/SYCL-CTS binaries artifact retention period.' + type: string + default: 1 + permissions: read-all jobs: @@ -42,22 +89,40 @@ jobs: environment: WindowsCILock env: ${{ fromJSON(inputs.env) }} steps: + - name: Detect hung tests + if: always() + shell: powershell + continue-on-error: true + run: | + Invoke-WebRequest -Uri "https://raw.githubusercontent.com/intel/llvm/refs/heads/sycl/devops/scripts/windows_detect_hung_tests.ps1" -OutFile "windows_detect_hung_tests.ps1" + powershell.exe -File windows_detect_hung_tests.ps1 + $exitCode = $LASTEXITCODE + Remove-Item -Path "windows_detect_hung_tests.ps1" + exit $exitCode + - uses: actions/checkout@v4 + with: + sparse-checkout: | + devops/actions + ref: ${{ inputs.devops_ref|| inputs.repo_ref || github.sha }} - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 with: arch: amd64 + - name: Setup oneAPI env + uses: ./devops/actions/setup_windows_oneapi_env + if: ${{ always() && !cancelled() && inputs.compiler == 'icx' }} - name: Set env run: | git config --system core.longpaths true git config --global core.autocrlf false echo "C:\Program Files\Git\usr\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - # TODO: use cached_checkout - - uses: actions/checkout@v4 + - name: Register cleanup after job is finished + uses: ./devops/actions/cleanup + - uses: ./devops/actions/cached_checkout + if: inputs.tests_selector == 'e2e' with: - persist-credentials: false - ref: ${{ inputs.ref || github.sha }} path: llvm - - name: Register cleanup after job is finished - uses: ./llvm/devops/actions/cleanup + ref: ${{ inputs.repo_ref || github.sha }} + cache_path: "D:\\\\github\\\\_work\\\\repo_cache\\\\" - name: Download compiler toolchain uses: actions/download-artifact@v4 with: @@ -71,36 +136,52 @@ jobs: - name: Setup SYCL toolchain run: | echo "PATH=$env:GITHUB_WORKSPACE\\install\\bin;$env:PATH" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + echo "LIB=$env:GITHUB_WORKSPACE\\install\\lib;$env:LIB" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - run: | sycl-ls - run: | sycl-ls --verbose - name: Configure E2E with Level Zero target + if: inputs.tests_selector == 'e2e' shell: cmd run: | mkdir build-e2e - cmake -GNinja -B build-e2e -S.\llvm\sycl\test-e2e -DSYCL_TEST_E2E_TARGETS="level_zero:gpu" -DCMAKE_CXX_COMPILER="clang++" -DLEVEL_ZERO_LIBS_DIR="D:\github\level-zero_win-sdk\lib" -DLEVEL_ZERO_INCLUDE="D:\github\level-zero_win-sdk\include" -DLLVM_LIT="..\llvm\llvm\utils\lit\lit.py" + cmake -GNinja -B build-e2e -S.\llvm\sycl\test-e2e -DSYCL_TEST_E2E_TARGETS="level_zero:gpu" -DCMAKE_CXX_COMPILER="clang++" -DLEVEL_ZERO_LIBS_DIR="D:\\github\\level-zero_win-sdk\\lib" -DLEVEL_ZERO_INCLUDE="D:\\github\\level-zero_win-sdk\\include" -DLLVM_LIT="..\\llvm\\llvm\\utils\\lit\\lit.py" - name: Run End-to-End tests + if: inputs.tests_selector == 'e2e' shell: bash run: | # Run E2E tests. + if [[ ${{inputs.compiler}} == 'icx' ]]; then + export LIT_FILTER_OUT="compile_on_win_with_mdd" + fi export LIT_OPTS="-v --no-progress-bar --show-unsupported --show-pass --show-xfail --max-time 3600 --time-tests ${{ inputs.extra_lit_opts }}" cmake --build build-e2e --target check-sycl-e2e - - name: Detect hung tests - shell: powershell + + - name: Run SYCL CTS Tests + if: inputs.tests_selector == 'cts' + uses: ./devops/actions/run-tests/windows/cts + with: + ref: ${{ inputs.tests_ref || 'main' }} + cts_exclude_ref: ${{ inputs.repo_ref }} + extra_cmake_args: ${{ inputs.extra_cmake_args }} + cts_testing_mode: ${{ inputs.cts_testing_mode }} + sycl_cts_artifact: ${{ inputs.sycl_cts_artifact }} + target_devices: ${{ inputs.target_devices }} + retention-days: ${{ inputs.artifact_retention_days }} + + - name: Detect hung tests if: always() + shell: powershell run: | - $exitCode = 0 - $hungTests = Get-Process | Where-Object { ($_.Path -match "llvm\\install") -or ($_.Path -match "llvm\\build-e2e") } - $hungTests | Foreach-Object { - $exitCode = 1 - echo "Test $($_.Path) hung!" - Stop-Process -Force $_ - } + Invoke-WebRequest -Uri "https://raw.githubusercontent.com/intel/llvm/refs/heads/sycl/devops/scripts/windows_detect_hung_tests.ps1" -OutFile "windows_detect_hung_tests.ps1" + powershell.exe -File windows_detect_hung_tests.ps1 + $exitCode = $LASTEXITCODE + Remove-Item -Path "windows_detect_hung_tests.ps1" exit $exitCode - name: Cleanup shell: cmd if: always() run: | rmdir /q /s install - rmdir /q /s build-e2e + if exist build-e2e rmdir /q /s build-e2e diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml new file mode 100644 index 0000000000000..26637cac9d5e3 --- /dev/null +++ b/.github/workflows/trivy.yml @@ -0,0 +1,37 @@ +# This workflow performs a trivy check of docker config files. + +name: Trivy + +on: + pull_request: + paths: + - 'devops/containers/**' + - 'devops/.trivyignore.yaml' + - '.github/workflows/trivy.yml' + workflow_dispatch: + +jobs: + build: + name: Trivy + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + sparse-checkout: devops + + # There is a github action, but for some reason it ignores ignore-file. + - name: Install Trivy + run: | + curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh + ./bin/trivy --version + + - name: Run Trivy vulnerability scanner + run: ./bin/trivy config --format json --output trivy-report.json --ignorefile=devops/.trivyignore.yaml devops/containers --exit-code 1 + + - name: Upload report artifact + uses: actions/upload-artifact@v4 + with: + name: trivy-report + path: trivy-report.json + retention-days: 3 diff --git a/devops/.trivyignore.yaml b/devops/.trivyignore.yaml new file mode 100644 index 0000000000000..f942ef6ba2a9d --- /dev/null +++ b/devops/.trivyignore.yaml @@ -0,0 +1,5 @@ +misconfigurations: + - id: AVD-DS-0001 + statement: "We use our own containers, no uncontrolled behavior is expected when the image is updated" + - id: AVD-DS-0026 + statement: "Our containers do not provide running services, but only preinstalled tools, there is not much value in adding HEALTHCHECK directives" diff --git a/devops/actions/cached_checkout/action.yml b/devops/actions/cached_checkout/action.yml index f2c25bcbca0a5..6f58f1de825bf 100644 --- a/devops/actions/cached_checkout/action.yml +++ b/devops/actions/cached_checkout/action.yml @@ -10,7 +10,7 @@ inputs: description: | Commit-ish to merge post-checkout if non-empty. Must be reachable from the default_branch input paramter. - default: 'FETCH_HEAD' + default: '' path: description: 'Path to checkout repo to' fetch-depth: @@ -34,6 +34,7 @@ runs: git pull --prune else git clone https://github.com/${{ inputs.repository }}.git . + git gc fi - name: Checkout env: diff --git a/devops/actions/run-tests/cts/action.yml b/devops/actions/run-tests/cts/action.yml new file mode 100644 index 0000000000000..93fd2770e166d --- /dev/null +++ b/devops/actions/run-tests/cts/action.yml @@ -0,0 +1,176 @@ +name: 'Run SYCL CTS tests on Linux' + +inputs: + ref: + description: "Commit SHA or branch to checkout tests" + required: true + cts_exclude_ref: + description: "Commit SHA or branch to checkout the cts_exclude_filter dir" + required: true + extra_cmake_args: + required: false + cts_testing_mode: + required: true + sycl_cts_artifact: + require: false + target_devices: + required: true + retention-days: + required: false + +runs: + using: "composite" + steps: + - name: Checkout cts_exclude_filter folder + uses: actions/checkout@v4 + with: + ref: ${{ inputs.cts_exclude_ref }} + path: cts_exclude + sparse-checkout: | + sycl/cts_exclude_filter + - name: Move sycl to root + shell: bash + run: | + mv cts_exclude/sycl . + rm -rf cts_exclude + - name: Checkout SYCL CTS tests + if: inputs.cts_testing_mode != 'run-only' + uses: ./devops/actions/cached_checkout + with: + path: khronos_sycl_cts + repository: 'KhronosGroup/SYCL-CTS' + ref: ${{ inputs.ref }} + cache_path: "/__w/repo_cache/" + - name: SYCL CTS GIT submodules init + if: inputs.cts_testing_mode != 'run-only' + shell: bash + run: | + git -C khronos_sycl_cts submodule update --init + - name: Build SYCL CTS tests + if: inputs.cts_testing_mode != 'run-only' + shell: bash + env: + CMAKE_EXTRA_ARGS: ${{ inputs.extra_cmake_args }} + run: | + cts_exclude_filter="" + # If CTS_TESTS_TO_BUILD is null - use filter + if [ -z "$CTS_TESTS_TO_BUILD" ]; then + if [ "${{ contains(inputs.cts_testing_mode, 'build-only') }}" = "true" ]; then + cts_exclude_filter=$PWD/sycl/cts_exclude_filter/compfails + elif [ "${{ contains(inputs.target_devices, 'opencl:cpu') }}" = "true" ]; then + cts_exclude_filter=$PWD/sycl/cts_exclude_filter/OCL_CPU + elif [ "${{ contains(inputs.target_devices, 'level_zero:gpu') }}" = "true" ]; then + cts_exclude_filter=$PWD/sycl/cts_exclude_filter/L0_GPU + fi + + # List excluded SYCL CTS categories: + # SYCL_CTS_EXCLUDE_TEST_CATEGORIES - Optional file specifying a list + # of test categories to be excluded from the build. + echo "::group::Excluded test categories" + cat $cts_exclude_filter + echo "::endgroup::" + fi + + cmake -GNinja -B./build-cts -S./khronos_sycl_cts -DCMAKE_CXX_COMPILER=$(which clang++) \ + -DSYCL_IMPLEMENTATION=DPCPP \ + -DSYCL_CTS_EXCLUDE_TEST_CATEGORIES="$cts_exclude_filter" \ + -DSYCL_CTS_ENABLE_OPENCL_INTEROP_TESTS=OFF \ + -DDPCPP_INSTALL_DIR="$(dirname $(which clang++))/.." \ + $CMAKE_EXTRA_ARGS + # Ignore errors so that if one category build fails others still have a + # chance to finish and be executed at the run stage. Note that + # "test_conformance" target skips building "test_all" executable. + ninja -C build-cts -k0 $( [ -n "$CTS_TESTS_TO_BUILD" ] && echo "$CTS_TESTS_TO_BUILD" || echo "test_conformance") + + - name: Pack SYCL-CTS binaries + if: always() && !cancelled() && inputs.cts_testing_mode == 'build-only' + shell: bash + run: tar -I 'zstd -9' -cf sycl_cts_bin.tar.zst -C ./build-cts/bin . + + - name: Upload SYCL-CTS binaries + if: always() && !cancelled() && inputs.cts_testing_mode == 'build-only' + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.sycl_cts_artifact }} + path: sycl_cts_bin.tar.zst + retention-days: ${{ inputs.retention-days }} + + - name: Download SYCL-CTS binaries + if: inputs.cts_testing_mode == 'run-only' + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.sycl_cts_artifact }} + + - name: Extract SYCL-CTS binaries + if: inputs.cts_testing_mode == 'run-only' + shell: bash + run: | + mkdir -p build-cts/bin + tar -I 'zstd' -xf sycl_cts_bin.tar.zst -C build-cts/bin + + - name: SYCL CTS List devices + # Proceed with execution even if the 'build' step did not succeed. + if: (always() && !cancelled()) && inputs.cts_testing_mode != 'build-only' + shell: bash + env: + ONEAPI_DEVICE_SELECTOR: ${{ inputs.target_devices }} + run: | + ./build-cts/bin/* --list-devices + + # If the suite was built on another machine then the build contains the full + # set of tests. We have special files to filter out some test categories, + # see "sycl/cts_exclude_filter/*". Each configuration has its own file, e.g. + # there is "cts_exclude_filter/OCL_CPU" for opencl:cpu device. Therefore, + # these files may differ from each other, so when there is a pre-built set of + # tests, we need to filter it according to the filter-file. + - name: Filter SYCL CTS test categories + if: inputs.cts_testing_mode == 'run-only' + shell: bash + run: | + cts_exclude_filter="" + if [ "${{ contains(inputs.target_devices, 'opencl:cpu') }}" = "true" ]; then + cts_exclude_filter=$PWD/sycl/cts_exclude_filter/OCL_CPU + elif [ "${{ contains(inputs.target_devices, 'level_zero:gpu') }}" = "true" ]; then + cts_exclude_filter=$PWD/sycl/cts_exclude_filter/L0_GPU + fi + + while IFS= read -r line; do + if [[ $line != \#* ]]; then + rm "./build-cts/bin/test_$line" + fi + done < "$cts_exclude_filter" + + - name: Run SYCL CTS tests + # Proceed with execution even if the previous two steps did not succeed. + if: (always() && !cancelled()) && inputs.cts_testing_mode != 'build-only' + env: + ONEAPI_DEVICE_SELECTOR: ${{ inputs.target_devices }} + # By-default GitHub actions execute the "run" shell script with -e option, + # so the execution terminates if any command returns a non-zero status. + # Since we're using a loop to run all test-binaries separately, some test + # may fail and terminate the execution. Setting "shell" value to override + # the default behavior. + # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#custom-shell + shell: bash {0} + run: | + # Run each test category separately so that + # - crash on one would not affect others + # - multiple tests could be run in parallel + mkdir logs + find build-cts/bin/ -type f -print | \ + xargs -t -I % -P 8 sh -c 'log=logs/$(basename %).log ; echo % >$log ; date >>$log ; timeout 60m % >>$log 2>&1 ; ret=$? ; echo "exit code: $ret" >>$log ; date >>$log ; exit $ret' + ret=$? + + for f in logs/* ; do + echo "::group::$f" + cat $f + echo "::endgroup::" + done + + echo "::group::Fails:" + grep 'exit code: [^0]' -r logs + echo "::endgroup::" + + grep 'exit code: [^0]' -r logs >> $GITHUB_STEP_SUMMARY + + exit $ret diff --git a/devops/actions/run-tests/e2e/action.yml b/devops/actions/run-tests/e2e/action.yml new file mode 100644 index 0000000000000..47fc75599ccdb --- /dev/null +++ b/devops/actions/run-tests/e2e/action.yml @@ -0,0 +1,83 @@ +name: 'Run SYCL E2E tests' + +inputs: + ref: + required: false + binaries_artifact: + required: false + testing_mode: + required: true + extra_cmake_args: + required: false + target_devices: + required: true + extra_lit_opts: + required: false + retention-days: + required: false + cxx_compiler: + required: false + + +runs: + using: "composite" + steps: + - name: Checkout E2E tests + uses: ./devops/actions/cached_checkout + with: + path: llvm + ref: ${{ inputs.ref || github.sha }} + cache_path: "/__w/repo_cache/" + + - name: Download E2E Binaries + if: inputs.testing_mode == 'run-only' + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.binaries_artifact }} + - name: Extract E2E Binaries + if: inputs.testing_mode == 'run-only' + shell: bash + run: | + mkdir build-e2e + tar -I 'zstd' -xf e2e_binaries.tar.zst -C build-e2e + + - name: Deduce E2E CMake options + if: inputs.testing_mode != 'run-only' + id: cmake_opts + shell: bash + env: + CMAKE_EXTRA_ARGS: ${{ inputs.extra_cmake_args }} + run: | + if [ -n "$CMAKE_EXTRA_ARGS" ]; then + echo "opts=$CMAKE_EXTRA_ARGS" >> $GITHUB_OUTPUT + fi + - name: Configure E2E tests + if: inputs.testing_mode != 'run-only' + shell: bash + run: | + cmake -GNinja -B./build-e2e -S./llvm/sycl/test-e2e -DCMAKE_CXX_COMPILER="${{ inputs.cxx_compiler || '$(which clang++)'}}" -DLLVM_LIT="$PWD/llvm/llvm/utils/lit/lit.py" ${{ steps.cmake_opts.outputs.opts }} + - name: SYCL End-to-end tests + shell: bash {0} + env: + LIT_OPTS: -v --no-progress-bar --show-unsupported --show-pass --show-xfail --max-time 3600 --time-tests --param print_features=True --param test-mode=${{ inputs.testing_mode }} --param sycl_devices=${{ inputs.target_devices }} ${{ inputs.extra_lit_opts }} + run: | + ninja -C build-e2e check-sycl-e2e > e2e.log 2>&1 + exit_code=$? + cat e2e.log + if [ $exit_code -ne 0 ]; then + awk '/^Failed Tests|Unexpectedly Passed Tests|Unresolved tests|Timed Out Tests|Testing Time/{flag=1}/FAILED: CMakeFiles/{flag=0}flag' e2e.log >> $GITHUB_STEP_SUMMARY + fi + exit $exit_code + + - name: Pack E2E binaries + if: ${{ always() && !cancelled() && inputs.binaries_artifact != '' && inputs.testing_mode != 'run-only'}} + shell: bash + run: | + tar -I 'zstd -9' -cf e2e_binaries.tar.zst -C ./build-e2e . + - name: Upload E2E binaries + if: ${{ always() && !cancelled() && inputs.binaries_artifact != '' && inputs.testing_mode != 'run-only'}} + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.binaries_artifact }} + path: e2e_binaries.tar.zst + retention-days: ${{ inputs.retention-days }} diff --git a/devops/actions/run-tests/windows/cts/action.yml b/devops/actions/run-tests/windows/cts/action.yml new file mode 100644 index 0000000000000..4d4d7aa7f9f46 --- /dev/null +++ b/devops/actions/run-tests/windows/cts/action.yml @@ -0,0 +1,178 @@ +name: 'Run SYCL CTS tests on Windows' + +inputs: + ref: + description: "Commit SHA or branch to checkout tests" + required: true + cts_exclude_ref: + description: "Commit SHA or branch to checkout the cts_exclude_filter dir" + required: true + extra_cmake_args: + required: false + cts_testing_mode: + required: true + sycl_cts_artifact: + require: false + target_devices: + required: true + retention-days: + required: false + +runs: + using: "composite" + steps: + - name: Checkout cts_exclude_filter folder + uses: actions/checkout@v4 + with: + ref: ${{ inputs.cts_exclude_ref }} + path: cts_exclude + sparse-checkout: | + sycl/cts_exclude_filter + - name: Move sycl to root + shell: bash + run: | + mv cts_exclude/sycl . + rm -rf cts_exclude + - name: Checkout SYCL CTS tests + if: inputs.cts_testing_mode != 'run-only' + uses: ./devops/actions/cached_checkout + with: + path: khronos_sycl_cts + repository: 'KhronosGroup/SYCL-CTS' + ref: ${{ inputs.ref }} + cache_path: "D:\\\\github\\\\_work\\\\repo_cache\\\\" + - name: SYCL CTS GIT submodules init + if: inputs.cts_testing_mode != 'run-only' + shell: bash + run: | + git -C khronos_sycl_cts submodule update --init + - name: Build SYCL CTS tests + if: inputs.cts_testing_mode != 'run-only' + shell: bash + env: + CMAKE_EXTRA_ARGS: ${{ inputs.extra_cmake_args }} + run: | + cts_exclude_filter="" + # If CTS_TESTS_TO_BUILD is null - use filter + if [ -z "$CTS_TESTS_TO_BUILD" ]; then + if [ "${{ contains(inputs.cts_testing_mode, 'build-only') }}" = "true" ]; then + cts_exclude_filter=$PWD/sycl/cts_exclude_filter/compfails + elif [ "${{ contains(inputs.target_devices, 'opencl:cpu') }}" = "true" ]; then + cts_exclude_filter=$PWD/sycl/cts_exclude_filter/OCL_CPU + elif [ "${{ contains(inputs.target_devices, 'level_zero:gpu') }}" = "true" ]; then + cts_exclude_filter=$PWD/sycl/cts_exclude_filter/L0_GPU + fi + + # List excluded SYCL CTS categories: + # SYCL_CTS_EXCLUDE_TEST_CATEGORIES - Optional file specifying a list + # of test categories to be excluded from the build. + echo "::group::Excluded test categories" + cat $cts_exclude_filter + echo "::endgroup::" + fi + + cmake -GNinja -B./build-cts -S./khronos_sycl_cts \ + -DSYCL_IMPLEMENTATION=DPCPP \ + -DSYCL_CTS_EXCLUDE_TEST_CATEGORIES="$cts_exclude_filter" \ + -DSYCL_CTS_ENABLE_OPENCL_INTEROP_TESTS=OFF \ + -DDPCPP_INSTALL_DIR="$(dirname $(which clang++))/.." \ + -DCMAKE_CXX_COMPILER=cl \ + -DCMAKE_BUILD_TYPE=Release \ + $CMAKE_EXTRA_ARGS + # Ignore errors so that if one category build fails others still have a + # chance to finish and be executed at the run stage. Note that + # "test_conformance" target skips building "test_all" executable. + ninja -C build-cts -k0 $( [ -n "$CTS_TESTS_TO_BUILD" ] && echo "$CTS_TESTS_TO_BUILD" || echo "test_conformance") + + - name: Pack SYCL-CTS binaries + if: always() && !cancelled() && inputs.cts_testing_mode == 'build-only' + shell: bash + run: tar -I 'zstd -9' -cf sycl_cts_bin.tar.zst -C ./build-cts/bin . + + - name: Upload SYCL-CTS binaries + if: always() && !cancelled() && inputs.cts_testing_mode == 'build-only' + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.sycl_cts_artifact }} + path: sycl_cts_bin.tar.zst + retention-days: ${{ inputs.retention-days }} + + - name: Download SYCL-CTS binaries + if: inputs.cts_testing_mode == 'run-only' + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.sycl_cts_artifact }} + + - name: Extract SYCL-CTS binaries + if: inputs.cts_testing_mode == 'run-only' + shell: bash + run: | + mkdir -p build-cts/bin + tar -I 'zstd' -xf sycl_cts_bin.tar.zst -C build-cts/bin + + - name: SYCL CTS List devices + # Proceed with execution even if the 'build' step did not succeed. + if: (always() && !cancelled()) && inputs.cts_testing_mode != 'build-only' + shell: bash + env: + ONEAPI_DEVICE_SELECTOR: ${{ inputs.target_devices }} + run: | + ./build-cts/bin/* --list-devices + + # If the suite was built on another machine then the build contains the full + # set of tests. We have special files to filter out some test categories, + # see "sycl/cts_exclude_filter/*". Each configuration has its own file, e.g. + # there is "cts_exclude_filter/OCL_CPU" for opencl:cpu device. Therefore, + # these files may differ from each other, so when there is a pre-built set of + # tests, we need to filter it according to the filter-file. + - name: Filter SYCL CTS test categories + if: inputs.cts_testing_mode == 'run-only' + shell: bash + run: | + cts_exclude_filter="" + if [ "${{ contains(inputs.target_devices, 'opencl:cpu') }}" = "true" ]; then + cts_exclude_filter=$PWD/sycl/cts_exclude_filter/OCL_CPU + elif [ "${{ contains(inputs.target_devices, 'level_zero:gpu') }}" = "true" ]; then + cts_exclude_filter=$PWD/sycl/cts_exclude_filter/L0_GPU + fi + + while IFS= read -r line; do + if [[ $line != \#* ]]; then + rm "./build-cts/bin/test_$line" + fi + done < "$cts_exclude_filter" + + - name: Run SYCL CTS tests + # Proceed with execution even if the previous two steps did not succeed. + if: (always() && !cancelled()) && inputs.cts_testing_mode != 'build-only' + env: + ONEAPI_DEVICE_SELECTOR: ${{ inputs.target_devices }} + # By-default GitHub actions execute the "run" shell script with -e option, + # so the execution terminates if any command returns a non-zero status. + # Since we're using a loop to run all test-binaries separately, some test + # may fail and terminate the execution. Setting "shell" value to override + # the default behavior. + # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#custom-shell + shell: bash {0} + run: | + # Run each test category separately so that + # - crash on one would not affect others + # - multiple tests could be run in parallel + mkdir logs + find build-cts/bin/ -type f -print | \ + xargs -t -I % -P 8 sh -c 'log=logs/$(basename %).log ; echo % >$log ; date >>$log ; timeout 60m % >>$log 2>&1 ; ret=$? ; echo "exit code: $ret" >>$log ; date >>$log ; exit $ret' + ret=$? + + for f in logs/* ; do + echo "::group::$f" + cat $f + echo "::endgroup::" + done + + echo "::group::Fails:" + grep 'exit code: [^0]' -r logs + echo "::endgroup::" + + grep 'exit code: [^0]' -r logs >> $GITHUB_STEP_SUMMARY + + exit $ret diff --git a/devops/actions/setup_linux_oneapi_env/action.yml b/devops/actions/setup_linux_oneapi_env/action.yml new file mode 100644 index 0000000000000..e54bd13d99b5e --- /dev/null +++ b/devops/actions/setup_linux_oneapi_env/action.yml @@ -0,0 +1,22 @@ +name: Linux setup oneAPI env + +runs: + using: "composite" + steps: + - name: Setup oneAPI env + shell: bash + run: | + sudo apt-get --fix-broken -y install + wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor \ + | sudo tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \ + sudo echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" \ + | sudo tee /etc/apt/sources.list.d/oneAPI.list && \ + sudo apt update && sudo apt-get -y install intel-oneapi-compiler-dpcpp-cpp-2025.0 + + env_before=$(env | sort) + source /opt/intel/oneapi/setvars.sh + env_after=$(env | sort) + changed_envvars=$(comm -13 <(echo "$env_before") <(echo "$env_after")) + while IFS= read -r line; do + echo "$line" >> $GITHUB_ENV + done <<< "$changed_envvars" diff --git a/devops/actions/setup_windows_oneapi_env/action.yml b/devops/actions/setup_windows_oneapi_env/action.yml new file mode 100644 index 0000000000000..7b7463f697f23 --- /dev/null +++ b/devops/actions/setup_windows_oneapi_env/action.yml @@ -0,0 +1,28 @@ +name: Windows setup oneAPI env + +runs: + using: "composite" + steps: + - name: Setup oneAPI env + shell: powershell + run: | + $batchFilePath = "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" + + $githubEnvFilePath = $env:GITHUB_ENV + + $envBefore = Get-ChildItem Env: | ForEach-Object { "$($_.Name)=$($_.Value)" } + + $envVars = & cmd.exe /c "call `"$batchFilePath`" && set" | Out-String + + $envAfter = $envVars -split "`r`n" | Where-Object { $_ -match "^(.*?)=(.*)$" } + + foreach ($envVar in $envAfter) { + if ($envVar -match "^(.*?)=(.*)$") { + $name = $matches[1] + $value = $matches[2] + $envBeforeVar = $envBefore | Where-Object { $_ -like "$name=*" } + if (-not $envBeforeVar -or $envBeforeVar -ne "$name=$value") { + Add-Content -Path $githubEnvFilePath -Value "$name=$value" + } + } + } diff --git a/devops/bandit.config b/devops/bandit.config new file mode 100644 index 0000000000000..4e501feef37ef --- /dev/null +++ b/devops/bandit.config @@ -0,0 +1,402 @@ + +### Bandit config file generated from: +# './bandit/bandit/cli/config_generator.py --out bandit.config' + +### This config may optionally select a subset of tests to run or skip by +### filling out the 'tests' and 'skips' lists given below. If no tests are +### specified for inclusion then it is assumed all tests are desired. The skips +### set will remove specific tests from the include set. This can be controlled +### using the -t/-s CLI options. Note that the same test ID should not appear +### in both 'tests' and 'skips', this would be nonsensical and is detected by +### Bandit at runtime. + +# Available tests: +# B101 : assert_used +# B102 : exec_used +# B103 : set_bad_file_permissions +# B104 : hardcoded_bind_all_interfaces +# B105 : hardcoded_password_string +# B106 : hardcoded_password_funcarg +# B107 : hardcoded_password_default +# B108 : hardcoded_tmp_directory +# B110 : try_except_pass +# B112 : try_except_continue +# B201 : flask_debug_true +# B301 : pickle +# B302 : marshal +# B303 : md5 +# B304 : ciphers +# B305 : cipher_modes +# B306 : mktemp_q +# B307 : eval +# B308 : mark_safe +# B310 : urllib_urlopen +# B311 : random +# B312 : telnetlib +# B313 : xml_bad_cElementTree +# B314 : xml_bad_ElementTree +# B315 : xml_bad_expatreader +# B316 : xml_bad_expatbuilder +# B317 : xml_bad_sax +# B318 : xml_bad_minidom +# B319 : xml_bad_pulldom +# B321 : ftplib +# B323 : unverified_context +# B324 : hashlib_new_insecure_functions +# B401 : import_telnetlib +# B402 : import_ftplib +# B403 : import_pickle +# B404 : import_subprocess +# B405 : import_xml_etree +# B406 : import_xml_sax +# B407 : import_xml_expat +# B408 : import_xml_minidom +# B409 : import_xml_pulldom +# B411 : import_xmlrpclib +# B412 : import_httpoxy +# B413 : import_pycrypto +# B501 : request_with_no_cert_validation +# B502 : ssl_with_bad_version +# B503 : ssl_with_bad_defaults +# B504 : ssl_with_no_version +# B505 : weak_cryptographic_key +# B506 : yaml_load +# B507 : ssh_no_host_key_verification +# B601 : paramiko_calls +# B602 : subprocess_popen_with_shell_equals_true +# B603 : subprocess_without_shell_equals_true +# B604 : any_other_function_with_shell_equals_true +# B605 : start_process_with_a_shell +# B606 : start_process_with_no_shell +# B607 : start_process_with_partial_path +# B608 : hardcoded_sql_expressions +# B609 : linux_commands_wildcard_injection +# B610 : django_extra_used +# B611 : django_rawsql_used +# B701 : jinja2_autoescape_false +# B702 : use_of_mako_templates +# B703 : django_mark_safe + +# (optional) list included test IDs here, eg '[B101, B406]': +# IPAS Required Checkers. Do not disable these +# Additional checkers may be added if desired +tests: + [ 'B301', 'B302', 'B303', 'B304', 'B305', 'B306', 'B308', 'B310', 'B311', 'B312', 'B313', 'B314', 'B315', 'B316', 'B317', 'B318', 'B319', 'B321', 'B323', 'B324', 'B401', 'B402', 'B403', 'B404', 'B405', 'B406', 'B407', 'B408', 'B409', 'B411', 'B412', 'B413'] + +# (optional) list skipped test IDs here, eg '[B101, B406]': +# The following checkers are not required but be added to tests list if desired +skips: + [ 'B101', 'B102', 'B103', 'B104', 'B105', 'B106', 'B107', 'B108', 'B110', 'B112', 'B201', 'B501', 'B502', 'B503', 'B504', 'B505', 'B506', 'B507', 'B601', 'B602', 'B603', 'B604', 'B605', 'B606', 'B607', 'B608', 'B609', 'B610', 'B611', 'B701', 'B702', 'B703'] + +# Exclude projects that are not part of SYCL (we don't build them / run any content) +exclude_dirs: ['bolt', 'cross-project-tests', 'libc/', 'lldb', 'mlir', 'openmp', 'polly', 'pstl', 'third-party'] + +### (optional) plugin settings - some test plugins require configuration data +### that may be given here, per-plugin. All bandit test plugins have a built in +### set of sensible defaults and these will be used if no configuration is +### provided. It is not necessary to provide settings for every (or any) plugin +### if the defaults are acceptable. + +any_other_function_with_shell_equals_true: + no_shell: + - os.execl + - os.execle + - os.execlp + - os.execlpe + - os.execv + - os.execve + - os.execvp + - os.execvpe + - os.spawnl + - os.spawnle + - os.spawnlp + - os.spawnlpe + - os.spawnv + - os.spawnve + - os.spawnvp + - os.spawnvpe + - os.startfile + shell: + - os.system + - os.popen + - os.popen2 + - os.popen3 + - os.popen4 + - popen2.popen2 + - popen2.popen3 + - popen2.popen4 + - popen2.Popen3 + - popen2.Popen4 + - commands.getoutput + - commands.getstatusoutput + subprocess: + - subprocess.Popen + - subprocess.call + - subprocess.check_call + - subprocess.check_output + - subprocess.run +assert_used: + skips: [] +hardcoded_tmp_directory: + tmp_dirs: + - /tmp + - /var/tmp + - /dev/shm +linux_commands_wildcard_injection: + no_shell: + - os.execl + - os.execle + - os.execlp + - os.execlpe + - os.execv + - os.execve + - os.execvp + - os.execvpe + - os.spawnl + - os.spawnle + - os.spawnlp + - os.spawnlpe + - os.spawnv + - os.spawnve + - os.spawnvp + - os.spawnvpe + - os.startfile + shell: + - os.system + - os.popen + - os.popen2 + - os.popen3 + - os.popen4 + - popen2.popen2 + - popen2.popen3 + - popen2.popen4 + - popen2.Popen3 + - popen2.Popen4 + - commands.getoutput + - commands.getstatusoutput + subprocess: + - subprocess.Popen + - subprocess.call + - subprocess.check_call + - subprocess.check_output + - subprocess.run +ssl_with_bad_defaults: + bad_protocol_versions: + - PROTOCOL_SSLv2 + - SSLv2_METHOD + - SSLv23_METHOD + - PROTOCOL_SSLv3 + - PROTOCOL_TLSv1 + - SSLv3_METHOD + - TLSv1_METHOD +ssl_with_bad_version: + bad_protocol_versions: + - PROTOCOL_SSLv2 + - SSLv2_METHOD + - SSLv23_METHOD + - PROTOCOL_SSLv3 + - PROTOCOL_TLSv1 + - SSLv3_METHOD + - TLSv1_METHOD +start_process_with_a_shell: + no_shell: + - os.execl + - os.execle + - os.execlp + - os.execlpe + - os.execv + - os.execve + - os.execvp + - os.execvpe + - os.spawnl + - os.spawnle + - os.spawnlp + - os.spawnlpe + - os.spawnv + - os.spawnve + - os.spawnvp + - os.spawnvpe + - os.startfile + shell: + - os.system + - os.popen + - os.popen2 + - os.popen3 + - os.popen4 + - popen2.popen2 + - popen2.popen3 + - popen2.popen4 + - popen2.Popen3 + - popen2.Popen4 + - commands.getoutput + - commands.getstatusoutput + subprocess: + - subprocess.Popen + - subprocess.call + - subprocess.check_call + - subprocess.check_output + - subprocess.run +start_process_with_no_shell: + no_shell: + - os.execl + - os.execle + - os.execlp + - os.execlpe + - os.execv + - os.execve + - os.execvp + - os.execvpe + - os.spawnl + - os.spawnle + - os.spawnlp + - os.spawnlpe + - os.spawnv + - os.spawnve + - os.spawnvp + - os.spawnvpe + - os.startfile + shell: + - os.system + - os.popen + - os.popen2 + - os.popen3 + - os.popen4 + - popen2.popen2 + - popen2.popen3 + - popen2.popen4 + - popen2.Popen3 + - popen2.Popen4 + - commands.getoutput + - commands.getstatusoutput + subprocess: + - subprocess.Popen + - subprocess.call + - subprocess.check_call + - subprocess.check_output + - subprocess.run +start_process_with_partial_path: + no_shell: + - os.execl + - os.execle + - os.execlp + - os.execlpe + - os.execv + - os.execve + - os.execvp + - os.execvpe + - os.spawnl + - os.spawnle + - os.spawnlp + - os.spawnlpe + - os.spawnv + - os.spawnve + - os.spawnvp + - os.spawnvpe + - os.startfile + shell: + - os.system + - os.popen + - os.popen2 + - os.popen3 + - os.popen4 + - popen2.popen2 + - popen2.popen3 + - popen2.popen4 + - popen2.Popen3 + - popen2.Popen4 + - commands.getoutput + - commands.getstatusoutput + subprocess: + - subprocess.Popen + - subprocess.call + - subprocess.check_call + - subprocess.check_output + - subprocess.run +subprocess_popen_with_shell_equals_true: + no_shell: + - os.execl + - os.execle + - os.execlp + - os.execlpe + - os.execv + - os.execve + - os.execvp + - os.execvpe + - os.spawnl + - os.spawnle + - os.spawnlp + - os.spawnlpe + - os.spawnv + - os.spawnve + - os.spawnvp + - os.spawnvpe + - os.startfile + shell: + - os.system + - os.popen + - os.popen2 + - os.popen3 + - os.popen4 + - popen2.popen2 + - popen2.popen3 + - popen2.popen4 + - popen2.Popen3 + - popen2.Popen4 + - commands.getoutput + - commands.getstatusoutput + subprocess: + - subprocess.Popen + - subprocess.call + - subprocess.check_call + - subprocess.check_output + - subprocess.run +subprocess_without_shell_equals_true: + no_shell: + - os.execl + - os.execle + - os.execlp + - os.execlpe + - os.execv + - os.execve + - os.execvp + - os.execvpe + - os.spawnl + - os.spawnle + - os.spawnlp + - os.spawnlpe + - os.spawnv + - os.spawnve + - os.spawnvp + - os.spawnvpe + - os.startfile + shell: + - os.system + - os.popen + - os.popen2 + - os.popen3 + - os.popen4 + - popen2.popen2 + - popen2.popen3 + - popen2.popen4 + - popen2.Popen3 + - popen2.Popen4 + - commands.getoutput + - commands.getstatusoutput + subprocess: + - subprocess.Popen + - subprocess.call + - subprocess.check_call + - subprocess.check_output + - subprocess.run +try_except_continue: + check_typed_exception: false +try_except_pass: + check_typed_exception: false +weak_cryptographic_key: + weak_key_size_dsa_high: 1024 + weak_key_size_dsa_medium: 2048 + weak_key_size_ec_high: 160 + weak_key_size_ec_medium: 224 + weak_key_size_rsa_high: 1024 + weak_key_size_rsa_medium: 2048 + diff --git a/devops/containers/nightly.Dockerfile b/devops/containers/nightly.Dockerfile new file mode 100644 index 0000000000000..7639f676c1c95 --- /dev/null +++ b/devops/containers/nightly.Dockerfile @@ -0,0 +1,18 @@ +ARG base_tag=alldeps +ARG base_image=ghcr.io/intel/llvm/ubuntu2404_intel_drivers + +FROM $base_image:$base_tag + +USER root + +COPY scripts/drivers_entrypoint.sh /drivers_entrypoint.sh +RUN mkdir -p /opt/sycl +ADD sycl_linux.tar.gz /opt/sycl/ + +ENV PATH /opt/sycl/bin:$PATH +ENV LD_LIBRARY_PATH /opt/sycl/lib:$LD_LIBRARY_PATH + +USER sycl + +ENTRYPOINT ["/bin/bash", "/drivers_entrypoint.sh"] + diff --git a/devops/containers/ubuntu2204_base.Dockerfile b/devops/containers/ubuntu2204_base.Dockerfile index 07bb343cae93c..d08b6025e9908 100644 --- a/devops/containers/ubuntu2204_base.Dockerfile +++ b/devops/containers/ubuntu2204_base.Dockerfile @@ -8,25 +8,14 @@ USER root COPY scripts/install_build_tools.sh /install.sh RUN /install.sh -# By default Ubuntu sets an arbitrary UID value, that is different from host -# system. When CI passes default UID value of 1001, some of LLVM tools fail to -# discover user home directory and fail a few LIT tests. Fixes UID and GID to -# 1001, that is used as default by GitHub Actions. -RUN groupadd -g 1001 sycl && useradd sycl -u 1001 -g 1001 -m -s /bin/bash -# Add sycl user to video/irc groups so that it can access GPU -RUN usermod -aG video sycl -RUN usermod -aG irc sycl - -# group 109 is required for sycl user to access PVC card. -RUN groupadd -g 109 render -RUN usermod -aG render sycl - -# Allow sycl user to run as sudo -RUN echo "sycl ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers +COPY scripts/create-sycl-user.sh /user-setup.sh +RUN /user-setup.sh COPY actions/cached_checkout /actions/cached_checkout COPY actions/cleanup /actions/cleanup COPY scripts/docker_entrypoint.sh /docker_entrypoint.sh COPY scripts/install_drivers.sh /opt/install_drivers.sh +USER sycl + ENTRYPOINT ["/docker_entrypoint.sh"] diff --git a/devops/containers/ubuntu2204_build.Dockerfile b/devops/containers/ubuntu2204_build.Dockerfile index 313b455dbc25b..1aa814aaa775c 100644 --- a/devops/containers/ubuntu2204_build.Dockerfile +++ b/devops/containers/ubuntu2204_build.Dockerfile @@ -24,23 +24,18 @@ gpg --dearmor | tee /etc/apt/keyrings/rocm.gpg > /dev/null && \ # Add rocm repo echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/6.1.1 jammy main" \ | tee --append /etc/apt/sources.list.d/rocm.list && \ -printf 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' | tee /etc/apt/preferences.d/rocm-pin-600 && \ -apt update +printf 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' | tee /etc/apt/preferences.d/rocm-pin-600 # Install the kernel driver -RUN apt install -yqq rocm-dev && \ +RUN apt update && apt install -yqq rocm-dev && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -# By default Ubuntu sets an arbitrary UID value, that is different from host -# system. When CI passes default UID value of 1001, some of LLVM tools fail to -# discover user home directory and fail a few LIT tests. Fixes UID and GID to -# 1001, that is used as default by GitHub Actions. -RUN groupadd -g 1001 sycl && useradd sycl -u 1001 -g 1001 -m -s /bin/bash -# Add sycl user to video/irc groups so that it can access GPU -RUN usermod -aG video sycl -RUN usermod -aG irc sycl +COPY scripts/create-sycl-user.sh /user-setup.sh +RUN /user-setup.sh COPY scripts/docker_entrypoint.sh /docker_entrypoint.sh +USER sycl + ENTRYPOINT ["/docker_entrypoint.sh"] diff --git a/devops/containers/ubuntu2204_intel_drivers.Dockerfile b/devops/containers/ubuntu2204_intel_drivers.Dockerfile index fb018f195a48f..b27aee1b633b0 100644 --- a/devops/containers/ubuntu2204_intel_drivers.Dockerfile +++ b/devops/containers/ubuntu2204_intel_drivers.Dockerfile @@ -5,7 +5,9 @@ FROM $base_image:$base_tag ENV DEBIAN_FRONTEND=noninteractive -ARG use_latest=true +ARG use_unstable_driver=true + +USER root RUN apt update && apt install -yqq wget @@ -16,7 +18,7 @@ COPY dependencies.json / RUN mkdir /runtimes ENV INSTALL_LOCATION=/runtimes RUN --mount=type=secret,id=github_token \ - if [ "$use_latest" = "true" ]; then \ + if [ "$use_unstable_driver" = "true" ]; then \ install_driver_opt=" --use-latest"; \ else \ install_driver_opt=" dependencies.json"; \ @@ -25,5 +27,7 @@ RUN --mount=type=secret,id=github_token \ COPY scripts/drivers_entrypoint.sh /drivers_entrypoint.sh +USER sycl + ENTRYPOINT ["/bin/bash", "/drivers_entrypoint.sh"] diff --git a/devops/containers/ubuntu2404_base.Dockerfile b/devops/containers/ubuntu2404_base.Dockerfile new file mode 100644 index 0000000000000..3cdad5b74366e --- /dev/null +++ b/devops/containers/ubuntu2404_base.Dockerfile @@ -0,0 +1,28 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +USER root + +# Install SYCL prerequisites +COPY scripts/install_build_tools.sh /install.sh +RUN /install.sh + +# libzstd-dev installed by default on Ubuntu 24.04 is not compiled with -fPIC flag. +# This causes linking errors when building SYCL runtime. +# Bug: https://github.com/intel/llvm/issues/15935 +# Workaround: build zstd from sources with -fPIC flag. +COPY scripts/build_zstd_1_5_6_ub24.sh /build_zstd_1_5_6_ub24.sh +RUN /build_zstd_1_5_6_ub24.sh + +COPY scripts/create-sycl-user.sh /user-setup.sh +RUN /user-setup.sh + +COPY actions/cached_checkout /actions/cached_checkout +COPY actions/cleanup /actions/cleanup +COPY scripts/docker_entrypoint.sh /docker_entrypoint.sh +COPY scripts/install_drivers.sh /opt/install_drivers.sh + +USER sycl + +ENTRYPOINT ["/docker_entrypoint.sh"] diff --git a/devops/containers/ubuntu2404_build.Dockerfile b/devops/containers/ubuntu2404_build.Dockerfile new file mode 100644 index 0000000000000..c659eabbced51 --- /dev/null +++ b/devops/containers/ubuntu2404_build.Dockerfile @@ -0,0 +1,49 @@ +FROM nvidia/cuda:12.6.3-devel-ubuntu24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +USER root + +# Install SYCL prerequisites +COPY scripts/install_build_tools.sh /install.sh +RUN /install.sh + +# libzstd-dev installed by default on Ubuntu 24.04 is not compiled with -fPIC flag. +# This causes linking errors when building SYCL runtime. +# Bug: https://github.com/intel/llvm/issues/15935 +# Workaround: build zstd from sources with -fPIC flag. +COPY scripts/build_zstd_1_5_6_ub24.sh /build_zstd_1_5_6_ub24.sh +RUN /build_zstd_1_5_6_ub24.sh + +SHELL ["/bin/bash", "-ec"] + +# Make the directory if it doesn't exist yet. +# This location is recommended by the distribution maintainers. +RUN mkdir --parents --mode=0755 /etc/apt/keyrings +# Download the key, convert the signing-key to a full +# keyring required by apt and store in the keyring directory +RUN wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | \ +gpg --dearmor | tee /etc/apt/keyrings/rocm.gpg > /dev/null && \ +# Add rocm repo +echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/amdgpu/6.3/ubuntu noble main" \ + | tee /etc/apt/sources.list.d/amdgpu.list && \ +echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/6.3 noble main" \ + | tee --append /etc/apt/sources.list.d/rocm.list && \ +echo -e 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' \ + | tee /etc/apt/preferences.d/rocm-pin-600 && \ +echo -e 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' \ + | tee /etc/apt/preferences.d/rocm-pin-600 +# Install the ROCM kernel driver +RUN apt update && apt install -yqq rocm-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +COPY scripts/create-sycl-user.sh /user-setup.sh +RUN /user-setup.sh + +COPY scripts/docker_entrypoint.sh /docker_entrypoint.sh + +USER sycl + +ENTRYPOINT ["/docker_entrypoint.sh"] + diff --git a/devops/containers/ubuntu2404_intel_drivers.Dockerfile b/devops/containers/ubuntu2404_intel_drivers.Dockerfile new file mode 100644 index 0000000000000..a0970f3900141 --- /dev/null +++ b/devops/containers/ubuntu2404_intel_drivers.Dockerfile @@ -0,0 +1,33 @@ +ARG base_tag=latest +ARG base_image=ghcr.io/intel/llvm/ubuntu2404_base + +FROM $base_image:$base_tag + +ENV DEBIAN_FRONTEND=noninteractive + +ARG use_unstable_driver=true + +USER root + +RUN apt update && apt install -yqq wget + +COPY scripts/get_release.py / +COPY scripts/install_drivers.sh / +COPY dependencies.json / + +RUN mkdir /runtimes +ENV INSTALL_LOCATION=/runtimes +RUN --mount=type=secret,id=github_token \ + if [ "$use_unstable_driver" = "true" ]; then \ + install_driver_opt=" --use-latest"; \ + else \ + install_driver_opt=" dependencies.json"; \ + fi && \ + GITHUB_TOKEN=$(cat /run/secrets/github_token) /install_drivers.sh $install_driver_opt --all + +COPY scripts/drivers_entrypoint.sh /drivers_entrypoint.sh + +USER sycl + +ENTRYPOINT ["/bin/bash", "/drivers_entrypoint.sh"] + diff --git a/devops/containers/ubuntu2404_intel_drivers_igc_dev.Dockerfile b/devops/containers/ubuntu2404_intel_drivers_igc_dev.Dockerfile new file mode 100644 index 0000000000000..25cb0ff9819ed --- /dev/null +++ b/devops/containers/ubuntu2404_intel_drivers_igc_dev.Dockerfile @@ -0,0 +1,28 @@ +ARG base_tag=latest +ARG base_image=ghcr.io/intel/llvm/ubuntu2404_base + +FROM $base_image:$base_tag + +ENV DEBIAN_FRONTEND=noninteractive + +USER root + +RUN apt update && apt install -yqq libllvm14 + +COPY scripts/get_release.py / +COPY scripts/install_drivers.sh / +COPY dependencies.json / +COPY dependencies-igc-dev.json / + +RUN mkdir /runtimes +ENV INSTALL_LOCATION=/runtimes +RUN --mount=type=secret,id=github_token \ + install_driver_opt="dependencies.json dependencies-igc-dev.json --use-dev-igc"; \ + GITHUB_TOKEN=$(cat /run/secrets/github_token) /install_drivers.sh $install_driver_opt --all + +COPY scripts/drivers_entrypoint.sh /drivers_entrypoint.sh + +USER sycl + +ENTRYPOINT ["/bin/bash", "/drivers_entrypoint.sh"] + diff --git a/devops/dependencies-igc-dev.json b/devops/dependencies-igc-dev.json index 37cb1ee1d97f7..0f1ce44811665 100644 --- a/devops/dependencies-igc-dev.json +++ b/devops/dependencies-igc-dev.json @@ -1,10 +1,10 @@ { "linux": { "igc_dev": { - "github_tag": "igc-dev-7dad678", - "version": "7dad678", - "updated_at": "2024-11-24T10:48:51Z", - "url": "https://api.github.com/repos/intel/intel-graphics-compiler/actions/artifacts/2229466354/zip", + "github_tag": "igc-dev-a9e1ef2", + "version": "a9e1ef2", + "updated_at": "2025-03-09T09:38:44Z", + "url": "https://api.github.com/repos/intel/intel-graphics-compiler/actions/artifacts/2717684926/zip", "root": "{DEPS_ROOT}/opencl/runtime/linux/oclgpu" } } diff --git a/devops/dependencies.json b/devops/dependencies.json index 755a1f10625fb..e2eaab57ae049 100644 --- a/devops/dependencies.json +++ b/devops/dependencies.json @@ -1,15 +1,15 @@ { "linux": { "compute_runtime": { - "github_tag": "24.39.31294.12", - "version": "24.39.31294.12", - "url": "https://github.com/intel/compute-runtime/releases/tag/24.39.31294.12", + "github_tag": "25.09.32961.5", + "version": "25.09.32961.5", + "url": "https://github.com/intel/compute-runtime/releases/tag/25.09.32961.5", "root": "{DEPS_ROOT}/opencl/runtime/linux/oclgpu" }, "igc": { - "github_tag": "igc-1.0.17791.9", - "version": "1.0.17791.9", - "url": "https://github.com/intel/intel-graphics-compiler/releases/tag/igc-1.0.17791.9", + "github_tag": "v2.8.3", + "version": "v2.8.3", + "url": "https://github.com/intel/intel-graphics-compiler/releases/tag/v2.8.3", "root": "{DEPS_ROOT}/opencl/runtime/linux/oclgpu" }, "cm": { @@ -19,9 +19,9 @@ "root": "{DEPS_ROOT}/opencl/runtime/linux/oclgpu" }, "level_zero": { - "github_tag": "v1.18.5", - "version": "v1.18.5", - "url": "https://github.com/oneapi-src/level-zero/releases/tag/v1.18.5", + "github_tag": "v1.20.2", + "version": "v1.20.2", + "url": "https://github.com/oneapi-src/level-zero/releases/tag/v1.20.2", "root": "{DEPS_ROOT}/opencl/runtime/linux/oclgpu" }, "tbb": { diff --git a/devops/scripts/build_zstd_1_5_6_ub24.sh b/devops/scripts/build_zstd_1_5_6_ub24.sh new file mode 100755 index 0000000000000..68a947dfb43a0 --- /dev/null +++ b/devops/scripts/build_zstd_1_5_6_ub24.sh @@ -0,0 +1,108 @@ +#!/bin/bash + +# Script to build and install zstd 1.5.6 on Ubuntu 24, with -fPIC flag. +# The default installation of zstd on Ubuntu 24 does not have -fPIC flag +# enabled, which is required for building DPC++ in shared libraries mode. + +# Function to check if the OS is Ubuntu 24 +check_os() { + . /etc/os-release + if [[ "$NAME" != "Ubuntu" || "$VERSION_ID" != "24.04" ]]; then + echo "Warning: This script has only been tested with Ubuntu 24." + fi +} + +# Function to install packages with or without sudo +install_packages() { + if [ "$USE_SUDO" = true ]; then + sudo apt-get update + sudo apt-get install -y build-essential wget + else + apt-get update + apt-get install -y build-essential wget + fi +} + +# Function to uninstall libzstd-dev if installed +uninstall_libzstd_dev() { + if dpkg -l | grep -q libzstd-dev; then + if [ "$USE_SUDO" = true ]; then + sudo apt-get remove -y libzstd-dev + else + apt-get remove -y libzstd-dev + fi + fi +} + +# Function to build a shared library by linking zstd static lib. +# This is used to verify that zstd is built correctly, with -fPIC flag. +build_test_program() { + cat < test_zstd.c + #include + int main() { + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + ZSTD_freeCCtx(cctx); + return 0; + } +EOF + + # Try to use zstd's static library with -fPIC + gcc test_zstd.c -lzstd -fPIC -shared + if [ $? -ne 0 ]; then + echo "zstd installation verification failed." + else + echo "zstd installation verification passed." + fi + + # There won't be a.out file if verification failed. + rm test_zstd.c a.out || true +} + +# Check the OS +check_os + +# Set USE_SUDO to true or false based on your preference +USE_SUDO=true + +# Install necessary build tools +install_packages + +# Uninstall libzstd-dev package if installed +uninstall_libzstd_dev + +# Define the version and URL for zstd +ZSTD_VERSION="1.5.6" +ZSTD_URL="https://github.com/facebook/zstd/releases/download/v$ZSTD_VERSION/zstd-$ZSTD_VERSION.tar.gz" + +# Create a directory for the source code +mkdir -p zstd_build +cd zstd_build + +# Download and extract zstd source code +wget $ZSTD_URL +tar -xzf zstd-$ZSTD_VERSION.tar.gz +cd zstd-$ZSTD_VERSION + +# Build zstd with -fPIC flag. +CFLAGS="-fPIC" CXXFLAGS="-fPIC" make +if [ $? -ne 0 ]; then + echo "Error: make failed." + exit 1 +fi + +# Install zstd. +if [ "$USE_SUDO" = true ]; then + sudo make install +else + make install +fi +if [ $? -ne 0 ]; then + echo "Error: make install failed." + exit 1 +fi + +# Verify zstd installation. +build_test_program + +# Clean up +rm -rf zstd_build diff --git a/devops/scripts/create-sycl-user.sh b/devops/scripts/create-sycl-user.sh new file mode 100755 index 0000000000000..7199f1eaf10cd --- /dev/null +++ b/devops/scripts/create-sycl-user.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -e + +USER_NAME=sycl + +# By default Ubuntu sets an arbitrary UID value, that is different from host +# system. When CI passes default UID value of 1001, some of LLVM tools fail to +# discover user home directory and fail a few LIT tests. Fixes UID and GID to +# 1001, that is used as default by GitHub Actions. +USER_ID=1001 + +groupadd -g $USER_ID $USER_NAME && useradd $USER_NAME -u $USER_ID -g $USER_ID -m -s /bin/bash +# Add user to video/irc groups so that it can access GPU +usermod -aG video $USER_NAME +usermod -aG irc $USER_NAME + +# group 109 is required for user to access PVC card. +groupadd -f -g 109 render +usermod -aG render $USER_NAME + +# Allow user to run as sudo (without a password) +echo "$USER_NAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers diff --git a/devops/scripts/docker_entrypoint.sh b/devops/scripts/docker_entrypoint.sh index f0e89244d013f..5fc44481d2355 100755 --- a/devops/scripts/docker_entrypoint.sh +++ b/devops/scripts/docker_entrypoint.sh @@ -1,8 +1,3 @@ #!/bin/bash -if [ -d "$GITHUB_WORKSPACE" ]; then - chown -R sycl:sycl $GITHUB_WORKSPACE - su sycl -fi - exec "$@" diff --git a/devops/scripts/install_build_tools.sh b/devops/scripts/install_build_tools.sh index 9c4b9d56999de..6c47fce4bdd06 100755 --- a/devops/scripts/install_build_tools.sh +++ b/devops/scripts/install_build_tools.sh @@ -7,10 +7,9 @@ apt update && apt install -yqq \ ccache \ git \ python3 \ - python3-distutils \ + python3-psutil \ python-is-python3 \ python3-pip \ - zstd \ ocl-icd-opencl-dev \ vim \ libffi-dev \ @@ -21,10 +20,18 @@ apt update && apt install -yqq \ zstd \ zip \ unzip \ + pigz \ jq \ curl \ libhwloc-dev \ - libzstd-dev - -pip3 install psutil + libzstd-dev \ + time +# To obtain latest release of spriv-tool. +# Same as what's done in SPRIV-LLVM-TRANSLATOR: +# https://github.com/KhronosGroup/SPIRV-LLVM-Translator/blob/cec12d6cf46306d0a015e883d5adb5a8200df1c0/.github/workflows/check-out-of-tree-build.yml#L59 +# pkg-config is required for llvm-spriv to detect spriv-tools installation. +. /etc/os-release +curl -L "https://packages.lunarg.com/lunarg-signing-key-pub.asc" | apt-key add - +echo "deb https://packages.lunarg.com/vulkan $VERSION_CODENAME main" | tee -a /etc/apt/sources.list +apt update && apt install -yqq spirv-tools pkg-config diff --git a/devops/scripts/install_drivers.sh b/devops/scripts/install_drivers.sh index 6efbc792d9ffc..01c2dde54d6d0 100755 --- a/devops/scripts/install_drivers.sh +++ b/devops/scripts/install_drivers.sh @@ -119,6 +119,8 @@ InstallIGFX () { # This can help us avoid using the risky force-depends-version option in dpkg command. # # Of course, this also installed the libopencl-clang so that we can copy and use later as a temporariy workaround. + IS_IGC_DEV=$(CheckIGCdevTag $IGCTAG) + UBUNTU_VER="u24\.04" get_release intel/intel-graphics-compiler $IGC_TAG \ | grep ".*deb" \ | wget -qi - @@ -126,16 +128,20 @@ InstallIGFX () { | grep -E ".*((deb)|(sum))" \ | wget -qi - # Perform the checksum conditionally and then get the release - sha256sum -c *.sum && \ + # Skip the ww45 checksum because the igc_dev driver was manually updated + # so the package versions don't exactly match. + if [ ! -f "ww45.sum" ]; then + sha256sum -c *.sum + fi get_release intel/cm-compiler $CM_TAG \ | grep ".*deb" \ | grep -v "u18" \ | wget -qi - get_release oneapi-src/level-zero $L0_TAG \ - | grep ".*u22\.04.*deb" \ + | grep ".*$UBUNTU_VER.*deb" \ | wget -qi - - dpkg -i *.deb && rm *.deb *.sum - IS_IGC_DEV=$(CheckIGCdevTag $IGCTAG) + dpkg -i --force-all *.deb && rm *.deb *.sum + mkdir -p /usr/local/lib/igc/ echo "$IGC_TAG" > /usr/local/lib/igc/IGCTAG.txt if [ "$IS_IGC_DEV" == "Yes" ]; then # Dev IGC deb package did not include libopencl-clang @@ -143,18 +149,21 @@ InstallIGFX () { # Backup and install it from release igc as a temporarily workaround # while we working to resolve the issue. echo "Backup libopencl-clang" - cp -d /usr/local/lib/libopencl-clang.so.14* . + cp -d /usr/local/lib/libopencl-clang2.so.14* . echo "Download IGC dev git hash $IGC_DEV_VER" get_pre_release_igfx $IGC_DEV_URL $IGC_DEV_VER echo "Install IGC dev git hash $IGC_DEV_VER" # New dev IGC packaged iga64 conflicting with iga64 from intel-igc-media # force overwrite to workaround it first. - dpkg -i --force-overwrite *.deb + dpkg -i --force-all *.deb echo "Install libopencl-clang" # Workaround only, will download deb and install with dpkg once fixed. - cp -d libopencl-clang.so.14* /usr/local/lib/ + cp -d libopencl-clang2.so.14* /usr/local/lib/ + rm /usr/local/lib/libigc.so /usr/local/lib/libigc.so.1* && \ + ln -s /usr/local/lib/libigc.so.2 /usr/local/lib/libigc.so && \ + ln -s /usr/local/lib/libigc.so.2 /usr/local/lib/libigc.so.1 echo "Clean up" - rm *.deb libopencl-clang.so.14* + rm *.deb libopencl-clang2.so.14* echo "$IGC_DEV_TAG" > /usr/local/lib/igc/IGCTAG.txt fi } diff --git a/devops/scripts/update_drivers.py b/devops/scripts/update_drivers.py index 4c3cbb791c851..41ea70aeaf089 100644 --- a/devops/scripts/update_drivers.py +++ b/devops/scripts/update_drivers.py @@ -17,7 +17,7 @@ def get_latest_workflow_runs(repo, workflow_name): + repo + "/actions/workflows/" + workflow_name - + ".yml/runs?status=success" + + ".yml/runs" ).read() return json.loads(action_runs)["workflow_runs"][0] @@ -38,7 +38,7 @@ def uplift_linux_igfx_driver(config, platform_tag, igc_dev_only): config[platform_tag]["igc_dev"]["version"] = igcdevver config[platform_tag]["igc_dev"]["updated_at"] = igc_dev["updated_at"] config[platform_tag]["igc_dev"]["url"] = get_artifacts_download_url( - "intel/intel-graphics-compiler", "IGC_Ubuntu22.04_llvm14_clang-" + igcdevver + "intel/intel-graphics-compiler", "IGC_Ubuntu24.04_llvm14_clang-" + igcdevver ) return config @@ -48,16 +48,18 @@ def uplift_linux_igfx_driver(config, platform_tag, igc_dev_only): config[platform_tag]['compute_runtime']['version'] = compute_runtime['tag_name'] config[platform_tag]['compute_runtime']['url'] = 'https://github.com/intel/compute-runtime/releases/tag/' + compute_runtime['tag_name'] - for a in compute_runtime['assets']: - if a['name'].endswith('.sum'): - deps = str(urlopen(a['browser_download_url']).read()) - m = re.search(r"intel-igc-core_([0-9\.]*)_amd64", deps) - if m is not None: - ver = m.group(1) - config[platform_tag]['igc']['github_tag'] = 'igc-' + ver - config[platform_tag]['igc']['version'] = ver - config[platform_tag]['igc']['url'] = 'https://github.com/intel/intel-graphics-compiler/releases/tag/igc-' + ver - break + m = re.search( + re.escape("https://github.com/intel/intel-graphics-compiler/releases/tag/") + + r"(v[\.0-9]+)", + compute_runtime["body"], + ) + if m is not None: + ver = m.group(1) + config[platform_tag]["igc"]["github_tag"] = ver + config[platform_tag]["igc"]["version"] = ver + config[platform_tag]["igc"]["url"] = ( + "https://github.com/intel/intel-graphics-compiler/releases/tag/" + ver + ) cm = get_latest_release('intel/cm-compiler') config[platform_tag]['cm']['github_tag'] = cm['tag_name'] diff --git a/devops/scripts/windows_detect_hung_tests.ps1 b/devops/scripts/windows_detect_hung_tests.ps1 new file mode 100644 index 0000000000000..c6942b07ae638 --- /dev/null +++ b/devops/scripts/windows_detect_hung_tests.ps1 @@ -0,0 +1,8 @@ +$exitCode = 0 +$hungTests = Get-Process | Where-Object { ($_.Path -match "llvm\\install") -or ($_.Path -match "llvm\\build-e2e") -or ($_.Path -match "llvm\\build") } +$hungTests | Foreach-Object { + $exitCode = 1 + echo "Test $($_.Path) hung!" + tskill $_.ID +} +exit $exitCode