From 2d28503b72d51c627e796f3508fdb3dfe96e3097 Mon Sep 17 00:00:00 2001 From: "Sarnie, Nick" Date: Tue, 12 Nov 2024 10:20:14 -0800 Subject: [PATCH 1/8] [CI] Automatically detect AMD architecture Signed-off-by: Sarnie, Nick --- .github/workflows/sycl-linux-run-tests.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml index bfe958448e759..d6d3463fc3926 100644 --- a/.github/workflows/sycl-linux-run-tests.yml +++ b/.github/workflows/sycl-linux-run-tests.yml @@ -284,11 +284,8 @@ jobs: echo "opts=$CMAKE_EXTRA_ARGS" >> $GITHUB_OUTPUT else if [ "${{ contains(inputs.target_devices, 'ext_oneapi_hip') }}" == "true" ]; then - if [ "${{ runner.name }}" == "cp-amd-runner" ]; then - echo 'opts=-DHIP_PLATFORM="AMD" -DAMD_ARCH="gfx1030"' >> $GITHUB_OUTPUT - else - echo 'opts=-DHIP_PLATFORM="AMD" -DAMD_ARCH="gfx1031"' >> $GITHUB_OUTPUT - fi + amd_arch="$( env ONEAPI_DEVICE_SELECTOR=hip:gpu sycl-ls --verbose 2>&1 | grep 'Architecture:' | sed 's|\s*Architecture: amd_gpu_||g' )" + echo 'opts=-DHIP_PLATFORM="AMD" -DAMD_ARCH='$amd_arch'' >> $GITHUB_OUTPUT else echo 'opts=' >> $GITHUB_OUTPUT fi From 9f48bd60f4280f94302c57a8f133a86a4041c29d Mon Sep 17 00:00:00 2001 From: "Sarnie, Nick" Date: Wed, 13 Nov 2024 11:57:09 -0800 Subject: [PATCH 2/8] do it in the testing infra instead Signed-off-by: Sarnie, Nick --- .github/workflows/sycl-linux-precommit.yml | 1 - .github/workflows/sycl-linux-run-tests.yml | 10 --- .../Matrix/joint_matrix_hip_gfx90a.cpp | 4 +- .../Matrix/joint_matrix_hip_half_gfx90a.cpp | 4 +- .../Matrix/runtime_query_hip_gfx90a.cpp | 4 +- sycl/test-e2e/lit.cfg.py | 76 +++++++++---------- 6 files changed, 44 insertions(+), 55 deletions(-) diff --git a/.github/workflows/sycl-linux-precommit.yml b/.github/workflows/sycl-linux-precommit.yml index a73324e33fec5..fcb8852844cbb 100644 --- a/.github/workflows/sycl-linux-precommit.yml +++ b/.github/workflows/sycl-linux-precommit.yml @@ -162,7 +162,6 @@ jobs: runner: '["Linux", "amdgpu"]' image: ghcr.io/intel/llvm/ubuntu2204_build:latest image_extra_opts: --device=/dev/dri --device=/dev/kfd - extra_cmake_args: -DHIP_PLATFORM="AMD" -DAMD_ARCH="gfx1031" - name: CUDA system runner: '["Linux", "cuda"]' image: ghcr.io/intel/llvm/ubuntu2204_build:latest diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml index d6d3463fc3926..73b2a1f336db7 100644 --- a/.github/workflows/sycl-linux-run-tests.yml +++ b/.github/workflows/sycl-linux-run-tests.yml @@ -21,9 +21,6 @@ on: type: string required: True extra_cmake_args: - description: | - If empty, then HIP_PLATFORM and AMD_ARCH would be automatically added - if inputs.target_devices contains 'ext_oneapi_hip' type: string required: False tests_selector: @@ -282,13 +279,6 @@ jobs: run: | if [ -n "$CMAKE_EXTRA_ARGS" ]; then echo "opts=$CMAKE_EXTRA_ARGS" >> $GITHUB_OUTPUT - else - if [ "${{ contains(inputs.target_devices, 'ext_oneapi_hip') }}" == "true" ]; then - amd_arch="$( env ONEAPI_DEVICE_SELECTOR=hip:gpu sycl-ls --verbose 2>&1 | grep 'Architecture:' | sed 's|\s*Architecture: amd_gpu_||g' )" - echo 'opts=-DHIP_PLATFORM="AMD" -DAMD_ARCH='$amd_arch'' >> $GITHUB_OUTPUT - else - echo 'opts=' >> $GITHUB_OUTPUT - fi fi - name: Configure E2E tests if: inputs.tests_selector == 'e2e' diff --git a/sycl/test-e2e/Matrix/joint_matrix_hip_gfx90a.cpp b/sycl/test-e2e/Matrix/joint_matrix_hip_gfx90a.cpp index 008b3499a949d..c126a96bcb2ab 100644 --- a/sycl/test-e2e/Matrix/joint_matrix_hip_gfx90a.cpp +++ b/sycl/test-e2e/Matrix/joint_matrix_hip_gfx90a.cpp @@ -6,10 +6,10 @@ // //===----------------------------------------------------------------------===// -// RUN: %{build} -fsycl -fsycl-targets=amd_gpu_gfx90a %s -o %t.out +// RUN: %clangxx -fsycl -fsycl-targets=amd_gpu_gfx90a %s -o %t.out // RUN: %{run} %t.out -// REQUIRES: gpu-amd-gfx90a +// REQUIRES: arch-amd_gpu_gfx90a #include "joint_matrix_hip_apply.hpp" #include "joint_matrix_hip_copy.hpp" diff --git a/sycl/test-e2e/Matrix/joint_matrix_hip_half_gfx90a.cpp b/sycl/test-e2e/Matrix/joint_matrix_hip_half_gfx90a.cpp index 737147d14dbb3..c39832cd64357 100644 --- a/sycl/test-e2e/Matrix/joint_matrix_hip_half_gfx90a.cpp +++ b/sycl/test-e2e/Matrix/joint_matrix_hip_half_gfx90a.cpp @@ -6,10 +6,10 @@ // //===----------------------------------------------------------------------===// -// RUN: %{build} -fsycl -fsycl-targets=amd_gpu_gfx90a %s -o %t.out +// RUN: %clangxx -fsycl -fsycl-targets=amd_gpu_gfx90a %s -o %t.out // RUN: %{run} %t.out -// REQUIRES: gpu-amd-gfx90a +// REQUIRES: arch-amd_gpu_gfx90a // REQUIRES: aspect-fp16 #include "joint_matrix_hip_apply.hpp" diff --git a/sycl/test-e2e/Matrix/runtime_query_hip_gfx90a.cpp b/sycl/test-e2e/Matrix/runtime_query_hip_gfx90a.cpp index 72592fbd0bec1..6cb9a6626a1f7 100644 --- a/sycl/test-e2e/Matrix/runtime_query_hip_gfx90a.cpp +++ b/sycl/test-e2e/Matrix/runtime_query_hip_gfx90a.cpp @@ -6,8 +6,8 @@ // //===----------------------------------------------------------------------===// -// REQUIRES: gpu-amd-gfx90a -// RUN: %{build} -Xsycl-target-backend=amdgcn-amd-amdhsa --offload-arch=gfx90a -o %t.out +// REQUIRES: arch-amd_gpu_gfx90a +// RUN: %clangxx -Xsycl-target-backend=amdgcn-amd-amdhsa --offload-arch=gfx90a -o %t.out // RUN: %{run} %t.out #include diff --git a/sycl/test-e2e/lit.cfg.py b/sycl/test-e2e/lit.cfg.py index ee91220fd3be3..b410e22fda77a 100644 --- a/sycl/test-e2e/lit.cfg.py +++ b/sycl/test-e2e/lit.cfg.py @@ -442,8 +442,6 @@ def open_check_file(file_name): ) sp = subprocess.check_output(cmd, text=True, shell=True) for line in sp.splitlines(): - if "gfx90a" in line: - config.available_features.add("gpu-amd-gfx90a") if not line.startswith("["): continue (backend, device) = line[1:].split("]")[0].split(":") @@ -540,42 +538,6 @@ def open_check_file(file_name): config.cuda_libs_dir = os.path.join(os.environ["CUDA_PATH"], r"lib64") config.cuda_include = os.path.join(os.environ["CUDA_PATH"], "include") -# FIXME: This needs to be made per-device as well, possibly with a helper. -if "hip:gpu" in config.sycl_devices and config.hip_platform == "AMD": - if not config.amd_arch: - lit_config.error( - "Cannot run tests for HIP without an offload-arch. Please " - + "specify one via the 'amd_arch' parameter or 'AMD_ARCH' CMake " - + "variable." - ) - llvm_config.with_system_environment("ROCM_PATH") - config.available_features.add("hip_amd") - arch_flag = ( - "-Xsycl-target-backend=amdgcn-amd-amdhsa --offload-arch=" + config.amd_arch - ) - config.substitutions.append( - ("%rocm_path", os.environ.get("ROCM_PATH", "/opt/rocm")) - ) -elif "hip:gpu" in config.sycl_devices and config.hip_platform == "NVIDIA": - config.available_features.add("hip_nvidia") - arch_flag = "" -else: - arch_flag = "" - -if lit_config.params.get("compatibility_testing", False): - config.substitutions.append(("%clangxx", " true ")) - config.substitutions.append(("%clang", " true ")) -else: - config.substitutions.append( - ( - "%clangxx", - " " + config.dpcpp_compiler + " " + config.cxx_flags + " " + arch_flag, - ) - ) - config.substitutions.append( - ("%clang", " " + config.dpcpp_compiler + " " + config.c_flags) - ) - config.substitutions.append(("%threads_lib", config.sycl_threads_lib)) if lit_config.params.get("ze_debug"): @@ -805,6 +767,44 @@ def open_check_file(file_name): else: config.intel_driver_ver[sycl_device] = {} +if "hip:gpu" in config.sycl_devices and config.hip_platform == "AMD": + if not config.amd_arch: + amd_arch_prefix = "arch-amd_gpu_" + amd_device_arch = [i for i in config.sycl_dev_features["hip:gpu"] if amd_arch_prefix in i] + if len(amd_device_arch) == 0: + lit_config.error( + "Cannot detect architecture for AMD HIP device, specify it explicitly" + ) + if len(amd_device_arch) > 1: + lit_config.warning("Multiple AMD HIP devices, using the architecture from the first one") + config.amd_arch = amd_device_arch[0].replace(amd_arch_prefix, "") + llvm_config.with_system_environment("ROCM_PATH") + config.available_features.add("hip_amd") + arch_flag = ( + "-Xsycl-target-backend=amdgcn-amd-amdhsa --offload-arch=" + config.amd_arch + ) + config.substitutions.append( + ("%rocm_path", os.environ.get("ROCM_PATH", "/opt/rocm")) + ) +elif "hip:gpu" in config.sycl_devices and config.hip_platform == "NVIDIA": + config.available_features.add("hip_nvidia") + arch_flag = "" +else: + arch_flag = "" + +if lit_config.params.get("compatibility_testing", False): + config.substitutions.append(("%clangxx", " true ")) + config.substitutions.append(("%clang", " true ")) +else: + config.substitutions.append( + ( + "%clangxx", + " " + config.dpcpp_compiler + " " + config.cxx_flags + " " + arch_flag, + ) + ) + config.substitutions.append( + ("%clang", " " + config.dpcpp_compiler + " " + config.c_flags) + ) # Set timeout for a single test try: import psutil From a01f87274e894aa7bfe3737116c4ad06de5d7316 Mon Sep 17 00:00:00 2001 From: "Sarnie, Nick" Date: Wed, 13 Nov 2024 12:58:43 -0800 Subject: [PATCH 3/8] fix UNSUPPORTED Signed-off-by: Sarnie, Nick --- sycl/test-e2e/lit.cfg.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sycl/test-e2e/lit.cfg.py b/sycl/test-e2e/lit.cfg.py index b410e22fda77a..17e69dd09f0b1 100644 --- a/sycl/test-e2e/lit.cfg.py +++ b/sycl/test-e2e/lit.cfg.py @@ -805,6 +805,11 @@ def open_check_file(file_name): config.substitutions.append( ("%clang", " " + config.dpcpp_compiler + " " + config.c_flags) ) + +# Update each device's features with anything added during HIP device handling +for sycl_device in config.sycl_devices: + config.sycl_dev_features[sycl_device] = config.sycl_dev_features[sycl_device].union(config.available_features) + # Set timeout for a single test try: import psutil From 8494ca54e541fe5b15fbec67794ececb910580c9 Mon Sep 17 00:00:00 2001 From: "Sarnie, Nick" Date: Wed, 13 Nov 2024 13:49:01 -0800 Subject: [PATCH 4/8] format Signed-off-by: Sarnie, Nick --- sycl/test-e2e/lit.cfg.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/sycl/test-e2e/lit.cfg.py b/sycl/test-e2e/lit.cfg.py index 17e69dd09f0b1..51598549f1dd5 100644 --- a/sycl/test-e2e/lit.cfg.py +++ b/sycl/test-e2e/lit.cfg.py @@ -770,13 +770,17 @@ def open_check_file(file_name): if "hip:gpu" in config.sycl_devices and config.hip_platform == "AMD": if not config.amd_arch: amd_arch_prefix = "arch-amd_gpu_" - amd_device_arch = [i for i in config.sycl_dev_features["hip:gpu"] if amd_arch_prefix in i] + amd_device_arch = [ + i for i in config.sycl_dev_features["hip:gpu"] if amd_arch_prefix in i + ] if len(amd_device_arch) == 0: lit_config.error( "Cannot detect architecture for AMD HIP device, specify it explicitly" ) if len(amd_device_arch) > 1: - lit_config.warning("Multiple AMD HIP devices, using the architecture from the first one") + lit_config.warning( + "Multiple AMD HIP devices, using the architecture from the first one" + ) config.amd_arch = amd_device_arch[0].replace(amd_arch_prefix, "") llvm_config.with_system_environment("ROCM_PATH") config.available_features.add("hip_amd") @@ -808,7 +812,9 @@ def open_check_file(file_name): # Update each device's features with anything added during HIP device handling for sycl_device in config.sycl_devices: - config.sycl_dev_features[sycl_device] = config.sycl_dev_features[sycl_device].union(config.available_features) + config.sycl_dev_features[sycl_device] = config.sycl_dev_features[sycl_device].union( + config.available_features + ) # Set timeout for a single test try: From 48b6502ecd2fff8d8c8748e07980a39e6f068553 Mon Sep 17 00:00:00 2001 From: "Sarnie, Nick" Date: Wed, 13 Nov 2024 13:50:28 -0800 Subject: [PATCH 5/8] add -fsycl to test Signed-off-by: Sarnie, Nick --- sycl/test-e2e/Matrix/runtime_query_hip_gfx90a.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sycl/test-e2e/Matrix/runtime_query_hip_gfx90a.cpp b/sycl/test-e2e/Matrix/runtime_query_hip_gfx90a.cpp index 6cb9a6626a1f7..90da6ef580341 100644 --- a/sycl/test-e2e/Matrix/runtime_query_hip_gfx90a.cpp +++ b/sycl/test-e2e/Matrix/runtime_query_hip_gfx90a.cpp @@ -7,7 +7,7 @@ //===----------------------------------------------------------------------===// // REQUIRES: arch-amd_gpu_gfx90a -// RUN: %clangxx -Xsycl-target-backend=amdgcn-amd-amdhsa --offload-arch=gfx90a -o %t.out +// RUN: %clangxx -fsycl -Xsycl-target-backend=amdgcn-amd-amdhsa --offload-arch=gfx90a -o %t.out // RUN: %{run} %t.out #include From 7db9bff023318e25febf28f29be703370e64df79 Mon Sep 17 00:00:00 2001 From: "Sarnie, Nick" Date: Wed, 13 Nov 2024 14:37:01 -0800 Subject: [PATCH 6/8] move to existing loop Signed-off-by: Sarnie, Nick --- .../Matrix/runtime_query_hip_gfx90a.cpp | 2 +- sycl/test-e2e/lit.cfg.py | 64 ++++++++----------- 2 files changed, 28 insertions(+), 38 deletions(-) diff --git a/sycl/test-e2e/Matrix/runtime_query_hip_gfx90a.cpp b/sycl/test-e2e/Matrix/runtime_query_hip_gfx90a.cpp index 90da6ef580341..ec9ecadf1ae16 100644 --- a/sycl/test-e2e/Matrix/runtime_query_hip_gfx90a.cpp +++ b/sycl/test-e2e/Matrix/runtime_query_hip_gfx90a.cpp @@ -7,7 +7,7 @@ //===----------------------------------------------------------------------===// // REQUIRES: arch-amd_gpu_gfx90a -// RUN: %clangxx -fsycl -Xsycl-target-backend=amdgcn-amd-amdhsa --offload-arch=gfx90a -o %t.out +// RUN: %clangxx -fsycl -fsycl-targets=amdgcn-amd-amdhsa -Xsycl-target-backend=amdgcn-amd-amdhsa --offload-arch=gfx90a %s -o %t.out // RUN: %{run} %t.out #include diff --git a/sycl/test-e2e/lit.cfg.py b/sycl/test-e2e/lit.cfg.py index 51598549f1dd5..da400d8431256 100644 --- a/sycl/test-e2e/lit.cfg.py +++ b/sycl/test-e2e/lit.cfg.py @@ -441,7 +441,7 @@ def open_check_file(file_name): "{} {}".format(config.run_launcher, sycl_ls) if config.run_launcher else sycl_ls ) sp = subprocess.check_output(cmd, text=True, shell=True) - for line in sp.splitlines(): + for line in sp.splitlines(): if not line.startswith("["): continue (backend, device) = line[1:].split("]")[0].split(":") @@ -761,41 +761,37 @@ def open_check_file(file_name): # Use short names for LIT rules. features.add(be) + if be == "hip" and config.hip_platform == "AMD": + if not config.amd_arch: + arch = "" + for a in architecture_feature: + arch = a + amd_arch_prefix = "arch-amd_gpu_" + if amd_arch_prefix not in arch or len(architecture_feature) != 1: + lit_config.error( + "Cannot detect architecture for AMD HIP device, specify it explicitly" + ) + config.amd_arch = arch.replace(amd_arch_prefix, "") + llvm_config.with_system_environment("ROCM_PATH") + config.available_features.add("hip_amd") + arch_flag = ( + "-Xsycl-target-backend=amdgcn-amd-amdhsa --offload-arch=" + config.amd_arch + ) + config.substitutions.append( + ("%rocm_path", os.environ.get("ROCM_PATH", "/opt/rocm")) + ) + elif be == "hip" and config.hip_platform == "NVIDIA": + config.available_features.add("hip_nvidia") + arch_flag = "" + else: + arch_flag = "" + config.sycl_dev_features[sycl_device] = features.union(config.available_features) if is_intel_driver: config.intel_driver_ver[sycl_device] = intel_driver_ver else: config.intel_driver_ver[sycl_device] = {} -if "hip:gpu" in config.sycl_devices and config.hip_platform == "AMD": - if not config.amd_arch: - amd_arch_prefix = "arch-amd_gpu_" - amd_device_arch = [ - i for i in config.sycl_dev_features["hip:gpu"] if amd_arch_prefix in i - ] - if len(amd_device_arch) == 0: - lit_config.error( - "Cannot detect architecture for AMD HIP device, specify it explicitly" - ) - if len(amd_device_arch) > 1: - lit_config.warning( - "Multiple AMD HIP devices, using the architecture from the first one" - ) - config.amd_arch = amd_device_arch[0].replace(amd_arch_prefix, "") - llvm_config.with_system_environment("ROCM_PATH") - config.available_features.add("hip_amd") - arch_flag = ( - "-Xsycl-target-backend=amdgcn-amd-amdhsa --offload-arch=" + config.amd_arch - ) - config.substitutions.append( - ("%rocm_path", os.environ.get("ROCM_PATH", "/opt/rocm")) - ) -elif "hip:gpu" in config.sycl_devices and config.hip_platform == "NVIDIA": - config.available_features.add("hip_nvidia") - arch_flag = "" -else: - arch_flag = "" - if lit_config.params.get("compatibility_testing", False): config.substitutions.append(("%clangxx", " true ")) config.substitutions.append(("%clang", " true ")) @@ -808,13 +804,7 @@ def open_check_file(file_name): ) config.substitutions.append( ("%clang", " " + config.dpcpp_compiler + " " + config.c_flags) - ) - -# Update each device's features with anything added during HIP device handling -for sycl_device in config.sycl_devices: - config.sycl_dev_features[sycl_device] = config.sycl_dev_features[sycl_device].union( - config.available_features - ) + ) # Set timeout for a single test try: From b666f7bb0597d6507900a69f09d822eff785df2e Mon Sep 17 00:00:00 2001 From: "Sarnie, Nick" Date: Wed, 13 Nov 2024 15:43:20 -0800 Subject: [PATCH 7/8] format Signed-off-by: Sarnie, Nick --- sycl/test-e2e/lit.cfg.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sycl/test-e2e/lit.cfg.py b/sycl/test-e2e/lit.cfg.py index da400d8431256..e0b39cbb27507 100644 --- a/sycl/test-e2e/lit.cfg.py +++ b/sycl/test-e2e/lit.cfg.py @@ -441,7 +441,7 @@ def open_check_file(file_name): "{} {}".format(config.run_launcher, sycl_ls) if config.run_launcher else sycl_ls ) sp = subprocess.check_output(cmd, text=True, shell=True) - for line in sp.splitlines(): + for line in sp.splitlines(): if not line.startswith("["): continue (backend, device) = line[1:].split("]")[0].split(":") @@ -770,7 +770,7 @@ def open_check_file(file_name): if amd_arch_prefix not in arch or len(architecture_feature) != 1: lit_config.error( "Cannot detect architecture for AMD HIP device, specify it explicitly" - ) + ) config.amd_arch = arch.replace(amd_arch_prefix, "") llvm_config.with_system_environment("ROCM_PATH") config.available_features.add("hip_amd") @@ -804,7 +804,7 @@ def open_check_file(file_name): ) config.substitutions.append( ("%clang", " " + config.dpcpp_compiler + " " + config.c_flags) - ) + ) # Set timeout for a single test try: From 75f90f9d84d93ce0f883d0ccfe2fb2f75aa04333 Mon Sep 17 00:00:00 2001 From: "Sarnie, Nick" Date: Thu, 14 Nov 2024 08:08:34 -0800 Subject: [PATCH 8/8] clean up set extraction Signed-off-by: Sarnie, Nick --- sycl/test-e2e/lit.cfg.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sycl/test-e2e/lit.cfg.py b/sycl/test-e2e/lit.cfg.py index e0b39cbb27507..f76a9935ea108 100644 --- a/sycl/test-e2e/lit.cfg.py +++ b/sycl/test-e2e/lit.cfg.py @@ -763,9 +763,8 @@ def open_check_file(file_name): if be == "hip" and config.hip_platform == "AMD": if not config.amd_arch: - arch = "" - for a in architecture_feature: - arch = a + # Guaranteed to be a single element in the set + arch = [x for x in architecture_feature][0] amd_arch_prefix = "arch-amd_gpu_" if amd_arch_prefix not in arch or len(architecture_feature) != 1: lit_config.error(