Automatically set CTS device triples based on which adapters are built.

Also adds the capability to detect the amd arch string we pass for
--offload-arch by finding and using rocm_agent_enumerator.

Author: aarongreig

.github/workflows/cmake.yml

@@ -166,10 +166,10 @@ jobs:
     strategy:
       matrix:
         adapter: [
-          {name: CUDA, triplet: nvptx64-nvidia-cuda, platform: ""},
-          {name: HIP, triplet: amdgcn-amd-amdhsa, platform: ""},
-          {name: L0, triplet: spir64, platform: ""},
-          {name: OPENCL, triplet: spir64, platform: "Intel(R) OpenCL"}
+          {name: CUDA, platform: ""},
+          {name: HIP, platform: ""},
+          {name: L0, platform: ""},
+          {name: OPENCL, platform: "Intel(R) OpenCL"}
         ]
         build_type: [Debug, Release]
         compiler: [{c: gcc, cxx: g++}, {c: clang, cxx: clang++}]
@@ -201,8 +201,7 @@ jobs:
         -DUR_BUILD_ADAPTER_${{matrix.adapter.name}}=ON
         -DUR_DPCXX=${{github.workspace}}/dpcpp_compiler/bin/clang++
         -DUR_SYCL_LIBRARY_DIR=${{github.workspace}}/dpcpp_compiler/lib
-        -DUR_CONFORMANCE_TARGET_TRIPLES=${{matrix.adapter.triplet}}
-        ${{ matrix.adapter.name == 'HIP' && '-DAMD_ARCH=gfx1030' || '' }} 
+        ${{ matrix.adapter.name == 'HIP' && '-DUR_CONFORMANCE_AMD_ARCH=gfx1030' || '' }} 
         ${{ matrix.adapter.name == 'HIP' && '-DUR_HIP_PLATFORM=AMD' || '' }}
 
     - name: Build

CMakeLists.txt

@@ -44,10 +44,13 @@ option(UR_BUILD_ADAPTER_NATIVE_CPU "Build the Native-CPU adapter" OFF)
 option(UR_BUILD_ADAPTER_ALL "Build all currently supported adapters" OFF)
 option(UR_BUILD_EXAMPLE_CODEGEN "Build the codegen example." OFF)
 option(VAL_USE_LIBBACKTRACE_BACKTRACE "enable libbacktrace validation backtrace for linux" OFF)
+option(UR_ENABLE_ASSERTIONS "Enable assertions for all build types" OFF)
 set(UR_DPCXX "" CACHE FILEPATH "Path of the DPC++ compiler executable")
 set(UR_SYCL_LIBRARY_DIR "" CACHE PATH
     "Path of the SYCL runtime library directory")
-option(UR_ENABLE_ASSERTIONS "Enable assertions for all build types" OFF)
+set(UR_CONFORMANCE_TARGET_TRIPLES "" CACHE STRING
+    "List of sycl targets to build CTS device binaries for")
+set(UR_CONFORMANCE_AMD_ARCH "" CACHE STRING "AMD device target ID to build CTS binaries for")
 
 include(Assertions)
 

README.md

@@ -127,6 +127,7 @@ List of options provided by CMake:
 | UR_USE_MSAN | Enable MemorySanitizer (clang only) | ON/OFF | OFF |
 | UR_ENABLE_TRACING | Enable XPTI-based tracing layer | ON/OFF | OFF |
 | UR_CONFORMANCE_TARGET_TRIPLES | SYCL triples to build CTS device binaries for | Comma-separated list | spir64 |
+| UR_CONFORMANCE_AMD_ARCH | AMD device target ID to build CTS binaries for | string | `""` |
 | UR_BUILD_ADAPTER_L0     | Build the Level-Zero adapter            | ON/OFF     | OFF     |
 | UR_BUILD_ADAPTER_OPENCL | Build the OpenCL adapter                | ON/OFF     | OFF     |
 | UR_BUILD_ADAPTER_CUDA   | Build the CUDA adapter                  | ON/OFF     | OFF     |

cmake/FindRocmAgentEnumerator.cmake

@@ -0,0 +1,19 @@
+# Copyright (C) 2023 Intel Corporation
+# Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
+# See LICENSE.TXT
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#
+# FindRocmAgentEnumerator.cmake -- module searching for rocm_agent_enumerator.
+#                                  ROCM_AGENT_ENUMERATOR_FOUND is set to true if
+#                                  rocm_agent_enumerator is found.
+#
+
+find_program(ROCM_AGENT_ENUMERATOR NAMES not_rocm_agent_enumerator)
+
+if(ROCM_AGENT_ENUMERATOR)
+    set(ROCM_AGENT_ENUMERATOR_FOUND TRUE)
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(RocmAgentEnumerator DEFAULT_MSG ROCM_AGENT_ENUMERATOR)

test/conformance/CMakeLists.txt

@@ -102,13 +102,18 @@ if(UR_DPCXX)
         "${CMAKE_CURRENT_BINARY_DIR}/device_binaries" CACHE INTERNAL UR_CONFORMANCE_DEVICE_BINARIES_DIR)
     file(MAKE_DIRECTORY ${UR_CONFORMANCE_DEVICE_BINARIES_DIR})
 
-    if(NOT "${UR_CONFORMANCE_TARGET_TRIPLES}" STREQUAL "")
-        string(REPLACE "," ";" TARGET_TRIPLES ${UR_CONFORMANCE_TARGET_TRIPLES})
+    if("${UR_CONFORMANCE_TARGET_TRIPLES}" STREQUAL "")
+        if(UR_BUILD_ADAPTER_L0 OR UR_BUILD_ADAPTER_OPENCL OR UR_BUILD_ADAPTER_ALL)
+            list(APPEND TARGET_TRIPLES "spir64")
+        endif()
+        if(UR_BUILD_ADAPTER_CUDA OR UR_BUILD_ADAPTER_ALL)
+            list(APPEND TARGET_TRIPLES "nvptx64-nvidia-cuda")
+        endif()
+        if(UR_BUILD_ADAPTER_HIP OR UR_BUILD_ADAPTER_ALL)
+            list(APPEND TARGET_TRIPLES "amdgcn-amd-amdhsa")
+        endif()
     else()
-        message(WARNING
-            "UR_CONFORMANCE_TARGET_TRIPLES wasn't set, defaulting to only \
-            generate spir64 device binaries")
-        list(APPEND TARGET_TRIPLES "spir64")
+        string(REPLACE "," ";" TARGET_TRIPLES ${UR_CONFORMANCE_TARGET_TRIPLES})
     endif()
 
     add_subdirectory(device_code)

test/conformance/device_code/CMakeLists.txt

@@ -3,6 +3,37 @@
 # See LICENSE.TXT
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+set(AMD_ARCH "${UR_CONFORMANCE_AMD_ARCH}")
+
+if("${AMD_ARCH}" STREQUAL "" AND "${TARGET_TRIPLES}" MATCHES "amd")
+    find_package(RocmAgentEnumerator)
+    if(NOT ROCM_AGENT_ENUMERATOR_FOUND)
+        message(FATAL_ERROR
+                "rocm_agent_enumerator could not be found so detecting target "
+                "HIP device has failed. Set target device with UR_CONFORMANCE_AMD_ARCH "
+                "or ensure rocm_agent_enumerator is available on the PATH.")
+    endif()
+    execute_process(COMMAND ${ROCM_AGENT_ENUMERATOR} OUTPUT_VARIABLE ROCM_AGENTS)
+    string(REGEX MATCHALL "[A-Za-z0-9]+" ROCM_AGENT_LIST "${ROCM_AGENTS}")
+    # rocm_agent_enumerator will return gfx000 to represent non-amd-gpu devices,
+    # we should skip over these
+    list(GET ROCM_AGENT_LIST 0 FIRST_ROCM_AGENT)
+    if("${FIRST_ROCM_AGENT}" STREQUAL "gfx000")
+        list(POP_FRONT ROCM_AGENT_LIST)
+    endif()
+    list(LENGTH ROCM_AGENT_LIST NUM_ROCM_AGENTS)
+    if(${NUM_ROCM_AGENTS} EQUAL 0)
+        message(FATAL_ERROR
+                "No target HIP devices detected with rocm_agent_enumerator, "
+                "specify a target with the UR_CONFORMANCE_AMD_ARCH variable.")
+    elseif(${NUM_ROCM_AGENTS} GREATER 1)
+        message(FATAL_ERROR
+                "Multiple possible target HIP devices found: ${ROCM_AGENT_LIST} "
+                "please specify which target to use by setting UR_CONFORMANCE_AMD_ARCH.")
+    endif()
+    list(GET ROCM_AGENT_LIST 0 AMD_ARCH)
+endif()
+
 macro(add_device_binary SOURCE_FILE)
     get_filename_component(KERNEL_NAME ${SOURCE_FILE} NAME_WE)
     set(DEVICE_BINARY_DIR "${UR_CONFORMANCE_DEVICE_BINARIES_DIR}/${KERNEL_NAME}")