Add CTS macro for marking tests as known failures in code

kbenzie · kbenzie · commit ba57ce94a14e · 2024-12-03T16:03:41.000Z
The new `UUR_KNOWN_FAILURE_ON` macro can be used to skip tests on
devices where the test is known to fail. This can be done for a devices
in an adapter or by substring match of the device name. For example:

```cpp
UUR_KNOWN_FAILURE_ON(uur::OpenCL{"Intel(R) UHD Graphics 770"});
```

This invocation is also used in a few places in order to have clean CTS
runs on machines with this iGPU.
diff --git a/test/conformance/kernel/urKernelSetArgValue.cpp b/test/conformance/kernel/urKernelSetArgValue.cpp
@@ -3,6 +3,7 @@
 // See LICENSE.TXT
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+#include "uur/known_failure.h"
 #include <uur/fixtures.h>
 
 struct urKernelSetArgValueTest : uur::urKernelTest {
@@ -45,6 +46,7 @@ TEST_P(urKernelSetArgValueTest, InvalidKernelArgumentIndex) {
 }
 
 TEST_P(urKernelSetArgValueTest, InvalidKernelArgumentSize) {
+    UUR_KNOWN_FAILURE_ON(uur::OpenCL{"Intel(R) UHD Graphics 770"});
     ASSERT_EQ_RESULT(UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_SIZE,
                      urKernelSetArgValue(kernel, 2, 0, nullptr, &arg_value));
 }
diff --git a/test/conformance/memory/urMemImageCreate.cpp b/test/conformance/memory/urMemImageCreate.cpp
@@ -2,6 +2,7 @@
 // Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
 // See LICENSE.TXT
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#include "uur/known_failure.h"
 #include <uur/fixtures.h>
 #include <uur/raii.h>
 
@@ -188,6 +189,7 @@ TEST_P(urMemImageCreateTest, InvalidNullPointerImageFormat) {
 }
 
 TEST_P(urMemImageCreateTest, InvalidSize) {
+    UUR_KNOWN_FAILURE_ON(uur::OpenCL{"Intel(R) UHD Graphics 770"});
 
     uur::raii::Mem image_handle = nullptr;
 
diff --git a/test/conformance/memory/urMemImageCreateWithImageFormatParam.cpp b/test/conformance/memory/urMemImageCreateWithImageFormatParam.cpp
@@ -2,6 +2,7 @@
 // Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
 // See LICENSE.TXT
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#include "uur/known_failure.h"
 #include <uur/fixtures.h>
 #include <vector>
 
@@ -90,6 +91,7 @@ UUR_TEST_SUITE_P(
     uur::deviceTestWithParamPrinter<ur_image_format_t>);
 
 TEST_P(urMemImageCreateTestWithImageFormatParam, Success) {
+    UUR_KNOWN_FAILURE_ON(uur::OpenCL{"Intel(R) UHD Graphics 770"});
     ur_image_channel_order_t channel_order =
         std::get<1>(GetParam()).channelOrder;
     ur_image_channel_type_t channel_type = std::get<1>(GetParam()).channelType;
diff --git a/test/conformance/program/urProgramGetFunctionPointer.cpp b/test/conformance/program/urProgramGetFunctionPointer.cpp
@@ -3,6 +3,7 @@
 // See LICENSE.TXT
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+#include "uur/known_failure.h"
 #include <uur/fixtures.h>
 
 struct urProgramGetFunctionPointerTest : uur::urProgramTest {
@@ -20,6 +21,7 @@ struct urProgramGetFunctionPointerTest : uur::urProgramTest {
 UUR_INSTANTIATE_KERNEL_TEST_SUITE_P(urProgramGetFunctionPointerTest);
 
 TEST_P(urProgramGetFunctionPointerTest, Success) {
+    UUR_KNOWN_FAILURE_ON(uur::OpenCL{"Intel(R) UHD Graphics 770"});
     void *function_pointer = nullptr;
     ASSERT_SUCCESS(urProgramGetFunctionPointer(
         device, program, function_name.data(), &function_pointer));
diff --git a/test/conformance/testing/include/uur/known_failure.h b/test/conformance/testing/include/uur/known_failure.h
@@ -0,0 +1,102 @@
+// Copyright (C) 2024 Intel Corporation
+// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
+// See LICENSE.TXT
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef UR_CONFORMANCE_INCLUDE_KNOWN_FAILURE_H_INCLUDED
+#define UR_CONFORMANCE_INCLUDE_KNOWN_FAILURE_H_INCLUDED
+
+#include "uur/utils.h"
+#include <string>
+#include <vector>
+
+namespace uur {
+struct DeviceMatcher {
+    DeviceMatcher(uint32_t adapterVersion, ur_platform_backend_t backend,
+                  std::vector<std::string> deviceNames)
+        : adapterVersion(adapterVersion), backend(backend),
+          deviceNames(deviceNames) {}
+
+    uint32_t adapterVersion;
+    ur_platform_backend_t backend;
+    std::vector<std::string> deviceNames;
+};
+
+struct OpenCL : DeviceMatcher {
+    OpenCL(std::initializer_list<std::string> il)
+        : DeviceMatcher(1, UR_PLATFORM_BACKEND_OPENCL, {il.begin(), il.end()}) {
+    }
+};
+
+struct LevelZero : DeviceMatcher {
+    LevelZero(std::initializer_list<std::string> il)
+        : DeviceMatcher(1, UR_PLATFORM_BACKEND_LEVEL_ZERO,
+                        {il.begin(), il.end()}) {}
+};
+
+struct LevelZeroV2 : DeviceMatcher {
+    LevelZeroV2(std::initializer_list<std::string> il)
+        : DeviceMatcher(2, UR_PLATFORM_BACKEND_LEVEL_ZERO,
+                        {il.begin(), il.end()}) {}
+};
+
+struct CUDA : DeviceMatcher {
+    CUDA(std::initializer_list<std::string> il)
+        : DeviceMatcher(1, UR_PLATFORM_BACKEND_CUDA, {il.begin(), il.end()}) {}
+};
+
+struct HIP : DeviceMatcher {
+    HIP(std::initializer_list<std::string> il)
+        : DeviceMatcher(1, UR_PLATFORM_BACKEND_HIP, {il.begin(), il.end()}) {}
+};
+
+struct NativeCPU : DeviceMatcher {
+    NativeCPU(std::initializer_list<std::string> il)
+        : DeviceMatcher(1, UR_PLATFORM_BACKEND_NATIVE_CPU,
+                        {il.begin(), il.end()}) {}
+};
+
+inline bool isKnownFailureOn(ur_adapter_handle_t adapter,
+                             ur_platform_handle_t platform,
+                             ur_device_handle_t device,
+                             std::vector<DeviceMatcher> deviceMatchers) {
+    for (const auto &deviceMatcher : deviceMatchers) {
+        uint32_t adapterVersion = 0;
+        urAdapterGetInfo(adapter, UR_ADAPTER_INFO_VERSION,
+                         sizeof(adapterVersion), &adapterVersion, nullptr);
+        ur_platform_backend_t backend = UR_PLATFORM_BACKEND_UNKNOWN;
+        uur::GetPlatformInfo<ur_platform_backend_t>(
+            platform, UR_PLATFORM_INFO_BACKEND, backend);
+        if (deviceMatcher.adapterVersion != adapterVersion &&
+            deviceMatcher.backend != backend) {
+            continue;
+        }
+        if (deviceMatcher.deviceNames.empty()) {
+            return true;
+        }
+        std::string deviceName;
+        uur::GetDeviceInfo<std::string>(device, UR_DEVICE_INFO_NAME,
+                                        deviceName);
+        for (const auto &unsupportedDeviceName : deviceMatcher.deviceNames) {
+            if (deviceName.find(unsupportedDeviceName) != std::string::npos) {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+} // namespace uur
+
+#define UUR_KNOWN_FAILURE_ON(...)                                              \
+    if (uur::isKnownFailureOn(adapter, platform, device, {__VA_ARGS__})) {     \
+        std::string platformName;                                              \
+        uur::GetPlatformInfo<std::string>(platform, UR_PLATFORM_INFO_NAME,     \
+                                          platformName);                       \
+        std::string deviceName;                                                \
+        uur::GetDeviceInfo<std::string>(device, UR_DEVICE_INFO_NAME,           \
+                                        deviceName);                           \
+        GTEST_SKIP() << "Known failure on: " << platformName << ", "           \
+                     << deviceName;                                            \
+    }
+
+#endif // UR_CONFORMANCE_INCLUDE_KNOWN_FAILURE_H_INCLUDED
diff --git a/test/conformance/testing/include/uur/utils.h b/test/conformance/testing/include/uur/utils.h
@@ -43,12 +43,12 @@ ur_result_t GetInfo(ObjectTy object, InfoTy info, Callable cb, T &out_value) {
 
     // special case for strings
     if constexpr (std::is_same_v<std::string, T>) {
-        std::vector<char> data(size);
-        result = cb(object, info, size, data.data(), nullptr);
+        std::string value(size, '\0');
+        result = cb(object, info, size, value.data(), nullptr);
         if (result != UR_RESULT_SUCCESS) {
             return result;
         }
-        out_value = std::string(data.data(), data.size());
+        out_value = value.substr(0, value.find_last_of('\0'));
         return UR_RESULT_SUCCESS;
     } else {
         if (size != sizeof(T)) {

Original file line number	Diff line number	Diff line change
`@@ -3,6 +3,7 @@`
`3`	`3`	`// See LICENSE.TXT`
`4`	`4`	`// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception`
`5`	`5`
	`6`	`+#include "uur/known_failure.h"`
`6`	`7`	`#include <uur/fixtures.h>`
`7`	`8`
`8`	`9`	`struct urKernelSetArgValueTest : uur::urKernelTest {`
`@@ -45,6 +46,7 @@ TEST_P(urKernelSetArgValueTest, InvalidKernelArgumentIndex) {`
`45`	`46`	`}`
`46`	`47`
`47`	`48`	`TEST_P(urKernelSetArgValueTest, InvalidKernelArgumentSize) {`
	`49`	`+ UUR_KNOWN_FAILURE_ON(uur::OpenCL{"Intel(R) UHD Graphics 770"});`
`48`	`50`	`ASSERT_EQ_RESULT(UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_SIZE,`
`49`	`51`	`urKernelSetArgValue(kernel, 2, 0, nullptr, &arg_value));`
`50`	`52`	`}`