[UR][Tests] Add a CMake option to limit the test devices count

and test platforms count in CTS.

Author: szadam

test/conformance/CMakeLists.txt

@@ -4,6 +4,8 @@
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 set(UR_CONFORMANCE_TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR})
+option(UR_TEST_DEVICES_COUNT "Count of devices on which CTS will be run" 1)
+option(UR_TEST_PLATFORMS_COUNT "Count of platforms on which CTS will be run" 1)
 
 function(add_test_adapter name adapter)
     set(TEST_TARGET_NAME test-${name})
@@ -12,7 +14,7 @@ function(add_test_adapter name adapter)
     add_test(NAME ${TEST_NAME}
         COMMAND ${CMAKE_COMMAND}
         -D TEST_FILE=${Python3_EXECUTABLE}
-        -D TEST_ARGS="${UR_CONFORMANCE_TEST_DIR}/cts_exe.py --test_command ${CMAKE_BINARY_DIR}/bin/${TEST_TARGET_NAME}"
+        -D TEST_ARGS="${UR_CONFORMANCE_TEST_DIR}/cts_exe.py --test_command ${CMAKE_BINARY_DIR}/bin/${TEST_TARGET_NAME} --test_devices_count=${UR_TEST_DEVICES_COUNT} --test_platforms_count=${UR_TEST_PLATFORMS_COUNT}"
         -D MODE=stdout
         -D MATCH_FILE=${CMAKE_CURRENT_SOURCE_DIR}/${name}_${adapter}.match
         -P ${PROJECT_SOURCE_DIR}/cmake/match.cmake

test/conformance/README.md

@@ -8,4 +8,11 @@ In the future, when all bugs are fixed, and the tests pass,
 this solution will no longer be necessary.
 When you fix any test, the match file must be updated
 Empty match files indicate that there are no failing tests
-in a particular group for the corresponding adapter.
+in a particular group for the corresponding adapter.
+
+## How to limit the test devices count
+
+To limit how many devices you want to run the CTS on,
+use CMake option UR_TEST_DEVICES_COUNT. If you want to run
+the tests on all available devices, set 0.
+The default value is 1.

test/conformance/cts_exe.py

@@ -20,9 +20,13 @@
 
     parser = ArgumentParser()
     parser.add_argument("--test_command", help="Ctest test case")
+    parser.add_argument("--test_devices_count", type=str, help="Number of devices on which tests will be run")
+    parser.add_argument("--test_platforms_count", type=str, help="Number of platforms on which tests will be run")
     args = parser.parse_args()
 
-    result = subprocess.Popen([args.test_command, '--gtest_brief=1'], stdout = subprocess.PIPE, text = True)  # nosec B603
+    result = subprocess.Popen([args.test_command, '--gtest_brief=1', f'--devices_count={args.test_devices_count}',
+                               f'--platforms_count={args.test_platforms_count}'],
+                               stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)  # nosec B603
 
     pat = re.compile(r'\[( )*FAILED( )*\]')
     output_list = []

test/conformance/source/environment.cpp

@@ -3,6 +3,7 @@
 // See LICENSE.TXT
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+#include <algorithm>
 #include <cstring>
 #include <fstream>
 
@@ -209,6 +210,22 @@ DevicesEnvironment::DevicesEnvironment(int argc, char **argv)
         error = "Could not find any devices associated with the platform";
         return;
     }
+    // Get the argument (test_devices_count) to limit test devices count.
+    u_long count_set = 0;
+    for (int i = 1; i < argc; ++i) {
+        if (std::strcmp(argv[i], "--test_devices_count") == 0 && i + 1 < argc) {
+            count_set = std::strtoul(argv[i + 1], nullptr, 10);
+            break;
+        }
+    }
+    // In case, the count_set is "0", the variable count will not be changed.
+    // The CTS will run on all devices.
+    if (count_set > (std::numeric_limits<uint32_t>::max)()) {
+        error = "Invalid test_devices_count argument";
+        return;
+    } else if (count_set > 0) {
+        count = (std::min)(count, static_cast<uint32_t>(count_set));
+    }
     devices.resize(count);
     if (urDeviceGet(platform, UR_DEVICE_TYPE_ALL, count, devices.data(),
                     nullptr)) {