[UR] Partially reapply "Move conformance tests over to lit" (#18382) (#18392)

This reverts commit 44ed9b7, but
excludes the problematic changes to jobs.

Author: RossBrunton

.github/workflows/ur-build-hw.yml

@@ -144,7 +144,6 @@ jobs:
         -DUR_DEVELOPER_MODE=ON
         -DUR_BUILD_TESTS=ON
         -DUR_BUILD_ADAPTER_${{matrix.adapter.name}}=ON
-        -DUR_CONFORMANCE_TEST_LOADER=${{ matrix.adapter.other_name != '' && 'ON' || 'OFF' }}
         ${{ matrix.adapter.other_name != '' && format('-DUR_BUILD_ADAPTER_{0}=ON', matrix.adapter.other_name) || '' }}
         -DUR_STATIC_LOADER=${{matrix.adapter.static_Loader}}
         -DUR_STATIC_ADAPTER_${{matrix.adapter.name}}=${{matrix.adapter.static_adapter}}
@@ -172,7 +171,8 @@ jobs:
     - name: Test adapters
       env:
         ZE_ENABLE_LOADER_DEBUG_TRACE: 1
-      run: ctest -C ${{matrix.build_type}} --test-dir build --output-on-failure -L "conformance" --timeout 600 -VV
+        LIT_OPTS: "--timeout 120"
+      run: cmake --build build -j $(nproc) -- check-unified-runtime-conformance
 
     - name: Get information about platform
       if: ${{ always() }}

unified-runtime/CMakeLists.txt

@@ -65,8 +65,7 @@ set(UR_SYCL_LIBRARY_DIR "" CACHE PATH
 set(UR_CONFORMANCE_TARGET_TRIPLES "" CACHE STRING
     "List of sycl targets to build CTS device binaries for")
 set(UR_CONFORMANCE_AMD_ARCH "" CACHE STRING "AMD device target ID to build CTS binaries for")
-option(UR_CONFORMANCE_ENABLE_MATCH_FILES "Enable CTS match files" ON)
-option(UR_CONFORMANCE_TEST_LOADER "Also test the loader in the conformance tests" OFF)
+set(UR_CONFORMANCE_SELECTOR "" CACHE STRING "If nonempty, the device selector for conformance tests")
 option(UR_USE_DEBUG_POSTFIX "Enable debug postfix 'd' for libraries" OFF)
 set(UR_ADAPTER_LEVEL_ZERO_SOURCE_DIR "" CACHE PATH
     "Path to external 'level_zero' adapter source dir")

unified-runtime/README.md

@@ -128,8 +128,7 @@ List of options provided by CMake:
 | UR_ENABLE_SANITIZER | Enable device sanitizer layer | ON/OFF | ON |
 | UR_CONFORMANCE_TARGET_TRIPLES | SYCL triples to build CTS device binaries for | Comma-separated list | spir64 |
 | UR_CONFORMANCE_AMD_ARCH | AMD device target ID to build CTS binaries for | string | `""` |
-| UR_CONFORMANCE_ENABLE_MATCH_FILES | Enable CTS match files | ON/OFF | ON |
-| UR_CONFORMANCE_TEST_LOADER | Additionally build and run "loader" tests for the CTS | ON/OFF | OFF |
+| UR_CONFORMANCE_SELECTOR | `ONEAPI_DEVICE_SELECTOR` for conformance testing | string | All enabled adapters |
 | UR_BUILD_ADAPTER_L0     | Build the Level-Zero adapter            | ON/OFF     | OFF     |
 | UR_BUILD_ADAPTER_OPENCL | Build the OpenCL adapter                | ON/OFF     | OFF     |
 | UR_BUILD_ADAPTER_CUDA   | Build the CUDA adapter                  | ON/OFF     | OFF     |

unified-runtime/scripts/core/CONTRIB.rst

@@ -272,14 +272,21 @@ Conformance tests *must* not make assumptions about the adapter under test.
 Tests fixtures or cases *must* query for support of optional features and skip
 testing if unsupported by the adapter.
 
-All tests in the Unified Runtime project are configured to use CTest to run. 
-All conformance tests have the ``conformance`` label attached to them which 
-allows them to be run independently. To run all the conformance tests, execute 
-the following command from the build directory.
+Conformance tests are ran as part of the main ``test`` target, but can also be
+executed using the `check-unified-runtime-conformance` target as follows:
 
 .. code-block:: console
      
-    ctest -L "conformance"
+    cmake --build build --target check-unified-runtime-conformance
+
+A specific device/adapter can be specified by using ``ONEAPI_DEVICE_SELECTOR``
+in the same way as in the `DPC++ compiler
+<https://github.com/intel/llvm/blob/sycl/sycl/doc/EnvironmentVariables.md#oneapi_device_selector>`_.
+
+A number of tests are skipped on certain adapters due to being known failures.
+To force all tests to run, including known failures,
+``UR_CTS_ALSO_RUN_KNOWN_FAILURES`` may be set to ``1`` as an environment
+variable.
 
 Experimental Features
 =====================

unified-runtime/scripts/core/INTRO.rst

@@ -417,12 +417,12 @@ CTS Environment Variables
 
 The following environment variables are used by the CTS runner and can be used to specify the platform that the test
 framework should run on. This can be used during development and testing to run CTS tests in case multiple platforms
-are available. If both filters are specified, then they both must match a platform for it to be selected. If there are
-no valid platforms, then the tests will fail. Command line arguments take priority over these variables.
+are available. Conformance tests will be skipped if there are no valid platforms.
 
-.. envvar:: UR_CTS_BACKEND
+.. envvar:: ONEAPI_DEVICE_SELECTOR
 
-    A (case insensitive) backend to force the test to use. For example, `opencl`, `level_zero`, `hip` and so on.
+   If ``ONEAPI_DEVICE_SELECTOR`` is set in the environment, only devices
+   matched by the selector will be tested.
 
 .. envvar:: UR_CTS_ALSO_RUN_KNOWN_FAILURES
 

unified-runtime/scripts/ctest_parser.py

@@ -31,8 +31,9 @@ configure_file(lit.site.cfg.py.in lit.site.cfg.py)
 add_custom_target(deps_check-unified-runtime)
 if(UR_STANDALONE_BUILD)
   add_custom_target(check-unified-runtime
-    COMMAND "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" -v
+    COMMAND "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" -sv
     DEPENDS deps_check-unified-runtime
+    USES_TERMINAL
   )
 else()
   # Use the LLVM method to add the test suite - this also registers it under `check-all`
@@ -60,7 +61,8 @@ function(add_ur_lit_testsuite suite)
 
   if(UR_STANDALONE_BUILD)
     add_custom_target(${TARGET}
-      COMMAND "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" -v
+      COMMAND "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" -sv
+      USES_TERMINAL
     )
   else()
     add_lit_testsuite(${TARGET} "Running Unified Runtime ${suite} tests"