[CI] Add "loader" support to conformance testing

This expands our CI to test the loader; the dispatcher that is
used when multiple adapters are availabe.

The old "run on hardware" jobs should behave the same (they have
the loader tests disabled), but there are new "combined" jobs
with OpenCL/Level Zero + Native CPU.

Closes: oneapi-src#2081

Author: RossBrunton

.github/workflows/build-hw-reusable.yml

@@ -7,10 +7,15 @@ on:
       adapter_name:
         required: true
         type: string
+      other_adapter_name:
+        required: false
+        type: string
+        default: ""
       runner_name:
         required: true
         type: string
       platform:
+        description: "Platform string, `UR_CTS_ADAPTER_PLATFORM` will be set to this."
         required: false
         type: string
         default: ""
@@ -39,9 +44,13 @@ jobs:
     if: github.repository == 'oneapi-src/unified-runtime'  # run only on upstream; forks won't have the HW
     strategy:
       matrix:
-        adapter: [
-          {name: "${{inputs.adapter_name}}", platform: "${{inputs.platform}}", static_Loader: "${{inputs.static_loader}}", static_adapter: "${{inputs.static_loader}}"},
-        ]
+        adapter: [{
+          name: "${{inputs.adapter_name}}",
+          other_name: "${{inputs.other_adapter_name}}",
+          platform: "${{inputs.platform}}",
+          static_Loader: "${{inputs.static_loader}}",
+          static_adapter: "${{inputs.static_loader}}"
+        }]
         build_type: [Debug, Release]
         compiler: [{c: gcc, cxx: g++}, {c: clang, cxx: clang++}]
         # TODO: The latest L0 loader segfaults when built with clang.
@@ -82,8 +91,9 @@ jobs:
         -DUR_ENABLE_TRACING=ON
         -DUR_DEVELOPER_MODE=ON
         -DUR_BUILD_TESTS=ON
-        -DUR_CONFORMANCE_TEST_LOADER=OFF
         -DUR_BUILD_ADAPTER_${{matrix.adapter.name}}=ON
+        -DUR_CONFORMANCE_TEST_LOADER=${{ matrix.adapter.other_name != '' && 'ON' || 'OFF' }}
+        ${{ matrix.adapter.other_name != '' && format('-DUR_BUILD_ADAPTER_{0}=ON', matrix.adapter.other_name) || '' }}
         -DUR_STATIC_LOADER=${{matrix.adapter.static_Loader}}
         -DUR_STATIC_ADAPTER_${{matrix.adapter.name}}=${{matrix.adapter.static_adapter}}
         -DUR_DPCXX=${{github.workspace}}/dpcpp_compiler/bin/clang++
@@ -103,6 +113,8 @@ jobs:
     - name: Test adapter specific
       working-directory: ${{github.workspace}}/build
       run: ctest -C ${{matrix.build_type}} --output-on-failure -L "adapter-specific" --timeout 180
+      # Don't run adapter specific tests when building multiple adapters
+      if: ${{ matrix.adapter.other_name == '' }}
 
     - name: Test adapters
       working-directory: ${{github.workspace}}/build

.github/workflows/cmake.yml

@@ -187,6 +187,24 @@ jobs:
       adapter_name: NATIVE_CPU
       runner_name: NATIVE_CPU
 
+  # Native CPU jobs are here to force the loader to be used (UR will not use the loader if there is only one target)
+  combined-opencl-native-cpu:
+    name: OpenCL + Native CPU (Loader)
+    uses: ./.github/workflows/build-hw-reusable.yml
+    with:
+      adapter_name: OPENCL
+      other_adapter_name: NATIVE_CPU
+      runner_name: OPENCL
+      platform: "OPENCL:Intel(R) OpenCL"
+
+  combined-level-zero-native-cpu:
+    name: Level Zero + Native CPU (Loader)
+    uses: ./.github/workflows/build-hw-reusable.yml
+    with:
+      adapter_name: L0
+      other_adapter_name: NATIVE_CPU
+      runner_name: L0
+
   e2e-level-zero:
     name: E2E L0
     permissions:

test/conformance/adapter/adapter_adapter_native_cpu.match

@@ -1,5 +1,6 @@
 {{NONDETERMINISTIC}}
-urAdapterGetLastErrorTest.Success
-urAdapterGetLastErrorTest.InvalidHandle
-urAdapterGetLastErrorTest.InvalidMessagePtr
-urAdapterGetLastErrorTest.InvalidErrorPtr
+# These pass when the adapter is launched by the loader
+{{OPT}}urAdapterGetLastErrorTest.Success
+{{OPT}}urAdapterGetLastErrorTest.InvalidHandle
+{{OPT}}urAdapterGetLastErrorTest.InvalidMessagePtr
+{{OPT}}urAdapterGetLastErrorTest.InvalidErrorPtr

test/conformance/event/event_adapter_native_cpu.match

@@ -1,4 +1,4 @@
-{{NONDETERMINISTIC}}
+# Note: This file is only for use with cts_exe.py
 urEventGetInfoTest.Success/SYCL_NATIVE_CPU___SYCL_Native_CPU__{{.*}}__UR_EVENT_INFO_COMMAND_QUEUE
 urEventGetInfoTest.Success/SYCL_NATIVE_CPU___SYCL_Native_CPU__{{.*}}__UR_EVENT_INFO_CONTEXT
 urEventGetInfoTest.Success/SYCL_NATIVE_CPU___SYCL_Native_CPU__{{.*}}__UR_EVENT_INFO_COMMAND_TYPE
@@ -31,3 +31,9 @@ urEventSetCallbackTest.AllStates/SYCL_NATIVE_CPU___SYCL_Native_CPU__{{.*}}
 urEventSetCallbackTest.EventAlreadyCompleted/SYCL_NATIVE_CPU___SYCL_Native_CPU__{{.*}}
 urEventSetCallbackNegativeTest.InvalidNullPointerCallback/SYCL_NATIVE_CPU___SYCL_Native_CPU__{{.*}}
 urEventSetCallbackNegativeTest.InvalidEnumeration/SYCL_NATIVE_CPU___SYCL_Native_CPU__{{.*}}
+
+# These crash when ran through the loader
+{{OPT}}urEventRetainTest.InvalidNullHandle/*
+{{OPT}}urEventReleaseTest.InvalidNullHandle/*
+{{OPT}}urEventGetNativeHandleTest.InvalidNullHandleEvent/*
+{{OPT}}urEventSetCallbackNegativeTest.InvalidNullHandleEvent/*

test/conformance/event/event_adapter_opencl.match

@@ -0,0 +1,3 @@
+# Note: This file is only for use with cts_exe.py, not the legacy match checker
+# These crash when ran through the loader
+{{OPT}}urEventSetCallbackTest.ValidateParameters/*