[SYCL] Implement coverage instrumentation for device code (#20206)

This PR extends Clang's source-based code coverage to work with SYCL
device code. It includes the following changes:

1. The `InstrProfilingLoweringPass` was updated to lower profiling
counters to SYCL device globals.
2. A new function was added to the compiler runtime to increment the
host-side profiling counters.
3. The SYCL runtime was updated to send the contents of the device-side
profiling counters to the new compiler runtime function when their
device global map entries are being freed.

This feature may not work correctly for functions that differ between
host and device due to the use of the `__SYCL_DEVICE_ONLY__` macro. In
such cases, it may not be possible to correlate the profiling counters
from the device to the host. Resolves #7803.

---------

Signed-off-by: Michael Aziz <[email protected]>
Co-authored-by: Steffen Larsen <[email protected]>

Author: 0x12CC

buildbot/configure.py

@@ -21,7 +21,7 @@ def do_configure(args, passthrough_args):
     if not os.path.isdir(abs_obj_dir):
         os.makedirs(abs_obj_dir)
 
-    llvm_external_projects = "sycl;llvm-spirv;opencl;xpti;xptifw"
+    llvm_external_projects = "sycl;llvm-spirv;opencl;xpti;xptifw;compiler-rt"
 
     # libdevice build requires a working SYCL toolchain, which is not the case
     # with macOS target right now.

clang/lib/Driver/ToolChains/Clang.cpp

@@ -5480,6 +5480,9 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
       CmdArgs.push_back("-fsycl-is-device");
       CmdArgs.push_back("-fdeclare-spirv-builtins");
 
+      // Set the atomic profile update flag to increment counters atomically.
+      CmdArgs.push_back("-fprofile-update=atomic");
+
       // Set O2 optimization level by default
       if (!Args.getLastArg(options::OPT_O_Group))
         CmdArgs.push_back("-O2");

clang/lib/Driver/ToolChains/SYCL.cpp

@@ -1575,11 +1575,7 @@ static ArrayRef<options::ID> getUnsupportedOpts() {
       options::OPT_fno_profile_generate, // -f[no-]profile-generate
       options::OPT_ftest_coverage,
       options::OPT_fno_test_coverage, // -f[no-]test-coverage
-      options::OPT_fcoverage_mapping,
-      options::OPT_coverage,             // --coverage
-      options::OPT_fno_coverage_mapping, // -f[no-]coverage-mapping
-      options::OPT_fprofile_instr_generate,
-      options::OPT_fprofile_instr_generate_EQ,
+      options::OPT_coverage,          // --coverage
       options::OPT_fprofile_arcs,
       options::OPT_fno_profile_arcs,           // -f[no-]profile-arcs
       options::OPT_fno_profile_instr_generate, // -f[no-]profile-instr-generate

clang/test/Driver/sycl-unsupported.cpp

@@ -19,13 +19,6 @@
 // RUN:    -DOPT_CC1=-debug-info-kind=line-tables-only \
 // RUN:    -check-prefixes=UNSUPPORTED_OPT_DIAG,UNSUPPORTED_OPT
 
-// RUN: %clangxx -fsycl -fprofile-instr-generate -### %s 2>&1 \
-// RUN:  | FileCheck %s -DARCH=spir64 -DOPT=-fprofile-instr-generate \
-// RUN:    -DOPT_CC1=-fprofile-instrument=clang \
-// RUN:    -check-prefixes=UNSUPPORTED_OPT_DIAG,UNSUPPORTED_OPT
-// RUN: %clangxx -fsycl -fcoverage-mapping \
-// RUN:          -fprofile-instr-generate -### %s 2>&1 \
-// RUN:  | FileCheck %s -DARCH=spir64 -DOPT=-fcoverage-mapping
 // RUN: %clangxx -fsycl -ftest-coverage -### %s 2>&1 \
 // RUN:  | FileCheck %s -DARCH=spir64 -DOPT=-ftest-coverage \
 // RUN:    -DOPT_CC1=-coverage-notes-file \
@@ -49,12 +42,6 @@
 // RUN:  | FileCheck %s -DARCH=spir64 -DOPT=--coverage \
 // RUN:    -DOPT_CC1=-coverage-notes-file \
 // RUN:    -check-prefixes=UNSUPPORTED_OPT_DIAG,UNSUPPORTED_OPT
-// Check to make sure our '-fsanitize=address' exception isn't triggered by a
-// different option
-// RUN: %clangxx -fsycl -fprofile-instr-generate=address -### %s 2>&1 \
-// RUN:  | FileCheck %s -DARCH=spir64 -DOPT=-fprofile-instr-generate=address \
-// RUN:    -DOPT_CC1=-fprofile-instrument=clang \
-// RUN:    -check-prefixes=UNSUPPORTED_OPT_DIAG,UNSUPPORTED_OPT
 
 // CHECK: ignoring '[[OPT]]' option as it is not currently supported for target '[[ARCH]]{{.*}}'; only supported for host compilation [-Woption-ignored]
 // CHECK-NOT: clang{{.*}} "-fsycl-is-device"{{.*}} "[[OPT]]{{.*}}"

compiler-rt/lib/profile/InstrProfilingRuntime.cpp

@@ -10,6 +10,22 @@ extern "C" {
 
 #include "InstrProfiling.h"
 
+void __sycl_increment_profile_counters(uint64_t FnHash, size_t NumCounters,
+                                       const uint64_t *Increments) {
+  for (const __llvm_profile_data *DataVar = __llvm_profile_begin_data();
+       DataVar < __llvm_profile_end_data(); DataVar++) {
+    if (DataVar->NameRef != FnHash || DataVar->NumCounters != NumCounters)
+      continue;
+
+    uint64_t *const Counters = reinterpret_cast<uint64_t *>(
+        reinterpret_cast<uintptr_t>(DataVar) +
+        reinterpret_cast<uintptr_t>(DataVar->CounterPtr));
+    for (size_t i = 0; i < NumCounters; i++)
+      Counters[i] += Increments[i];
+    break;
+  }
+}
+
 static int RegisterRuntime() {
   __llvm_profile_initialize();
 #ifdef _AIX

llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp

@@ -1002,6 +1002,9 @@ bool InstrLowerer::lower() {
   if (!NeedsRuntimeHook && ContainsProfiling)
     emitRuntimeHook();
 
+  if (M.getTargetTriple().isSPIR())
+    return true;
+
   emitRegistration();
   emitUses();
   emitInitialization();
@@ -1116,6 +1119,18 @@ GlobalVariable *InstrLowerer::getOrCreateBiasVar(StringRef VarName) {
 }
 
 Value *InstrLowerer::getCounterAddress(InstrProfCntrInstBase *I) {
+  if (M.getTargetTriple().isSPIR()) {
+    auto *Counters = getOrCreateRegionCounters(I);
+    IRBuilder<> Builder(I);
+    auto *Addr = Builder.CreateLoad(PointerType::get(M.getContext(), 1),
+                                    Counters, "pgocount.addr");
+    const std::uint64_t Index = I->getIndex()->getZExtValue();
+    if (Index == 0)
+      return Addr;
+    auto *Offset = Builder.getInt64(Index * sizeof(std::uint64_t));
+    return Builder.CreatePtrAdd(Addr, Offset, "pgocount.offset");
+  }
+
   auto *Counters = getOrCreateRegionCounters(I);
   IRBuilder<> Builder(I);
 
@@ -1657,6 +1672,22 @@ InstrLowerer::getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc) {
 GlobalVariable *
 InstrLowerer::createRegionCounters(InstrProfCntrInstBase *Inc, StringRef Name,
                                    GlobalValue::LinkageTypes Linkage) {
+  if (M.getTargetTriple().isSPIR()) {
+    uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
+    auto &Ctx = M.getContext();
+    auto *PtrTy = PointerType::get(Ctx, 1);
+    auto *IntTy = Type::getInt64Ty(Ctx);
+    auto *StructTy = StructType::get(Ctx, {PtrTy, IntTy});
+    GlobalVariable *GV = new GlobalVariable(
+        M, StructTy, false, Linkage, Constant::getNullValue(StructTy), Name);
+    const std::uint64_t FnHash = IndexedInstrProf::ComputeHash(
+        getPGOFuncNameVarInitializer(Inc->getName()));
+    const std::string FnName = std::string{"__profc_"} + std::to_string(FnHash);
+    GV->addAttribute("sycl-unique-id", FnName);
+    GV->addAttribute("sycl-device-global-size", Twine(NumCounters * 8).str());
+    return GV;
+  }
+
   uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
   auto &Ctx = M.getContext();
   GlobalVariable *GV;

llvm/test/Instrumentation/InstrProfiling/coverage_sycl.ll

@@ -0,0 +1,29 @@
+; RUN: opt < %s -passes=instrprof -S | FileCheck %s
+
+target triple = "spir64-unknown-unknown"
+
+@__profn_foo = private constant [3 x i8] c"foo"
+; CHECK: @__profc_foo = private global { ptr addrspace(1), i64 } zeroinitializer, section "__llvm_prf_cnts", comdat #0
+; CHECK: @__profd_foo = private global { i64, i64, i64, i64, ptr, ptr, i32, [3 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 sub (i64 ptrtoint (ptr @__profc_foo to i64)
+@__profn_bar = private constant [3 x i8] c"bar"
+; CHECK: @__profc_bar = private global { ptr addrspace(1), i64 } zeroinitializer, section "__llvm_prf_cnts", comdat #1
+; CHECK: @__profd_bar = private global { i64, i64, i64, i64, ptr, ptr, i32, [3 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 sub (i64 ptrtoint (ptr @__profc_bar to i64)
+
+; CHECK: @__llvm_prf_nm = {{.*}} section "__llvm_prf_names"
+
+define void @_Z3foov() {
+  call void @llvm.instrprof.cover(ptr @__profn_foo, i64 12345678, i32 1, i32 0)
+  ; CHECK: %pgocount.addr = load ptr addrspace(1), ptr @__profc_foo, align 8
+  ; CHECK: store i8 0, ptr addrspace(1) %pgocount.addr, align 1
+  ret void
+}
+
+%class.A = type { ptr }
+define dso_local void @_Z3barv(ptr nocapture nonnull align 8 %0) unnamed_addr #0 align 2 {
+  call void @llvm.instrprof.cover(ptr @__profn_bar, i64 87654321, i32 1, i32 0)
+  ; CHECK: %pgocount.addr = load ptr addrspace(1), ptr @__profc_bar, align 8
+  ; CHECK: store i8 0, ptr addrspace(1) %pgocount.addr, align 1
+  ret void
+}
+
+declare void @llvm.instrprof.cover(ptr, i64, i32, i32)

sycl/CMakeLists.txt

@@ -389,13 +389,16 @@ add_custom_target(sycl-compiler
           clang-offload-extract
           clang-offload-packager
           clang-linker-wrapper
+          compiler-rt
           file-table-tform
           llc
           llvm-ar
           llvm-foreach
           llvm-spirv
           llvm-link
           llvm-objcopy
+          llvm-profdata
+          llvm-cov
           spirv-to-ir-wrapper
           sycl-post-link
           opencl-aot

sycl/doc/design/DeviceCodeCoverage.md

@@ -0,0 +1,71 @@
+# Design for Device-side Code Coverage
+
+## Overview
+
+This document describes the design and implementation of device-side code coverage for SYCL, extending Clang's source-based code coverage to support device code. The approach leverages the existing SYCL device global infrastructure, as detailed in the [DeviceGlobal.md](DeviceGlobal.md) design document, to enable collection and aggregation of coverage data from device kernels.
+
+## Design Details
+
+### Profiling Counter Representation
+
+Profiling counters for code coverage are lowered by the compiler as device globals. Specifically, the `InstrProfilingLoweringPass` is modified so that, when targeting SPIR-V, coverage counters are represented as pointers to USM buffers, matching the representation of other SYCL device globals. This indirection allows counters to be relocatable and managed consistently with other device-side global variables.
+
+Each counter is annotated with a unique identifier (`sycl-unique-id`) of the form `__profc_<fn_hash>`, where `<fn_hash>` is a 64-bit unsigned integer uniquely identifying the instrumented function. The counter's size is also recorded via the `sycl-device-global-size` attribute. These attributes ensure that counters are discoverable and manageable by the SYCL runtime and integration headers/footers.
+
+The profile counter device global is represented as an array of 8-byte integers (`std::uint64_t`). The number of elements in this array corresponds to the number of regions in the function being instrumented, where a region typically represents a distinct code branch or block. The size of the device global variable is therefore determined by multiplying the number of regions by eight bytes, and this value is recorded in the `sycl-device-global-size` attribute for use by the runtime and integration logic.
+
+### Integration with Device Global Infrastructure
+
+The device global infrastructure, as described in [DeviceGlobal.md](DeviceGlobal.md), provides mechanisms for mapping host and device instances of global variables, managing their lifetimes, and facilitating data transfer. Device-side coverage counters are treated as a special class of device globals:
+
+- They use the shared allocation type rather than the device allocation type for the underlying USM memory.
+- They do not have corresponding `device_global` declarations in host code.
+- Their lifetime and cleanup are managed via the device global map, with integration footer code ensuring registration and deregistration.
+
+### Runtime Handling and Data Aggregation
+
+When a device global entry corresponding to a coverage counter is released (e.g., when a device image is unloaded), the SYCL runtime aggregates the values from the device-side counter into the equivalent host-side counter. Equivalence is determined by matching both the `<fn_hash>` and the number of counter regions. If no matching host-side counter exists—typically due to differences in code between host and device caused by the `__SYCL_DEVICE_ONLY__` macro—the device-side counter values are discarded.
+
+The aggregation is performed by invoking a new function in the compiler runtime, `__sycl_increment_profile_counters`, which is weakly linked to accommodate optional runtime availability. This function accepts the `<fn_hash>`, the number of regions, and the increment values, and updates the host-side counters accordingly. At program exit, the final profile data reflects the sum of host and device coverage counters.
+
+### Compiler and Runtime Changes
+
+#### Compiler Frontend
+
+- The lowering pass for coverage counters is updated to emit device globals with the appropriate attributes and indirection.
+- Integration headers and footers are updated to register device global counters with the runtime, using the unique identifier and size.
+
+#### SYCL Runtime
+
+- Device globals with IDs matching the `__profc_<fn_hash>` pattern are recognized as coverage counters.
+- USM allocation and management for counters is handled as for other device globals, but without host-side declarations.
+- Upon cleanup, device-side counter values are aggregated into host-side counters via the runtime API.
+
+#### Compiler Runtime
+
+- The new function `__sycl_increment_profile_counters` is introduced to update host-side counters.
+- The function is weakly linked to allow for optional inclusion.
+
+### Limitations and Considerations
+
+- The feature is currently implemented only for SPIR-V targets; CUDA and HIP backends are not supported.
+- Devices lacking support for device globals cannot utilize device-side code coverage.
+- Differences in code between host and device (e.g., due to `__SYCL_DEVICE_ONLY__`) may prevent aggregation of coverage data for some functions.
+- The design relies on the robustness of the device global infrastructure for correct mapping and lifetime management.
+
+## Relationship to Device Global Design
+
+This feature is built upon the mechanisms described in [DeviceGlobal.md](DeviceGlobal.md), including:
+
+- Use of unique string identifiers (`sycl-unique-id`) for mapping and management.
+- USM-based allocation and zero-initialization of device-side storage.
+- Integration header/footer registration for host-device correlation.
+- Runtime database for device global management and lookup.
+
+The code coverage counters are a specialized use case of device globals, with additional logic for aggregation and profile generation.
+
+## References
+
+- [Implementation design for SYCL device globals](DeviceGlobal.md)
+- [Clang Source-based Code Coverage](https://clang.llvm.org/docs/SourceBasedCodeCoverage.html)
+- [SYCL Specification](https://registry.khronos.org/SYCL/specs/sycl-2020/html/sycl-2020.html)

sycl/doc/index.rst

@@ -40,6 +40,7 @@ Design Documents for the oneAPI DPC++ Compiler
    design/ParallelForRangeRounding
    design/SYCLInstrumentationUsingXPTI
    design/ITTAnnotations
+   design/DeviceCodeCoverage
    design/DeviceGlobal
    design/CompileTimeProperties
    design/HostPipes