[SYCL] Implement coverage instrumentation for device code

buildbot/configure.py

@@ -21,7 +21,7 @@ def do_configure(args, passthrough_args):
     if not os.path.isdir(abs_obj_dir):
         os.makedirs(abs_obj_dir)
 
-    llvm_external_projects = "sycl;llvm-spirv;opencl;xpti;xptifw"
+    llvm_external_projects = "sycl;llvm-spirv;opencl;xpti;xptifw;compiler-rt"
 
     # libdevice build requires a working SYCL toolchain, which is not the case
     # with macOS target right now.

clang/lib/Driver/ToolChains/SYCL.cpp

@@ -1574,11 +1574,7 @@ static ArrayRef<options::ID> getUnsupportedOpts() {
       options::OPT_fno_profile_generate, // -f[no-]profile-generate
       options::OPT_ftest_coverage,
       options::OPT_fno_test_coverage, // -f[no-]test-coverage
-      options::OPT_fcoverage_mapping,
-      options::OPT_coverage,             // --coverage
-      options::OPT_fno_coverage_mapping, // -f[no-]coverage-mapping
-      options::OPT_fprofile_instr_generate,
-      options::OPT_fprofile_instr_generate_EQ,
+      options::OPT_coverage,          // --coverage
       options::OPT_fprofile_arcs,
       options::OPT_fno_profile_arcs,           // -f[no-]profile-arcs
       options::OPT_fno_profile_instr_generate, // -f[no-]profile-instr-generate

clang/lib/Frontend/CompilerInvocation.cpp

@@ -5377,6 +5377,8 @@ bool CompilerInvocation::CreateFromArgsImpl(
   if (LangOpts.SYCLIsDevice) {
     // Set the triple of the host for SYCL device compile.
     Res.getTargetOpts().HostTriple = Res.getFrontendOpts().AuxTriple;
+    // Set the atomic profile update flag to increment counters atomically.
+    Res.getCodeGenOpts().AtomicProfileUpdate = true;
     // If specified, create empty integration header files for now.
     CreateEmptyFile(LangOpts.SYCLIntHeader);
     CreateEmptyFile(LangOpts.SYCLIntFooter);

clang/test/Driver/sycl-unsupported.cpp

@@ -19,13 +19,6 @@
 // RUN:    -DOPT_CC1=-debug-info-kind=line-tables-only \
 // RUN:    -check-prefixes=UNSUPPORTED_OPT_DIAG,UNSUPPORTED_OPT
 
-// RUN: %clangxx -fsycl -fprofile-instr-generate -### %s 2>&1 \
-// RUN:  | FileCheck %s -DARCH=spir64 -DOPT=-fprofile-instr-generate \
-// RUN:    -DOPT_CC1=-fprofile-instrument=clang \
-// RUN:    -check-prefixes=UNSUPPORTED_OPT_DIAG,UNSUPPORTED_OPT
-// RUN: %clangxx -fsycl -fcoverage-mapping \
-// RUN:          -fprofile-instr-generate -### %s 2>&1 \
-// RUN:  | FileCheck %s -DARCH=spir64 -DOPT=-fcoverage-mapping
 // RUN: %clangxx -fsycl -ftest-coverage -### %s 2>&1 \
 // RUN:  | FileCheck %s -DARCH=spir64 -DOPT=-ftest-coverage \
 // RUN:    -DOPT_CC1=-coverage-notes-file \
@@ -49,12 +42,6 @@
 // RUN:  | FileCheck %s -DARCH=spir64 -DOPT=--coverage \
 // RUN:    -DOPT_CC1=-coverage-notes-file \
 // RUN:    -check-prefixes=UNSUPPORTED_OPT_DIAG,UNSUPPORTED_OPT
-// Check to make sure our '-fsanitize=address' exception isn't triggered by a
-// different option
-// RUN: %clangxx -fsycl -fprofile-instr-generate=address -### %s 2>&1 \
-// RUN:  | FileCheck %s -DARCH=spir64 -DOPT=-fprofile-instr-generate=address \
-// RUN:    -DOPT_CC1=-fprofile-instrument=clang \
-// RUN:    -check-prefixes=UNSUPPORTED_OPT_DIAG,UNSUPPORTED_OPT
 
 // CHECK: ignoring '[[OPT]]' option as it is not currently supported for target '[[ARCH]]{{.*}}'; only supported for host compilation [-Woption-ignored]
 // CHECK-NOT: clang{{.*}} "-fsycl-is-device"{{.*}} "[[OPT]]{{.*}}"

compiler-rt/lib/profile/InstrProfilingRuntime.cpp

@@ -10,6 +10,22 @@ extern "C" {
 
 #include "InstrProfiling.h"
 
+void __sycl_increment_profile_counters(uint64_t FnHash, size_t NumCounters,
+                                       const uint64_t *Increments) {
+  for (const __llvm_profile_data *DataVar = __llvm_profile_begin_data();
+       DataVar < __llvm_profile_end_data(); DataVar++) {
+    if (DataVar->NameRef != FnHash || DataVar->NumCounters != NumCounters)
+      continue;
+
+    uint64_t *const Counters = reinterpret_cast<uint64_t *>(
+        reinterpret_cast<uintptr_t>(DataVar) +
+        reinterpret_cast<uintptr_t>(DataVar->CounterPtr));
+    for (size_t i = 0; i < NumCounters; i++)
+      Counters[i] += Increments[i];
+    break;
+  }
+}
+
 static int RegisterRuntime() {
   __llvm_profile_initialize();
 #ifdef _AIX

llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp

@@ -1002,6 +1002,9 @@ bool InstrLowerer::lower() {
   if (!NeedsRuntimeHook && ContainsProfiling)
     emitRuntimeHook();
 
+  if (M.getTargetTriple().isSPIR())
+    return true;
+
   emitRegistration();
   emitUses();
   emitInitialization();
@@ -1116,6 +1119,22 @@ GlobalVariable *InstrLowerer::getOrCreateBiasVar(StringRef VarName) {
 }
 
 Value *InstrLowerer::getCounterAddress(InstrProfCntrInstBase *I) {
+  if (M.getTargetTriple().isSPIR()) {
+    auto *Counters = getOrCreateRegionCounters(I);
+    IRBuilder<> Builder(I);
+    auto *Addr = Builder.CreateLoad(PointerType::get(M.getContext(), 1),
+                                    Counters, "pgocount.addr");
+    const std::uint64_t Index = I->getIndex()->getZExtValue();
+    if (Index > 0) {
+      auto *Offset = Builder.getInt64(I->getIndex()->getZExtValue() *
+                                      sizeof(std::uint64_t));
+      auto *AddrWithOffset =
+          Builder.CreatePtrAdd(Addr, Offset, "pgocount.offset");
+      return AddrWithOffset;
+    }
+    return Addr;
+  }
+
   auto *Counters = getOrCreateRegionCounters(I);
   IRBuilder<> Builder(I);
 
@@ -1657,6 +1676,23 @@ InstrLowerer::getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc) {
 GlobalVariable *
 InstrLowerer::createRegionCounters(InstrProfCntrInstBase *Inc, StringRef Name,
                                    GlobalValue::LinkageTypes Linkage) {
+  if (M.getTargetTriple().isSPIR()) {
+    uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
+    auto &Ctx = M.getContext();
+    GlobalVariable *GV;
+    auto *PtrTy = PointerType::get(Ctx, 1);
+    auto *IntTy = Type::getInt64Ty(Ctx);
+    auto *StructTy = StructType::get(Ctx, {PtrTy, IntTy});
+    GV = new GlobalVariable(M, StructTy, false, Linkage,
+                            Constant::getNullValue(StructTy), Name);
+    const std::uint64_t FnHash = IndexedInstrProf::ComputeHash(
+        getPGOFuncNameVarInitializer(Inc->getName()));
+    const std::string FnName = std::string{"__profc_"} + std::to_string(FnHash);
+    GV->addAttribute("sycl-unique-id", FnName);
+    GV->addAttribute("sycl-device-global-size", Twine(NumCounters * 8).str());
+    return GV;
+  }
+
   uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
   auto &Ctx = M.getContext();
   GlobalVariable *GV;

sycl/CMakeLists.txt

@@ -389,13 +389,16 @@ add_custom_target(sycl-compiler
           clang-offload-extract
           clang-offload-packager
           clang-linker-wrapper
+          compiler-rt
           file-table-tform
           llc
           llvm-ar
           llvm-foreach
           llvm-spirv
           llvm-link
           llvm-objcopy
+          llvm-profdata
+          llvm-cov
           spirv-to-ir-wrapper
           sycl-post-link
           opencl-aot

sycl/doc/design/DeviceCodeCoverage.md

@@ -0,0 +1,69 @@
+# Design for Device-side Code Coverage
+
+## Overview
+
+This document describes the design and implementation of device-side code coverage for SYCL, extending Clang's source-based code coverage to support device code. The approach leverages the existing SYCL device global infrastructure, as detailed in the [DeviceGlobal.md](DeviceGlobal.md) design document, to enable collection and aggregation of coverage data from device kernels.
+
+## Design Details
+
+### Profiling Counter Representation
+
+Profiling counters for code coverage are lowered by the compiler as device globals. Specifically, the `InstrProfilingLoweringPass` is modified so that, when targeting SPIR-V, coverage counters are represented as pointers to USM buffers, matching the representation of other SYCL device globals. This indirection allows counters to be relocatable and managed consistently with other device-side global variables.
+
+Each counter is annotated with a unique identifier (`sycl-unique-id`) of the form `__profc_<fn_hash>`, where `<fn_hash>` is a 64-bit unsigned integer uniquely identifying the instrumented function. The counter's size is also recorded via the `sycl-device-global-size` attribute. These attributes ensure that counters are discoverable and manageable by the SYCL runtime and integration headers/footers.
+
+### Integration with Device Global Infrastructure
+
+The device global infrastructure, as described in [DeviceGlobal.md](DeviceGlobal.md), provides mechanisms for mapping host and device instances of global variables, managing their lifetimes, and facilitating data transfer. Device-side coverage counters are treated as a special class of device globals:
+
+- They use the shared allocation type rather than the device allocation type for the underlying USM memory.
+- They do not have corresponding `device_global` declarations in host code.
+- Their lifetime and cleanup are managed via the device global map, with integration footer code ensuring registration and deregistration.
+
+### Runtime Handling and Data Aggregation
+
+When a device global entry corresponding to a coverage counter is released (e.g., when a device image is unloaded), the SYCL runtime aggregates the values from the device-side counter into the equivalent host-side counter. Equivalence is determined by matching both the `<fn_hash>` and the number of counter regions. If no matching host-side counter exists—typically due to differences in code between host and device caused by the `__SYCL_DEVICE_ONLY__` macro—the device-side counter values are discarded.
+
+The aggregation is performed by invoking a new function in the compiler runtime, `__sycl_increment_profile_counters`, which is weakly linked to accommodate optional runtime availability. This function accepts the `<fn_hash>`, the number of regions, and the increment values, and updates the host-side counters accordingly. At program exit, the final profile data reflects the sum of host and device coverage counters.
+
+### Compiler and Runtime Changes
+
+#### Compiler Frontend
+
+- The lowering pass for coverage counters is updated to emit device globals with the appropriate attributes and indirection.
+- Integration headers and footers are updated to register device global counters with the runtime, using the unique identifier and size.
+
+#### SYCL Runtime
+
+- Device globals with IDs matching the `__profc_<fn_hash>` pattern are recognized as coverage counters.
+- USM allocation and management for counters is handled as for other device globals, but without host-side declarations.
+- Upon cleanup, device-side counter values are aggregated into host-side counters via the runtime API.
+
+#### Compiler Runtime
+
+- The new function `__sycl_increment_profile_counters` is introduced to update host-side counters.
+- The function is weakly linked to allow for optional inclusion.
+
+### Limitations and Considerations
+
+- The feature is currently implemented only for SPIR-V targets; CUDA and HIP backends are not supported.
+- Devices lacking support for device globals cannot utilize device-side code coverage.
+- Differences in code between host and device (e.g., due to `__SYCL_DEVICE_ONLY__`) may prevent aggregation of coverage data for some functions.
+- The design relies on the robustness of the device global infrastructure for correct mapping and lifetime management.
+
+## Relationship to Device Global Design
+
+This feature is built upon the mechanisms described in [DeviceGlobal.md](DeviceGlobal.md), including:
+
+- Use of unique string identifiers (`sycl-unique-id`) for mapping and management.
+- USM-based allocation and zero-initialization of device-side storage.
+- Integration header/footer registration for host-device correlation.
+- Runtime database for device global management and lookup.
+
+The code coverage counters are a specialized use case of device globals, with additional logic for aggregation and profile generation.
+
+## References
+
+- [Implementation design for SYCL device globals](DeviceGlobal.md)
+- [Clang Source-based Code Coverage](https://clang.llvm.org/docs/SourceBasedCodeCoverage.html)
+- [SYCL Specification](https://registry.khronos.org/SYCL/specs/sycl-2020/html/sycl-2020.html)

sycl/doc/index.rst

@@ -40,6 +40,7 @@ Design Documents for the oneAPI DPC++ Compiler
    design/ParallelForRangeRounding
    design/SYCLInstrumentationUsingXPTI
    design/ITTAnnotations
+   design/DeviceCodeCoverage
    design/DeviceGlobal
    design/CompileTimeProperties
    design/HostPipes

sycl/source/detail/context_impl.cpp

@@ -128,6 +128,11 @@ context_impl::~context_impl() {
       if (DGEntry != nullptr)
         DGEntry->removeAssociatedResources(this);
     }
+    // Free all profile counter USM allocations associated with this context.
+    for (DeviceGlobalMapEntry *DGEntry :
+         detail::ProgramManager::getInstance()
+             .getProfileCounterDeviceGlobalEntries(this))
+      DGEntry->cleanupProfileCounter(this);
     MCachedLibPrograms.clear();
     // TODO catch an exception and put it to list of asynchronous exceptions
     getAdapter().call_nocheck<UrApiKind::urContextRelease>(MContext);

sycl/source/detail/device_global_map.hpp

@@ -75,7 +75,10 @@ class DeviceGlobalMap {
         //       cannot be set until registration happens.
         auto EntryUPtr = std::make_unique<DeviceGlobalMapEntry>(
             DeviceGlobal->Name, Img, TypeSize, DeviceImageScopeDecorated);
-        MDeviceGlobals.emplace(DeviceGlobal->Name, std::move(EntryUPtr));
+        auto NewEntry =
+            MDeviceGlobals.emplace(DeviceGlobal->Name, std::move(EntryUPtr));
+        if (NewEntry.first->second->isProfileCounter())
+          MProfileCounterDeviceGlobals.push_back(NewEntry.first->second.get());
       }
     }
   }
@@ -114,6 +117,8 @@ class DeviceGlobalMap {
     auto EntryUPtr =
         std::make_unique<DeviceGlobalMapEntry>(UniqueId, DeviceGlobalPtr);
     auto NewEntry = MDeviceGlobals.emplace(UniqueId, std::move(EntryUPtr));
+    if (NewEntry.first->second->isProfileCounter())
+      MProfileCounterDeviceGlobals.push_back(NewEntry.first->second.get());
     MPtr2DeviceGlobal.insert({DeviceGlobalPtr, NewEntry.first->second.get()});
   }
 
@@ -154,6 +159,11 @@ class DeviceGlobalMap {
     }
   }
 
+  std::vector<DeviceGlobalMapEntry *> getProfileCounterEntries() {
+    std::lock_guard<std::mutex> DeviceGlobalsGuard(MDeviceGlobalsMutex);
+    return MProfileCounterDeviceGlobals;
+  }
+
   const std::unordered_map<const void *, DeviceGlobalMapEntry *>
   getPointerMap() const {
     return MPtr2DeviceGlobal;
@@ -177,6 +187,9 @@ class DeviceGlobalMap {
       MDeviceGlobals;
   std::unordered_map<const void *, DeviceGlobalMapEntry *> MPtr2DeviceGlobal;
 
+  // List of profile counter device globals.
+  std::vector<DeviceGlobalMapEntry *> MProfileCounterDeviceGlobals;
+
   /// Protects MDeviceGlobals and MPtr2DeviceGlobal.
   std::mutex MDeviceGlobalsMutex;
 };