Merge pull request #45368 from fwyzard/Alpaka_CachingAllocator_debug_141x

Fix CachingAllocator debug for non-async operations

Author: cmsbuild

Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py

@@ -1102,10 +1102,12 @@ def setup_(self, step, stepName, stepDict, k, properties):
     digi = {
         # customize the ECAL Local Reco part of the HLT menu for Alpaka
         '--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka
+        '--customise' : 'HeterogeneousCore/AlpakaServices/customiseAlpakaServiceMemoryFilling.customiseAlpakaServiceMemoryFilling',
     },
     reco = {
         '-s': 'RAW2DIGI:RawToDigi_ecalOnly,RECO:reconstruction_ecalOnly,VALIDATION:@ecalOnlyValidation,DQM:@ecalOnly',
-        '--procModifiers': 'alpaka'
+        '--procModifiers': 'alpaka',
+        '--customise' : 'HeterogeneousCore/AlpakaServices/customiseAlpakaServiceMemoryFilling.customiseAlpakaServiceMemoryFilling',
     },
     harvest = {
         '-s': 'HARVESTING:@ecalOnlyValidation+@ecal'
@@ -1276,10 +1278,12 @@ def setup_(self, step, stepName, stepDict, k, properties):
 upgradeWFs['PatatrackHCALOnlyAlpakaValidation'] = PatatrackWorkflow(
     digi = {
         '--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka
+        '--customise' : 'HeterogeneousCore/AlpakaServices/customiseAlpakaServiceMemoryFilling.customiseAlpakaServiceMemoryFilling',
     },
     reco = {
         '-s': 'RAW2DIGI:RawToDigi_hcalOnly,RECO:reconstruction_hcalOnly,VALIDATION:@hcalOnlyValidation,DQM:@hcalOnly+@hcal2Only',
-        '--procModifiers': 'alpaka'
+        '--procModifiers': 'alpaka',
+        '--customise' : 'HeterogeneousCore/AlpakaServices/customiseAlpakaServiceMemoryFilling.customiseAlpakaServiceMemoryFilling',
     },
     harvest = {
         '-s': 'HARVESTING:@hcalOnlyValidation'
@@ -1294,10 +1298,12 @@ def setup_(self, step, stepName, stepDict, k, properties):
 upgradeWFs['PatatrackHCALOnlyGPUandAlpakaValidation'] = PatatrackWorkflow(
     digi = {
         '--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka
+        '--customise' : 'HeterogeneousCore/AlpakaServices/customiseAlpakaServiceMemoryFilling.customiseAlpakaServiceMemoryFilling',
     },
     reco = {
-        '-s' : 'RAW2DIGI:RawToDigi_hcalOnly,RECO:reconstruction_hcalOnlyLegacy+reconstruction_hcalOnly,VALIDATION:@hcalOnlyValidation+pfClusterHBHEOnlyAlpakaComparisonSequence,DQM:@hcalOnly+@hcal2Only+hcalOnlyOfflineSourceSequenceAlpaka',
-        '--procModifiers': 'alpaka'
+        '-s': 'RAW2DIGI:RawToDigi_hcalOnly,RECO:reconstruction_hcalOnlyLegacy+reconstruction_hcalOnly,VALIDATION:@hcalOnlyValidation+pfClusterHBHEOnlyAlpakaComparisonSequence,DQM:@hcalOnly+@hcal2Only+hcalOnlyOfflineSourceSequenceAlpaka',
+        '--procModifiers': 'alpaka',
+        '--customise' : 'HeterogeneousCore/AlpakaServices/customiseAlpakaServiceMemoryFilling.customiseAlpakaServiceMemoryFilling',
     },
     harvest = {
         '-s': 'HARVESTING:@hcalOnlyValidation'
@@ -1329,11 +1335,13 @@ def setup_(self, step, stepName, stepDict, k, properties):
 upgradeWFs['PatatrackFullRecoAlpaka'] = PatatrackWorkflow(
     digi = {
         '--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka
+        '--customise' : 'HeterogeneousCore/AlpakaServices/customiseAlpakaServiceMemoryFilling.customiseAlpakaServiceMemoryFilling',
     },
     reco = {
         # skip the @pixelTrackingOnlyValidation which cannot run together with the full reconstruction
         '-s': 'RAW2DIGI:RawToDigi+RawToDigi_pixelOnly,L1Reco,RECO:reconstruction+reconstruction_pixelTrackingOnly,RECOSIM,PAT,VALIDATION:@standardValidation+@miniAODValidation,DQM:@standardDQM+@ExtraHLT+@miniAODDQM+@pixelTrackingOnlyDQM',
-        '--procModifiers': 'alpaka,pixelNtupletFit'
+        '--procModifiers': 'alpaka,pixelNtupletFit',
+        '--customise' : 'HeterogeneousCore/AlpakaServices/customiseAlpakaServiceMemoryFilling.customiseAlpakaServiceMemoryFilling',
     },
     harvest = {
         # skip the @pixelTrackingOnlyDQM harvesting
@@ -1646,16 +1654,15 @@ def setup_(self, step, stepName, stepDict, k, properties):
     offset = 0.597,
 )
 
-
-# Alpaka workflows
-
 upgradeWFs['PatatrackPixelOnlyAlpaka'] = PatatrackWorkflow(
     digi = {
         '--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka
+        '--customise' : 'HeterogeneousCore/AlpakaServices/customiseAlpakaServiceMemoryFilling.customiseAlpakaServiceMemoryFilling',
     },
     reco = {
         '-s': 'RAW2DIGI:RawToDigi_pixelOnly,RECO:reconstruction_pixelTrackingOnly,VALIDATION:@pixelTrackingOnlyValidation,DQM:@pixelTrackingOnlyDQM',
-        '--procModifiers': 'alpaka'
+        '--procModifiers': 'alpaka',
+        '--customise' : 'HeterogeneousCore/AlpakaServices/customiseAlpakaServiceMemoryFilling.customiseAlpakaServiceMemoryFilling',
     },
     harvest = {
         '-s': 'HARVESTING:@trackingOnlyValidation+@pixelTrackingOnlyDQM'
@@ -1667,10 +1674,12 @@ def setup_(self, step, stepName, stepDict, k, properties):
 upgradeWFs['PatatrackPixelOnlyAlpakaValidation'] = PatatrackWorkflow(
     digi = {
         '--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka
+        '--customise' : 'HeterogeneousCore/AlpakaServices/customiseAlpakaServiceMemoryFilling.customiseAlpakaServiceMemoryFilling',
     },
     reco = {
         '-s': 'RAW2DIGI:RawToDigi_pixelOnly,RECO:reconstruction_pixelTrackingOnly,VALIDATION:@pixelTrackingOnlyValidation,DQM:@pixelTrackingOnlyDQM',
-        '--procModifiers': 'alpakaValidation'
+        '--procModifiers': 'alpakaValidation',
+        '--customise' : 'HeterogeneousCore/AlpakaServices/customiseAlpakaServiceMemoryFilling.customiseAlpakaServiceMemoryFilling',
     },
     harvest = {
         '-s': 'HARVESTING:@trackingOnlyValidation+@pixelTrackingOnlyDQM'

HeterogeneousCore/AlpakaCore/README.md

@@ -358,13 +358,8 @@ There are a few different options for using Alpaka-based modules in the CMSSW co
 In all cases the configuration must load the necessary `ProcessAccelerator` objects (see below) For accelerators used in production, these are aggregated in `Configuration.StandardSequences.Accelerators_cff`. The `runTheMatrix.py` handles the loading of this `Accelerators_cff` automatically. The HLT menus also load the necessary `ProcessAccelerator`s.
 ```python
 ## Load explicitly
-# One ProcessAccelerator for each accelerator technology
+# One ProcessAccelerator for each accelerator technology, plus a generic one for Alpaka
 process.load("Configuration.StandardSequences.Accelerators_cff")
-
-# And one ProcessAccelerator for Alpaka
-# (eventually to be absorbed to Accelerators_cff)
-process.load("HeterogeneousCore.AlpakaCore.ProcessAcceleratorAlpaka_cfi")
-
 ```
 
 ### Explicit module type (non-portable)

HeterogeneousCore/AlpakaInterface/interface/CachingAllocator.h

@@ -122,8 +122,10 @@ namespace cms::alpakatools {
     explicit CachingAllocator(
         Device const& device,
         AllocatorConfig const& config,
-        bool reuseSameQueueAllocations,  // reuse non-ready allocations if they are in the same queue as the new one;
-                                         // this is safe only if all memory operations are scheduled in the same queue
+        bool reuseSameQueueAllocations,  // Reuse non-ready allocations if they are in the same queue as the new one;
+                                         // this is safe only if all memory operations are scheduled in the same queue.
+                                         // In particular, this is not safe if the memory will be accessed without using
+                                         // any queue, like host memory accessed directly or with immediate operations.
         bool debug = false)
         : device_(device),
           binGrowth_(config.binGrowth),
@@ -175,6 +177,22 @@ namespace cms::alpakatools {
       return cachedBytes_;
     }
 
+    // Fill a memory buffer with the specified bye value.
+    // If the underlying device is the host and the allocator is configured to support immediate
+    // (non queue-ordered) operations, fill the memory synchronously using std::memset.
+    // Otherwise, let the alpaka queue schedule the operation.
+    //
+    // This is not used for deallocation/caching, because the memory may still be in use until the
+    // corresponding event is reached.
+    void immediateOrAsyncMemset(Queue queue, Buffer buffer, uint8_t value) {
+      // host-only
+      if (std::is_same_v<Device, alpaka::DevCpu> and not reuseSameQueueAllocations_) {
+        std::memset(buffer.data(), value, alpaka::getExtentProduct(buffer) * sizeof(alpaka::Elem<Buffer>));
+      } else {
+        alpaka::memset(queue, buffer, value);
+      }
+    }
+
     // Allocate given number of bytes on the current device associated to given queue
     void* allocate(size_t bytes, Queue queue) {
       // create a block descriptor for the requested allocation
@@ -187,15 +205,15 @@ namespace cms::alpakatools {
       if (tryReuseCachedBlock(block)) {
         // fill the re-used memory block with a pattern
         if (fillReallocations_) {
-          alpaka::memset(*block.queue, *block.buffer, fillReallocationValue_);
+          immediateOrAsyncMemset(*block.queue, *block.buffer, fillReallocationValue_);
         } else if (fillAllocations_) {
-          alpaka::memset(*block.queue, *block.buffer, fillAllocationValue_);
+          immediateOrAsyncMemset(*block.queue, *block.buffer, fillAllocationValue_);
         }
       } else {
         allocateNewBlock(block);
         // fill the newly allocated memory block with a pattern
         if (fillAllocations_) {
-          alpaka::memset(*block.queue, *block.buffer, fillAllocationValue_);
+          immediateOrAsyncMemset(*block.queue, *block.buffer, fillAllocationValue_);
         }
       }
 

HeterogeneousCore/AlpakaServices/python/customiseAlpakaServiceMemoryFilling.py

@@ -0,0 +1,46 @@
+import FWCore.ParameterSet.Config as cms
+
+def customiseAlpakaServiceMemoryFilling(process):
+  # load all variants of the AlpakaService
+  # ProcessAcceleratorAlpaka will take care of removing the unused ones
+
+  process.load('HeterogeneousCore.AlpakaServices.AlpakaServiceSerialSync_cfi')
+
+  # load the CUDAService and the AlpakaService for the CUDA backend, if available
+  try:
+      process.load('HeterogeneousCore.CUDAServices.CUDAService_cfi')
+      process.load('HeterogeneousCore.AlpakaServices.AlpakaServiceCudaAsync_cfi')
+  except:
+      pass
+
+  # load the ROCmService and the AlpakaService for the ROCm backend, if available
+  try:
+      process.load('HeterogeneousCore.ROCmServices.ROCmService_cfi')
+      process.load('HeterogeneousCore.AlpakaServices.AlpakaServiceROCmAsync_cfi')
+  except:
+      pass
+
+  # enable junk memory filling for all AlpakaServices
+  for name in process.services_():
+    if name.startswith('AlpakaService'):
+      service = getattr(process, name)
+      # host allocator
+      service.hostAllocator.fillAllocations = True
+      service.hostAllocator.fillAllocationValue = 0xB4
+      service.hostAllocator.fillReallocations = True
+      service.hostAllocator.fillReallocationValue = 0x78
+      service.hostAllocator.fillDeallocations = True
+      service.hostAllocator.fillDeallocationValue = 0x4B
+      service.hostAllocator.fillCaches = True
+      service.hostAllocator.fillCacheValue = 0x87
+      # device allocator
+      service.deviceAllocator.fillAllocations = True
+      service.deviceAllocator.fillAllocationValue = 0xA5
+      service.deviceAllocator.fillReallocations = True
+      service.deviceAllocator.fillReallocationValue = 0x69
+      service.deviceAllocator.fillDeallocations = True
+      service.deviceAllocator.fillDeallocationValue = 0x5A
+      service.deviceAllocator.fillCaches = True
+      service.deviceAllocator.fillCacheValue = 0x96
+
+  return process