Merge pull request #47269 from hqucms/dev/hgc-scaling-test

[HGCAL] Fix the implementation of scaling test in HGCalRecHitProducers

Author: cmsbuild

RecoLocalCalo/HGCalRecAlgos/plugins/alpaka/HGCalRecHitCalibrationAlgorithms.dev.cc

@@ -122,9 +122,6 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
                                                                 HGCalCalibParamDevice const& device_calib,
                                                                 HGCalConfigParamDevice const& device_config) const {
     LogDebug("HGCalRecHitCalibrationAlgorithms") << "\n\nINFO -- Start of calibrate\n\n" << std::endl;
-    LogDebug("HGCalRecHitCalibrationAlgorithms")
-        << "N blocks: " << n_blocks_ << "\tN threads: " << n_threads_ << std::endl;
-    auto grid = make_workdiv<Acc1D>(n_blocks_, n_threads_);
 
     LogDebug("HGCalRecHitCalibrationAlgorithms") << "\n\nINFO -- Copying the digis to the device\n\n" << std::endl;
     HGCalDigiDevice device_digis(host_digis.view().metadata().size(), queue);
@@ -134,6 +131,16 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
         << "\n\nINFO -- Allocating rechits buffer and initiating values" << std::endl;
     HGCalRecHitDevice device_recHits(device_digis.view().metadata().size(), queue);
 
+    // number of items per group
+    uint32_t items = n_threads_;
+    // use as many groups as needed to cover the whole problem
+    uint32_t groups = divide_up_by(device_digis.view().metadata().size(), items);
+    // map items to
+    //   - threads with a single element per thread on a GPU backend
+    //   - elements within a single thread on a CPU backend
+    auto grid = make_workdiv<Acc1D>(groups, items);
+    LogDebug("HGCalRecHitCalibrationAlgorithms") << "N groups: " << groups << "\tN items: " << items << std::endl;
+
     alpaka::exec<Acc1D>(queue,
                         grid,
                         HGCalRecHitCalibrationKernel_flagRecHits{},

RecoLocalCalo/HGCalRecAlgos/plugins/alpaka/HGCalRecHitProducers.cc

@@ -33,6 +33,9 @@
 #include "CondFormats/HGCalObjects/interface/alpaka/HGCalCalibrationParameterDevice.h"
 #include "RecoLocalCalo/HGCalRecAlgos/interface/alpaka/HGCalRecHitCalibrationAlgorithms.h"
 
+// flag to assist the computational performance test
+// #define HGCAL_PERF_TEST
+
 namespace ALPAKA_ACCELERATOR_NAMESPACE {
 
   using namespace cms::alpakatools;
@@ -47,11 +50,11 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
     edm::ESWatcher<HGCalElectronicsMappingRcd> calibWatcher_;
     edm::ESWatcher<HGCalModuleConfigurationRcd> configWatcher_;
     const edm::EDGetTokenT<hgcaldigi::HGCalDigiHost> digisToken_;
-    device::ESGetToken<hgcalrechit::HGCalCalibParamDevice, HGCalModuleConfigurationRcd> calibToken_;
+    edm::ESGetToken<hgcalrechit::HGCalCalibParamHost, HGCalModuleConfigurationRcd> calibToken_;
     device::ESGetToken<hgcalrechit::HGCalConfigParamDevice, HGCalModuleConfigurationRcd> configToken_;
     const device::EDPutToken<hgcalrechit::HGCalRecHitDevice> recHitsToken_;
     const HGCalRecHitCalibrationAlgorithms calibrator_;
-    int n_hits_scale;
+    const int n_hits_scale;
   };
 
   HGCalRecHitsProducer::HGCalRecHitsProducer(const edm::ParameterSet& iConfig)
@@ -61,7 +64,13 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
         configToken_{esConsumes(iConfig.getParameter<edm::ESInputTag>("configSource"))},
         recHitsToken_{produces()},
         calibrator_{iConfig.getParameter<int>("n_blocks"), iConfig.getParameter<int>("n_threads")},
-        n_hits_scale{iConfig.getParameter<int>("n_hits_scale")} {}
+        n_hits_scale{iConfig.getParameter<int>("n_hits_scale")} {
+#ifndef HGCAL_PERF_TEST
+    if (n_hits_scale > 1) {
+      throw cms::Exception("RuntimeError") << "Build with `HGCAL_PERF_TEST` flag to activate `n_hits_scale`.";
+    }
+#endif
+  }
 
   void HGCalRecHitsProducer::fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
     edm::ParameterSetDescription desc;
@@ -78,7 +87,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
     auto& queue = iEvent.queue();
 
     // Read digis
-    auto const& deviceCalibParamProvider = iSetup.getData(calibToken_);
+    auto const& hostCalibParamProvider = iSetup.getData(calibToken_);
     auto const& deviceConfigParamProvider = iSetup.getData(configToken_);
     auto const& hostDigisIn = iEvent.get(digisToken_);
 
@@ -87,28 +96,53 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
       if (calibWatcher_.check(iSetup)) {
         for (int i = 0; i < deviceConfigParamProvider.view().metadata().size(); i++) {
           log << "idx = " << i << ", "
-              << "gain = " << deviceConfigParamProvider.view()[i].gain() << ","
-              << "ADC_ped = " << deviceCalibParamProvider.view()[i].ADC_ped() << ", "
-              << "CM_slope = " << deviceCalibParamProvider.view()[i].CM_slope() << ", "
-              << "CM_ped = " << deviceCalibParamProvider.view()[i].CM_ped() << ", "
-              << "BXm1_slope = " << deviceCalibParamProvider.view()[i].BXm1_slope() << ", ";
+              << "gain = " << deviceConfigParamProvider.view()[i].gain() << ", ";
+        }
+        for (int i = 0; i < hostCalibParamProvider.view().metadata().size(); i++) {
+          log << "idx = " << i << ", "
+              << "ADC_ped = " << hostCalibParamProvider.view()[i].ADC_ped() << ", "
+              << "CM_slope = " << hostCalibParamProvider.view()[i].CM_slope() << ", "
+              << "CM_ped = " << hostCalibParamProvider.view()[i].CM_ped() << ", "
+              << "BXm1_slope = " << hostCalibParamProvider.view()[i].BXm1_slope() << ", ";
         }
       }
     });
 
-    int oldSize = hostDigisIn.view().metadata().size();
-    int newSize = oldSize * n_hits_scale;
+#ifdef HGCAL_PERF_TEST
+    uint32_t oldSize = hostDigisIn.view().metadata().size();
+    uint32_t newSize = oldSize * (n_hits_scale > 0 ? (unsigned)n_hits_scale : 1);
     auto hostDigis = HGCalDigiHost(newSize, queue);
+    auto hostCalibParam = HGCalCalibParamHost(newSize, queue);
     // TODO: replace with memcp ?
-    for (int i = 0; i < newSize; i++) {
+    for (uint32_t i = 0; i < newSize; i++) {
       hostDigis.view()[i].tctp() = hostDigisIn.view()[i % oldSize].tctp();
       hostDigis.view()[i].adcm1() = hostDigisIn.view()[i % oldSize].adcm1();
       hostDigis.view()[i].adc() = hostDigisIn.view()[i % oldSize].adc();
       hostDigis.view()[i].tot() = hostDigisIn.view()[i % oldSize].tot();
       hostDigis.view()[i].toa() = hostDigisIn.view()[i % oldSize].toa();
       hostDigis.view()[i].cm() = hostDigisIn.view()[i % oldSize].cm();
       hostDigis.view()[i].flags() = hostDigisIn.view()[i % oldSize].flags();
+
+      hostCalibParam.view()[i].ADC_ped() = hostCalibParamProvider.view()[i % oldSize].ADC_ped();
+      hostCalibParam.view()[i].Noise() = hostCalibParamProvider.view()[i % oldSize].Noise();
+      hostCalibParam.view()[i].CM_slope() = hostCalibParamProvider.view()[i % oldSize].CM_slope();
+      hostCalibParam.view()[i].CM_ped() = hostCalibParamProvider.view()[i % oldSize].CM_ped();
+      hostCalibParam.view()[i].BXm1_slope() = hostCalibParamProvider.view()[i % oldSize].BXm1_slope();
+      hostCalibParam.view()[i].TOTtoADC() = hostCalibParamProvider.view()[i % oldSize].TOTtoADC();
+      hostCalibParam.view()[i].TOT_ped() = hostCalibParamProvider.view()[i % oldSize].TOT_ped();
+      hostCalibParam.view()[i].TOT_lin() = hostCalibParamProvider.view()[i % oldSize].TOT_lin();
+      hostCalibParam.view()[i].TOT_P0() = hostCalibParamProvider.view()[i % oldSize].TOT_P0();
+      hostCalibParam.view()[i].TOT_P1() = hostCalibParamProvider.view()[i % oldSize].TOT_P1();
+      hostCalibParam.view()[i].TOT_P2() = hostCalibParamProvider.view()[i % oldSize].TOT_P2();
+      hostCalibParam.view()[i].TOAtops() = hostCalibParamProvider.view()[i % oldSize].TOAtops();
+      hostCalibParam.view()[i].MIPS_scale() = hostCalibParamProvider.view()[i % oldSize].MIPS_scale();
+      hostCalibParam.view()[i].valid() = hostCalibParamProvider.view()[i % oldSize].valid();
     }
+#else
+    const auto& hostDigis = hostDigisIn;
+    const auto& hostCalibParam = hostCalibParamProvider;
+#endif
+
     LogDebug("HGCalRecHitsProducer") << "Loaded host digis: " << hostDigis.view().metadata().size();  //<< std::endl;
 
     LogDebug("HGCalRecHitsProducer") << "\n\nINFO -- calling calibrate method";  //<< std::endl;
@@ -118,13 +152,24 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
     auto start = std::chrono::steady_clock::now();
 #endif
 
-    auto recHits = calibrator_.calibrate(queue, hostDigis, deviceCalibParamProvider, deviceConfigParamProvider);
+    LogDebug("HGCalRecHitsProducer") << "\n\nINFO -- Copying the calib to the device\n\n" << std::endl;
+    HGCalCalibParamDevice deviceCalibParam(hostCalibParam.view().metadata().size(), queue);
+    alpaka::memcpy(queue, deviceCalibParam.buffer(), hostCalibParam.const_buffer());
+
+#ifdef HGCAL_PERF_TEST
+    auto tmpRecHits = calibrator_.calibrate(queue, hostDigis, deviceCalibParam, deviceConfigParamProvider);
+    HGCalRecHitDevice recHits(oldSize, queue);
+    alpaka::memcpy(queue, recHits.buffer(), tmpRecHits.const_buffer(), oldSize);
+#else
+    auto recHits = calibrator_.calibrate(queue, hostDigis, deviceCalibParam, deviceConfigParamProvider);
+#endif
 
 #ifdef EDM_ML_DEBUG
     alpaka::wait(queue);
     auto stop = std::chrono::steady_clock::now();
     std::chrono::duration<float> elapsed = stop - start;
-    LogDebug("HGCalRecHitsProducer") << "Time spent calibrating: " << elapsed.count();  //<< std::endl;
+    LogDebug("HGCalRecHitsProducer") << "Time spent calibrating " << hostDigis.view().metadata().size()
+                                     << " digis: " << elapsed.count();  //<< std::endl;
 #endif
 
     LogDebug("HGCalRecHitsProducer") << "\n\nINFO -- storing rec hits in the event";  //<< std::endl;