Merge pull request #47124 from Jakub-Gajownik/ecal-phase2-reco-gpu-alpaka-integration-150x

Ecal Phase2 Weights-based Reconstruction, Alpaka integration

Author: cmsbuild

Configuration/PyReleaseValidation/python/relval_gpu.py

@@ -36,6 +36,7 @@
 # mc Run4   
 # no PU
 #           Alpaka pixel-only:                                  TTbar: quadruplets any backend, any backend vs cpu validation, profiling, triplets      
+#           Alpaka ECAL-only development:                       TTbar: any backend
 #           Alpaka pixel-only:                                  Single Nu E10: any backend
 #           Alpaka LST-only:                                    TTbar: TRK-only w/ 2 iterations and LST building on any backend
 # with PU
@@ -70,6 +71,7 @@
 
            # Run4, Alpaka-based noPU
            29634.402, 29634.403, 29634.404, 29634.406, 29634.704,
+           29634.612,
            29661.402,
 
            # Run4, Alpaka-based PU

Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py

@@ -2118,10 +2118,13 @@ def condition(self, fragment, stepList, key, hasHarvest):
     offset = 0.61,
 )
 
-# ECAL Phase 2 workflow running on CPU or GPU (if available)
-upgradeWFs['ecalDevelGPU'] = UpgradeWorkflow_ecalDevel(
-    reco = {'--procModifiers': 'gpu'},
-    suffix = '_ecalDevelGPU',
+# ECAL Phase 2 workflow running on CPU or GPU with Alpaka code
+upgradeWFs['ecalDevelAlpaka'] = UpgradeWorkflow_ecalDevel(
+    reco = {
+        '--procModifiers': 'alpaka',
+        '--customise' : 'HeterogeneousCore/AlpakaServices/customiseAlpakaServiceMemoryFilling.customiseAlpakaServiceMemoryFilling'
+    },
+    suffix = '_ecalDevelAlpaka',
     offset = 0.612,
 )
 

RecoLocalCalo/EcalRecProducers/plugins/EcalUncalibRecHitSoAToLegacy.cc

@@ -32,16 +32,21 @@ class EcalUncalibRecHitSoAToLegacy : public edm::global::EDProducer<> {
 void EcalUncalibRecHitSoAToLegacy::fillDescriptions(edm::ConfigurationDescriptions &confDesc) {
   edm::ParameterSetDescription desc;
 
-  desc.add<edm::InputTag>("inputCollectionEB",
-                          edm::InputTag("ecalMultiFitUncalibRecHitPortable", "EcalUncalibRecHitsEB"));
   desc.add<std::string>("outputLabelEB", "EcalUncalibRecHitsEB");
   desc.ifValue(edm::ParameterDescription<bool>("isPhase2", false, true),
                false >> (edm::ParameterDescription<edm::InputTag>(
+                             "inputCollectionEB",
+                             edm::InputTag("ecalMultiFitUncalibRecHitPortable", "EcalUncalibRecHitsEB"),
+                             true) and
+                         edm::ParameterDescription<edm::InputTag>(
                              "inputCollectionEE",
                              edm::InputTag("ecalMultiFitUncalibRecHitPortable", "EcalUncalibRecHitsEE"),
                              true) and
                          edm::ParameterDescription<std::string>("outputLabelEE", "EcalUncalibRecHitsEE", true)) or
-                   true >> edm::EmptyGroupDescription());
+                   true >> (edm::ParameterDescription<edm::InputTag>(
+                               "inputCollectionEB",
+                               edm::InputTag("ecalUncalibRecHitPhase2Portable", "EcalUncalibRecHitsEB"),
+                               true)));
   confDesc.add("ecalUncalibRecHitSoAToLegacy", desc);
 }
 

RecoLocalCalo/EcalRecProducers/plugins/alpaka/EcalPhase2DigiToPortableProducer.cc

@@ -0,0 +1,85 @@
+#include "DataFormats/EcalDigi/interface/EcalDigiCollections.h"
+#include "DataFormats/EcalDigi/interface/EcalDigiPhase2HostCollection.h"
+#include "DataFormats/EcalDigi/interface/alpaka/EcalDigiPhase2DeviceCollection.h"
+#include "FWCore/MessageLogger/interface/MessageLogger.h"
+#include "FWCore/Utilities/interface/EDGetToken.h"
+#include "FWCore/Utilities/interface/EDPutToken.h"
+#include "HeterogeneousCore/AlpakaCore/interface/alpaka/global/EDProducer.h"
+#include "HeterogeneousCore/AlpakaCore/interface/alpaka/Event.h"
+#include "HeterogeneousCore/AlpakaCore/interface/alpaka/EventSetup.h"
+#include "HeterogeneousCore/AlpakaCore/interface/alpaka/MakerMacros.h"
+#include "HeterogeneousCore/AlpakaInterface/interface/config.h"
+
+namespace ALPAKA_ACCELERATOR_NAMESPACE {
+
+  class EcalPhase2DigiToPortableProducer : public global::EDProducer<> {
+  public:
+    explicit EcalPhase2DigiToPortableProducer(edm::ParameterSet const &ps);
+    ~EcalPhase2DigiToPortableProducer() override = default;
+    static void fillDescriptions(edm::ConfigurationDescriptions &descriptions);
+
+    void produce(edm::StreamID sid, device::Event &event, device::EventSetup const &setup) const override;
+
+  private:
+    const edm::EDGetTokenT<EBDigiCollectionPh2> inputDigiToken_;
+    const edm::EDPutTokenT<EcalDigiPhase2HostCollection> outputDigiHostToken_;
+  };
+
+  void EcalPhase2DigiToPortableProducer::fillDescriptions(edm::ConfigurationDescriptions &descriptions) {
+    edm::ParameterSetDescription desc;
+
+    desc.add<edm::InputTag>("BarrelDigis", edm::InputTag("simEcalUnsuppressedDigis", ""));
+    desc.add<std::string>("digisLabelEB", "ebDigis");
+
+    descriptions.addWithDefaultLabel(desc);
+  }
+
+  EcalPhase2DigiToPortableProducer::EcalPhase2DigiToPortableProducer(edm::ParameterSet const &ps)
+      : EDProducer(ps),
+        inputDigiToken_{consumes(ps.getParameter<edm::InputTag>("BarrelDigis"))},
+        outputDigiHostToken_{produces(ps.getParameter<std::string>("digisLabelEB"))} {}
+
+  void EcalPhase2DigiToPortableProducer::produce(edm::StreamID sid,
+                                                 device::Event &event,
+                                                 device::EventSetup const &setup) const {
+    //input data from event
+    const auto &inputDigis = event.get(inputDigiToken_);
+
+    const uint32_t size = inputDigis.size();
+
+    //create host and device Digi collections of required size
+    EcalDigiPhase2HostCollection digisHostColl{static_cast<int32_t>(size), event.queue()};
+    auto digisHostCollView = digisHostColl.view();
+
+    //iterate over digis
+    uint32_t i = 0;
+    for (const auto &inputDigi : inputDigis) {
+      const unsigned int nSamples = inputDigi.size();
+      //assign id to host collection
+      digisHostCollView.id()[i] = inputDigi.id();
+      if (nSamples > ecalPh2::sampleSize) {
+        edm::LogError("size_mismatch") << "Number of input samples (" << nSamples
+                                       << ") larger than the maximum sample size (" << ecalPh2::sampleSize
+                                       << "). Ignoring the excess samples.";
+      }
+      //iterate over sample in digi, make sure the size of the input is not larger than the max sample size in Phase 2, if smaller set to 0
+      for (unsigned int sample = 0; sample < ecalPh2::sampleSize; ++sample) {
+        if (sample < nSamples) {
+          //get samples from input digi
+          EcalLiteDTUSample thisSample = inputDigi[sample];
+          //assign adc data to host collection
+          digisHostCollView.data()[i][sample] = thisSample.raw();
+        } else {
+          digisHostCollView.data()[i][sample] = 0;
+        }
+      }
+      ++i;
+    }
+    digisHostCollView.size() = i;
+
+    //emplace device collection in the event
+    event.emplace(outputDigiHostToken_, std::move(digisHostColl));
+  }
+}  // namespace ALPAKA_ACCELERATOR_NAMESPACE
+
+DEFINE_FWK_ALPAKA_MODULE(EcalPhase2DigiToPortableProducer);

RecoLocalCalo/EcalRecProducers/plugins/alpaka/EcalUncalibRecHitPhase2WeightsAlgoPortable.dev.cc

@@ -0,0 +1,80 @@
+#include <alpaka/alpaka.hpp>
+
+#include "DataFormats/EcalDigi/interface/alpaka/EcalDigiPhase2DeviceCollection.h"
+#include "DataFormats/EcalDigi/interface/EcalDataFrame_Ph2.h"
+#include "DataFormats/EcalDigi/interface/EcalDigiCollections.h"
+#include "DataFormats/EcalRecHit/interface/alpaka/EcalUncalibratedRecHitDeviceCollection.h"
+#include "DataFormats/EcalRecHit/interface/EcalUncalibratedRecHit.h"
+
+#include "HeterogeneousCore/AlpakaInterface/interface/config.h"
+#include "HeterogeneousCore/AlpakaInterface/interface/traits.h"
+#include "HeterogeneousCore/AlpakaInterface/interface/workdivision.h"
+
+#include "EcalUncalibRecHitPhase2WeightsAlgoPortable.h"
+#include "EcalUncalibRecHitPhase2WeightsStruct.h"
+
+namespace ALPAKA_ACCELERATOR_NAMESPACE::ecal::weights {
+  using namespace cms::alpakatools;
+
+  class Phase2WeightsKernel {
+  public:
+    ALPAKA_FN_ACC void operator()(Acc1D const& acc,
+                                  EcalUncalibRecHitPhase2Weights const* weightsObj,
+                                  EcalDigiPhase2DeviceCollection::ConstView digisDev,
+                                  EcalUncalibratedRecHitDeviceCollection::View uncalibratedRecHitsDev) const {
+      constexpr int nsamples = ecalPh2::sampleSize;
+      auto const nchannels = digisDev.size();
+      // one thread sets the output collection size scalar
+      if (once_per_grid(acc)) {
+        uncalibratedRecHitsDev.size() = digisDev.size();
+      }
+
+      auto const* weightsdata = weightsObj->weights.data();
+      auto const* timeWeightsdata = weightsObj->timeWeights.data();
+      //divide the grid into uniform elements
+      for (auto tx : uniform_elements(acc, nchannels)) {
+        bool g1 = false;
+        const auto& digi = digisDev[tx].data();
+        auto recHit = uncalibratedRecHitsDev[tx];
+        recHit.amplitude() = 0;
+        recHit.jitter() = 0;
+        for (int s = 0; s < nsamples; ++s) {
+          const auto sample = digi[s];
+          const auto trace =
+              (static_cast<float>(ecalLiteDTU::adc(sample))) * ecalPh2::gains[ecalLiteDTU::gainId(sample)];
+          recHit.amplitude() += (trace * weightsdata[s]);
+          recHit.jitter() += (trace * timeWeightsdata[s]);
+          if (ecalLiteDTU::gainId(sample) == 1)
+            g1 = true;
+          recHit.outOfTimeAmplitudes()[s] = 0.;
+        }
+        recHit.amplitudeError() = 1.0f;
+        recHit.id() = digisDev.id()[tx];
+        recHit.flags() = 0;
+        recHit.pedestal() = 0.;
+        recHit.jitterError() = 0.;
+        recHit.chi2() = 0.;
+        recHit.aux() = 0;
+        if (g1) {
+          recHit.flags() = 0x1 << EcalUncalibratedRecHit::kHasSwitchToGain1;
+        }
+      }  //if within nchannels
+    }  //kernel}
+  };
+
+  void phase2Weights(EcalDigiPhase2DeviceCollection const& digis,
+                     EcalUncalibratedRecHitDeviceCollection& uncalibratedRecHits,
+                     EcalUncalibRecHitPhase2Weights const* weightsObj,
+                     Queue& queue) {
+    // use 64 items per group (arbitrary value, a reasonable starting point)
+    uint32_t items = 64;
+    // use as many groups as needed to cover the whole problem
+    uint32_t groups = divide_up_by(digis->metadata().size(), items);
+    //create the work division
+    auto workDiv = make_workdiv<Acc1D>(groups, items);
+    //launch the kernel
+    alpaka::exec<Acc1D>(
+        queue, workDiv, Phase2WeightsKernel{}, weightsObj, digis.const_view(), uncalibratedRecHits.view());
+  }
+
+}  // namespace ALPAKA_ACCELERATOR_NAMESPACE::ecal::weights

RecoLocalCalo/EcalRecProducers/plugins/alpaka/EcalUncalibRecHitPhase2WeightsAlgoPortable.h

@@ -0,0 +1,19 @@
+#ifndef RecoLocalCalo_EcalRecProducers_plugins_alpaka_EcalUncalibRecHitPhase2WeightsAlgoPortable_h
+#define RecoLocalCalo_EcalRecProducers_plugins_alpaka_EcalUncalibRecHitPhase2WeightsAlgoPortable_h
+
+#include "DataFormats/EcalDigi/interface/alpaka/EcalDigiPhase2DeviceCollection.h"
+#include "DataFormats/EcalRecHit/interface/alpaka/EcalUncalibratedRecHitDeviceCollection.h"
+
+#include "DataFormats/EcalDigi/interface/EcalDataFrame_Ph2.h"
+#include "EcalUncalibRecHitPhase2WeightsStruct.h"
+
+namespace ALPAKA_ACCELERATOR_NAMESPACE::ecal::weights {
+
+  void phase2Weights(EcalDigiPhase2DeviceCollection const &digis,
+                     EcalUncalibratedRecHitDeviceCollection &uncalibratedRecHits,
+                     EcalUncalibRecHitPhase2Weights const *weightsObj,
+                     Queue &queue);
+
+}  //namespace ALPAKA_ACCELERATOR_NAMESPACE::ecal::weights
+
+#endif  // RecoLocalCalo_EcalRecProducers_plugins_EcalUncalibRecHitPhase2WeightsAlgoPortable_h

RecoLocalCalo/EcalRecProducers/plugins/alpaka/EcalUncalibRecHitPhase2WeightsProducerPortable.cc

@@ -0,0 +1,142 @@
+#include <array>
+#include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
+#include "FWCore/ParameterSet/interface/ParameterSet.h"
+#include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
+#include "FWCore/Framework/interface/Frameworkfwd.h"
+#include "FWCore/Utilities/interface/StreamID.h"
+#include "FWCore/Utilities/interface/InputTag.h"
+
+#include "HeterogeneousCore/AlpakaInterface/interface/config.h"
+#include "HeterogeneousCore/AlpakaCore/interface/alpaka/Event.h"
+#include "HeterogeneousCore/AlpakaCore/interface/alpaka/EventSetup.h"
+#include "HeterogeneousCore/AlpakaCore/interface/alpaka/EDPutToken.h"
+#include "HeterogeneousCore/AlpakaCore/interface/alpaka/global/EDProducer.h"
+#include "HeterogeneousCore/AlpakaCore/interface/alpaka/MakerMacros.h"
+#include "HeterogeneousCore/AlpakaCore/interface/MoveToDeviceCache.h"
+
+#include "DataFormats/EcalDigi/interface/EcalDataFrame_Ph2.h"
+#include "DataFormats/EcalDigi/interface/EcalConstants.h"
+#include "DataFormats/EcalDigi/interface/alpaka/EcalDigiPhase2DeviceCollection.h"
+#include "DataFormats/EcalDigi/interface/EcalDigiPhase2HostCollection.h"
+#include "DataFormats/EcalRecHit/interface/EcalUncalibratedRecHitHostCollection.h"
+#include "DataFormats/EcalRecHit/interface/alpaka/EcalUncalibratedRecHitDeviceCollection.h"
+#include "DataFormats/Portable/interface/PortableObject.h"
+
+#include "EcalUncalibRecHitPhase2WeightsAlgoPortable.h"
+#include "EcalUncalibRecHitPhase2WeightsStruct.h"
+
+namespace ALPAKA_ACCELERATOR_NAMESPACE {
+  class EcalUncalibRecHitPhase2WeightsProducerPortable : public global::EDProducer<> {
+  public:
+    explicit EcalUncalibRecHitPhase2WeightsProducerPortable(edm::ParameterSet const &ps);
+    ~EcalUncalibRecHitPhase2WeightsProducerPortable() override = default;
+    static void fillDescriptions(edm::ConfigurationDescriptions &);
+
+    void produce(edm::StreamID sid, device::Event &, device::EventSetup const &) const override;
+
+  private:
+    using InputProduct = EcalDigiPhase2DeviceCollection;
+    const device::EDGetToken<InputProduct> digisToken_;  //both tokens stored on the device
+    using OutputProduct = EcalUncalibratedRecHitDeviceCollection;
+    const device::EDPutToken<OutputProduct> uncalibratedRecHitsToken_;
+
+    // class data member
+    cms::alpakatools::MoveToDeviceCache<Device, PortableHostObject<EcalUncalibRecHitPhase2Weights>> weightsCache_;
+  };
+
+  // constructor with initialisation of elements
+  EcalUncalibRecHitPhase2WeightsProducerPortable::EcalUncalibRecHitPhase2WeightsProducerPortable(
+      const edm::ParameterSet &ps)
+      : EDProducer(ps),
+        digisToken_{consumes(ps.getParameter<edm::InputTag>("digisLabelEB"))},
+        uncalibratedRecHitsToken_{produces(ps.getParameter<std::string>("uncalibratedRecHitsLabelEB"))},
+        weightsCache_(PortableHostObject<EcalUncalibRecHitPhase2Weights>(
+            cms::alpakatools::host(), [](const edm::ParameterSet &ps) {
+              EcalUncalibRecHitPhase2Weights weights;
+              const auto amp_weights = ps.getParameter<std::vector<double>>("weights");
+              const auto timeWeights = ps.getParameter<std::vector<double>>("timeWeights");
+              for (unsigned int i = 0; i < ecalPh2::sampleSize; ++i) {
+                if (i < amp_weights.size()) {
+                  weights.weights[i] = static_cast<float>(amp_weights[i]);
+                } else {
+                  weights.weights[i] = 0;
+                }
+                if (i < timeWeights.size()) {
+                  weights.timeWeights[i] = static_cast<float>(timeWeights[i]);
+                } else {
+                  weights.timeWeights[i] = 0;
+                }
+              }
+              return weights;
+            }(ps))) {}
+
+  void EcalUncalibRecHitPhase2WeightsProducerPortable::fillDescriptions(edm::ConfigurationDescriptions &descriptions) {
+    edm::ParameterSetDescription desc;
+
+    desc.add<std::string>("uncalibratedRecHitsLabelEB", "EcalUncalibRecHitsEB");
+    //The weights values below should be kept up to date with those on the CPU version of this module
+    //stored in RecoLocalCalo/EcalRecProducers/plugins/EcalUncalibRecHitPhase2WeightsProducer.cc
+    desc.add<std::vector<double>>("weights",
+                                  {-0.121016,
+                                   -0.119899,
+                                   -0.120923,
+                                   -0.0848959,
+                                   0.261041,
+                                   0.509881,
+                                   0.373591,
+                                   0.134899,
+                                   -0.0233605,
+                                   -0.0913195,
+                                   -0.112452,
+                                   -0.118596,
+                                   -0.121737,
+                                   -0.121737,
+                                   -0.121737,
+                                   -0.121737});
+    desc.add<std::vector<double>>("timeWeights",
+                                  {0.429452,
+                                   0.442762,
+                                   0.413327,
+                                   0.858327,
+                                   4.42324,
+                                   2.04369,
+                                   -3.42426,
+                                   -4.16258,
+                                   -2.36061,
+                                   -0.725371,
+                                   0.0727267,
+                                   0.326005,
+                                   0.402035,
+                                   0.404287,
+                                   0.434207,
+                                   0.422775});
+
+    desc.add<edm::InputTag>("digisLabelEB", edm::InputTag("ecalPhase2DigiToPortableProducer", "ebDigis"));
+
+    descriptions.addWithDefaultLabel(desc);
+  }
+
+  void EcalUncalibRecHitPhase2WeightsProducerPortable::produce(edm::StreamID sid,
+                                                               device::Event &event,
+                                                               const device::EventSetup &setup) const {
+    //get the device collection of digis
+    auto const &digis = event.get(digisToken_);
+
+    //get size of digis
+    const uint32_t size = digis->metadata().size();
+
+    //allocate output product on the device
+    OutputProduct uncalibratedRecHits{static_cast<int32_t>(size), event.queue()};
+
+    //do not run the algo if there are no digis
+    if (size > 0) {
+      auto const &weightsObj = weightsCache_.get(event.queue());
+      //launch the asynchronous work
+      ecal::weights::phase2Weights(digis, uncalibratedRecHits, weightsObj.const_data(), event.queue());
+    }
+    //put the output collection into the event
+    event.emplace(uncalibratedRecHitsToken_, std::move(uncalibratedRecHits));
+  }
+
+}  //namespace ALPAKA_ACCELERATOR_NAMESPACE
+DEFINE_FWK_ALPAKA_MODULE(EcalUncalibRecHitPhase2WeightsProducerPortable);