Allow specifying different track source for TPC map extraction (AliceO2Group#12992)

* Allow specifying different track source for TPC map extraction

Track type(s) A can be used to extract residuals for vDrift calibration
and track type(s) B can be use to extract residuals for the creation
of the map. E.g. use ITS-TPC-TRD-TOF tracks for vDrift and ITS-TPC
for the map creation.

* Make sure mAddTracksForMapPerTF are taken from best possible seeds

Author: martenole

Detectors/GlobalTrackingWorkflow/tpcinterpolationworkflow/include/TPCInterpolationWorkflow/TPCInterpolationSpec.h

@@ -38,7 +38,7 @@ namespace tpc
 class TPCInterpolationDPL : public Task
 {
  public:
-  TPCInterpolationDPL(std::shared_ptr<o2::globaltracking::DataRequest> dr, o2::dataformats::GlobalTrackID::mask_t src, std::shared_ptr<o2::base::GRPGeomRequest> gr, bool useMC, bool processITSTPConly, bool sendTrackData, bool debugOutput) : mDataRequest(dr), mSources(src), mGGCCDBRequest(gr), mUseMC(useMC), mProcessITSTPConly(processITSTPConly), mSendTrackData(sendTrackData), mDebugOutput(debugOutput) {}
+  TPCInterpolationDPL(std::shared_ptr<o2::globaltracking::DataRequest> dr, o2::dataformats::GlobalTrackID::mask_t src, o2::dataformats::GlobalTrackID::mask_t srcMap, std::shared_ptr<o2::base::GRPGeomRequest> gr, bool useMC, bool processITSTPConly, bool sendTrackData, bool debugOutput) : mDataRequest(dr), mSources(src), mSourcesMap(srcMap), mGGCCDBRequest(gr), mUseMC(useMC), mProcessITSTPConly(processITSTPConly), mSendTrackData(sendTrackData), mDebugOutput(debugOutput) {}
   ~TPCInterpolationDPL() override = default;
   void init(InitContext& ic) final;
   void run(ProcessingContext& pc) final;
@@ -53,6 +53,7 @@ class TPCInterpolationDPL : public Task
   o2::tpc::VDriftHelper mTPCVDriftHelper{};
   const o2::itsmft::TopologyDictionary* mITSDict = nullptr; ///< cluster patterns dictionary
   o2::dataformats::GlobalTrackID::mask_t mSources{};        ///< which input sources are configured
+  o2::dataformats::GlobalTrackID::mask_t mSourcesMap{};     ///< possible subset of mSources specifically for map creation
   bool mUseMC{false}; ///< MC flag
   bool mProcessITSTPConly{false}; ///< should also tracks without outer point (ITS-TPC only) be processed?
   bool mProcessSeeds{false};      ///< process not only most complete track, but also its shorter parts
@@ -64,7 +65,7 @@ class TPCInterpolationDPL : public Task
 };
 
 /// create a processor spec
-framework::DataProcessorSpec getTPCInterpolationSpec(o2::dataformats::GlobalTrackID::mask_t srcCls, o2::dataformats::GlobalTrackID::mask_t srcVtx, o2::dataformats::GlobalTrackID::mask_t srcTrk, bool useMC, bool processITSTPConly, bool sendTrackData, bool debugOutput);
+framework::DataProcessorSpec getTPCInterpolationSpec(o2::dataformats::GlobalTrackID::mask_t srcCls, o2::dataformats::GlobalTrackID::mask_t srcVtx, o2::dataformats::GlobalTrackID::mask_t srcTrk, o2::dataformats::GlobalTrackID::mask_t srcTrkMap, bool useMC, bool processITSTPConly, bool sendTrackData, bool debugOutput);
 
 } // namespace tpc
 } // namespace o2

Detectors/GlobalTrackingWorkflow/tpcinterpolationworkflow/src/TPCInterpolationSpec.cxx

@@ -52,6 +52,9 @@ void TPCInterpolationDPL::init(InitContext& ic)
   mSlotLength = ic.options().get<uint32_t>("sec-per-slot");
   mProcessSeeds = ic.options().get<bool>("process-seeds");
   mMatCorr = ic.options().get<int>("matCorrType");
+  if (mProcessSeeds && mSources != mSourcesMap) {
+    LOG(fatal) << "process-seeds option is not compatible with using different track sources for vDrift and map extraction";
+  }
 }
 
 void TPCInterpolationDPL::updateTimeDependentParams(ProcessingContext& pc)
@@ -63,7 +66,7 @@ void TPCInterpolationDPL::updateTimeDependentParams(ProcessingContext& pc)
     initOnceDone = true;
     // other init-once stuff
     const auto& param = SpacePointsCalibConfParam::Instance();
-    mInterpolation.init(mSources);
+    mInterpolation.init(mSources, mSourcesMap);
     if (mProcessITSTPConly) {
       mInterpolation.setProcessITSTPConly();
     }
@@ -74,10 +77,10 @@ void TPCInterpolationDPL::updateTimeDependentParams(ProcessingContext& pc)
     if (nTracksPerTfMax > 0) {
       LOGP(info, "We will stop processing tracks after validating {} tracks per TF, since we want to accumulate {} tracks for a slot with {} TFs",
            nTracksPerTfMax, param.maxTracksPerCalibSlot, nTfs);
-      if (param.additionalTracksITSTPC > 0) {
-        int nITSTPCadd = param.additionalTracksITSTPC / nTfs;
-        LOGP(info, "In addition up to {} ITS-TPC tracks are processed per TF", nITSTPCadd);
-        mInterpolation.setAddITSTPCTracksPerTF(nITSTPCadd);
+      if (param.additionalTracksMap > 0) {
+        int nTracksAdditional = param.additionalTracksMap / nTfs;
+        LOGP(info, "In addition up to {} additional tracks are processed per TF", nTracksAdditional);
+        mInterpolation.setAddTracksForMapPerTF(nTracksAdditional);
       }
     } else if (nTracksPerTfMax < 0) {
       LOG(info) << "The number of processed tracks per TF is not limited";
@@ -155,7 +158,7 @@ void TPCInterpolationDPL::endOfStream(EndOfStreamContext& ec)
        mTimer.CpuTime(), mTimer.RealTime(), mTimer.Counter() - 1);
 }
 
-DataProcessorSpec getTPCInterpolationSpec(GTrackID::mask_t srcCls, GTrackID::mask_t srcVtx, GTrackID::mask_t srcTrk, bool useMC, bool processITSTPConly, bool sendTrackData, bool debugOutput)
+DataProcessorSpec getTPCInterpolationSpec(GTrackID::mask_t srcCls, GTrackID::mask_t srcVtx, GTrackID::mask_t srcTrk, GTrackID::mask_t srcTrkMap, bool useMC, bool processITSTPConly, bool sendTrackData, bool debugOutput)
 {
   auto dataRequest = std::make_shared<DataRequest>();
   std::vector<OutputSpec> outputs;
@@ -196,7 +199,7 @@ DataProcessorSpec getTPCInterpolationSpec(GTrackID::mask_t srcCls, GTrackID::mas
     "tpc-track-interpolation",
     dataRequest->inputs,
     outputs,
-    AlgorithmSpec{adaptFromTask<TPCInterpolationDPL>(dataRequest, srcTrk, ggRequest, useMC, processITSTPConly, sendTrackData, debugOutput)},
+    AlgorithmSpec{adaptFromTask<TPCInterpolationDPL>(dataRequest, srcTrk, srcTrkMap, ggRequest, useMC, processITSTPConly, sendTrackData, debugOutput)},
     Options{
       {"matCorrType", VariantType::Int, 2, {"material correction type (definition in Propagator.h)"}},
       {"sec-per-slot", VariantType::UInt32, 600u, {"number of seconds per calibration time slot (put 0 for infinite slot length)"}},

Detectors/GlobalTrackingWorkflow/tpcinterpolationworkflow/src/tpc-interpolation-workflow.cxx

@@ -39,6 +39,7 @@ void customize(std::vector<o2::framework::ConfigParamSpec>& workflowOptions)
     {"disable-mc", VariantType::Bool, false, {"disable MC propagation even if available"}},
     {"vtx-sources", VariantType::String, std::string{GID::ALL}, {"comma-separated list of sources used for the vertex finding"}},
     {"tracking-sources", VariantType::String, std::string{GID::ALL}, {"comma-separated list of sources to use for track inter-/extrapolation"}},
+    {"tracking-sources-map-extraction", VariantType::String, std::string{GID::ALL}, {"can be subset of \"tracking-sources\""}},
     {"send-track-data", VariantType::Bool, false, {"Send also the track information to the aggregator"}},
     {"debug-output", VariantType::Bool, false, {"Dump extended tracking information for debugging"}},
     {"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings ..."}}};
@@ -69,14 +70,24 @@ WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
   GID::mask_t allowedSources = GID::getSourcesMask("ITS-TPC,ITS-TPC-TRD,ITS-TPC-TOF,ITS-TPC-TRD-TOF");
   GID::mask_t srcVtx = allowedSources & GID::getSourcesMask(configcontext.options().get<std::string>("vtx-sources"));
   GID::mask_t srcTracks = allowedSources & GID::getSourcesMask(configcontext.options().get<std::string>("tracking-sources"));
+  GID::mask_t srcTracksMap = allowedSources & GID::getSourcesMask(configcontext.options().get<std::string>("tracking-sources-map-extraction"));
   if (srcTracks.count() > srcVtx.count()) {
     LOGP(error, "More sources configured for inter-/extrapolation: {} than for vertexing: {}. Additional sources will be ignored", GID::getSourcesNames(srcTracks), GID::getSourcesNames(srcVtx));
     srcTracks &= srcVtx;
   }
+  srcTracksMap &= srcVtx;
+  if (((srcTracksMap | srcTracks) ^ srcTracks).any()) {
+    LOGP(fatal, "tracking-sources-map-extraction ({}) must be a subset of tracking-sources ({}).", GID::getSourcesNames(srcTracksMap), GID::getSourcesNames(srcTracks));
+  } else if (srcTracksMap != srcTracks) {
+    LOGP(info, "Will extract residual from different track types. For vDrift from {} and for distortion map from {}", GID::getSourcesNames(srcTracks), GID::getSourcesNames(srcTracksMap));
+  } else {
+    LOGP(info, "Only a single track source is defined for residuals extraction: {}", GID::getSourcesNames(srcTracks));
+  }
   LOG(debug) << "Data sources for inter-/extrapolation: " << GID::getSourcesNames(srcTracks);
   // check first if ITS-TPC tracks were specifically requested from command line
   bool processITSTPConly = srcTracks[GID::ITSTPC];
   srcTracks |= GID::getSourcesMask("ITS,TPC,ITS-TPC"); // now add them in any case
+  srcTracksMap |= GID::getSourcesMask("ITS,TPC,ITS-TPC");
   srcVtx |= srcTracks;
   GID::mask_t srcClusters = srcTracks;
   if (srcTracks[GID::ITSTPCTRD] || srcTracks[GID::ITSTPCTRDTOF]) {
@@ -94,7 +105,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
   auto sendTrackData = configcontext.options().get<bool>("send-track-data");
   auto debugOutput = configcontext.options().get<bool>("debug-output");
 
-  specs.emplace_back(o2::tpc::getTPCInterpolationSpec(srcClusters, srcVtx, srcTracks, useMC, processITSTPConly, sendTrackData, debugOutput));
+  specs.emplace_back(o2::tpc::getTPCInterpolationSpec(srcClusters, srcVtx, srcTracks, srcTracksMap, useMC, processITSTPConly, sendTrackData, debugOutput));
   if (!configcontext.options().get<bool>("disable-root-output")) {
     specs.emplace_back(o2::tpc::getTPCResidualWriterSpec(sendTrackData, debugOutput));
   }

Detectors/TPC/calibration/SpacePoints/include/SpacePoints/SpacePointsCalibConfParam.h

@@ -25,8 +25,8 @@ namespace tpc
 // These are configurable params for the TPC space point calibration
 struct SpacePointsCalibConfParam : public o2::conf::ConfigurableParamHelper<SpacePointsCalibConfParam> {
 
-  int maxTracksPerCalibSlot = 3'500'000; ///< the number of tracks which is required to obtain an average correction map
-  int additionalTracksITSTPC = 2'000'000; ///< will be added to maxTracksPerCalibSlot for track sample with uniform acceptance (no PHOS hole)
+  int maxTracksPerCalibSlot = 500'000; ///< the number of tracks which is required to obtain an average correction map
+  int additionalTracksMap = 3'500'000; ///< will be added to maxTracksPerCalibSlot for track sample with uniform acceptance (no PHOS hole)
 
   // define track cuts for track interpolation
   int minTPCNCls = 70;             ///< min number of TPC clusters

Detectors/TPC/calibration/SpacePoints/include/SpacePoints/TrackInterpolation.h

@@ -189,13 +189,13 @@ class TrackInterpolation
   // -------------------------------------- processing functions --------------------------------------------------
 
   /// Initialize everything, set the requested track sources
-  void init(o2::dataformats::GlobalTrackID::mask_t src);
+  void init(o2::dataformats::GlobalTrackID::mask_t src, o2::dataformats::GlobalTrackID::mask_t srcMap);
 
   /// Check if input track passes configured cuts
   bool isInputTrackAccepted(const o2::dataformats::GlobalTrackID& gid, const o2::globaltracking::RecoContainer::GlobalIDSet& gidTable, const o2::dataformats::PrimaryVertex& pv) const;
 
   /// For given vertex track source which is not in mSourcesConfigured find the seeding source which is enabled
-  o2::dataformats::GlobalTrackID::Source findValidSource(o2::dataformats::GlobalTrackID::Source src) const;
+  o2::dataformats::GlobalTrackID::Source findValidSource(const o2::dataformats::GlobalTrackID::mask_t mask, const o2::dataformats::GlobalTrackID::Source src) const;
 
   /// Prepare input track sample (not relying on CreateTracksVariadic functionality)
   void prepareInputTrackSample(const o2::globaltracking::RecoContainer& inp);
@@ -252,8 +252,8 @@ class TrackInterpolation
   /// Sets the maximum number of tracks to be processed (successfully) per TF
   void setMaxTracksPerTF(int n) { mMaxTracksPerTF = n; }
 
-  /// In addition to mMaxTracksPerTF up to the set number of ITS-TPC only tracks can be processed
-  void setAddITSTPCTracksPerTF(int n) { mAddTracksITSTPC = n; }
+  /// In addition to mMaxTracksPerTF up to the set number of additional tracks can be processed
+  void setAddTracksForMapPerTF(int n) { mAddTracksForMapPerTF = n; }
 
   /// Enable full output
   void setDumpTrackPoints() { mDumpTrackPoints = true; }
@@ -288,11 +288,13 @@ class TrackInterpolation
   float mSqrtS{13600.f};                             ///< centre of mass energy set from LHC IF
   MatCorrType mMatCorr{MatCorrType::USEMatCorrNONE}; ///< if material correction should be done
   int mMaxTracksPerTF{-1};                           ///< max number of tracks to be processed per TF (-1 means there is no limit)
-  int mAddTracksITSTPC{0};                           ///< number of ITS-TPC tracks which can be processed in addition to mMaxTracksPerTF
+  int mAddTracksForMapPerTF{0};                      ///< in case residuals from different track types are used for vDrift calibration and map creation this defines the statistics for the latter
   bool mDumpTrackPoints{false};                      ///< dump also track points in ITS, TRD and TOF
   bool mProcessSeeds{false};                         ///< in case for global tracks also their shorter parts are processed separately
   bool mProcessITSTPConly{false};                    ///< flag, whether or not to extrapolate ITS-only through TPC
-  o2::dataformats::GlobalTrackID::mask_t mSourcesConfigured; ///< keep only the matches here, not the individual detector contributors
+  o2::dataformats::GlobalTrackID::mask_t mSourcesConfigured;    ///< the track sources taken into account for extra-/interpolation
+  o2::dataformats::GlobalTrackID::mask_t mSourcesConfiguredMap; ///< possible subset of mSourcesConfigured
+  bool mSingleSourcesConfigured{true};                          ///< whether mSourcesConfigured == mSourcesConfiguredMap
 
   // input
   const o2::globaltracking::RecoContainer* mRecoCont = nullptr;                            ///< input reco container

Detectors/TPC/calibration/SpacePoints/src/TrackInterpolation.cxx

@@ -38,7 +38,7 @@ using namespace o2::tpc;
 using GTrackID = o2::dataformats::GlobalTrackID;
 using DetID = o2::detectors::DetID;
 
-void TrackInterpolation::init(o2::dataformats::GlobalTrackID::mask_t src)
+void TrackInterpolation::init(o2::dataformats::GlobalTrackID::mask_t src, o2::dataformats::GlobalTrackID::mask_t srcMap)
 {
   // perform initialization
   LOG(info) << "Start initializing TrackInterpolation";
@@ -58,13 +58,16 @@ void TrackInterpolation::init(o2::dataformats::GlobalTrackID::mask_t src)
   mParams = &SpacePointsCalibConfParam::Instance();
 
   mSourcesConfigured = src;
+  mSourcesConfiguredMap = srcMap;
+  mSingleSourcesConfigured = (mSourcesConfigured == mSourcesConfiguredMap);
   mTrackTypes.insert({GTrackID::ITSTPC, 0});
   mTrackTypes.insert({GTrackID::ITSTPCTRD, 1});
   mTrackTypes.insert({GTrackID::ITSTPCTOF, 2});
   mTrackTypes.insert({GTrackID::ITSTPCTRDTOF, 3});
 
   mInitDone = true;
-  LOGP(info, "Done initializing TrackInterpolation. Configured track input: {}", GTrackID::getSourcesNames(mSourcesConfigured));
+  LOGP(info, "Done initializing TrackInterpolation. Configured track input: {}. Track input specifically for map: {}",
+       GTrackID::getSourcesNames(mSourcesConfigured), mSingleSourcesConfigured ? "identical" : GTrackID::getSourcesNames(mSourcesConfiguredMap));
 }
 
 bool TrackInterpolation::isInputTrackAccepted(const GTrackID& gid, const o2::globaltracking::RecoContainer::GlobalIDSet& gidTable, const o2::dataformats::PrimaryVertex& pv) const
@@ -114,19 +117,19 @@ bool TrackInterpolation::isInputTrackAccepted(const GTrackID& gid, const o2::glo
   return true;
 }
 
-GTrackID::Source TrackInterpolation::findValidSource(GTrackID::Source src) const
+GTrackID::Source TrackInterpolation::findValidSource(const GTrackID::mask_t mask, const GTrackID::Source src) const
 {
-  LOGP(debug, "Trying to find valid source for {}", GTrackID::getSourcesNames(src));
+  LOGP(debug, "Trying to find valid source for {} in {}", GTrackID::getSourceName(src), GTrackID::getSourcesNames(mask));
   if (src == GTrackID::ITSTPCTRDTOF) {
-    if (mSourcesConfigured[GTrackID::ITSTPCTRD]) {
+    if (mask[GTrackID::ITSTPCTRD]) {
       return GTrackID::ITSTPCTRD;
-    } else if (mSourcesConfigured[GTrackID::ITSTPC]) {
+    } else if (mask[GTrackID::ITSTPC]) {
       return GTrackID::ITSTPC;
     } else {
       return GTrackID::NSources;
     }
   } else if (src == GTrackID::ITSTPCTRD || src == GTrackID::ITSTPCTOF) {
-    if (mSourcesConfigured[GTrackID::ITSTPC]) {
+    if (mask[GTrackID::ITSTPC]) {
       return GTrackID::ITSTPC;
     } else {
       return GTrackID::NSources;
@@ -169,9 +172,9 @@ void TrackInterpolation::prepareInputTrackSample(const o2::globaltracking::RecoC
         }
         auto gidTable = mRecoCont->getSingleDetectorRefs(vid);
         if (!mSourcesConfigured[is]) {
-          auto src = findValidSource(static_cast<GTrackID::Source>(vid.getSource()));
+          auto src = findValidSource(mSourcesConfigured, static_cast<GTrackID::Source>(vid.getSource()));
           if (src == GTrackID::ITSTPCTRD || src == GTrackID::ITSTPC) {
-            LOGP(debug, "Found valid source {}", GTrackID::getSourcesNames(src));
+            LOGP(debug, "prepareInputTrackSample: Found valid source {}", GTrackID::getSourceName(src));
             vid = gidTable[src];
             gidTable = mRecoCont->getSingleDetectorRefs(vid);
           } else {
@@ -252,19 +255,33 @@ void TrackInterpolation::process()
   trackIndices.insert(trackIndices.end(), mTrackIndices[mTrackTypes[GTrackID::ITSTPC]].begin(), mTrackIndices[mTrackTypes[GTrackID::ITSTPC]].end());
 
   int nSeeds = mSeeds.size();
-  int maxOutputTracks = (mMaxTracksPerTF >= 0) ? mMaxTracksPerTF + mAddTracksITSTPC : nSeeds;
+  int maxOutputTracks = (mMaxTracksPerTF >= 0) ? mMaxTracksPerTF + mAddTracksForMapPerTF : nSeeds;
   mTrackData.reserve(maxOutputTracks);
   mClRes.reserve(maxOutputTracks * param::NPadRows);
   bool maxTracksReached = false;
   for (int iSeed = 0; iSeed < nSeeds; ++iSeed) {
-    if (mMaxTracksPerTF >= 0 && mTrackDataCompact.size() >= mMaxTracksPerTF + mAddTracksITSTPC) {
+    if (mMaxTracksPerTF >= 0 && mTrackDataCompact.size() >= mMaxTracksPerTF + mAddTracksForMapPerTF) {
       LOG(info) << "Maximum number of tracks per TF reached. Skipping the remaining " << nSeeds - iSeed << " tracks.";
       break;
     }
     int seedIndex = trackIndices[iSeed];
     if (mParams->enableTrackDownsampling && !isTrackSelected(mSeeds[seedIndex])) {
       continue;
     }
+    if (!mSingleSourcesConfigured && !mSourcesConfiguredMap[mGIDs[seedIndex].getSource()]) {
+      auto src = findValidSource(mSourcesConfiguredMap, static_cast<GTrackID::Source>(mGIDs[seedIndex].getSource()));
+      if (src == GTrackID::ITSTPCTRD || src == GTrackID::ITSTPC) {
+        LOGP(debug, "process: Found valid source {}", GTrackID::getSourceName(src));
+        mGIDs.push_back(mGIDtables[seedIndex][src]);
+        mGIDtables.push_back(mRecoCont->getSingleDetectorRefs(mGIDs.back()));
+        mTrackTimes.push_back(mTrackTimes[seedIndex]);
+        mSeeds.push_back(mSeeds[seedIndex]);
+      }
+    }
+    if (mMaxTracksPerTF >= 0 && mTrackDataCompact.size() >= mMaxTracksPerTF) {
+      LOG(debug) << "We already have reached mMaxTracksPerTF, but we continue to create seeds until mAddTracksForMapPerTF is also reached";
+      continue;
+    }
     if (mGIDs[seedIndex].includesDet(DetID::TRD) || mGIDs[seedIndex].includesDet(DetID::TOF)) {
       interpolateTrack(seedIndex);
       if (mProcessSeeds) {
@@ -283,8 +300,16 @@ void TrackInterpolation::process()
       extrapolateTrack(seedIndex);
     }
   }
+  if (mSeeds.size() > nSeeds) {
+    LOGP(info, "Up to {} tracks out of {} additional seeds will be processed", mAddTracksForMapPerTF, mSeeds.size() - nSeeds);
+  }
   for (int iSeed = nSeeds; iSeed < (int)mSeeds.size(); ++iSeed) {
+    if (!mProcessSeeds && mAddTracksForMapPerTF > 0 && mTrackDataCompact.size() >= mMaxTracksPerTF + mAddTracksForMapPerTF) {
+      LOG(info) << "Maximum number of additional tracks per TF reached. Skipping the remaining " << mSeeds.size() - iSeed << " tracks.";
+      break;
+    }
     // this loop will only be entered in case mProcessSeeds is set
+    LOGP(debug, "Processing additional track {}", mGIDs[iSeed].asString());
     if (mGIDs[iSeed].includesDet(DetID::TRD) || mGIDs[iSeed].includesDet(DetID::TOF)) {
       interpolateTrack(iSeed);
     } else {