Merge branch 'AliceO2Group:master' into master

Author: sashingo

.github/workflows/codeowner-self-approval.yml

@@ -25,7 +25,7 @@ jobs:
 
     steps:
       - name: Install dependencies
-        run: pip install codeowners PyGithub
+        run: pip install --break-system-packages codeowners PyGithub
 
       # Approve the PR, if the author is only editing files owned by themselves.
       - name: Auto-approve PR if permitted

Common/TableProducer/centralityTable.cxx

@@ -249,18 +249,21 @@ struct CentralityTable {
     /* check the previous run number */
     auto bc = collision.bc_as<BCsWithTimestampsAndRun2Infos>();
     if (bc.runNumber() != mRunNumber) {
+      mRunNumber = bc.runNumber(); // mark that this run has been attempted already regardless of outcome
       LOGF(debug, "timestamp=%llu", bc.timestamp());
       TList* callst = nullptr;
       if (ccdbConfig.reconstructionPass.value == "") {
-        callst = ccdb->getForTimeStamp<TList>(ccdbConfig.ccdbPath, bc.timestamp());
+        callst = ccdb->getForRun<TList>(ccdbConfig.ccdbPath, bc.runNumber());
       } else if (ccdbConfig.reconstructionPass.value == "metadata") {
         std::map<std::string, std::string> metadata;
         metadata["RecoPassName"] = metadataInfo.get("RecoPassName");
-        callst = ccdb->getSpecific<TList>(ccdbConfig.ccdbPath, bc.timestamp(), metadata);
+        LOGF(info, "Loading CCDB for reconstruction pass (from metadata): %s", metadataInfo.get("RecoPassName"));
+        callst = ccdb->getSpecificForRun<TList>(ccdbConfig.ccdbPath, bc.runNumber(), metadata);
       } else {
         std::map<std::string, std::string> metadata;
         metadata["RecoPassName"] = ccdbConfig.reconstructionPass.value;
-        callst = ccdb->getSpecific<TList>(ccdbConfig.ccdbPath, bc.timestamp(), metadata);
+        LOGF(info, "Loading CCDB for reconstruction pass (from provided argument): %s", ccdbConfig.reconstructionPass.value);
+        callst = ccdb->getSpecificForRun<TList>(ccdbConfig.ccdbPath, bc.runNumber(), metadata);
       }
 
       Run2V0MInfo.mCalibrationStored = false;
@@ -350,15 +353,11 @@ struct CentralityTable {
             LOGF(fatal, "Calibration information from CL1 multiplicity for run %d corrupted", bc.runNumber());
           }
         }
-        if (Run2V0MInfo.mCalibrationStored || Run2V0AInfo.mCalibrationStored || Run2SPDTksInfo.mCalibrationStored || Run2SPDClsInfo.mCalibrationStored || Run2CL0Info.mCalibrationStored || Run2CL1Info.mCalibrationStored) {
-          mRunNumber = bc.runNumber();
-        }
       } else {
         if (!ccdbConfig.doNotCrashOnNull) { // default behaviour: crash
           LOGF(fatal, "Centrality calibration is not available in CCDB for run=%d at timestamp=%llu", bc.runNumber(), bc.timestamp());
         } else { // only if asked: continue filling with non-valid values (105)
           LOGF(info, "Centrality calibration is not available in CCDB for run=%d at timestamp=%llu, will fill tables with dummy values", bc.runNumber(), bc.timestamp());
-          mRunNumber = bc.runNumber();
         }
       }
     }
@@ -473,6 +472,7 @@ struct CentralityTable {
       /* check the previous run number */
       auto bc = collision.template bc_as<BCsWithTimestamps>();
       if (bc.runNumber() != mRunNumber) {
+        mRunNumber = bc.runNumber(); // mark that this run has been attempted already regardless of outcome
         LOGF(info, "timestamp=%llu, run number=%d", bc.timestamp(), bc.runNumber());
         TList* callst = nullptr;
         // Check if the ccdb path is a root file
@@ -485,15 +485,17 @@ struct CentralityTable {
           }
         } else {
           if (ccdbConfig.reconstructionPass.value == "") {
-            callst = ccdb->getForTimeStamp<TList>(ccdbConfig.ccdbPath, bc.timestamp());
+            callst = ccdb->getForRun<TList>(ccdbConfig.ccdbPath, bc.runNumber());
           } else if (ccdbConfig.reconstructionPass.value == "metadata") {
             std::map<std::string, std::string> metadata;
             metadata["RecoPassName"] = metadataInfo.get("RecoPassName");
-            callst = ccdb->getSpecific<TList>(ccdbConfig.ccdbPath, bc.timestamp(), metadata);
+            LOGF(info, "Loading CCDB for reconstruction pass (from metadata): %s", metadataInfo.get("RecoPassName"));
+            callst = ccdb->getSpecificForRun<TList>(ccdbConfig.ccdbPath, bc.runNumber(), metadata);
           } else {
             std::map<std::string, std::string> metadata;
             metadata["RecoPassName"] = ccdbConfig.reconstructionPass.value;
-            callst = ccdb->getSpecific<TList>(ccdbConfig.ccdbPath, bc.timestamp(), metadata);
+            LOGF(info, "Loading CCDB for reconstruction pass (from provided argument): %s", ccdbConfig.reconstructionPass.value);
+            callst = ccdb->getSpecificForRun<TList>(ccdbConfig.ccdbPath, bc.runNumber(), metadata);
           }
         }
 
@@ -555,13 +557,11 @@ struct CentralityTable {
                 break;
             }
           }
-          mRunNumber = bc.runNumber();
         } else {
           if (!ccdbConfig.doNotCrashOnNull) { // default behaviour: crash
             LOGF(fatal, "Centrality calibration is not available in CCDB for run=%d at timestamp=%llu", bc.runNumber(), bc.timestamp());
           } else { // only if asked: continue filling with non-valid values (105)
             LOGF(info, "Centrality calibration is not available in CCDB for run=%d at timestamp=%llu, will fill tables with dummy values", bc.runNumber(), bc.timestamp());
-            mRunNumber = bc.runNumber();
           }
         }
       }

Common/TableProducer/multiplicityExtraTable.cxx

@@ -41,6 +41,15 @@ struct MultiplicityExtraTable {
   Configurable<float> minFT0CforBCTable{"minFT0CforBCTable", 25.0f, "Minimum FT0C amplitude to fill BC table to reduce data"};
   Configurable<bool> saveOnlyBCsWithCollisions{"saveOnlyBCsWithCollisions", true, "save only BCs with collisions in them"};
 
+  Configurable<float> bcTableFloatPrecision{"bcTableFloatPrecision", 0.1, "float precision in bc table for data reduction"};
+
+  float tru(float value)
+  {
+    if (bcTableFloatPrecision < 1e-4)
+      return value; // make sure nothing bad happens in case zero (best precision)
+    return bcTableFloatPrecision * std::round(value / bcTableFloatPrecision) + 0.5f * bcTableFloatPrecision;
+  };
+
   // needed for downscale
   unsigned int randomSeed = 0;
 
@@ -231,9 +240,10 @@ struct MultiplicityExtraTable {
 
       bc2mult(bc2multArray[bc.globalIndex()]);
       multBC(
-        multFT0A, multFT0C, posZFT0, posZFT0valid, multFV0A,
-        multFDDA, multFDDC, multZNA, multZNC, multZEM1,
-        multZEM2, multZPA, multZPC, Tvx, isFV0OrA,
+        tru(multFT0A), tru(multFT0C),
+        tru(posZFT0), posZFT0valid, tru(multFV0A),
+        tru(multFDDA), tru(multFDDC), tru(multZNA), tru(multZNC), tru(multZEM1),
+        tru(multZEM2), tru(multZPA), tru(multZPC), Tvx, isFV0OrA,
         multFV0TriggerBits, multFT0TriggerBits, multFDDTriggerBits, multBCTriggerMask, collidingBC,
         bc.flags());
     }

Common/TableProducer/multiplicityTable.cxx

@@ -379,15 +379,17 @@ struct MultiplicityTable {
         if (bc.runNumber() != mRunNumber) {
           mRunNumber = bc.runNumber(); // mark this run as at least tried
           if (ccdbConfig.reconstructionPass.value == "") {
-            lCalibObjects = ccdb->getForTimeStamp<TList>(ccdbConfig.ccdbPath, bc.timestamp());
+            lCalibObjects = ccdb->getForRun<TList>(ccdbConfig.ccdbPath, mRunNumber);
           } else if (ccdbConfig.reconstructionPass.value == "metadata") {
             std::map<std::string, std::string> metadata;
             metadata["RecoPassName"] = metadataInfo.get("RecoPassName");
-            lCalibObjects = ccdb->getSpecific<TList>(ccdbConfig.ccdbPath, bc.timestamp(), metadata);
+            LOGF(info, "Loading CCDB for reconstruction pass (from metadata): %s", metadataInfo.get("RecoPassName"));
+            lCalibObjects = ccdb->getSpecificForRun<TList>(ccdbConfig.ccdbPath, mRunNumber, metadata);
           } else {
             std::map<std::string, std::string> metadata;
             metadata["RecoPassName"] = ccdbConfig.reconstructionPass.value;
-            lCalibObjects = ccdb->getSpecific<TList>(ccdbConfig.ccdbPath, bc.timestamp(), metadata);
+            LOGF(info, "Loading CCDB for reconstruction pass (from provided argument): %s", ccdbConfig.reconstructionPass.value);
+            lCalibObjects = ccdb->getSpecificForRun<TList>(ccdbConfig.ccdbPath, mRunNumber, metadata);
           }
 
           if (lCalibObjects) {

DPG/Tasks/AOTTrack/qaImpPar.cxx

@@ -112,6 +112,7 @@ struct QaImpactPar {
   Configurable<bool> keepAllTracksPVrefit{"keepAllTracksPVrefit", false, "Keep all tracks for PV refit (for debug)"};
   Configurable<bool> use_customITSHitMap{"use_customITSHitMap", false, "Use custom ITS hitmap selection"};
   Configurable<int> customITShitmap{"customITShitmap", 0, "Custom ITS hitmap (consider the binary representation)"};
+  Configurable<int> customITShitmap_exclude{"customITShitmap_exclude", 0, "Custom ITS hitmap of layers to be excluded (consider the binary representation)"};
   Configurable<int> n_customMinITShits{"n_customMinITShits", 0, "Minimum number of layers crossed by a track among those in \"customITShitmap\""};
   Configurable<bool> custom_forceITSTPCmatching{"custom_forceITSTPCmatching", false, "Consider or not only ITS-TPC macthed tracks when using custom ITS hitmap"};
 
@@ -135,9 +136,9 @@ struct QaImpactPar {
 
   /// Histogram registry (from o2::framework)
   HistogramRegistry histograms{"HistogramsImpParQA"};
-  bool isPIDPionApplied = ((nSigmaTPCPionMin > -10.001 && nSigmaTPCPionMax < 10.001) || (nSigmaTOFPionMin > -10.001 && nSigmaTOFPionMax < 10.001));
-  bool isPIDKaonApplied = ((nSigmaTPCKaonMin > -10.001 && nSigmaTPCKaonMax < 10.001) || (nSigmaTOFKaonMin > -10.001 && nSigmaTOFKaonMax < 10.001));
-  bool isPIDProtonApplied = ((nSigmaTPCProtonMin > -10.001 && nSigmaTPCProtonMax < 10.001) || (nSigmaTOFProtonMin > -10.001 && nSigmaTOFProtonMax < 10.001));
+  bool isPIDPionApplied;
+  bool isPIDKaonApplied;
+  bool isPIDProtonApplied;
 
   // Needed for PV refitting
   Service<o2::ccdb::BasicCCDBManager> ccdb;
@@ -239,7 +240,7 @@ struct QaImpactPar {
     // }
     mRunNumber = -1;
 
-    /// Custom cut selection objects
+    /// Custom cut selection objects - ITS layers that must be present
     std::set<uint8_t> set_customITShitmap; // = {};
     if (use_customITSHitMap) {
       for (int index_ITSlayer = 0; index_ITSlayer < 7; index_ITSlayer++) {
@@ -258,6 +259,24 @@ struct QaImpactPar {
 
       selector_ITShitmap.SetRequireHitsInITSLayers(n_customMinITShits, set_customITShitmap);
     }
+    /// Custom cut selection objects - ITS layers that must be absent
+    std::set<uint8_t> set_customITShitmap_exclude; // = {};
+    if (use_customITSHitMap) {
+      for (int index_ITSlayer = 0; index_ITSlayer < 7; index_ITSlayer++) {
+        if ((customITShitmap_exclude & (1 << index_ITSlayer)) > 0) {
+          set_customITShitmap_exclude.insert(static_cast<uint8_t>(index_ITSlayer));
+        }
+      }
+      LOG(info) << "### customITShitmap_exclude: " << customITShitmap_exclude;
+      LOG(info) << "### set_customITShitmap_exclude.size(): " << set_customITShitmap_exclude.size();
+      LOG(info) << "### ITS layers to be excluded: ";
+      for (std::set<uint8_t>::iterator it = set_customITShitmap_exclude.begin(); it != set_customITShitmap_exclude.end(); it++) {
+        LOG(info) << "Layer " << static_cast<int>(*it) << " ";
+      }
+      LOG(info) << "############";
+
+      selector_ITShitmap.SetRequireNoHitsInITSLayers(set_customITShitmap_exclude);
+    }
 
     // tracks
     const AxisSpec trackPtAxis{binningPt, "#it{p}_{T} (GeV/#it{c})"};
@@ -303,14 +322,17 @@ struct QaImpactPar {
       histograms.add("Reco/h4ImpParPulls", "", kTHnSparseD, {trackPtAxis, trackImpParRPhiPullsAxis, trackEtaAxis, trackPhiAxis, trackPDGAxis, trackChargeAxis, axisVertexNumContrib, trackIsPvContrib});
       histograms.add("Reco/h4ImpParZPulls", "", kTHnSparseD, {trackPtAxis, trackImpParZPullsAxis, trackEtaAxis, trackPhiAxis, trackPDGAxis, trackChargeAxis, axisVertexNumContrib, trackIsPvContrib});
     }
+    isPIDPionApplied = ((nSigmaTPCPionMin > -10.001 && nSigmaTPCPionMax < 10.001) || (nSigmaTOFPionMin > -10.001 && nSigmaTOFPionMax < 10.001));
     if (isPIDPionApplied) {
       histograms.add("Reco/h4ImpPar_Pion", "", kTHnSparseD, {trackPtAxis, trackImpParRPhiAxis, trackEtaAxis, trackPhiAxis, trackPDGAxis, trackChargeAxis, axisVertexNumContrib, trackIsPvContrib});
       histograms.add("Reco/h4ImpParZ_Pion", "", kTHnSparseD, {trackPtAxis, trackImpParZAxis, trackEtaAxis, trackPhiAxis, trackPDGAxis, trackChargeAxis, axisVertexNumContrib, trackIsPvContrib});
     }
+    isPIDKaonApplied = ((nSigmaTPCKaonMin > -10.001 && nSigmaTPCKaonMax < 10.001) || (nSigmaTOFKaonMin > -10.001 && nSigmaTOFKaonMax < 10.001));
     if (isPIDKaonApplied) {
       histograms.add("Reco/h4ImpPar_Kaon", "", kTHnSparseD, {trackPtAxis, trackImpParRPhiAxis, trackEtaAxis, trackPhiAxis, trackPDGAxis, trackChargeAxis, axisVertexNumContrib, trackIsPvContrib});
       histograms.add("Reco/h4ImpParZ_Kaon", "", kTHnSparseD, {trackPtAxis, trackImpParZAxis, trackEtaAxis, trackPhiAxis, trackPDGAxis, trackChargeAxis, axisVertexNumContrib, trackIsPvContrib});
     }
+    isPIDProtonApplied = ((nSigmaTPCProtonMin > -10.001 && nSigmaTPCProtonMax < 10.001) || (nSigmaTOFProtonMin > -10.001 && nSigmaTOFProtonMax < 10.001));
     if (isPIDProtonApplied) {
       histograms.add("Reco/h4ImpPar_Proton", "", kTHnSparseD, {trackPtAxis, trackImpParRPhiAxis, trackEtaAxis, trackPhiAxis, trackPDGAxis, trackChargeAxis, axisVertexNumContrib, trackIsPvContrib});
       histograms.add("Reco/h4ImpParZ_Proton", "", kTHnSparseD, {trackPtAxis, trackImpParZAxis, trackEtaAxis, trackPhiAxis, trackPDGAxis, trackChargeAxis, axisVertexNumContrib, trackIsPvContrib});

EventFiltering/macros/uploadOTSobjects.C

@@ -16,6 +16,7 @@
 #include <string>
 #include <map>
 #include <array>
+#include <cmath>
 
 #include "TFile.h"
 #include "TGrid.h"
@@ -26,16 +27,23 @@
 
 #include "CCDB/BasicCCDBManager.h"
 #include "EventFiltering/ZorroHelper.h"
+#include "CommonConstants/LHCConstants.h"
 
-void uploadOTSobjects(std::string inputList, std::string passName, bool useAlien)
+constexpr uint32_t chunkSize = 1000000;
+
+void uploadOTSobjects(std::string inputList, std::string passName, bool useAlien, bool chunkedProcessing)
 {
   const std::string kBaseCCDBPath = "Users/m/mpuccio/EventFiltering/OTS/";
   std::string baseCCDBpath = passName.empty() ? kBaseCCDBPath : kBaseCCDBPath + passName + "/";
+  if (chunkedProcessing) {
+    baseCCDBpath += "Chunked/";
+  }
   if (useAlien) {
     TGrid::Connect("alien://");
   }
   o2::ccdb::CcdbApi api;
   api.init("http://alice-ccdb.cern.ch");
+  auto& ccdb = o2::ccdb::BasicCCDBManager::instance();
 
   std::ifstream file(inputList.data());
   std::string path;
@@ -47,9 +55,11 @@ void uploadOTSobjects(std::string inputList, std::string passName, bool useAlien
     const int runNumber = std::stoi(runString);
     metadata["runNumber"] = runString;
     std::pair<int64_t, int64_t> duration = o2::ccdb::BasicCCDBManager::getRunDuration(api, runNumber);
-    duration.first -= 10000;   // subtract 3 minutes from the run start
-    duration.second += 180000; // add 3 minutes to the run duration
+    duration.first -= 60000;  // subtract 1 minutes from the run start
+    duration.second += 60000; // add 1 minutes to the run duration
     std::cout << ">>> Begin - end timestamps for the upload: " << duration.first << " - " << duration.second << std::endl;
+    auto ctp = ccdb.getForTimeStamp<std::vector<Long64_t>>("CTP/Calib/OrbitReset", duration.first / 2 + duration.second / 2);
+    auto orbitResetTimestamp = (*ctp)[0];
     path = useAlien ? "alien://" + path : path;
     std::unique_ptr<TFile> scalersFile{TFile::Open((path + "/AnalysisResults_fullrun.root").data(), "READ")};
     TH1* scalers = static_cast<TH1*>(scalersFile->Get("central-event-filter-task/scalers/mScalers"));
@@ -85,13 +95,48 @@ void uploadOTSobjects(std::string inputList, std::string passName, bool useAlien
         }
       }
     }
-    api.storeAsTFileAny(&zorroHelpers, baseCCDBpath + "ZorroHelpers", metadata, duration.first, duration.second);
+    std::sort(zorroHelpers.begin(), zorroHelpers.end(), [](const ZorroHelper& a, const ZorroHelper& b) {
+      return a.bcAOD < b.bcAOD;
+    });
+    if (!chunkedProcessing) {
+      api.storeAsTFileAny(&zorroHelpers, baseCCDBpath + "ZorroHelpers", metadata, duration.first, duration.second);
+      std::cout << std::endl;
+    } else {
+      uint32_t helperIndex{0};
+      auto startTS = duration.first;
+      int64_t endTS = 0;
+      while (helperIndex < zorroHelpers.size()) {
+        std::vector<ZorroHelper> chunk;
+        uint32_t endIndex = helperIndex + chunkSize;
+        while (true) {
+          if (endIndex >= zorroHelpers.size() - 2) {
+            endTS = duration.second;
+            chunk.insert(chunk.begin(), zorroHelpers.begin() + helperIndex, zorroHelpers.end());
+            break;
+          }
+          auto bcEnd{zorroHelpers[endIndex].bcAOD > zorroHelpers[endIndex].bcEvSel ? zorroHelpers[endIndex].bcAOD : zorroHelpers[endIndex].bcEvSel};
+          const auto& nextHelper = zorroHelpers[endIndex + 1];
+          auto bcNext{nextHelper.bcAOD < nextHelper.bcEvSel ? nextHelper.bcAOD : nextHelper.bcEvSel};
+          if (bcNext - bcEnd > 2001 / o2::constants::lhc::LHCBunchSpacingMUS) { /// ensure a gap of 2ms between chunks
+            chunk.insert(chunk.begin(), zorroHelpers.begin() + helperIndex, zorroHelpers.begin() + endIndex + 1);
+            endTS = (orbitResetTimestamp + int64_t(bcEnd * o2::constants::lhc::LHCBunchSpacingNS * 1e-3)) / 1000 + 1;
+            break;
+          }
+          bcEnd = nextHelper.bcAOD > nextHelper.bcEvSel ? nextHelper.bcAOD : nextHelper.bcEvSel;
+          endIndex++;
+        }
+        std::cout << ">>> Chunk " << helperIndex << " - " << helperIndex + chunk.size() << " : " << startTS << " - " << endTS << " \t" << (endTS - startTS) * 1.e-3 << std::endl;
+        api.storeAsTFileAny(&zorroHelpers, baseCCDBpath + "ZorroHelpers", metadata, startTS, endTS);
+        startTS = endTS + 1;
+        helperIndex += chunk.size();
+      }
+    }
   }
 }
 
-void uploadOTSobjects(std::string periodName)
+void uploadOTSobjects(std::string periodName, bool chunkedProcessing)
 {
   int year = 2000 + std::stoi(periodName.substr(3, 2));
   gSystem->Exec(Form("alien_find /alice/data/%i/%s/ ctf_skim_full/AnalysisResults_fullrun.root | sed 's:/AnalysisResults_fullrun\\.root::' > list_%s.txt", year, periodName.data(), periodName.data()));
-  uploadOTSobjects(Form("list_%s.txt", periodName.data()), "", true);
+  uploadOTSobjects(Form("list_%s.txt", periodName.data()), "", true, chunkedProcessing);
 }