Merge d16c8a9 into sapling-pr-archive-ktf

Author: ktf

Detectors/Align/Workflow/src/BarrelAlignmentSpec.cxx

@@ -265,7 +265,7 @@ void BarrelAlignmentSpec::updateTimeDependentParams(ProcessingContext& pc)
       prevField = newField;
       if (mDetMask[DetID::TPC]) {
         mTPCParam.reset(new o2::gpu::GPUParam);
-        mTPCParam->SetDefaults(o2::base::Propagator::Instance()->getNominalBz());
+        mTPCParam->SetDefaults(o2::base::Propagator::Instance()->getNominalBz(), false);
         mController->setTPCParam(mTPCParam.get());
       }
     }

Framework/AnalysisSupport/src/Plugin.cxx

@@ -117,6 +117,24 @@ struct RunSummary : o2::framework::ServicePlugin {
               LOGP(info, "Last file opened: {}", lastFileRead);
             }
           }
+          for (size_t li = 0; li < metrics.metricLabels.size(); ++li) {
+            MetricLabel const&label = metrics.metricLabels[li];
+            if (strcmp(label.label, "aod-file-open-info") != 0) {
+              continue;
+            }
+            MetricInfo const&metric = metrics.metrics[li];
+            auto &files = metrics.stringMetrics[metric.storeIdx];
+            if (metric.filledMetrics) {
+              LOGP(info, "### Files opened stats ###");
+            }
+            std::string lastFileRead;
+            for (size_t fi = 0; fi < metric.filledMetrics; ++fi) {
+              lastFileRead = files[fi % files.size()].data;
+            }
+            if (lastFileRead.empty() == false) {
+              LOGP(info, "Last file opened: {}", lastFileRead);
+            }
+          }
         } },
       .kind = ServiceKind::Serial};
   }

Framework/Core/include/Framework/AnalysisDataModel.h

@@ -1776,7 +1776,7 @@ DECLARE_SOA_COLUMN(Chi2NDF, chi2NDF, float);                            //! chi^
 DECLARE_SOA_COLUMN(PsiPair, psiPair, float);                            //! Psi pair
 DECLARE_SOA_COLUMN(DCAr, dcaR, float);                                  //! DCA in radial direction
 DECLARE_SOA_COLUMN(DCAz, dcaZ, float);                                  //! DCA in z direction
-DECLARE_SOA_COLUMN(Mass, mass, float);                                  //! mass of the conversion. Do NOT use for cut!
+DECLARE_SOA_COLUMN(MassInMeV, mass, float);                             //! mass of the conversion in MeV. Do NOT use for cut!
 } // namespace oftv0
 namespace pmd
 {
@@ -1824,7 +1824,7 @@ DECLARE_SOA_TABLE(Run2OTFV0s, "AOD", "Run2OTFV0", //! Run 2 V0 on the fly table
                   run2::oftv0::X, run2::oftv0::Y, run2::oftv0::Z,
                   run2::oftv0::Chi2NDF, run2::oftv0::PsiPair,
                   run2::oftv0::DCAr, run2::oftv0::DCAz,
-                  run2::oftv0::Mass);
+                  run2::oftv0::MassInMeV);
 
 using Run2OTFV0 = Run2OTFV0s::iterator;
 

Framework/Core/include/Framework/TableBuilder.h

@@ -753,11 +753,12 @@ auto makeEmptyTable()
   return b.finalize();
 }
 
-template <typename... Cs>
-auto makeEmptyTable(const char* name, framework::pack<Cs...> p)
+
+template <PackLike P>
+auto makeEmptyTable(const char* name)
 {
   TableBuilder b;
-  [[maybe_unused]] auto writer = b.cursor(p);
+  [[maybe_unused]] auto writer = b.cursor(P{});
   b.setLabel(name);
   return b.finalize();
 }
@@ -772,7 +773,7 @@ auto spawner(std::shared_ptr<arrow::Table> const& fullTable, const char* name, o
 {
   using placeholders_pack_t = typename o2::aod::MetadataTrait<D>::metadata::placeholders_pack_t;
   if (fullTable->num_rows() == 0) {
-    return makeEmptyTable(name, placeholders_pack_t{});
+    return makeEmptyTable<placeholders_pack_t>(name);
   }
   return spawnerHelper(fullTable, schema, framework::pack_size(placeholders_pack_t{}), projectors, name, projector);
 }
@@ -791,7 +792,7 @@ auto spawner(std::shared_ptr<arrow::Table> const& fullTable, const char* name, e
 {
   using expression_pack_t = typename o2::aod::MetadataTrait<D>::metadata::expression_pack_t;
   if (fullTable->num_rows() == 0) {
-    return makeEmptyTable(name, expression_pack_t{});
+    return makeEmptyTable<expression_pack_t>(name);
   }
   return spawnerHelper(fullTable, schema, framework::pack_size(expression_pack_t{}), projectors, name, projector);
 }
@@ -805,12 +806,12 @@ auto spawner(std::vector<std::shared_ptr<arrow::Table>>&& tables, const char* na
 }
 
 template <typename... C>
-auto spawner(framework::pack<C...>, std::vector<std::shared_ptr<arrow::Table>>&& tables, const char* name, expressions::Projector* projectors, std::shared_ptr<gandiva::Projector>& projector, std::shared_ptr<arrow::Schema> const& schema)
+auto spawner(framework::pack<C...> columns, std::vector<std::shared_ptr<arrow::Table>>&& tables, const char* name, expressions::Projector* projectors, std::shared_ptr<gandiva::Projector>& projector, std::shared_ptr<arrow::Schema> const& schema)
 {
   std::array<const char*, 1> labels{"original"};
   auto fullTable = soa::ArrowHelpers::joinTables(std::move(tables), std::span<const char* const>{labels});
   if (fullTable->num_rows() == 0) {
-    return makeEmptyTable(name, framework::pack<C...>{});
+    return makeEmptyTable<decltype(columns)>(name);
   }
   return spawnerHelper(fullTable, schema, sizeof...(C), projectors, name, projector);
 }

Framework/Foundation/include/Framework/Pack.h

@@ -30,6 +30,9 @@ constexpr std::size_t pack_size(pack<Ts...> const&)
   return sizeof...(Ts);
 }
 
+template <typename P>
+concept PackLike = requires (P &p) { o2::framework::pack_size(p); };
+
 template <std::size_t I, typename T>
 struct pack_element;
 

GPU/GPUTracking/Base/GPUParam.cxx

@@ -30,7 +30,7 @@ using namespace o2::gpu;
 
 #include "utils/qconfigrtc.h"
 
-void GPUParam::SetDefaults(float solenoidBz)
+void GPUParam::SetDefaults(float solenoidBz, bool assumeConstantBz)
 {
   memset((void*)this, 0, sizeof(*this));
   new (&rec) GPUSettingsRec;
@@ -83,8 +83,7 @@ void GPUParam::SetDefaults(float solenoidBz)
   }
 #endif
 
-  par.dAlpha = 0.349066f;
-  UpdateBzOnly(solenoidBz);
+  UpdateBzOnly(solenoidBz, assumeConstantBz);
   par.dodEdx = 0;
 
   constexpr float plusZmin = 0.0529937;
@@ -102,36 +101,30 @@ void GPUParam::SetDefaults(float solenoidBz)
     if (tmp >= GPUCA_NSECTORS / 4) {
       tmp -= GPUCA_NSECTORS / 2;
     }
-    SectorParam[i].Alpha = 0.174533f + par.dAlpha * tmp;
+    SectorParam[i].Alpha = 0.174533f + dAlpha * tmp;
     SectorParam[i].CosAlpha = CAMath::Cos(SectorParam[i].Alpha);
     SectorParam[i].SinAlpha = CAMath::Sin(SectorParam[i].Alpha);
-    SectorParam[i].AngleMin = SectorParam[i].Alpha - par.dAlpha / 2.f;
-    SectorParam[i].AngleMax = SectorParam[i].Alpha + par.dAlpha / 2.f;
+    SectorParam[i].AngleMin = SectorParam[i].Alpha - dAlpha / 2.f;
+    SectorParam[i].AngleMax = SectorParam[i].Alpha + dAlpha / 2.f;
   }
 
-  par.assumeConstantBz = false;
-  par.toyMCEventsFlag = false;
   par.continuousTracking = false;
   continuousMaxTimeBin = 0;
   tpcCutTimeBin = 0;
-  par.debugLevel = 0;
   par.earlyTpcTransform = false;
 }
 
 void GPUParam::UpdateSettings(const GPUSettingsGRP* g, const GPUSettingsProcessing* p, const GPURecoStepConfiguration* w, const GPUSettingsRecDynamic* d)
 {
   if (g) {
-    UpdateBzOnly(g->solenoidBzNominalGPU);
-    par.assumeConstantBz = g->constBz;
-    par.toyMCEventsFlag = g->homemadeEvents;
+    UpdateBzOnly(g->solenoidBzNominalGPU, g->constBz);
     par.continuousTracking = g->grpContinuousMaxTimeBin != 0;
     continuousMaxTimeBin = g->grpContinuousMaxTimeBin == -1 ? GPUSettings::TPC_MAX_TF_TIME_BIN : g->grpContinuousMaxTimeBin;
     tpcCutTimeBin = g->tpcCutTimeBin;
   }
   par.earlyTpcTransform = rec.tpc.forceEarlyTransform == -1 ? (!par.continuousTracking) : rec.tpc.forceEarlyTransform;
   qptB5Scaler = CAMath::Abs(bzkG) > 0.1f ? CAMath::Abs(bzkG) / 5.006680f : 1.f; // Repeat here, since passing in g is optional
   if (p) {
-    par.debugLevel = p->debugLevel;
     UpdateRun3ClusterErrors(p->param.tpcErrorParamY, p->param.tpcErrorParamZ);
   }
   if (w) {
@@ -145,12 +138,12 @@ void GPUParam::UpdateSettings(const GPUSettingsGRP* g, const GPUSettingsProcessi
   }
 }
 
-void GPUParam::UpdateBzOnly(float newSolenoidBz)
+void GPUParam::UpdateBzOnly(float newSolenoidBz, bool assumeConstantBz)
 {
   bzkG = newSolenoidBz;
   bzCLight = bzkG * o2::gpu::gpu_common_constants::kCLight;
   polynomialField.Reset();
-  if (par.assumeConstantBz) {
+  if (assumeConstantBz) {
     GPUTPCGMPolynomialFieldManager::GetPolynomialField(GPUTPCGMPolynomialFieldManager::kUniform, bzkG, polynomialField);
   } else {
     GPUTPCGMPolynomialFieldManager::GetPolynomialField(bzkG, polynomialField);
@@ -160,7 +153,7 @@ void GPUParam::UpdateBzOnly(float newSolenoidBz)
 
 void GPUParam::SetDefaults(const GPUSettingsGRP* g, const GPUSettingsRec* r, const GPUSettingsProcessing* p, const GPURecoStepConfiguration* w)
 {
-  SetDefaults(g->solenoidBzNominalGPU);
+  SetDefaults(g->solenoidBzNominalGPU, g->constBz);
   if (r) {
     rec = *r;
     if (rec.fitPropagateBzOnly == -1) {

GPU/GPUTracking/Base/GPUParam.h

@@ -47,6 +47,8 @@ namespace internal
 {
 template <class T, class S>
 struct GPUParam_t {
+  static constexpr float dAlpha = 0.349066f;
+
   T rec;
   S par;
 
@@ -77,10 +79,10 @@ struct GPUParam_t {
 struct GPUParam : public internal::GPUParam_t<GPUSettingsRec, GPUSettingsParam> {
 
 #ifndef GPUCA_GPUCODE
-  void SetDefaults(float solenoidBz);
+  void SetDefaults(float solenoidBz, bool assumeConstantBz);
   void SetDefaults(const GPUSettingsGRP* g, const GPUSettingsRec* r = nullptr, const GPUSettingsProcessing* p = nullptr, const GPURecoStepConfiguration* w = nullptr);
   void UpdateSettings(const GPUSettingsGRP* g, const GPUSettingsProcessing* p = nullptr, const GPURecoStepConfiguration* w = nullptr, const GPUSettingsRecDynamic* d = nullptr);
-  void UpdateBzOnly(float newSolenoidBz);
+  void UpdateBzOnly(float newSolenoidBz, bool assumeConstantBz);
   void UpdateRun3ClusterErrors(const float* yErrorParam, const float* zErrorParam);
 #endif
 
@@ -92,7 +94,7 @@ struct GPUParam : public internal::GPUParam_t<GPUSettingsRec, GPUSettingsParam>
     if (iSector >= GPUCA_NSECTORS / 4) {
       iSector -= GPUCA_NSECTORS / 2;
     }
-    return 0.174533f + par.dAlpha * iSector;
+    return 0.174533f + dAlpha * iSector;
   }
   GPUd() float GetClusterErrorSeeding(int32_t yz, int32_t type, float zDiff, float angle2, float unscaledMult) const;
   GPUd() void GetClusterErrorsSeeding2(uint8_t sector, int32_t row, float z, float sinPhi, float DzDs, float time, float& ErrY2, float& ErrZ2) const;

GPU/GPUTracking/Base/GPUReconstructionTimeframe.cxx

@@ -25,14 +25,13 @@
 #include "TPCFastTransform.h"
 #include "CorrectionMapsHelper.h"
 #include "GPUO2DataTypes.h"
+#include "GPUSettings.h"
 
 #include <cstdio>
 #include <exception>
 #include <memory>
 #include <cstring>
 
-#include "utils/qconfig.h"
-
 using namespace o2::gpu;
 
 namespace o2::gpu

GPU/GPUTracking/Base/cuda/GPUReconstructionCUDA.cu

@@ -125,34 +125,25 @@ int32_t GPUReconstructionCUDA::InitDevice_Runtime()
     }
     std::vector<bool> devicesOK(count, false);
     std::vector<size_t> devMemory(count, 0);
-    bool contextCreated = false;
+    std::vector<bool> contextCreated(count, false);
     for (int32_t i = 0; i < count; i++) {
       if (GetProcessingSettings().debugLevel >= 4) {
         GPUInfo("Examining device %d", i);
       }
       size_t free, total;
-#ifndef __HIPCC__ // CUDA
-      if (GPUChkErrI(cudaInitDevice(i, 0, 0))) {
-#else // HIP
-      if (GPUChkErrI(hipSetDevice(i))) {
-#endif
+      if (GPUChkErrI(cudaSetDevice(i))) {
         if (GetProcessingSettings().debugLevel >= 4) {
           GPUWarning("Couldn't create context for device %d. Skipping it.", i);
         }
         continue;
       }
-      contextCreated = true;
+      contextCreated[i] = true;
       if (GPUChkErrI(cudaMemGetInfo(&free, &total))) {
         if (GetProcessingSettings().debugLevel >= 4) {
           GPUWarning("Error obtaining CUDA memory info about device %d! Skipping it.", i);
         }
-        GPUChkErr(cudaDeviceReset());
         continue;
       }
-      if (count > 1) {
-        GPUChkErr(cudaDeviceReset());
-        contextCreated = false;
-      }
       if (GetProcessingSettings().debugLevel >= 4) {
         GPUInfo("Obtained current memory usage for device %d", i);
       }
@@ -212,13 +203,20 @@ int32_t GPUReconstructionCUDA::InitDevice_Runtime()
         bestDevice = GetProcessingSettings().deviceNum;
       }
     }
-    if (noDevice) {
-      if (contextCreated) {
+    for (int32_t i = 0; i < count; i++) {
+      if (contextCreated[i] && (noDevice || i != bestDevice)) {
+        GPUChkErrI(cudaSetDevice(i));
         GPUChkErrI(cudaDeviceReset());
       }
+    }
+    if (noDevice) {
       return (1);
     }
     mDeviceId = bestDevice;
+    if (GPUChkErrI(cudaSetDevice(mDeviceId))) {
+      GPUError("Could not set CUDA Device!");
+      return (1);
+    }
 
     GPUChkErrI(cudaGetDeviceProperties(&deviceProp, mDeviceId));
 
@@ -262,15 +260,6 @@ int32_t GPUReconstructionCUDA::InitDevice_Runtime()
     }
 #endif
 
-#ifndef __HIPCC__ // CUDA
-    if (contextCreated == 0 && GPUChkErrI(cudaInitDevice(mDeviceId, 0, 0))) {
-#else // HIP
-    if (contextCreated == 0 && GPUChkErrI(hipSetDevice(mDeviceId))) {
-#endif
-      GPUError("Could not set CUDA Device!");
-      return (1);
-    }
-
 #ifndef __HIPCC__ // CUDA
     if (GPUChkErrI(cudaDeviceSetLimit(cudaLimitStackSize, GPUCA_GPU_STACK_SIZE))) {
       GPUError("Error setting CUDA stack size");
@@ -420,8 +409,10 @@ void GPUReconstructionCUDA::genAndLoadRTC()
       mInternals->kernelModules.emplace_back(std::make_unique<CUmodule>());
       GPUChkErr(cuModuleLoad(mInternals->kernelModules.back().get(), (filename + "_" + std::to_string(i) + mRtcBinExtension).c_str()));
     }
-    remove((filename + "_" + std::to_string(i) + mRtcSrcExtension).c_str());
-    remove((filename + "_" + std::to_string(i) + mRtcBinExtension).c_str());
+    if (!GetProcessingSettings().rtctech.keepTempFiles) {
+      remove((filename + "_" + std::to_string(i) + mRtcSrcExtension).c_str());
+      remove((filename + "_" + std::to_string(i) + mRtcBinExtension).c_str());
+    }
   }
   if (GetProcessingSettings().rtctech.runTest == 2) {
     return;

GPU/GPUTracking/DataCompression/GPUTPCCompressionTrackModel.cxx

@@ -27,7 +27,6 @@ GPUd() void GPUTPCCompressionTrackModel::Init(float x, float y, float z, float a
 {
   mProp.SetMaterialTPC();
   mProp.SetMaxSinPhi(GPUCA_MAX_SIN_PHI);
-  mProp.SetToyMCEventsFlag(false);
   mProp.SetSeedingErrors(true); // Larger errors for seeds, better since we don't start with good hypothesis
   mProp.SetFitInProjections(true);
   mProp.SetPropagateBzOnly(true);