Added FWIO/RNTupleTempTests items

  - based on IOPool/Tests
  - includes changes needed to make tests work

Author: Dr15Jones

FWIO/RNTupleTempInput/bin/CollUtil.cc

@@ -16,6 +16,7 @@
 #include "TObject.h"
 #include "TTree.h"
 #include "ROOT/RNTupleReader.hxx"
+#include "ROOT/RNTupleDescriptor.hxx"
 
 #include <iomanip>
 #include <iostream>
@@ -71,7 +72,6 @@ namespace edm::rntuple_temp {
       }
     }
     Long64_t branchCompressedSizes(TBranch *branch) { return branch->GetZipBytes("*"); }
-
   }  // namespace
 
   void printBranchNames(TTree *tree) {
@@ -100,6 +100,53 @@ namespace edm::rntuple_temp {
     }
   }
 
+  namespace {
+    Long64_t storageForField(ROOT::RFieldDescriptor const &iField, ROOT::RNTupleDescriptor const &iDesc) {
+      Long64_t storage = 0;
+      for (auto &col : iDesc.GetColumnIterable(iField)) {
+        if (col.IsAliasColumn()) {
+          continue;
+        }
+        auto id = col.GetPhysicalId();
+
+        for (auto &cluster : iDesc.GetClusterIterable()) {
+          auto columnRange = cluster.GetColumnRange(id);
+          if (columnRange.IsSuppressed()) {
+            continue;
+          }
+          const auto &pageRange = cluster.GetPageRange(id);
+          for (const auto &page : pageRange.GetPageInfos()) {
+            storage += page.GetLocator().GetNBytesOnStorage();
+          }
+        }
+      }
+      return storage;
+    }
+
+    Long64_t storageForFieldAndSubFields(ROOT::RFieldDescriptor const &iField, ROOT::RNTupleDescriptor const &iDesc) {
+      Long64_t storage = storageForField(iField, iDesc);
+      for (auto const &sub : iDesc.GetFieldIterable(iField)) {
+        storage += storageForFieldAndSubFields(sub, iDesc);
+      }
+      return storage;
+    }
+  }  // namespace
+
+  void printFieldNames(ROOT::RNTupleReader &reader) {
+    auto &desc = reader.GetDescriptor();
+    auto topLevel = desc.GetTopLevelFields();
+    unsigned int index = 0;
+    for (auto const &topField : topLevel) {
+      Long64_t storage = storageForFieldAndSubFields(topField, desc);
+      std::cout << "Field " << index << " of " << desc.GetName() << " RNTuple: " << topField.GetFieldName()
+                << " Bytes On Storage = " << storage << std::endl;
+
+      ++index;
+    }
+  }
+
+  void longFieldPrint(ROOT::RNTupleReader &reader) { reader.PrintInfo(ROOT::ENTupleInfo::kStorageDetails); }
+
   namespace {
     class BranchBasketBytes {
     public:
@@ -283,6 +330,9 @@ namespace edm::rntuple_temp {
     processClusters(tr, BasketPrinter{}, branchName);
   }
 
+  void clusterPrint(ROOT::RNTupleReader &) {}
+  void pagePrint(ROOT::RNTupleReader &, std::string const &) {}
+
   std::string getUuid(ROOT::RNTupleReader *uuidTree) {
     FileID fid;
     auto entry = uuidTree->GetModel().CreateEntry();

FWIO/RNTupleTempInput/bin/CollUtil.h

@@ -25,6 +25,12 @@ namespace edm::rntuple_temp {
   void printUuids(ROOT::RNTupleReader *uuidTree);
   void printEventLists(TFile *tfl);
   void printEventsInLumis(TFile *tfl);
+
+  void printFieldNames(ROOT::RNTupleReader &tree);
+  void longFieldPrint(ROOT::RNTupleReader &tr);
+  void clusterPrint(ROOT::RNTupleReader &tr);
+  void pagePrint(ROOT::RNTupleReader &tr, const std::string &branchName);
+
 }  // namespace edm::rntuple_temp
 
 #endif

FWIO/RNTupleTempInput/bin/EdmRNTupleTempFileUtil.cpp

@@ -24,6 +24,7 @@
 
 #include "TFile.h"
 #include "TError.h"
+#include "TTree.h"
 #include "ROOT/RNTuple.hxx"
 #include "ROOT/RNTupleReader.hxx"
 
@@ -267,27 +268,53 @@ int main(int argc, char* argv[]) {
       }
 
       if (print or printBranchDetails or printClusters or not printBaskets.empty()) {
-        TTree* printTree = (TTree*)tfile->Get(selectedTree.c_str());
-        if (printTree == nullptr) {
-          std::cout << "Tree " << selectedTree << " appears to be missing. Could not find it in the file.\n";
+        auto container = tfile->GetObjectUnchecked(selectedTree.c_str());
+        if (container == nullptr) {
+          std::cout << "RNTuple " << selectedTree << " appears to be missing. Could not find it in the file.\n";
           std::cout << "Exiting\n";
           return 1;
         }
-        // Print out each tree
-        if (print) {
-          edm::rntuple_temp::printBranchNames(printTree);
-        }
+        auto* printRNTuple = tfile->Get<ROOT::RNTuple>(selectedTree.c_str());
+        if (printRNTuple) {
+          auto reader = ROOT::RNTupleReader::Open(*printRNTuple);
+          if (print) {
+            edm::rntuple_temp::printFieldNames(*reader);
+          }
 
-        if (printBranchDetails) {
-          edm::rntuple_temp::longBranchPrint(printTree);
-        }
+          if (printBranchDetails) {
+            edm::rntuple_temp::longFieldPrint(*reader);
+          }
 
-        if (printClusters) {
-          edm::rntuple_temp::clusterPrint(printTree);
-        }
+          if (printClusters) {
+            edm::rntuple_temp::clusterPrint(*reader);
+          }
+
+          if (not printBaskets.empty()) {
+            edm::rntuple_temp::pagePrint(*reader, printBaskets);
+          }
+        } else {
+          TTree* printTree = (TTree*)tfile->Get(selectedTree.c_str());
+          if (printTree == nullptr) {
+            std::cout << "Tree " << selectedTree << " appears to be missing. Could not find it in the file.\n";
+            std::cout << "Exiting\n";
+            return 1;
+          }
+          // Print out each tree
+          if (print) {
+            edm::rntuple_temp::printBranchNames(printTree);
+          }
+
+          if (printBranchDetails) {
+            edm::rntuple_temp::longBranchPrint(printTree);
+          }
+
+          if (printClusters) {
+            edm::rntuple_temp::clusterPrint(printTree);
+          }
 
-        if (not printBaskets.empty()) {
-          edm::rntuple_temp::basketPrint(printTree, printBaskets);
+          if (not printBaskets.empty()) {
+            edm::rntuple_temp::basketPrint(printTree, printBaskets);
+          }
         }
       }
 

FWIO/RNTupleTempInput/src/RootFile.cc

@@ -246,7 +246,9 @@ namespace edm::rntuple_temp {
     metaDataEntry->BindRawPtr(poolNames::indexIntoFileBranchName(), &indexIntoFile_);
 
     storedProcessBlockHelper_ = std::make_unique<StoredProcessBlockHelper>();
-    if (inputType == InputType::Primary) {
+    if (inputType == InputType::Primary and
+        metaDataReader->GetModel().GetFieldNames().end() !=
+            metaDataReader->GetModel().GetFieldNames().find(poolNames::processBlockHelperBranchName())) {
       metaDataEntry->BindRawPtr(poolNames::processBlockHelperBranchName(), storedProcessBlockHelper_.get());
     }
 
@@ -478,9 +480,13 @@ namespace edm::rntuple_temp {
       processingOrderMerge(*processHistoryRegistry_, processingOrder);
       newReg->setProcessOrder(processingOrder);
 
-      // freeze the product registry
-      newReg->setFrozen(inputType != InputType::Primary);
-      productRegistry_.reset(newReg.release());
+      if (not processingOrder.empty()) {
+        // freeze the product registry
+        newReg->setFrozen(inputType != InputType::Primary);
+        productRegistry_.reset(newReg.release());
+      } else {
+        productRegistry_ = std::make_shared<ProductRegistry>();
+      }
     }
 
     // Set up information from the product registry.
@@ -541,6 +547,7 @@ namespace edm::rntuple_temp {
 
   RootFile::~RootFile() {}
 
+  bool RootFile::empty() const { return runTree_.entries() == 0; }
   void RootFile::readEntryDescriptionTree(EntryDescriptionMap& entryDescriptionMap, InputType inputType) {
     // Called only for old format files.
     // We use a smart pointer so the tree will be deleted after use, and not kept for the life of the file.

FWIO/RNTupleTempInput/src/RootFile.h

@@ -157,6 +157,8 @@ namespace edm {
       std::array<bool, NumBranchTypes> const& hasNewlyDroppedBranch() const { return hasNewlyDroppedBranch_; }
       bool branchListIndexesUnchanged() const { return branchListIndexesUnchanged_; }
       bool modifiedIDs() const { return daqProvenanceHelper_.get() != nullptr; }
+      //Are there no data stored in the file?
+      bool empty() const;
       std::shared_ptr<FileBlock> createFileBlock();
       void updateFileBlock(FileBlock&);
 

FWIO/RNTupleTempOutput/src/RootOutputFile.cc

@@ -589,7 +589,9 @@ namespace edm::rntuple_temp {
       model->AddField(setupBranchIDListRegistry());
       model->AddField(setupThinnedAssociationsHelper());
       model->AddField(setupProductDependencies());
-      model->AddField(setupProcessBlockHelper());
+      if (!om_->outputProcessBlockHelper().processesWithProcessBlockProducts().empty()) {
+        model->AddField(setupProcessBlockHelper());
+      }
     }
 
     auto writeOptions = ROOT::RNTupleWriteOptions();

FWIO/RNTupleTempTests/test/AssociationMapReadTest_cfg.py

@@ -0,0 +1,31 @@
+import FWCore.ParameterSet.Config as cms
+
+process = cms.Process("READ")
+
+process.load("FWCore.MessageLogger.MessageLogger_cfi")
+
+process.MessageLogger.cerr.enableStatistics = False
+
+
+process.load("FWCore.Framework.test.cmsExceptionsFatal_cff")
+
+process.source = cms.Source("RNTupleTempSource",
+    fileNames = cms.untracked.vstring(
+        'file:AssociationMapTest.root'
+    )
+)
+
+process.test = cms.EDAnalyzer("AssociationMapAnalyzer",
+    inputTag1 = cms.InputTag("intvec1"),
+    inputTag2 = cms.InputTag("intvec2"),
+    associationMapTag1 = cms.InputTag("associationMapProducer"),
+    associationMapTag2 = cms.InputTag("associationMapProducer", "twoArg"),
+    associationMapTag3 = cms.InputTag("associationMapProducer"),
+    associationMapTag4 = cms.InputTag("associationMapProducer", "handleArg"),
+    associationMapTag5 = cms.InputTag("associationMapProducer"),
+    associationMapTag6 = cms.InputTag("associationMapProducer"),
+    associationMapTag7 = cms.InputTag("associationMapProducer"),
+    associationMapTag8 = cms.InputTag("associationMapProducer", "twoArg")
+)
+
+process.p = cms.Path(process.test)

FWIO/RNTupleTempTests/test/AssociationMapTest_cfg.py

@@ -0,0 +1,54 @@
+import FWCore.ParameterSet.Config as cms
+
+process = cms.Process("PROD")
+
+process.load("FWCore.MessageLogger.MessageLogger_cfi")
+
+process.MessageLogger.cerr.enableStatistics = False
+
+
+process.load("FWCore.Framework.test.cmsExceptionsFatal_cff")
+
+process.maxEvents = cms.untracked.PSet(
+    input = cms.untracked.int32(3)
+)
+
+process.source = cms.Source("EmptySource")
+
+process.intvec1 = cms.EDProducer("IntVectorProducer",
+    count = cms.int32(9),
+    ivalue = cms.int32(11),
+    delta = cms.int32(1)
+)
+
+process.intvec2 = cms.EDProducer("IntVectorProducer",
+    count = cms.int32(9),
+    ivalue = cms.int32(21),
+    delta = cms.int32(1)
+)
+
+process.associationMapProducer = cms.EDProducer("AssociationMapProducer",
+    inputTag1 = cms.InputTag("intvec1"),
+    inputTag2 = cms.InputTag("intvec2")
+)
+
+process.test = cms.EDAnalyzer("AssociationMapAnalyzer",
+    inputTag1 = cms.InputTag("intvec1"),
+    inputTag2 = cms.InputTag("intvec2"),
+    associationMapTag1 = cms.InputTag("associationMapProducer"),
+    associationMapTag2 = cms.InputTag("associationMapProducer", "twoArg"),
+    associationMapTag3 = cms.InputTag("associationMapProducer"),
+    associationMapTag4 = cms.InputTag("associationMapProducer", "handleArg"),
+    associationMapTag5 = cms.InputTag("associationMapProducer"),
+    associationMapTag6 = cms.InputTag("associationMapProducer"),
+    associationMapTag7 = cms.InputTag("associationMapProducer"),
+    associationMapTag8 = cms.InputTag("associationMapProducer", "twoArg")
+)
+
+process.out = cms.OutputModule("RNTupleTempOutputModule",
+    fileName = cms.untracked.string('AssociationMapTest.root')
+)
+
+process.p = cms.Path(process.intvec1 * process.intvec2 * process.associationMapProducer * process.test)
+
+process.e = cms.EndPath(process.out)