Test provenance of dropped products

Only provenance of dropped ancestors are stored in ROOT files. Using untracked parameter for InputTag keeps the ProcessHistory the same so there is only one Run in the workflow.

Author: Dr15Jones

FWCore/Framework/interface/OccurrenceForOutput.h

@@ -64,6 +64,7 @@ namespace edm {
     Handle<PROD> getHandle(EDGetTokenT<PROD> token) const;
 
     Provenance getProvenance(BranchID const& theID) const;
+    StableProvenance const& getStableProvenance(BranchID const& ithID) const;
 
     void getAllProvenance(std::vector<Provenance const*>& provenances) const;
 

FWCore/Framework/src/OccurrenceForOutput.cc

@@ -29,6 +29,10 @@ namespace edm {
     return provRecorder_.principal().getProvenance(bid);
   }
 
+  StableProvenance const& OccurrenceForOutput::getStableProvenance(BranchID const& bid) const {
+    return provRecorder_.principal().getStableProvenance(bid);
+  }
+
   void OccurrenceForOutput::getAllProvenance(std::vector<Provenance const*>& provenances) const {
     provRecorder_.principal().getAllProvenance(provenances);
   }

FWCore/Framework/test/stubs/ToyIntProducers.cc

@@ -458,13 +458,23 @@ namespace edmtest {
       for (auto const& label : labels) {
         tokens_.emplace_back(consumes(label));
       }
+      {
+        auto const& labels = p.getUntrackedParameter<std::vector<edm::InputTag>>("untrackedLabels");
+        for (auto const& label : labels) {
+          tokens_.emplace_back(consumes(label));
+        }
+      }
     }
     void produce(edm::StreamID, edm::Event& e, edm::EventSetup const& c) const override;
 
     static void fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
       edm::ParameterSetDescription desc;
       desc.addUntracked<unsigned int>("onlyGetOnEvent", 0u);
       desc.add<std::vector<edm::InputTag>>("labels");
+      desc.addUntracked<std::vector<edm::InputTag>>("untrackedLabels", {})
+          ->setComment(
+              "Using this can change the stored ProductRegistry for the same ProcessHistory if this is the only module "
+              "that depends on these labels.");
       descriptions.addDefault(desc);
     }
 

FWCore/Integration/test/BuildFile.xml

@@ -43,6 +43,7 @@
   <test name="TestIntegrationEmptyRootFile" command="run_TestEmptyRootFile.sh"/>
   <test name="TestStdProducts" command="cmsRun ${LOCALTOP}/src/FWCore/Integration/test/testStdProducts_cfg.py"/>
   <test name="TestPostInsertProducer" command="cmsRun ${LOCALTOP}/src/FWCore/Integration/test/testPostInsertProducer_cfg.py"/>
+  <test name="TestFWCoreIntegrationProvenance" command ="provenance_test.sh"/>
 
   <bin file="ProcessConfiguration_t.cpp,ProcessHistory_t.cpp" name="TestIntegrationDataFormatsProvenance">
     <use name="FWCore/ParameterSet"/>

FWCore/Integration/test/provenance_check_cfg.py

@@ -0,0 +1,13 @@
+import FWCore.ParameterSet.Config as cms
+
+process = cms.Process("READ")
+
+from IOPool.Input.modules import PoolSource
+
+process.source = PoolSource(fileNames = ["file:prov.root", "file:prov_extra.root"])
+
+from FWCore.Modules.modules import ProvenanceCheckerOutputModule, AsciiOutputModule
+process.out = ProvenanceCheckerOutputModule()
+process.prnt = AsciiOutputModule(verbosity = 2, allProvenance=True)
+
+process.e = cms.EndPath(process.out+process.prnt)

FWCore/Integration/test/provenance_prod_cfg.py

@@ -0,0 +1,41 @@
+import FWCore.ParameterSet.Config as cms
+from argparse import ArgumentParser
+
+parser = ArgumentParser(description='Write streamer output file for provenance read test')
+parser.add_argument("--consumeProd2", help="add an extra producer to the job and drop on output", action="store_true")
+args = parser.parse_args()
+
+
+process = cms.Process("OUTPUT")
+
+from FWCore.Modules.modules import EmptySource
+
+runNumber = 1
+eventNumber = 1
+if args.consumeProd2:
+    eventNumber = 2
+
+process.source = EmptySource(firstRun = runNumber, firstEvent = eventNumber )
+
+from FWCore.Framework.modules import AddIntsProducer, IntProducer
+
+process.one = IntProducer(ivalue=1)
+process.two = IntProducer(ivalue=2)
+process.sum = AddIntsProducer(labels=['one'])
+process.t = cms.Task(process.one, process.two, process.sum)
+
+baseOutFileName = "prov"
+if args.consumeProd2 :
+    process.sum.untrackedLabels = ['two']
+    baseOutFileName += "_extra"
+
+
+from IOPool.Output.modules import PoolOutputModule
+
+process.out = PoolOutputModule(fileName = baseOutFileName+".root",
+                               outputCommands = ["drop *", "keep *_sum_*_*"])
+
+from FWCore.Modules.modules import AsciiOutputModule
+process.prnt = AsciiOutputModule(verbosity = 2, allProvenance = True)
+process.e = cms.EndPath(process.out+process.prnt, process.t)
+process.maxEvents.input = 1

FWCore/Integration/test/provenance_test.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+
+function die { echo $1: status $2 ;  exit $2; }
+
+#The two jobs will have different ProductRegistries in their output files but have the same ProcessHistory.
+# The ProductRegistry just differ because the internal dependencies between the data products is different
+# and PoolOutputModule only stores provenance of 'dropped' data products IFF they are parents of a kept product. 
+# The check makes sure the provenance in the ProductRegistry is properly updated when the new file is read
+cmsRun ${SCRAM_TEST_PATH}/provenance_prod_cfg.py || die 'Failed in provenance_prod_cfg.py' $?
+cmsRun ${SCRAM_TEST_PATH}/provenance_prod_cfg.py  --consumeProd2 || die 'Failed in provenance_prod_cfg.py  --consumeProd2' $?
+cmsRun ${SCRAM_TEST_PATH}/provenance_check_cfg.py || die 'Failed test of provenance' $?

FWCore/Modules/src/AsciiOutputModule.cc

@@ -34,14 +34,16 @@ namespace edm {
     int prescale_;
     int verbosity_;
     int counter_;
+    bool allProvenance_;
   };
 
   AsciiOutputModule::AsciiOutputModule(ParameterSet const& pset)
       : global::OutputModuleBase(pset),
         global::OutputModule<>(pset),
         prescale_(pset.getUntrackedParameter<unsigned int>("prescale")),
         verbosity_(pset.getUntrackedParameter<unsigned int>("verbosity")),
-        counter_(0) {
+        counter_(0),
+        allProvenance_(pset.getUntrackedParameter<bool>("allProvenance")) {
     if (prescale_ == 0)
       prescale_ = 1;
   }
@@ -81,6 +83,29 @@ namespace edm {
         auto const& prov = e.getProvenance(desc.originalBranchID());
         LogAbsolute("AsciiOut") << prov;
 
+        if (verbosity_ > 2) {
+          ProductDescription const& desc2 = prov.productDescription();
+          std::string const& process = desc2.processName();
+          std::string const& label = desc2.moduleLabel();
+          ProcessHistory const& processHistory = e.processHistory();
+
+          for (ProcessConfiguration const& pc : processHistory) {
+            if (pc.processName() == process) {
+              ParameterSetID const& psetID = pc.parameterSetID();
+              pset::Registry const* psetRegistry = pset::Registry::instance();
+              ParameterSet const* processPset = psetRegistry->getMapped(psetID);
+              if (processPset) {
+                if (desc.isAlias()) {
+                  LogAbsolute("AsciiOut") << "Alias PSet\n" << processPset->getParameterSet(desc.moduleLabel());
+                }
+                LogAbsolute("AsciiOut") << processPset->getParameterSet(label) << "\n";
+              }
+            }
+          }
+        }
+      } else if (allProvenance_) {
+        auto const& prov = e.getStableProvenance(desc.originalBranchID());
+        LogAbsolute("AsciiOut") << prov;
         if (verbosity_ > 2) {
           ProductDescription const& desc2 = prov.productDescription();
           std::string const& process = desc2.processName();
@@ -115,6 +140,8 @@ namespace edm {
             "1: event ID and timestamp only\n"
             "2: provenance for each kept product\n"
             ">2: PSet and provenance for each kept product");
+    desc.addUntracked("allProvenance", false)
+        ->setComment("when printing provenance info, also print stable provenance of non-kept data products.");
     OutputModule::fillDescription(desc);
     descriptions.add("asciiOutput", desc);
   }