Merge pull request cms-sw#33555 from christopheralanwest/alca-run-dependent-mc_12_0_X

Make run-dependent MC configurable with cmsDriver options

Author: cmsbuild

CalibCalorimetry/EcalLaserCorrection/plugins/EcalLaserCorrectionServiceMC.cc

@@ -22,12 +22,16 @@
 #include "CondFormats/DataRecord/interface/EcalLaserAPDPNRatiosMCRcd.h"
 #include "CondFormats/DataRecord/interface/EcalLinearCorrectionsRcd.h"
 
+#include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
+#include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
+
 class EcalLaserCorrectionServiceMC : public edm::ESProducer {
 public:
   EcalLaserCorrectionServiceMC(const edm::ParameterSet&);
   ~EcalLaserCorrectionServiceMC() override;
 
   std::shared_ptr<EcalLaserDbService> produce(const EcalLaserDbRecordMC&);
+  static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);
 
 private:
   using HostType = edm::ESProductHost<EcalLaserDbService,
@@ -64,6 +68,10 @@ EcalLaserCorrectionServiceMC::~EcalLaserCorrectionServiceMC() {}
 //
 // member functions
 //
+void EcalLaserCorrectionServiceMC::fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
+  edm::ParameterSetDescription desc;
+  descriptions.add("EcalLaserCorrectionServiceMC", desc);
+}
 
 // ------------ method called to produce the data  ------------
 std::shared_ptr<EcalLaserDbService> EcalLaserCorrectionServiceMC::produce(const EcalLaserDbRecordMC& record) {

Configuration/Applications/python/ConfigBuilder.py

@@ -82,6 +82,8 @@ class Options:
 defaultOptions.fast=False
 defaultOptions.runsAndWeightsForMC = None
 defaultOptions.runsScenarioForMC = None
+defaultOptions.runsAndWeightsForMCIntegerWeights = None
+defaultOptions.runsScenarioForMCIntegerWeights = None
 defaultOptions.runUnscheduled = False
 defaultOptions.timeoutOutput = False
 defaultOptions.nThreads = '1'
@@ -496,6 +498,30 @@ def filesFromOption(self):
             self.additionalCommands.append('import SimGeneral.Configuration.ThrowAndSetRandomRun as ThrowAndSetRandomRun')
             self.additionalCommands.append('ThrowAndSetRandomRun.throwAndSetRandomRun(process.source,%s)'%(self.runsAndWeights))
 
+        # modify source in case of run-dependent MC (Run-3 method)
+        self.runsAndWeightsInt=None
+        if self._options.runsAndWeightsForMCIntegerWeights or self._options.runsScenarioForMCIntegerWeights:
+            if not self._options.isMC :
+                raise Exception("options --runsAndWeightsForMCIntegerWeights and --runsScenarioForMCIntegerWeights are only valid for MC")
+            if self._options.runsAndWeightsForMCIntegerWeights:
+                self.runsAndWeightsInt = eval(self._options.runsAndWeightsForMCIntegerWeights)
+            else:
+                from Configuration.StandardSequences.RunsAndWeights import RunsAndWeights
+                if isinstance(RunsAndWeights[self._options.runsScenarioForMCIntegerWeights], str):
+                    __import__(RunsAndWeights[self._options.runsScenarioForMCIntegerWeights])
+                    self.runsAndWeightsInt = sys.modules[RunsAndWeights[self._options.runsScenarioForMCIntegerWeights]].runProbabilityDistribution
+                else:
+                    self.runsAndWeightsInt = RunsAndWeights[self._options.runsScenarioForMCIntegerWeights]
+
+        if self.runsAndWeightsInt:
+            if not self._options.relval:
+                raise Exception("--relval option required when using --runsAndWeightsInt")
+            if 'DATAMIX' in self._options.step:
+                from SimGeneral.Configuration.LumiToRun import lumi_to_run
+                total_events, events_per_job  = self._options.relval.split(',')
+                lumi_to_run_mapping = lumi_to_run(self.runsAndWeightsInt, int(total_events), int(events_per_job))
+                self.additionalCommands.append("process.source.firstLuminosityBlockForEachRun = cms.untracked.VLuminosityBlockID(*[cms.LuminosityBlockID(x,y) for x,y in " + str(lumi_to_run_mapping) + "])")
+
         return
 
     def addOutput(self):

Configuration/Applications/python/Options.py

@@ -89,7 +89,7 @@
                   default=False)
 
 parser.add_option("--runsAndWeightsForMC",
-                  help="Assign run numbers to MC source according to relatieve weights. [(run1,weight1),...,(runN,weightN)])",
+                  help="Assign run numbers to MC source according to relative weights. [(run1,weight1),...,(runN,weightN)])",
                   default=None,
                   dest="runsAndWeightsForMC")
 
@@ -98,6 +98,16 @@
                   default=None,
                   dest="runsScenarioForMC")
 
+parser.add_option("--runsAndWeightsForMCIntegerWeights",
+                  help="Assign run numbers to MC source according to relative weights where weighting is determined by the number of times the run number appears. [(run1,run2,...,runN)])",
+                  default=None,
+                  dest="runsAndWeightsForMCIntegerWeights")
+
+parser.add_option("--runsScenarioForMCIntegerWeights",
+                  help="Load a scenario to set run numbers in MC with integer IOV weights.)",
+                  default=None,
+                  dest="runsScenarioForMCIntegerWeights")
+
 parser.add_option("--runUnscheduled",
                   help="Automatically convert configuration to run unscheduled the EDProducers/EDFilters that were scheduled",
                   action="store_true",

Configuration/ProcessModifiers/python/runDependent_cff.py

@@ -0,0 +1,7 @@
+import FWCore.ParameterSet.Config as cms
+
+# This modifier is for configuration changes specific to run-dependent MC.
+# In order to preserve the possibily of multi-year MC, only one run-dependent MC
+# modifier should exist and be used for all years.
+
+runDependent =  cms.Modifier()

Configuration/PyReleaseValidation/python/relval_premix.py

@@ -48,7 +48,7 @@
 workflows[250202.18]=['',['TTbar_13UP18','DIGIPRMXUP18_PU25','RECOPRMXUP18_PU25','HARVESTUP18_PU25']]
 workflows[250202.181]=['',['TTbar_13UP18','PREMIXUP18_PU25','DIGIPRMXLOCALUP18_PU25','RECOPRMXUP18_PU25','HARVESTUP18_PU25']]
 workflows[250202.182]=['',['TTbar_13UP18_RD','DIGIPRMXUP18_PU25_RD','RECOPRMXUP18_PU25_RD','HARVESTUP18_PU25_RD']]
-workflows[250202.183]=['',['TTbar_13UP18_RD','DIGIPRMXUP18_PU25_RD_IB','RECOPRMXUP18_PU25_RD','HARVESTUP18_PU25_RD']]
+workflows[250202.183]=['',['TTbar_13UP18_RD_IB','DIGIPRMXUP18_PU25_RD_IB','RECOPRMXUP18_PU25_RD','HARVESTUP18_PU25_RD']]
 workflows[250203.18]=['',['H125GGgluonfusion_13UP18','DIGIPRMXUP18_PU25','RECOPRMXUP18_PU25','HARVESTUP18_PU25']]
 workflows[250204.18]=['',['QQH1352T_13UP18','DIGIPRMXUP18_PU25','RECOPRMXUP18_PU25','HARVESTUP18_PU25']]
 workflows[250205.18]=['',['ZTT_13UP18','DIGIPRMXUP18_PU25','RECOPRMXUP18_PU25','HARVESTUP18_PU25']]

Configuration/PyReleaseValidation/python/relval_steps.py

@@ -1454,13 +1454,17 @@ def lhegensim2018(fragment,howMuch):
     return merge([{'cfg':fragment},howMuch,step1LHEGenSimUp2018Default])
 
 # Run-Dependent MC
+kevents_rd = 9
+events_per_job_rd = 50
 def gen2018RD(fragment,howMuch):
     global step1Up2018Defaults
-    return merge([{'cfg':fragment},howMuch,{'--conditions':'auto:phase1_2018_realistic_rd','--customise_commands': "\"process.source.numberEventsInLuminosityBlock=cms.untracked.uint32(5) \""},step1Up2018Defaults])
+    return merge([{'cfg':fragment},howMuch,{'--conditions':'auto:phase1_2018_realistic_rd','--customise_commands': "\"process.source.numberEventsInLuminosityBlock=cms.untracked.uint32(" + str(events_per_job_rd) + ") \""},step1Up2018Defaults])
 
-steps['ZEE_13UP18_RD']=gen2018RD('ZEE_13TeV_TuneCUETP8M1_cfi',Kby(10,50))
-steps['ZMM_13UP18_RD']=gen2018RD('ZMM_13TeV_TuneCUETP8M1_cfi',Kby(10,50))
-steps['TTbar_13UP18_RD']=gen2018RD('TTbar_13TeV_TuneCUETP8M1_cfi',Kby(10,50))
+steps['ZEE_13UP18_RD']=gen2018RD('ZEE_13TeV_TuneCUETP8M1_cfi',Kby(kevents_rd,events_per_job_rd))
+steps['ZMM_13UP18_RD']=gen2018RD('ZMM_13TeV_TuneCUETP8M1_cfi',Kby(kevents_rd,events_per_job_rd))
+steps['TTbar_13UP18_RD']=gen2018RD('TTbar_13TeV_TuneCUETP8M1_cfi',Kby(kevents_rd,events_per_job_rd))
+steps['TTbar_13UP18_RD_IB']=gen2018RD('TTbar_13TeV_TuneCUETP8M1_cfi',Kby(kevents_rd,events_per_job_rd))
+steps['TTbar_13UP18_RD_IB']['--customise_commands'] = "\"process.source.numberEventsInLuminosityBlock=cms.untracked.uint32(5)\""
 
 steps['TTbar012Jets_NLO_Mad_py8_Evt_13']=lhegensim('Configuration/Generator/python/TTbar012Jets_5f_NLO_FXFX_Madgraph_LHE_13TeV_cfi.py',Kby(9,50))
 steps['GluGluHToZZTo4L_M125_Pow_py8_Evt_13']=lhegensim('Configuration/Generator/python/GGHZZ4L_JHUGen_Pow_NNPDF30_LHE_13TeV_cfi.py', Kby(9,50))
@@ -1797,16 +1801,17 @@ def lhegensim2018ml(fragment,howMuch):
 steps['DIGIPRMXLOCALUP18_PU25']=merge([digiPremixLocalPileupUp2018Defaults25ns])
 
 # Run-Dependent MC: DIGI step; 17 is a dummy test; 2018 with 2000 lumis
-steps['DIGIPRMXUP17_PU25_RD']=merge([digiPremixUp2017Defaults25ns, { '--customise_commands':"\"process.EcalLaserCorrectionServiceMC = cms.ESProducer('EcalLaserCorrectionServiceMC') \\n process.GlobalTag.toGet = cms.VPSet( cms.PSet(     record = cms.string('EcalLaserAPDPNRatiosMCRcd'),  tag = cms.string('EcalLaserAPDPNRatios_UL_2017_mc'),  connect = cms.string('frontier://FrontierProd/CMS_CONDITIONS') ) )  \\n process.mixData.workers.ecal.timeDependent=True\"" } ])
+steps['DIGIPRMXUP17_PU25_RD']=merge([{'--procModifiers':'premix_stage2,runDependent'}, digiPremixUp2017Defaults25ns, { '--customise_commands':"\"process.GlobalTag.toGet = cms.VPSet( cms.PSet(     record = cms.string('EcalLaserAPDPNRatiosMCRcd'),  tag = cms.string('EcalLaserAPDPNRatios_UL_2017_mc'),  connect = cms.string('frontier://FrontierProd/CMS_CONDITIONS') ) )\""} ])
 
 digiPremixRD2018 = {
-    '--pileup_input':'das:/RelValPREMIXUP18_PU25/%s/PREMIX'%baseDataSetRelease[24]
+    '--pileup_input':'das:/RelValPREMIXUP18_PU25/%s/PREMIX'%baseDataSetRelease[24],
+    '--procModifiers':'premix_stage2,runDependent'
 }
 
-steps['DIGIPRMXUP18_PU25_RD']=merge([digiPremixRD2018, {'--conditions':'auto:phase1_2018_realistic_rd', '--customise_commands':"\"process.EcalLaserCorrectionServiceMC = cms.ESProducer('EcalLaserCorrectionServiceMC') \\n process.mixData.workers.ecal.timeDependent=True \\n process.source.firstLuminosityBlockForEachRun = cms.untracked.VLuminosityBlockID(*[cms.LuminosityBlockID(x,y) for x,y in ((315257, 1), (316082, 222), (316720, 445), (317527, 668), (320917, 890), (321414, 1112), (321973, 1334), (322492, 1556), (324245, 1779))]) \""}, digiPremixUp2018Defaults25ns])
+steps['DIGIPRMXUP18_PU25_RD']=merge([{'--conditions':'auto:phase1_2018_realistic_rd', '--relval': str(kevents_rd*1000) + ',' + str(events_per_job_rd), '--runsScenarioForMCIntegerWeights': 'Run2018_Equal_Lumi_Integer_Weights'}, digiPremixRD2018, digiPremixUp2018Defaults25ns])
 
 # configuration to simulate cross run number boundary in IB, given 5 events per lumi
-steps['DIGIPRMXUP18_PU25_RD_IB']=merge([digiPremixRD2018, {'--conditions':'auto:phase1_2018_realistic_rd', '--customise_commands':"\"process.EcalLaserCorrectionServiceMC = cms.ESProducer('EcalLaserCorrectionServiceMC') \\n process.mixData.workers.ecal.timeDependent=True \\n process.source.setRunNumberForEachLumi = cms.untracked.vuint32(315257,316083) \""}, digiPremixUp2018Defaults25ns])
+steps['DIGIPRMXUP18_PU25_RD_IB']=merge([{'--conditions':'auto:phase1_2018_realistic_rd', '--customise_commands':"\"process.source.setRunNumberForEachLumi = cms.untracked.vuint32(315257,316083) \"", '--procModifiers':'premix_stage2,runDependent'}, digiPremixRD2018, digiPremixUp2018Defaults25ns])
 
 premixProd25ns = {'-s'             : 'DIGI,DATAMIX,L1,DIGI2RAW,HLT:@relval2016',
                  '--eventcontent' : 'PREMIXRAW',
@@ -2457,7 +2462,7 @@ def gen2021HiMix(fragment,howMuch):
 steps['RECOPRMXUP18_PU25_L1TMuDQM']=merge([{'-s':'RAW2DIGI,L1Reco,RECO,RECOSIM,EI,PAT,VALIDATION:@standardValidationNoHLT+@miniAODValidation,DQM:@standardDQMFakeHLT+@miniAODDQM+@L1TMuon'},steps['RECOPRMXUP18_PU25']])
 
 #Run-Dependent RECO
-step_RECO18_RD = {'--conditions':'auto:phase1_2018_realistic_rd', '--customise_commands':"\"process.EcalLaserCorrectionServiceMC = cms.ESProducer('EcalLaserCorrectionServiceMC') \\n \""}
+step_RECO18_RD = {'--conditions':'auto:phase1_2018_realistic_rd'}
 steps['RECOPRMXUP18_PU25_RD']=merge([step_RECO18_RD, steps['RECOPRMXUP18_PU25']])
 steps['RECOPRMXUP18_PU25_L1TEgDQM_RD']=merge([step_RECO18_RD, steps['RECOPRMXUP18_PU25_L1TEgDQM']])
 steps['RECOPRMXUP18_PU25_L1TMuDQM_RD']=merge([step_RECO18_RD, steps['RECOPRMXUP18_PU25_L1TMuDQM']])
@@ -2981,7 +2986,7 @@ def gen2021HiMix(fragment,howMuch):
 steps['HARVESTUP18_PU25_L1TMuDQM']=steps['HARVESTUP18_L1TMuDQM']
 
 #Run-Dependent harvesting; using MultiRun harvesting + force RunNumber=1 for GUI MC
-step_harvest_MRH_RD = {'--harvesting':'AtJobEnd','--customise_commands':'"process.dqmSaver.forceRunNumber = 1"'}
+step_harvest_MRH_RD = {'--harvesting':'AtJobEnd', '--procModifiers':'runDependent'}
 steps['HARVESTUP18_PU25_RD']=merge([step_harvest_MRH_RD,steps['HARVESTUP18']])
 steps['HARVESTUP18_PU25_L1TEgDQM_RD']=merge([step_harvest_MRH_RD,steps['HARVESTUP18_L1TEgDQM']])
 steps['HARVESTUP18_PU25_L1TMuDQM_RD']=merge([step_harvest_MRH_RD,steps['HARVESTUP18_L1TMuDQM']])

Configuration/StandardSequences/python/AdditionalConditions_cff.py

@@ -2,6 +2,7 @@
 
 # the following are needed for non PoolDBESSources
 from CalibCalorimetry.EcalLaserCorrection.ecalLaserCorrectionService_cfi import *
+from CalibCalorimetry.EcalLaserCorrection.EcalLaserCorrectionServiceMC_cfi import *
 from CalibCalorimetry.HcalPlugins.Hcal_Conditions_forGlobalTag_cff import *
 from CalibCalorimetry.CastorCalib.CastorDbProducer_cfi import *
 from CalibTracker.Configuration.Tracker_DependentRecords_forGlobalTag_nofakes_cff import *

Configuration/StandardSequences/python/DQMSaverAtJobEnd_cff.py

@@ -6,6 +6,8 @@
 # needed output                                                                                                                                                                                                                              
 from DQMServices.Components.DQMEnvironment_cfi import *
 
+# modifications for run-dependent MC
+from Configuration.ProcessModifiers.runDependent_cff import runDependent
 
 dqmSaver.convention = 'Offline'
 dqmSaver.workflow = '/Global/CMSSW_X_Y_Z/RECO'
@@ -17,6 +19,8 @@
 
 DQMSaver = cms.Sequence(dqmSaver)
 
+runDependent.toModify(dqmSaver, forceRunNumber = 1)
+
 # configuration is modified as a side effect, this is just a placeholder
 # to allow using this file as a customisation for cmsDriver.
 def customise(process):

Configuration/StandardSequences/python/RunsAndWeights.py

@@ -1,4 +1,5 @@
 RunsAndWeights = {
     'Run2012_AB_C_D_oneRunPerEra' : 'SimGeneral.Configuration.RunsAndWeights_Run2012_AB_C_D_oneRunPerEra',
-    'Run2018_ABCD' : 'SimGeneral.Configuration.RunsAndWeights_Run2018_ABCD'
+    'Run2018_ABCD' : 'SimGeneral.Configuration.RunsAndWeights_Run2018_ABCD',
+    'Run2018_Equal_Lumi_Integer_Weights' : 'SimGeneral.Configuration.RunsAndWeights_Run2018_Equal_Lumi_Integer_Weights'
     }

SimGeneral/Configuration/python/LumiToRun.py

@@ -0,0 +1,10 @@
+def lumi_to_run(runs, events_in_sample, events_per_job):
+    '''Print tuple for use in firstLuminosityBlockForEachRun'''
+    n_iovs = len(runs)
+    n_lumis = events_in_sample // events_per_job
+    if n_lumis % n_iovs != 0:
+        raise Exception('n_lumis should be evenly divisible by n_iovs.')
+    pairs = []
+    for i, run in enumerate(runs):
+        pairs.append((run, 1 + i*n_lumis//n_iovs))
+    return tuple(pairs)