Skip to content

Commit acb7e14

Browse files
kpedro88kpedro88
committed
reorg for better full-fast separation, handle harvest
1 parent 1862a94 commit acb7e14

File tree

1 file changed

+74
-70
lines changed

1 file changed

+74
-70
lines changed

‎Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py‎

Lines changed: 74 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -2520,29 +2520,12 @@ def setup_(self, step, stepName, stepDict, k, properties):
25202520
# just copy steps
25212521
stepDict[stepName][k] = merge([stepDict[step][k]])
25222522
def setupPU_(self, step, stepName, stepDict, k, properties):
2523-
# setup for stage 1
2524-
if "GenSim" in stepName:
2525-
stepNamePmx = stepName.replace('GenSim','Premix')
2526-
if not stepNamePmx in stepDict: stepDict[stepNamePmx] = {}
2527-
# avoid overwriting fastsim alternative
2528-
if not k in stepDict[stepNamePmx]:
2529-
stepDict[stepNamePmx][k] = merge([
2530-
{
2531-
'-s': 'GEN,SIM,DIGI:pdigi_valid',
2532-
'--datatier': 'PREMIX',
2533-
'--eventcontent': 'PREMIX',
2534-
'--procModifiers': 'premix_stage1'
2535-
},
2536-
stepDict[stepName][k]
2537-
])
2538-
if "ProdLike" in self.suffix:
2539-
stepDict[stepNamePmx][k] = merge([{'-s': 'GEN,SIM,DIGI'},stepDict[stepNamePmx][k]])
2540-
# setup for stage 1 fastsim
2541-
elif "Gen" in stepName:
2542-
stepNamePmx = stepName.replace('Gen','Premix')
2543-
if not stepNamePmx in stepDict: stepDict[stepNamePmx] = {}
2544-
# avoid overwriting fullsim alternative
2545-
if not k in stepDict[stepNamePmx]:
2523+
# fastsim version
2524+
if 'FS' in k:
2525+
# setup for stage 1 fastsim
2526+
if "Gen" in stepName:
2527+
stepNamePmx = stepName.replace('Gen','Premix')
2528+
if not stepNamePmx in stepDict: stepDict[stepNamePmx] = {}
25462529
stepDict[stepNamePmx][k] = merge([
25472530
{
25482531
'-s': 'GEN,SIM,RECOBEFMIX,DIGI:pdigi_valid',
@@ -2556,65 +2539,84 @@ def setupPU_(self, step, stepName, stepDict, k, properties):
25562539
if "ProdLike" in self.suffix:
25572540
# todo
25582541
pass
2559-
# setup for stage 2
2560-
elif "Digi" in step or "Reco" in step:
2561-
# go back to non-PU step version
2562-
d = merge([stepDict[self.getStepName(step)][k]])
2563-
if d is None: return
2564-
if "Digi" in step:
2542+
# setup for stage 2 fastsim
2543+
elif "FastSimRun3" in step:
2544+
# go back to non-PU step version
2545+
d = merge([stepDict[self.getStepName(step)][k]])
2546+
if d is None: return
25652547
tmpsteps = []
25662548
for s in d["-s"].split(","):
25672549
if s == "DIGI" or "DIGI:" in s:
25682550
tmpsteps.extend([s, "DATAMIX"])
25692551
else:
25702552
tmpsteps.append(s)
25712553
d = merge([{"-s" : ",".join(tmpsteps),
2572-
"--datamix" : "PreMix",
2573-
"--procModifiers": "premix_stage2"},
2554+
"--datamix" : "PreMix"},
25742555
d])
2575-
# for combined stage1+stage2
2576-
if "_PMXS1S2" in self.suffix:
2577-
d = merge([digiPremixLocalPileup, d])
2578-
elif "Reco" in step:
25792556
if "--procModifiers" in d:
25802557
d["--procModifiers"] += ",premix_stage2"
25812558
else:
25822559
d["--procModifiers"] = "premix_stage2"
2583-
stepDict[stepName][k] = d
2584-
# setup for stage 2 fastsim
2585-
elif "FastSimRun3" in step:
2586-
# go back to non-PU step version
2587-
d = merge([stepDict[self.getStepName(step)][k]])
2588-
if d is None: return
2589-
tmpsteps = []
2590-
for s in d["-s"].split(","):
2591-
if s == "DIGI" or "DIGI:" in s:
2592-
tmpsteps.extend([s, "DATAMIX"])
2593-
else:
2594-
tmpsteps.append(s)
2595-
d = merge([{"-s" : ",".join(tmpsteps),
2596-
"--datamix" : "PreMix"},
2597-
d])
2598-
if "--procModifiers" in d:
2599-
d["--procModifiers"] += ",premix_stage2"
2600-
else:
2601-
d["--procModifiers"] = "premix_stage2"
2602-
# for combined stage1+stage2
2603-
if "_PMXS1S2" in self.suffix:
2604-
d = merge([digiPremixLocalPileup, d])
2605-
stepDict[stepName][k] = d
2606-
# separate nano step now only used in ProdLike workflows for Run3/Phase2
2607-
elif "Nano"==step:
2608-
# go back to non-PU step version
2609-
d = merge([stepDict[self.getStepName(step)][k]])
2610-
if "_PMXS1S2" in self.suffix and "--filein" in d:
2611-
filein = d["--filein"]
2612-
m = re.search("step(?P<ind>\\d+)_", filein)
2613-
if m:
2614-
d["--filein"] = filein.replace(m.group(), "step%d_"%(int(m.group("ind"))+1))
2615-
stepDict[stepName][k] = d
2616-
# run2/3 WFs use Nano (not NanoPU) in PU WF
2617-
stepDict[self.getStepName(step)][k] = merge([d])
2560+
# for combined stage1+stage2
2561+
if "_PMXS1S2" in self.suffix:
2562+
d = merge([digiPremixLocalPileup, d])
2563+
stepDict[stepName][k] = d
2564+
elif "HARVESTFastRun3" in step:
2565+
# increment input step number
2566+
stepDict[stepName][k] = merge([{'--filein':'file:step3_inDQM.root'},stepDict[stepName][k]])
2567+
else:
2568+
# setup for stage 1
2569+
if "GenSim" in stepName:
2570+
stepNamePmx = stepName.replace('GenSim','Premix')
2571+
if not stepNamePmx in stepDict: stepDict[stepNamePmx] = {}
2572+
stepDict[stepNamePmx][k] = merge([
2573+
{
2574+
'-s': 'GEN,SIM,DIGI:pdigi_valid',
2575+
'--datatier': 'PREMIX',
2576+
'--eventcontent': 'PREMIX',
2577+
'--procModifiers': 'premix_stage1'
2578+
},
2579+
stepDict[stepName][k]
2580+
])
2581+
if "ProdLike" in self.suffix:
2582+
stepDict[stepNamePmx][k] = merge([{'-s': 'GEN,SIM,DIGI'},stepDict[stepNamePmx][k]])
2583+
# setup for stage 2
2584+
elif "Digi" in step or "Reco" in step:
2585+
# go back to non-PU step version
2586+
d = merge([stepDict[self.getStepName(step)][k]])
2587+
if d is None: return
2588+
if "Digi" in step:
2589+
tmpsteps = []
2590+
for s in d["-s"].split(","):
2591+
if s == "DIGI" or "DIGI:" in s:
2592+
tmpsteps.extend([s, "DATAMIX"])
2593+
else:
2594+
tmpsteps.append(s)
2595+
d = merge([{"-s" : ",".join(tmpsteps),
2596+
"--datamix" : "PreMix",
2597+
"--procModifiers": "premix_stage2"},
2598+
d])
2599+
# for combined stage1+stage2
2600+
if "_PMXS1S2" in self.suffix:
2601+
d = merge([digiPremixLocalPileup, d])
2602+
elif "Reco" in step:
2603+
if "--procModifiers" in d:
2604+
d["--procModifiers"] += ",premix_stage2"
2605+
else:
2606+
d["--procModifiers"] = "premix_stage2"
2607+
stepDict[stepName][k] = d
2608+
# separate nano step now only used in ProdLike workflows for Run3/Phase2
2609+
elif "Nano"==step:
2610+
# go back to non-PU step version
2611+
d = merge([stepDict[self.getStepName(step)][k]])
2612+
if "_PMXS1S2" in self.suffix and "--filein" in d:
2613+
filein = d["--filein"]
2614+
m = re.search("step(?P<ind>\\d+)_", filein)
2615+
if m:
2616+
d["--filein"] = filein.replace(m.group(), "step%d_"%(int(m.group("ind"))+1))
2617+
stepDict[stepName][k] = d
2618+
# run2/3 WFs use Nano (not NanoPU) in PU WF
2619+
stepDict[self.getStepName(step)][k] = merge([d])
26182620
def condition(self, fragment, stepList, key, hasHarvest):
26192621
if not 'PU' in key:
26202622
return False
@@ -2655,6 +2657,7 @@ def workflow_(self, workflows, num, fragment, stepList, key):
26552657
'RecoNanoFakeHLT',
26562658
'Nano',
26572659
'FastSimRun3',
2660+
'HARVESTFastRun3',
26582661
],
26592662
suffix = '_PMXS2',
26602663
offset = 0.98,
@@ -2677,6 +2680,7 @@ def workflow_(self, workflows, num, fragment, stepList, key):
26772680
'RecoNanoFakeHLT',
26782681
'Nano',
26792682
'FastSimRun3',
2683+
'HARVESTFastRun3',
26802684
],
26812685
suffix = '_PMXS1S2',
26822686
offset = 0.99,

0 commit comments

Comments
 (0)