Merge pull request #48281 from AdrianoDee/data_2025_wfs

Adding 2025 Data Wfs

Author: cmsbuild

Configuration/PyReleaseValidation/python/relval_data_highstats.py

@@ -11,6 +11,34 @@
 offset_pd = 0.001 # less than 100 pds per year
 offset_events = 0.0001 # less than 10 event setups (10k,50k,150k,250k,500k,1M)
 
+## 2025
+base_wf = 2025.0
+for e_n,era in enumerate(eras_2025):
+    for p_n,pd in enumerate(pds_2025):
+        for e_key,evs in event_steps_dict.items(): 
+            wf_number = base_wf
+            wf_number = wf_number + offset_era * e_n
+            wf_number = wf_number + offset_pd * p_n
+            wf_number = wf_number + offset_events * evs 
+            wf_number = round(wf_number,6)
+
+            ## ZeroBias has its own RECO and HARVESTING setup
+            ## ScoutingPFMonitor has its own HLT, RECO and HARVESTING setup
+            recoharv = hlt = '' 
+            if 'ZeroBias' in pd:
+                recoharv = 'ZB_'
+            elif 'ScoutingPFMonitor' in pd:
+                hlt = recoharv = 'ScoutingPFMonitor_'
+            
+            recosetup = 'RECONANORUN3_' + recoharv + 'reHLT_2025' 
+            
+            y = str(int(base_wf))
+            
+            ## this is because ParkingDouble* PDs would end up with a too long name for the submission infrastructure
+            step_name = 'Run' + pd.replace('ParkingDouble','Park2') + era.split('Run')[1] + '_' + e_key 
+
+            workflows[wf_number] = ['',[step_name,'HLTDR3_' + hlt + y,'RECONANORUN3_' + recoharv + 'reHLT_'+y,'HARVESTRUN3_' + recoharv + y]]
+
 ## 2024
 base_wf = 2024.0
 for e_n,era in enumerate(eras_2024):
@@ -25,7 +53,7 @@
             ## Here we use JetMET1 PD to run the TeVJet skims
             skim = 'TeVJet' if pd == 'JetMET1' else ''
 
-            ## ZeroBias have their own HARVESTING
+            ## ZeroBias has its own RECO and HARVESTING setup
             suff = 'ZB_' if 'ZeroBias' in pd else ''
 
             # Running C,D,E with the offline GT.
@@ -34,7 +62,10 @@
             recosetup = recosetup if era[-1] > 'E' else recosetup + '_Offline'
 
             y = str(int(base_wf))
+
+            ## this is because ParkingDouble* PDs would end up with a too long name for the submission infrastructure
             step_name = 'Run' + pd.replace('ParkingDouble','Park2') + era.split('Run')[1] + skim + '_' + e_key
+
             workflows[wf_number] = ['',[step_name,'HLTDR3_' + y,'RECONANORUN3_' + suff + 'reHLT_'+y,'HARVESTRUN3_' + suff + y]]
 
 ## 2023
@@ -47,7 +78,10 @@
             wf_number = wf_number + offset_pd * p_n
             wf_number = wf_number + offset_events * evs 
             wf_number = round(wf_number,6)
+
+            ## this is because ParkingDouble* PDs would end up with a too long name for the submission infrastructure
             step_name = 'Run' + pd.replace('ParkingDouble','Park2') + era.split('Run')[1] + '_' + e_key
+
             y = str(int(base_wf)) + 'B' if '2023B' in era else str(int(base_wf))
             suff = 'ZB_' if 'ZeroBias' in step_name else ''
             workflows[wf_number] = ['',[step_name,'HLTDR3_' + y,'RECONANORUN3_' + suff + 'reHLT_'+y,'HARVESTRUN3_' + suff + y]]

Configuration/PyReleaseValidation/python/relval_steps.py

@@ -654,6 +654,21 @@
 # or (if available) from eos. the number of events limits 
 # the files used as input
 
+###2025 
+pds_2025  = ['BTagMu', 'DisplacedJet', 'EGamma0', 'HcalNZS', 'JetMET0', 'Muon0', 'MuonEG', 'NoBPTX', 'ParkingDoubleMuonLowMass0', 'ParkingHH', 'ParkingLLP', 'ParkingSingleMuon0', 'ParkingVBF0', 'Tau', 'ZeroBias','JetMET1','ScoutingPFMonitor']
+eras_2025 = ['Run2025B', 'Run2025C','Run2025D','Run2025E', 'Run2025F','Run2025G','Run2025H','Run2025I']
+for era in eras_2025:
+    for pd in pds_2025:
+        dataset = "/" + pd + "/" + era
+        skim = ''
+
+        dataset = dataset + '-v1/RAW' 
+        
+        for e_key,evs in event_steps_dict.items():
+            step_name = "Run" + pd.replace("ParkingDouble","Park2") + era.split("Run")[1] + skim + "_" + e_key 
+            steps[step_name] = {'INPUT':InputInfo(dataSet=dataset,label=era.split("Run")[1],events=int(evs*1e6), skimEvents=True, location='STD')}
+
+
 ###2024 
 ## N.B. here we use JetMet0 as "starndard" PD and JetMET1 for the TeVJet skims 
 pds_2024  = ['BTagMu', 'DisplacedJet', 'EGamma0', 'HcalNZS', 'JetMET0', 'Muon0', 'MuonEG', 'NoBPTX', 'ParkingDoubleMuonLowMass0', 'ParkingHH', 'ParkingLLP', 'ParkingSingleMuon0', 'ParkingVBF0', 'Tau', 'ZeroBias','JetMET1']
@@ -2306,12 +2321,19 @@ def lhegensim2018ml(fragment,howMuch):
 
 steps['HLTDR3_2024']=merge( [ {'-s':'L1REPACK:Full,HLT:@%s'%hltKey2025,},{'--conditions':'auto:run3_hlt_relval'},{'--era' : 'Run3_2024'},steps['HLTD'] ] )
 
+steps['HLTDR3_2025']=merge( [ {'-s':'L1REPACK:Full,HLT:@%s'%hltKey2025,},{'--conditions':'auto:run3_hlt_relval'},{'--era' : 'Run3_2025'},steps['HLTD'] ] )
+
 steps['HLTDR3_ScoutingPFMonitor_2024']=merge( [ {'-s':'L1REPACK:Full,HLT:@%s'%hltKey2025,},
                                                 {'--conditions':'auto:run3_hlt_relval'},
                                                 {'--era' : 'Run3_2024'},
                                                 {'--filein' : '/store/group/dpg_trigger/comm_trigger/TriggerStudiesGroup/Scouting/Run3/ScoutingPFMonitor/300684ed-1a51-474f-8c4f-b3bf1e1f5044_skimmed.root'},
                                                 steps['HLTD'] ] )
 
+steps['HLTDR3_ScoutingPFMonitor_2025']=merge( [ {'-s':'L1REPACK:Full,HLT:@%s'%hltKey2025,},
+                                                 {'--conditions':'auto:run3_hlt_relval'},
+                                                 {'--era' : 'Run3_2025'},
+                                                 steps['HLTD'] ] )
+
 steps['HLTDR3_HI2023ARawprime']=merge([{'-s':'L1REPACK:Full,HLT:HIon'},
                                        {'--conditions':'auto:run3_hlt_HIon'},
                                        {'--era' : 'Run3_pp_on_PbPb_approxSiStripClusters_2023'},
@@ -2867,6 +2889,7 @@ def lhegensim2018ml(fragment,howMuch):
 steps['RECODR3_reHLT_2023']=merge([{'--conditions':'auto:run3_data_relval', '--hltProcess':'reHLT'},steps['RECODR3_2023']])
 steps['RECODR3_reHLT_2023B']=merge([{'--conditions':'auto:run3_data_relval', '--hltProcess':'reHLT'},steps['RECODR3']])
 steps['RECODR3_reHLT_2024']=merge([{'--conditions':'auto:run3_data_prompt_relval', '--hltProcess':'reHLT'},steps['RECODR3_2024']])
+steps['RECODR3_reHLT_2025']=merge([{'--conditions':'auto:run3_data_prompt_relval', '--hltProcess':'reHLT'},steps['RECODR3_2025']])
 # Added to run with the offline GT on few 2024 Eras.
 # Could be removed once 2025 wfs are in and we'll test the online GT with them
 steps['RECODR3_reHLT_2024_Offline']=merge([{'--conditions':'auto:run3_data_relval', '--hltProcess':'reHLT'},steps['RECODR3_2024']])
@@ -3280,7 +3303,11 @@ def gen2024HiMix(fragment,howMuch):
 steps['RECONANORUN3_reHLT_2024_Offline']=merge([{'-s':'RAW2DIGI,L1Reco,RECO,PAT,NANO,DQM:@standardDQM+@miniAODDQM+@nanoAODDQM','--datatier':'RECO,MINIAOD,NANOAOD,DQMIO','--eventcontent':'RECO,MINIAOD,NANOEDMAOD,DQM'},steps['RECODR3_reHLT_2024_Offline']])
 steps['RECONANORUN3_ZB_reHLT_2024_Offline']=merge([{'-s':'RAW2DIGI,L1Reco,RECO,PAT,NANO,DQM:@rerecoZeroBias+@miniAODDQM+@nanoAODDQM'},steps['RECONANORUN3_reHLT_2024_Offline']])
 
+steps['RECONANORUN3_reHLT_2025']=merge([{'-s':'RAW2DIGI,L1Reco,RECO,PAT,NANO,DQM:@standardDQM+@miniAODDQM+@nanoAODDQM','--datatier':'RECO,MINIAOD,NANOAOD,DQMIO','--eventcontent':'RECO,MINIAOD,NANOEDMAOD,DQM'},steps['RECODR3_reHLT_2025']])
+steps['RECONANORUN3_ZB_reHLT_2025']=merge([{'-s':'RAW2DIGI,L1Reco,RECO,PAT,NANO,DQM:@rerecoZeroBias+@miniAODDQM+@nanoAODDQM'},steps['RECONANORUN3_reHLT_2025']])
+
 steps['RECONANORUN3_ScoutingPFMonitor_reHLT_2024']=merge([{'-s':'RAW2DIGI,L1Reco,RECO,PAT,NANO,DQM:@standardDQM+@miniAODDQM+@nanoAODDQM+@hltScouting'},steps['RECONANORUN3_reHLT_2024']])
+steps['RECONANORUN3_ScoutingPFMonitor_reHLT_2025']=merge([{'-s':'RAW2DIGI,L1Reco,RECO,PAT,NANO,DQM:@standardDQM+@miniAODDQM+@nanoAODDQM+@hltScouting'},steps['RECONANORUN3_reHLT_2025']])
 
 steps['AODNANORUN3_reHLT_2024']=merge([{'-s':'RAW2DIGI,L1Reco,RECO,PAT,NANO,DQM:@standardDQM+@miniAODDQM+@nanoAODDQM','--datatier':'AOD,MINIAOD,NANOAOD,DQMIO','--eventcontent':'AOD,MINIAOD,NANOEDMAOD,DQM'},steps['RECODR3_reHLT_2024']])
 
@@ -4092,7 +4119,11 @@ def gen2024HiMix(fragment,howMuch):
 steps['HARVESTRUN3_ZB_2024']=merge([{'--era':'Run3_2024', '-s':'HARVESTING:@rerecoZeroBias+@miniAODDQM+@nanoAODDQM'},steps['HARVESTDRUN3']])
 steps['HARVESTRUN3_2024']=merge([{'--era':'Run3_2024', '-s':'HARVESTING:@standardDQM+@miniAODDQM+@nanoAODDQM'},steps['HARVESTDRUN3']])
 steps['HARVESTRUN3_ScoutingPFMonitor_2024']=merge([{'--era':'Run3_2024', '-s':'HARVESTING:@standardDQM+@miniAODDQM+@nanoAODDQM+@hltScouting'},steps['HARVESTDRUN3']])
-
+# 2025
+steps['HARVESTRUN3_ZB_2025']=merge([{'--era':'Run3_2025', '-s':'HARVESTING:@rerecoZeroBias+@miniAODDQM+@nanoAODDQM'},steps['HARVESTDRUN3']])
+steps['HARVESTRUN3_2025']=merge([{'--era':'Run3_2025', '-s':'HARVESTING:@standardDQM+@miniAODDQM+@nanoAODDQM'},steps['HARVESTDRUN3']])
+steps['HARVESTRUN3_ScoutingPFMonitor_2025']=merge([{'--era':'Run3_2025', '-s':'HARVESTING:@standardDQM+@miniAODDQM+@nanoAODDQM+@hltScouting'},steps['HARVESTDRUN3']])
+# HI
 steps['HARVESTRUN3_HI2023A']=merge([{'--era':'Run3_pp_on_PbPb_approxSiStripClusters_2023', '-s':'HARVESTING:@standardDQM+@miniAODDQM'},steps['HARVESTRUN3_2022']])
 
 steps['HARVESTRUN3_pixelTrackingOnly'] = merge([ {'-s':'HARVESTING:@pixelTrackingOnlyDQM'}, steps['HARVESTRUN3_2023']])

Configuration/PyReleaseValidation/scripts/das-up-to-nevents.py

@@ -149,17 +149,17 @@ def no_intersection():
         if os.path.isdir(cert_path):
             json_list = os.listdir(cert_path)
             if len(json_list) == 0:
-                web_fallback == True
-            json_list = [c for c in json_list if "Golden" in c and "era" not in c]
-            json_list = [c for c in json_list if c.startswith("Cert_C") and c.endswith("json")]
+                web_fallback == True 
+            json_list = [c for c in json_list if "golden" in c.lower() and "era" not in c.lower()]
+            json_list = [c for c in json_list if c.lower().startswith("cert_c") and c.endswith("json")]
         else:
             web_fallback = True
         ## ... if not we go to the website
         if web_fallback:
             cert_url = base_cert_url + cert_type + "/"
             json_list = get_url_clean(cert_url).split("\n")
-            json_list = [c for c in json_list if "Golden" in c and "era" not in c and "Cert_C" in c]
-            json_list = [[cc for cc in c.split(" ") if cc.startswith("Cert_C") and cc.endswith("json")][0] for c in json_list]
+            json_list = [c for c in json_list if "golden" in c.lower() and "era" not in c.lower() and "cert_c" in c.lower()]
+            json_list = [[cc for cc in c.split(" ") if cc.lower().startswith("cert_c") and cc.endswith("json")][0] for c in json_list]
 
         # the larger the better, assuming file naming schema 
         # Cert_X_RunStart_RunFinish_Type.json