Lighter das-up-to-nevents when running tests, moving limited matrix data wfs to fixed events

Author: AdrianoDee

Configuration/PyReleaseValidation/python/relval_steps.py

@@ -2340,11 +2340,11 @@ def lhegensim2018ml(fragment,howMuch):
 
 steps['HLTDR3_2023B']=merge( [ {'-s':'L1REPACK:Full,HLT:@%s'%hltKey2023,},{'--conditions':'auto:run3_hlt_relval'},{'--era' : 'Run3'},steps['HLTD'] ] )
 
-steps['HLTDR3_2024']=merge( [ {'-s':'L1REPACK:Full,HLT:@%s'%hltKey2025,},{'--conditions':'auto:run3_hlt_relval'},{'--era' : 'Run3_2024'},steps['HLTD'] ] )
+steps['HLTDR3_2024']=merge( [ {'-s':'L1REPACK:Full,HLT:@%s'%hltKey2024,},{'--conditions':'auto:run3_hlt_relval'},{'--era' : 'Run3_2024'},steps['HLTD'] ] )
 
 steps['HLTDR3_2025']=merge( [ {'-s':'L1REPACK:Full,HLT:@%s'%hltKey2025,},{'--conditions':'auto:run3_hlt_relval'},{'--era' : 'Run3_2025'},steps['HLTD'] ] )
 
-steps['HLTDR3_ScoutingPFMonitor_2024']=merge( [ {'-s':'L1REPACK:Full,HLT:@%s'%hltKey2025,},
+steps['HLTDR3_ScoutingPFMonitor_2024']=merge( [ {'-s':'L1REPACK:Full,HLT:@%s'%hltKey2024,},
                                                 {'--conditions':'auto:run3_hlt_relval'},
                                                 {'--era' : 'Run3_2024'},
                                                 {'--filein' : '/store/group/dpg_trigger/comm_trigger/TriggerStudiesGroup/Scouting/Run3/ScoutingPFMonitor/300684ed-1a51-474f-8c4f-b3bf1e1f5044_skimmed.root'},

Configuration/PyReleaseValidation/scripts/das-up-to-nevents.py

@@ -37,6 +37,7 @@ def get_lumi_ranges(i):
     return result
 
 def das_do_command(cmd):
+    print( "Running DAS command: %s"%cmd) ##TODO: remove me
     out = subprocess.check_output(cmd, shell=True, executable="/bin/bash").decode('utf8')
     return out.split("\n")
 
@@ -197,30 +198,38 @@ def no_intersection():
         if (len(golden_data_runs)==0):
             no_intersection()
 
+        if testing:
+            golden_data_runs = golden_data_runs[:1] # take only the first run
         # building the dataframe, cleaning for bad lumis
         golden_data_runs_tocheck = golden_data_runs
 
-        if testing or args.precheck:
+        if args.precheck and not testing:
             golden_data_runs_tocheck = []
             # Here we check run per run.
             # This implies more dasgoclient queries, but smaller outputs
             # useful when running the IB/PR tests not to have huge
             # query results that have to be cached.
-
             sum_events = 0
-
             for r in golden_data_runs: 
                 sum_events = sum_events + int(das_run_events_data(dataset,r))
                 golden_data_runs_tocheck.append(r)
                 if events > 0 and sum_events > events:
                     break
-
             das_opt = "run in %s"%(str([int(g) for g in golden_data_runs_tocheck]))
-     
-    df = das_lumi_data(dataset,opt=das_opt).merge(das_file_data(dataset,opt=das_opt),on="file",how="inner") # merge file informations with run and lumis
+            
+        if testing:
+            golden_data_runs_tocheck = golden_data_runs[:1] # take only the first run
+            # in testing mode we just take the first file
+            das_opt = "run=%s"%(golden_data_runs_tocheck[0])
+    
+    if not testing:
+        df = das_lumi_data(dataset,opt=das_opt).merge(das_file_data(dataset,opt=das_opt),on="file",how="inner") # merge file informations with run and lumis
+    else:
+        df = das_lumi_data(dataset,opt=das_opt)
+
     df["lumis"] = [[int(ff) for ff in f.replace("[","").replace("]","").split(",")] for f in df.lumis.values]
 
-    if not args.nogolden:
+    if not args.nogolden and not testing:
 
         df_rs = []
         for r in golden_data_runs_tocheck:
@@ -247,13 +256,15 @@ def no_intersection():
     df.loc[:,"max_lumi"] = [max(f) for f in df.lumis]
     df = df.sort_values(["run","min_lumi","max_lumi"])
 
+    if testing:
+        df = df.head(1) # take only the first file
     if site is not None:
         df = df.merge(das_file_site(dataset,site),on="file",how="inner")
 
     if args.pandas:
         df.to_csv(dataset.replace("/","")+".csv")
 
-    if events > 0:
+    if events > 0 and not testing:
         df = df[df["events"] <= events] #jump too big files
         df.loc[:,"sum_evs"] = df.loc[:,"events"].cumsum()
         df = df[df["sum_evs"] < events]

Configuration/PyReleaseValidation/scripts/runTheMatrix.py

@@ -99,7 +99,7 @@ def runSelected(opt):
             12846.0,    # RelValZEE_13                  2024
             13034.0,    # RelValTTbar_14TeV             2024 PU = Run3_Flat55To75_PoissonOOTPU
             16834.0,    # RelValTTbar_14TeV             2025
-            17034.0,    # RelValTTbar_14TeV		2025 PU = Run3_Flat55To75_PoissonOOTPU
+            17034.0,    # RelValTTbar_14TeV		        2025 PU = Run3_Flat55To75_PoissonOOTPU
             14034.0,    # RelValTTbar_14TeV             Run3_2023_FastSim
             14234.0,    # RelValTTbar_14TeV             Run3_2023_FastSim   PU = Run3_Flat55To75_PoissonOOTPU
             2500.3001,   # RelValTTbar_14TeV             NanoAOD from existing MINI
@@ -110,20 +110,24 @@ def runSelected(opt):
             139.001,    # Run2021  MinimumBias                  Commissioning2021
 
             # 2022
-            140.045,    # Run2022C JetHT
+            2022.106001,     # Run2022C JetHT
 
             # 2023
-            141.042,    # Run2023D ZeroBias
+            2023.211001,     # Run2023D ZeroBias
 
             # 2024
-            145.014,      # Run2024B ZeroBias
-            145.104,      # Run2024C JetMet0
-            145.202,      # Run2024D EGamma0
-            145.301,      # Run2024E DisplacedJet
-            145.408,      # Run2024F ParkingDoubleMuonLowMass0
-            145.500,      # Run2024G BTagMu
-            145.604,      # Run2024H JetMET0
-            145.713,      # Run2024I Tau
+            2024.014001,      # Run2024B ZeroBias
+            2024.204001,      # Run2024C JetMet0
+            2024.202001,      # Run2024D EGamma0
+            2024.301001,      # Run2024E DisplacedJet
+            2024.408001,      # Run2024F ParkingDoubleMuonLowMass0
+            2024.500001,      # Run2024G BTagMu
+            2024.504001,      # Run2024H JetMET0
+            2024.713001,      # Run2024I Tau
+
+            # 2025
+            2025.014001,     # Run2025B ZeroBias
+            2025.104001,     # Run2025C JetMet0
         ],
 
         'phase2' : [