Merge pull request #48149 from andreypz/test-rebase-22May

Update DT calibration workflows to work with newest CMSSW releases

Author: cmsbuild

CalibMuon/DTCalibration/python/PlottingTools/drawHistoAllChambers.py

@@ -1,23 +1,30 @@
 import ROOT
 
-def drawHisto(histo,title,ymin,ymax,option="HISTOP",draw=True):
+def drawHisto(histo,yaxis_title,ymin,ymax,option="HISTOP",draw=True):
+    ROOT.gStyle.SetPadRightMargin(0.02)
+    ROOT.gStyle.SetPadLeftMargin(0.07)
+    
     histo.SetStats(0)
     histo.SetLineWidth(3)
     histo.SetMarkerStyle(20)
     histo.SetMarkerSize(0.9)
     histo.GetYaxis().SetRangeUser(ymin,ymax)
-    histo.GetYaxis().SetTitle(title)
+    histo.GetYaxis().SetTitleOffset(1.)
+    histo.GetYaxis().SetDecimals(1)
+    histo.GetYaxis().SetMaxDigits(4)
+    histo.GetYaxis().SetTitle(yaxis_title)
     histo.GetXaxis().SetLabelSize(0.04)
     histo.GetXaxis().SetTickLength(0.)
     histo.LabelsOption("d","X")
- 
+    
     fillColor = 0
     canvas = None 
     if draw:
-        canvas = ROOT.TCanvas("c_" + histo.GetName())
+        canvas = ROOT.TCanvas("c_" + histo.GetName(), "Default Canvas", 1000, 500)
         canvas.SetGridy()
         canvas.SetFillColor(fillColor)
-    if draw: histo.Draw(option)
+        histo.Draw(option)
+        
 
     linesWh = {}
     linesSt = {}

CalibMuon/DTCalibration/python/PlottingTools/plotResiduals.py

@@ -2,12 +2,22 @@
 from .fitResidual import fitResidual
 from .drawHistoAllChambers import drawHisto
 
-def plot(fileName,sl,dir='DQMData/Run 1/DT/Run summary/DTCalibValidation',option="HISTOPE1",draw=True):
+# This script is not to be run standalone.
+# Use CalibMuon/DTCalibration/scripts/allPlotsResiduals.py instead.
 
-    mean_ymin = -0.02
-    mean_ymax =  0.02
-    sig_ymin = 0.
-    sig_ymax = 0.07
+def plot(fileName,sl,dir='DTResiduals', run='1',option="HISTOPE1",draw=True):
+
+    if run in ['390106','390170']:
+        # These are cosmic Runs
+        mean_ymin = -0.15
+        mean_ymax =  0.15
+        sig_ymin = 0.
+        sig_ymax = 0.15
+    else:
+        mean_ymin = -0.02
+        mean_ymax =  0.02
+        sig_ymin = 0.
+        sig_ymax = 0.07
 
     slType = sl
     slStr = "SL%d" % slType
@@ -22,23 +32,23 @@ def plot(fileName,sl,dir='DQMData/Run 1/DT/Run summary/DTCalibValidation',option
     stations = (1,2,3,4)
 
     # (Wh-2 MB1 Sec1 ... Wh-2 MB1 Sec12 ... Wh-1 MB1 Sec1 ... Wh-1 MB1 Sec12 ...)
-    # (Wh-2 MB2 Sec1 ... Wh-2 MB2 Sec12 ... Wh-1 MB2 Sec1 ... Wh-1 MB1 Sec12 ...) ...  
+    # (Wh-2 MB2 Sec1 ... Wh-2 MB2 Sec12 ... Wh-1 MB2 Sec1 ... Wh-1 MB1 Sec12 ...) ...
     nBins = 250
     if slType == 2: nBins = 180
-    histoMean = ROOT.TH1F("h_ResMeanAll","Mean of residuals",nBins,0,nBins)
-    histoSigma = ROOT.TH1F("h_ResSigmaAll","Sigma of residuals",nBins,0,nBins)
+    histoMean = ROOT.TH1F("h_ResMeanAll","Mean of residuals for Run=%s at %s" % (run,slStr),nBins,0,nBins)
+    histoSigma = ROOT.TH1F("h_ResSigmaAll","Sigma of residuals for Run=%s at %s" % (run,slStr),nBins,0,nBins)
     for st in stations:
         nSectors = 12
         if st == 4: nSectors = 14
-        if st == 4 and slType == 2: continue 
+        if st == 4 and slType == 2: continue
         if verbose: print("Station",st)
         for wh in wheels:
-            if verbose: print("Wheel",wh) 
+            if verbose: print("Wheel",wh)
             for sec in range(1,nSectors+1):
                 if verbose: print("Sector",sec)
                 # Get histogram
-                histoName = "%s/Wheel%d/Station%d/Sector%d/hResDist_STEP3_W%d_St%d_Sec%d_%s" % (dir,wh,st,sec,wh,st,sec,slStr) 
-                print("Accessing",histoName)
+                histoName = "%s/Wheel%d/Station%d/Sector%d/hResDist_STEP3_W%d_St%d_Sec%d_%s" % (dir,wh,st,sec,wh,st,sec,slStr)
+                if verbose: print("Accessing",histoName)
                 histo = file.Get(histoName)
                 (histo,fitFunc) = fitResidual(histo,nSigmas,verbose)
                 fitMean = fitFunc.GetParameter(1)
@@ -52,16 +62,16 @@ def plot(fileName,sl,dir='DQMData/Run 1/DT/Run summary/DTCalibValidation',option
                 histoMean.SetBinError(binHistoNew,fitMeanErr)
                 histoSigma.SetBinContent(binHistoNew,fitSigma)
                 histoSigma.SetBinError(binHistoNew,fitSigmaErr)
-  
+
                 if sec == 1:
                     label = "Wheel %d" % wh
-                    if wh == -2: label += " MB%d" % st  
-                    histoMean.GetXaxis().SetBinLabel(binHistoNew,label) 
+                    if wh == -2: label += " MB%d" % st
+                    histoMean.GetXaxis().SetBinLabel(binHistoNew,label)
                     histoSigma.GetXaxis().SetBinLabel(binHistoNew,label)
 
-    objectsMean = drawHisto(histoMean,title="Mean of residuals (cm)",
+    objectsMean = drawHisto(histoMean,yaxis_title="Mean of residuals (cm)",
                                       ymin=mean_ymin,ymax=mean_ymax,option=option,draw=draw)
-    objectsSigma = drawHisto(histoSigma,title="Sigma of residuals (cm)",
-                                        ymin=sig_ymin,ymax=sig_ymax,option=option,draw=draw)
+    objectsSigma = drawHisto(histoSigma,yaxis_title="Sigma of residuals (cm)",ymin=sig_ymin,ymax=sig_ymax,option=option,draw=draw)
+
 
     return (objectsMean,objectsSigma)

CalibMuon/DTCalibration/python/PlottingTools/plotResidualsCosmics.py

@@ -5,7 +5,7 @@ def binNumber(station,sector):
     start = (station - 1)*12
     return start + sector
 
-def plot(fileName,sl,option="HISTOP",draw=True):
+def plot(fileName,sl,run="1",option="HISTOP",draw=True):
 
     slType = sl
     slStr = "SL%d" % slType
@@ -21,14 +21,14 @@ def plot(fileName,sl,option="HISTOP",draw=True):
     histosWheel = {}
     for wh in wheels:
         histoName = 'Wheel%d_%s_VDrift' % (wh,slStr)
-        print("Accessing",histoName)
+        #print("Accessing",histoName)
         histosWheel[wh] = file.Get(histoName)
 
     # (Wh-2 MB1 Sec1 ... Wh-2 MB1 Sec12 ... Wh-1 MB1 Sec1 ... Wh-1 MB1 Sec12 ...)
     # (Wh-2 MB2 Sec1 ... Wh-2 MB2 Sec12 ... Wh-1 MB2 Sec1 ... Wh-1 MB1 Sec12 ...) ...  
     nBins = 250
     if slType == 2: nBins = 180
-    histo = ROOT.TH1F("h_VDriftAll","VDrift",nBins,0,nBins)
+    histo = ROOT.TH1F("h_VDriftAll_SL%s"%slStr,"VDrift for Run=%s at %s" % (run,slStr),nBins,0,nBins)
     for st in stations:
         nSectors = 12
         if st == 4: nSectors = 14 
@@ -53,9 +53,9 @@ def plot(fileName,sl,option="HISTOP",draw=True):
                     if wh == -2: label += " MB%d" % st  
                     histo.GetXaxis().SetBinLabel(binHistoNew,label) 
 
-    objects = drawHisto(histo,
-                        title="v_{drift} (#mum/ns)",
-                        ymin=53,ymax=57,option=option,draw=draw)
+        objects = drawHisto(histo,
+                            yaxis_title="v_{drift} (#mum/ns)",
+                            ymin=52,ymax=57,option=option,draw=draw)
 
     return objects
 

CalibMuon/DTCalibration/python/PlottingTools/plotVDriftFromHistos.py

@@ -50,16 +50,18 @@ def check_crabtask(self):
         print(self.crab_config_filepath)
         task = self.crabFunctions.CrabTask(crab_config = self.crab_config_filepath,
                                             initUpdate = False)
-        
+
         if self.options.no_exec:
             log.info("Nothing to check in no-exec mode")
             return True
         for n_check in range(self.options.max_checks):
             task.update()
             if task.state in ( "COMPLETED"):
-                print("Crab task complete. Getting output locally")
-                output_path = os.path.join( self.local_path, "unmerged_results" )
-                self.get_output_files(task, output_path)
+                print("Crab task is complete. You can run the next step now.")
+                #output_path = os.path.join( self.local_path, "unmerged_results" )
+                #self.get_output_files(task, output_path)
+                #self.get_output_files(task)
+                #print("Finished with get_output_files()")
                 return True
             if task.state in ("SUBMITFAILED", "FAILED"):
                 print("Crab task failed")
@@ -83,19 +85,24 @@ def check_crabtask(self):
             sys.stdout.write("\r")
             sys.stdout.write("".join([" " for i in range(tools.getTerminalSize()[0])]))
             sys.stdout.write("\r")
-            prompt_text = "Check (%d/%d). Task state: %s (%s). Press q and enter to stop checks: " % (n_check,
-                self.options.max_checks, task.state, jobinfos)
+            prompt_text = "Check (%d/%d). Task state: %s (%s). \nPress q and enter to stop checks: " \
+                % (n_check, self.options.max_checks, task.state, jobinfos)
             user_input = tools.stdinWait(prompt_text, "", self.options.check_interval)
-            if user_input in ("q","Q"):
+            if user_input in ["q","Q"]:
                 return False
         print("Task not completed after %d checks (%d minutes)" % ( self.options.max_checks,
             int( self.options.check_interval / 60. )))
         return False
 
     def voms_proxy_time_left(self):
-        process = subprocess.Popen(['voms-proxy-info', '-timeleft'],
-                                   stdout=subprocess.PIPE)
+        log.debug("Checking voms_proxy time left")
+        process = subprocess.Popen('voms-proxy-info -timeleft',
+                                   stdout = subprocess.PIPE,
+                                   stderr = subprocess.PIPE,
+                                   shell=True
+                                   )
         stdout = process.communicate()[0]
+
         if process.returncode != 0:
             return 0
         else:
@@ -200,8 +207,7 @@ def fill_options_from_crab_config(self):
 
     @property
     def crab(self):
-        """ Retuns a CrabController instance from cache or creates new
-           on on first call """
+        """ Retuns a CrabController instance from cache or creates new one if it is a first call """
         if self._crab is None:
             if self.cert_info.voGroup:
                 self._crab = self.crabFunctions.CrabController(voGroup = self.cert_info.voGroup)
@@ -212,11 +218,13 @@ def crab(self):
     @property
     def cert_info(self):
         if not self._cert_info:
+            log.debug("No cert info yet. Will try to get it.")
             if not self.voms_proxy_time_left() > 0:
-                warn_msg = "No valid proxy, a default proxy without a specific"
-                warn_msg = "VOGroup will be used"
-                self.voms_proxy_create()
+                warn_msg = "No valid proxy, a default proxy without a specific VOGroup will be used"
                 log.warning(warn_msg)
+                log.debug("Trying to create voms_proxy")
+                self.voms_proxy_create()
+                log.debug("... voms_proxy is created")
             self._cert_info = self.crabFunctions.CertInfo()
         return self._cert_info
 

CalibMuon/DTCalibration/python/Workflow/CrabHelper.py

@@ -34,6 +34,7 @@ class CrabController():
     # @param self: A previously defined logger. Crab log messages will use this logger as their parent logger.
     def __init__(self, debug=0, logger = None , workingArea = None, voGroup = None, username = None):
 
+        #print("Initializing CrabController()")
         setConsoleLogLevel(LOGLEVEL_MUTE)
         self.debug = debug
         if workingArea is not None:
@@ -186,10 +187,13 @@ def status(self,name):
     # @param crabArgs A list of arguments for crab beginning with the command
     def callCrabCommand( self, crabArgs ):
         crabCommandProcessArgs = (self.crab_q, crabArgs)
+        print("Will run the callCrabCommand:", crabArgs)
         p = Process(target=crabCommandProcess, args=(crabCommandProcessArgs))
         p.start()
+        #print("The (multi)Process() started")
         res = self.crab_q.get()
         p.join()
+        print("The Crab command Process has finished.")
         return res
 
     ## Call crab getlog
@@ -340,12 +344,13 @@ def crabCommandProcess(q,crabCommandArgs):
 
 class CertInfo:
     def __init__( self ):
+        # print("Running CertInfo() init")
         p = subprocess.Popen("voms-proxy-info  --fqan",
                               stdout = subprocess.PIPE,
                               stderr = subprocess.PIPE,
                               shell=True)
         stdout, stderr = p.communicate()
-        print(stdout)
+        # print("Grid Certificate:", stdout)
         if p.returncode != 0:
             self.vo = ""
             self.voGroup = ""
@@ -364,8 +369,7 @@ def __init__( self ):
             except:
                 self.voRole = ""
 
-## Class for a single CrabRequest
-#e
+
 # This class represents one crab3 task/request
 class CrabTask:
 
@@ -398,6 +402,7 @@ def __init__(self,
                 raise IOError("File %s not found" % crab_config )
             self.name = crab_config
             self.name = self.crabConfig.General.requestName
+            
         self.uuid = uuid.uuid4()
         #~ self.lock = multiprocessing.Lock()
         #setup logging

CalibMuon/DTCalibration/python/Workflow/Crabtools/crabFunctions.py

@@ -36,12 +36,12 @@ def has_crab3_env(self):
         return True
 
     def setup_crab_env(self):
+        log.info('Setting up crab environment')
         # following
-        #http://.com/questions/3503719/emulating-bash-source-in-python
+        # http://stackoverflow.com/questions/3503719/emulating-bash-source-in-python
         command = ['bash', '-c', 'unset module;source /cvmfs/cms.cern.ch/crab3/crab.sh && env']
         proc = subprocess.Popen(command, executable = '/bin/bash', stdout = subprocess.PIPE)
 
-        print('setting up crab')
         for line in proc.stdout:
           (key, _, value) = line.partition(b"=")
           os.environ[key.decode()] = value.decode().replace("\n","")