Merge pull request #44977 from AdrianoDee/fix_RelMon_piecharts

cmsbuild · web-flow · commit fa09b428a67a · 2024-06-12T03:35:18.000+02:00
Fix RelMon Piecharts using `Chart.js`
diff --git a/Utilities/RelMon/python/definitions.py b/Utilities/RelMon/python/definitions.py
@@ -197,3 +197,11 @@
           ("!(mu20+|wzMu20+|jet20+)","Btag@1"))
 data_pattern_blist_pairs=()
 
+## colors for gauge
+from  matplotlib.colors import LinearSegmentedColormap
+gauge_cmap=LinearSegmentedColormap.from_list('rg',["r", "orange","y","lime"], N=256) 
+
+## cms logo
+cms_logo_url = "https://cms-docdb.cern.ch/cgi-bin/PublicDocDB/RetrieveFile?docid=3045&amp;filename=CMSlogo_color_label_1024_May2014.png&amp;version=3"
+
+
diff --git a/Utilities/RelMon/python/directories2html.py b/Utilities/RelMon/python/directories2html.py
@@ -24,6 +24,8 @@
 
 import os
 import hashlib
+import random
+from matplotlib.colors import to_hex
 
 if "RELMON_SA" in os.environ:
   from .dirstructure import Comparison,Directory
@@ -81,9 +83,12 @@ def fairy_url_single(run,sample,version,plot_path,tier,draw_opts="",h=250,w=200)
 def get_page_header(directory=None, standalone=False, additional_header=""):
   style_location="/cms-service-reldqm"
   if standalone:
-    style_location = "http://cms-service-reldqm.web.cern.ch/" + style_location +"/"
+    style_location = "https://raw.githubusercontent.com/cms-PdmV/RelMonService2/77c534ec93401ca5de222ac62a6422f02389dafc/report_website/" #RelMonService2 
   javascripts=''
   style=''
+  tablestyle=''
+  thead_h = 400 
+  wrapper_h = 1500
   if directory!=None and len(directory.comparisons)>0:
     meta=directory.meta
     style='img.fail {border:1px solid #ff0000;}\n'+\
@@ -94,18 +99,34 @@ def get_page_header(directory=None, standalone=False, additional_header=""):
           'a.black_link:hover {color: #737373}\n'+\
           'a.black_link:visited {color: #333333}\n'+\
           'a.black_link:active {color: #333333}\n'
+    ## fixed first row and first column table
+    
+    wrapper_h = min(thead_h + (70 * len(directory.comparisons)),1800)
+
+  tablestyle = '\n.wrapper { overflow: auto; height: %dpx;} \n'%(wrapper_h)
+  tablestyle += 'table {  position: relative; border-collapse: separate; border-spacing: 0;} \n'
+  tablestyle += 'table { position: relative; border-collapse: separate; border-spacing: 0;} \n'
+  tablestyle += 'table th, table td { width: 50px; padding: 5px; background-color: white;} \n'
+  tablestyle += 'table th { position: sticky; top: 0; z-index: 2; height: %dpx;} \n'%(thead_h)
+  tablestyle += 'table th:nth-child(1) { left:0; z-index:3;} \n'
+  tablestyle += '.sticky-col { position: sticky; background-color: #C9FFD1  ; width: 200px; left:0}\n'
+  tablestyle += '.center_head { position: absolute; top: 50%; left: 50%;} \n'
+  tablestyle += '.vertical_head {top: 60%; -webkit-transform:  translateX(-50%) translateY(-50%) rotate(-90deg); -moz-transform: translateX(-50%) translateY(-50%) rotate(-90deg);} \n'
   javascripts=""
+
   
-  
+
   html='<html>'+\
        '<head>'+\
        '<title>RelMon Summary</title>'+\
-       '<link rel="stylesheet" href="%s/style/blueprint/screen.css" type="text/css" media="screen, projection">'%style_location+\
-       '<link rel="stylesheet" href="%s/style/blueprint/print.css" type="text/css" media="print">'%style_location+\
-       '<link rel="stylesheet" href="%s/style/blueprint/plugins/fancy-type/screen.css" type="text/css" media="screen, projection">'%style_location+\
+       '<script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.js"></script>' + \
+       '<link rel="stylesheet" href="%s/screen.css" type="text/css" media="screen, projection">'%style_location+\
+       '<link rel="stylesheet" href="%s/print.css" type="text/css" media="print">'%style_location+\
+       '<link rel="stylesheet" href="%s/fancy-type-screen.css" type="text/css" media="screen, projection">'%style_location+\
        '<style type="text/css">'+\
        '.rotation {display: block;-webkit-transform: rotate(-90deg);-moz-transform: rotate(-90deg); }'+\
        '%s'%style+\
+       '%s'%tablestyle +\
        '</style>'+\
        '%s'%javascripts+\
        '%s'%additional_header+\
@@ -123,10 +144,7 @@ def get_page_footer():
 #-------------------------------------------------------------------------------
 
 def get_title_section(directory, hashing_flag, standalone, depth=2):
-  if standalone:
-      cms_logo_url = "http://cms-service-reldqm.web.cern.ch/cms-service-reldqm/style/CMS.gif"
-  else:
-      cms_logo_url = "cms-service-reldqm/style/CMS.gif"
+  
   mother_name=basename(directory.mother_dir)
   mother_file_name=""
   if depth==1:
@@ -212,7 +230,7 @@ def get_subdirs_section(directory, hashing_flag):
     html+='</div>'
     
     html+='<div class="span-6 last">'
-    html+='<a href="%s"><img src="%s" class="top right"></a>'%(link,subdir.get_summary_chart_ajax(150,100))
+    html+= subdir.get_piechart_js(120,link)
     html+='</div>'
     
     html+='<hr>'
@@ -263,7 +281,7 @@ def get_summary_section(directory,matrix_page=True):
   html+='</div>'
         
   html+='<div class="span-7 colborder">'+\
-        '<img src="%s" class="top right">'%directory.get_summary_chart_ajax(200,200)+\
+        '%s'%(directory.get_piechart_js(150)) +\
         '</div>'+\
         '<div class="span-9 last">'
   if matrix_page:
@@ -491,27 +509,35 @@ def directory2html(directory, hashing, standalone, depth=0):
 
   #chdir(old_cwd)
 
-#-------------------------------------------------------------------------------
+def build_gauge_js(rate,w=100,minrate=.80,add_rate=False):
+
+  color = to_hex(gauge_cmap((rate-minrate)/(1.0-minrate)))
+  font_size = int(w/9)
+  gauge_max = 1. - rate
+
+  name = random.getrandbits(64) # just a random has for the canvas
+  html = "" 
+  html += '<canvas id="%s" style="width:100%%;max-width:%d"></canvas>'%(name,w)
+
+  # "gauge" chart
+  html += '<script> new Chart("%s",'%(name) 
+  html += '{ type: "doughnut",'
 
-def build_gauge(total_success_rate,minrate=.80,small=False,escaped=False):
-  total_success_rate_scaled=(total_success_rate-minrate)
-  total_success_rate_scaled_repr=total_success_rate_scaled/(1-minrate)
-  if total_success_rate_scaled_repr<0:
-    total_success_rate_scaled_repr=0
-  size_s="200x100"
-  if small:
-    size_s="40x30"
-  #print "Total success rate %2.2f and scaled %2.2f "%(total_success_rate,total_success_rate_scaled)
-  gauge_link ="https://chart.googleapis.com/chart?chs=%s&cht=gom"%size_s
-  gauge_link+="&chd=t:%2.1f"%(total_success_rate_scaled_repr*100.)
-  if not small:
-    gauge_link+="&chxt=x,y&chxl=0:|%2.1f%%|1:|%i%%|%i%%|100%%"%(total_success_rate*100,minrate*100.,(1+minrate)*50)
-    gauge_link+="&chma=10,10,10,0"
-  img_tag= '<img src="%s">'%gauge_link
-  if escaped:
-    img_tag=html.escape(img_tag)    
-  return img_tag
+  # data
+  html += 'data: {'
+  html += 'labels: ["Success", "Failure"],'
+  html += 'datasets: [{ backgroundColor: ["%s", "#C7C7C7"],'%(color)
+  html += 'data: [%f,%f]}] },'%(rate,gauge_max)
 
+  #options
+  html += 'options: { responsive: true, rotation: -3.1415926536, circumference: 3.1415926536 ,' ## in radiants
+  html += ' legend: { display: false }, tooltips: {enabled: false}, hover: {mode: null}},'
+  html += '}); </script>'
+  if add_rate:
+    html += '<div style="width: 100%%; position: relative; left: %d; margin-top: -%dpx; font-align: center">'%(int(2*w/7),font_size)
+    html += '<span style="color: %s; font-family: courier; font-size:  %dpx;">%.2f%%</span></div>'%(color,font_size,rate*100.)
+
+  return html
 #-------------------------------------------------------------------------------
 
 def get_aggr_pairs_info(dir_dict,the_aggr_pairs=[]):
@@ -614,7 +640,7 @@ def make_categories_summary(dir_dict,aggregation_rules):
     html+='</div>'
     
     html+='<div class="span-6 last">'
-    html+=build_gauge(average_success_rate)
+    html+=build_gauge_js(average_success_rate,add_rate=True)
     html+='</div>'
     
     html+='<hr>'
@@ -648,7 +674,7 @@ def make_twiki_table(dir_dict,aggregation_rules):
 
   for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
     #print cat_name,present_subdirs,total_weight,average_success_rate
-    html+=build_gauge(average_success_rate,small=True,escaped=True)
+    html+=build_gauge_js(average_success_rate,w=40)
     html+=" | "    
   
   html+='</div> <a href="#top">Top...</a>'
@@ -691,7 +717,7 @@ def make_barchart_summary(dir_dict,name="the_chart",title="DQM directory",the_ag
                          });
       }
     </script>
-    """%(name,40*counter,title)
+    """%(name,35*counter,title)
   return script
 
 
@@ -802,7 +828,6 @@ def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_
   page_html+=make_categories_summary(dir_dict,aggregation_rules)
 
   # Make the Directories chart
-  page_html+='<div class="span-24"><h2 class="alt"><a name="detailed_barchart">Detailed Barchart</a></h2></div>'
   page_html+='<div id="dir_chart"></div> <a href="#top">Top...</a><hr>'
   
   # Barbarian vertical space. Suggestions are welcome
@@ -811,59 +836,61 @@ def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_
 
  
   # Prepare the table
-  page_html+='<div class="span-24"><h2 class="alt"><a name="summary_table">Summary Table</a></h2></div>'
+  page_html+='<div class="span-24"><h2 class="alt"><a name="summary_table">Summary Table</a></h2> <h4> <span class="alt"> (scrollable) </span> </h4> </div>'
 
-  for i in range(5):
-    page_html+='<div class="span-24"><p></p></div>\n'
-    
+  div_width= min(len(dir_dict.keys()) * 70 + 500,1500) #80 px per column + 200 for the first column
+  page_html+='<div class="wrapper" style = "width: %dpx;">'%(div_width)
   page_html+="""
-        <table border="1" >
+        <table>
+        <thead>
           <tr>
-          <td> </td>          
+          <th> <p class = "vertical_head center_head"></p> </th>        
   """
   
   # First row with samples
   page_html+="""
-          <td><div class="span-1"><p class="rotation" style="alt"><b>Summary</b></p></div></td>"""
+          <th> <p class = "vertical_head center_head">Summary</p></th>"""
 
   sorted_samples=sorted(dir_dict.keys())
   for sample in sorted_samples:
-    sample_nick=sample
-    ## For runs: put only the number after the _
-    #if "_" in sample:
-      #run_number=sample.split("_")[-1]      
-      #if (not run_number.isalpha()) and len(run_number)>=6:
-    #sample_nick=run_number
-      
+    if "_" in sample and "Data" not in sample:
+      sample_nick="_".join(sample.split("X_")[0].split("_")[:-1]) 
+      # Cleaning for MC: just the fragment
+    elif "Data" in sample and "RelVal" in sample:
+      sample_nick = "".join([sample.split("_")[0],sample.split("RelVal")[-1]])
+      # Cleaning for Data: PD + Era + Run
+    else:
+      sample_nick = sample
       
     page_html+="""
-          <td><div class="span-1"><p class="rotation" style="">%s</p></div></td>"""%sample_nick
-  page_html+="          </tr>\n"
+          <th> <p class = "vertical_head center_head">%s</th></p>"""%sample_nick
+  page_html+="</tr> \n </thead> \n </tbody> \n"
 
 
  # FIRST ROW
  # Now the summaries  at the beginning of the table
   page_html+="<tr>"
-  page_html+='<td  style="background-color:white;"><div class="span-1">'
+  page_html+='<td class="sticky-col">'
   
-  page_html+='<b>Summary</b></div></td>'
-  page_html+='<td style="background-color:white;" class = "colborder" ><div class="span-1"><img src="%s" alt="%s"></div></td>'%(global_dir.get_summary_chart_ajax(55,55),get_pie_tooltip(global_dir))
+  page_html+='<b>Summary</b></td>'
+  page_html+='<td><div class="span-1"> %s </div></td>'%(global_dir.get_piechart_js(50))
   for sample in sorted_samples:
     col=dir_dict[sample]
     # check if the directory was a top one or not
     summary_page_name="RelMonSummary.html"
     if col.name!="":
       summary_page_name=hash_name(col.name, hashing_flag)+".html"
-    img_link=col.get_summary_chart_ajax(55,55)
-    page_html+='<td  style="background-color:white;"><div class="span-1">'
-    page_html+='<a href="%s/%s"><img src="%s" title="%s"></a></div></td>' %(sample,summary_page_name,img_link,get_pie_tooltip(col))
+    title = get_pie_tooltip(col)
+    chart = col.get_piechart_js(50,sample+"/"+summary_page_name)
+    page_html+='<td>'
+    page_html+='%s </a></td>' %(chart)
   page_html+="</tr>"
 
   # Now the content
   for subdir_name in all_subdirs:  
 
     page_html+='          <tr>\n'
-    page_html+='          <td style="background-color:white;">%s</td>\n' %subdir_name  
+    page_html+='          <td class="sticky-col" style="font-weight:bold;">%s</td>\n' %subdir_name  
 
     row_summary=Directory("row_summary","")
     sample_counter=0
@@ -878,9 +905,10 @@ def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_
 
     # one first row for the summary!
     row_summary.calcStats()
-    img_link=row_summary.get_summary_chart_ajax(55,55)
-    page_html+='<td  style="background-color:white;"><div class="span-1">'
-    page_html+='<img src="%s" title="%s"></div></td>' %(img_link,get_pie_tooltip(row_summary))
+    title = get_pie_tooltip(col)
+    chart = row_summary.get_piechart_js(50)
+    page_html+='<td><div>'#<div class="span-1">'
+    page_html+= chart + '</div></td>'
 
     for sample in sorted_samples:
       sample_counter+=1      
@@ -903,30 +931,27 @@ def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_
         summary_page=join(sample,"%s.html"%(hash_name(directory.name+subdir_name,hashing_flag)))
       dir_is_there=subdir_name in subdirs_dict
 
-      img_link="https://chart.googleapis.com/chart?cht=p3&chco=C0C0C0&chs=50x50&chd=t:1"
-      img_tooltip="N/A"
-      if dir_is_there:
-        #row_summary.subdirs.append(subdirs_dict[subdir_name])
-        img_link=subdirs_dict[subdir_name].get_summary_chart_ajax(50,50)
-        img_tooltip=get_pie_tooltip(subdirs_dict[subdir_name])
+      img_link="https://upload.wikimedia.org/wikipedia/commons/a/a8/Circle_Davys-Grey_Solid.svg"
+      page_html+='<td><div class="span-1">'
 
-      page_html+='<td  style="background-color:white;"><div class="span-1">'
-      if dir_is_there:
-        page_html+='<a href="%s">'%(summary_page)
-      page_html+='<img src="%s" title="%s" height=50 width=50>' %(img_link,img_tooltip)
       if dir_is_there:
-        page_html+='</a>'
+        #row_summary.subdirs.append(subdirs_dict[subdir_name])
+        chart=subdirs_dict[subdir_name].get_piechart_js(50,summary_page)
+        page_html+='%s'%chart
+      else:
+        page_html+='<img src="%s" title="%s" height=50 width=50>' %(img_link,"Unavailable")
       page_html+='</div></td>' 
 
     page_html+="          </tr>\n"        
 
 
 
-  page_html+='</table> <a href="#top">Top...</a><hr>'
+  page_html+='</tbody> </table> </div> <a href="#top">Top...</a><hr>'
 
   page_html+=get_rank_section(global_dir)
 
-  page_html+=make_twiki_table(dir_dict,aggregation_rules_twiki)
+  #page_html+=make_twiki_table(dir_dict,aggregation_rules_twiki) 
+  # ^ commenting out for the moment, not really useful nor used
 
   page_html+=get_page_footer()
   return page_html  
diff --git a/Utilities/RelMon/python/dirstructure.py b/Utilities/RelMon/python/dirstructure.py
@@ -15,6 +15,7 @@
 from copy import deepcopy
 from os import chdir,getcwd,listdir,makedirs,rmdir
 from os.path import exists,join
+import random
 
 import sys
 argv=sys.argv
@@ -167,19 +168,45 @@ def get_subdirs_names(self):
       subdirnames.append(subdir.name)
     return subdirnames
 
-  def get_summary_chart_ajax(self,w=400,h=300):
-    """Emit the ajax to build a pie chart using google apis...
+  def get_piechart_js(self,w=400,link=None):
+
+    """
+    Build the HTML snippet to render a piechart with chart.js
     """
-    url = "https://chart.googleapis.com/chart?"
-    url+= "cht=p3" # Select the 3d chart
-    #url+= "&chl=Success|Null|Fail" # give labels
-    url+= "&chco=00FF00|FFFF00|FF0000|7A7A7A" # give colours to labels
-    url+= "&chs=%sx%s" %(w,h)
-    #url+= "&chtt=%s" %self.name
-    url+= "&chd=t:%.2f,%.2f,%.2f,%.2f"%(self.get_success_rate(),self.get_null_rate(),self.get_fail_rate(),self.get_skiped_rate())
+    if self.get_success_rate()>=99.9: # if the success rate is very high let's make the page lighter 
+      img_link = "https://raw.githubusercontent.com/cms-PdmV/RelMonService2/5ee98db210c0898fd34b4deac3653fa2bdff269b/report_website/lime_circle.png"
+      html ='<img src="%s" height=%d width=%d>' %(img_link,w,w)
+      if link is not None:
+        html = '<a href="%s"> %s </a>' %(link,html) 
+      return html
+
+    name = random.getrandbits(64) # just a random has for the canvas
+    html = "" 
+    html += '<canvas id="%s" height=%d width=%d></canvas>'%(name,w,w)
+    # piechart
+    html += '<script> new Chart("%s",'%(name) 
+    html += '{ type: "pie",'
+
+    # data
+    html += 'data: {'
+    html += 'labels: ["Success", "Null" , "Failure", "Skipped"],'
+    html += 'datasets: [{ backgroundColor: ["lime","yellow","red","grey"],'
+    html += 'data: [%.2f,%.2f,%.2f,%.2f]}] },'%(self.get_success_rate(),self.get_null_rate(),self.get_fail_rate(),self.get_skiped_rate())
     
-    return url
+    #display options
+    html += 'options: { '
+
+    if link is not None:
+      html += 'onClick : function(event) { window.open("%s", "_blank");},'%(link)
+  
 
+    html +='legend: { display: false }, responsive : false, hover: {mode: null}, tooltips: {enabled: false}' 
+    #tooltips: {enabled: false}, hover: {mode: null},'
+
+    html += '}}); </script>'
+
+    return html
+  
   def print_report(self,indent="",verbose=False):
     if len(indent)==0:
       self.calcStats(make_pie=False)
