Merge pull request #48961 from gartung/gartung-PerfTools-AllocMonitor-2

Make Circles plots for resources reported by Module Alloc Monitor service.

Author: cmsbuild

Configuration/Applications/python/ConfigBuilder.py

@@ -76,6 +76,7 @@ class Options:
 defaultOptions.profile = None
 defaultOptions.heap_profile = None
 defaultOptions.maxmem_profile = None
+defaultOptions.alloc_monitor = None
 defaultOptions.isRepacked = False
 defaultOptions.restoreRNDSeeds = False
 defaultOptions.donotDropOnInput = ''

Configuration/Applications/python/Options.py

@@ -440,6 +440,12 @@
                             action="store_true",
                             dest="maxmem_profile")
 
+expertSettings.add_argument("--alloc_monitor",
+                            help="Add necessary LD_PRELOAD for PerfTools/AllocMonitor",
+                            default=False,
+                            action="store_true",
+                            dest="alloc_monitor")
+
 expertSettings.add_argument("--io",
                             help="Create a json file with io informations",
                             default=None,

Configuration/Applications/python/cmsDriverOptions.py

@@ -258,6 +258,11 @@ def OptionsFromItems(items):
             raise Exception("--maxmem_profile and --prefix are incompatible")
         options.prefix = "env LD_PRELOAD=libPerfToolsAllocMonitorPreload.so:libPerfToolsMaxMemoryPreload.so "
 
+    if options.alloc_monitor:
+        if options.prefix:
+            raise Exception("--alloc_monitor and --prefix are incompatible")
+        options.prefix = "env LD_PRELOAD=libPerfToolsAllocMonitorPreload.so"
+
     # If an "era" argument was supplied make sure it is one of the valid possibilities
     if options.era :
         from Configuration.StandardSequences.Eras import eras

PerfTools/AllocMonitor/python/ModuleAllocMonitor.py

@@ -0,0 +1,6 @@
+import FWCore.ParameterSet.Config as cms
+def customise(process):
+    process.ModuleAllocMonitor = cms.Service("ModuleAllocMonitor",
+                                   fileName=cms.untracked.string("moduleAllocMonitor.log")
+                                   )
+    return(process)

PerfTools/AllocMonitor/scripts/edmModuleAllocJsonToCircles.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+import json
+transitionTypes = [
+    "construction",
+    "begin job",
+    "begin stream",
+    "global begin run",
+    "stream begin run",
+    "global begin luminosity block",
+    "stream begin luminosity block",
+    "event",
+]
+allocTypes = ["added", "nAlloc", "nDealloc", "maxTemp", "max1Alloc"]
+
+def processModuleTransition(moduleLabel, moduleType, moduleInfo, transitionType, moduleTransition):
+        moduleTransition[moduleLabel] = {"cpptype": moduleType, "allocs": []}
+        for entry in moduleInfo:
+            if entry["transition"] == transitionType:
+                moduleTransition[moduleLabel]["allocs"].append(entry.get("alloc",{}))
+        moduleTransition[moduleLabel]["nTransitions"] = len(moduleTransition[moduleLabel]["allocs"])
+
+def formatToCircles(moduleTransitions):
+    modules_dict = {}
+    doc = {
+       "modules": [],
+       "resources": [],
+       "total": {}
+    }
+    for transitionType in transitionTypes:
+        doc["resources"] += [
+            {
+               "name": f"added {transitionType}",
+               "description": f"{transitionType}: added memory (average)",
+               "title": f"{transitionType}: Amount of memory added to the process at the end of the transition",
+               "unit": "kB"
+            },
+            {
+
+               "name": f"nAlloc {transitionType}",
+               "description": f"{transitionType}: num allocs (average)",
+               "title": f"{transitionType}: Number of allocations during the transition",
+               "unit": ""
+            },
+            {
+               "name": f"nDealloc {transitionType}",
+               "description": f"{transitionType}: num deallocs (average)",
+               "title": f"{transitionType}: Number of deallocations during the transition",
+               "unit": ""
+            },
+            {
+               "name": f"maxTemp {transitionType}",
+               "description": f"{transitionType}: maximum temporary memory (average)",
+               "title": f"{transitionType}: Maximum temporary memory during the transition",
+               "unit": "kB"
+            },
+            {
+               "name": f"max1Alloc {transitionType}",
+               "description": f"{transitionType}: largest single allocation (average)",
+               "title": f"{transitionType}: Largest single allocation during the transition",
+               "unit": "kB"
+            },
+        ]
+    # The circles code uses the "events" field to normalize the values between files with different number of events
+    # Here we set it to 1 for the total events because the total is already normalized per transition
+        doc["total"]["events"] = 1
+        doc["total"]["label"] = "Job"
+        doc["total"]["type"] = "Job"
+        for allocType in allocTypes:
+            doc["total"][f"{allocType} {transitionType}"] = 0
+
+    for transitionType, moduleTransition in moduleTransitions.items():
+        for label, info in moduleTransition.items():
+            allocs = info.get("allocs", [])
+            if not label in modules_dict:
+                modules_dict[label] = {
+                    "label": info.get("label", label),
+                    "type": info.get("cpptype", "unknown")
+                }
+            added = 0
+            nAlloc = 0
+            nDealloc = 0
+            maxTemp = 0
+            max1Alloc = 0
+            for alloc in allocs:
+                added += alloc.get("added", 0)
+                nAlloc += alloc.get("nAlloc", 0)
+                nDealloc += alloc.get("nDealloc", 0)
+                maxTemp += alloc.get("maxTemp", 0)
+                max1Alloc += alloc.get("max1Alloc", 0)
+            ntransitions = moduleTransitions[transitionType][label]["nTransitions"]
+            if ntransitions > 0:
+                modules_dict[label][f"nAlloc {transitionType}"] = nAlloc/ntransitions
+                modules_dict[label][f"added {transitionType}"] = (added/ntransitions)/1024
+                modules_dict[label][f"maxTemp {transitionType}"] = (maxTemp/ntransitions)/1024
+                modules_dict[label][f"nDealloc {transitionType}"] = nDealloc/ntransitions
+                modules_dict[label][f"max1Alloc {transitionType}"] = (max1Alloc/ntransitions)/1024
+            else:
+                modules_dict[label][f"nAlloc {transitionType}"] = nAlloc
+                modules_dict[label][f"added {transitionType}"] = (added)/1024
+                modules_dict[label][f"maxTemp {transitionType}"] = (maxTemp)/1024
+                modules_dict[label][f"nDealloc {transitionType}"] = nDealloc
+                modules_dict[label][f"max1Alloc {transitionType}"] = max1Alloc/1024
+            doc["total"][f"nAlloc {transitionType}"] += modules_dict[label][f"nAlloc {transitionType}"]
+            doc["total"][f"nDealloc {transitionType}"] += modules_dict[label][f"nDealloc {transitionType}"]
+            doc["total"][f"maxTemp {transitionType}"] += modules_dict[label][f"maxTemp {transitionType}"]
+            doc["total"][f"added {transitionType}"] += modules_dict[label][f"added {transitionType}"]
+            doc["total"][f"max1Alloc {transitionType}"] += modules_dict[label][f"max1Alloc {transitionType}"]
+
+    for key in sorted(modules_dict.keys()):
+        module = modules_dict[key]
+        module["events"] = moduleTransitions['event'][key].get("nTransitions")
+        doc["modules"].append(module)
+
+    return doc
+
+def main(args):
+    import sys
+    doc = json.load(args.filename)
+    moduleTypes = doc['cpptypes']
+    moduleTransitions = dict()
+    for transition in transitionTypes:
+        moduleTransition = dict()
+        processModuleTransition("source", "PoolSource", doc["source"], transition, moduleTransition)
+        for moduleLabel, moduleInfo in doc["modules"].items():
+            processModuleTransition(moduleLabel, moduleTypes[moduleLabel], moduleInfo, transition, moduleTransition)
+        moduleTransitions[transition] = moduleTransition
+
+    json.dump(formatToCircles(moduleTransitions), sys.stdout, indent=2)
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser(description='Convert the JSON output of edmModuleAllocMonitorAnalyze.py to JSON for Circles')
+    parser.add_argument('filename',
+                        type=argparse.FileType('r'), # open file
+                        help='file to process')
+    args = parser.parse_args()
+    main(args)

PerfTools/AllocMonitor/scripts/edmModuleAllocMonitorAnalyze.py

@@ -620,9 +620,9 @@ def jsonInfo(self, syncs, temp, data):
         start = temp.findTime("source", self.transition, self.index)
         #we do not know the sync yet so have to wait until the framework transition
         if self.transition in [ Phase.construction, Phase.getNextTransition, Phase.destruction, Phase.openFile]:
-            data.insert( "source" , "sourceType", start, self.time, self.transition, self.index, (0,) , Activity.process, self.allocInfo)
+            data.insert( "source" , "PoolSource", start, self.time, self.transition, self.index, (0,) , Activity.process, self.allocInfo)
         else:
-            data.insert( "source" , "sourceType", start, self.time, self.transition, self.index, self.index , Activity.process, self.allocInfo)
+            data.insert( "source" , "PoolSource", start, self.time, self.transition, self.index, self.index , Activity.process, self.allocInfo)
     def jsonVisInfo(self,  data):
         index = self.index
         if self.transition == Phase.Event:
@@ -812,7 +812,7 @@ def jsonVisInfo(self,  data):
         return self._postJsonVis(data, self.allocInfo)
     def jsonInfo(self, syncs, temp, data):
         start = temp.findTime(self.moduleInfo._name+'source', self.transition, self.index)
-        data.insert( "source" , "sourceType", start, self.time, self.transition, self.index, syncs.get(self.transition, self.index) , Activity.delayedGet, self.allocInfo)
+        data.insert( "source" , "PoolSource", start, self.time, self.transition, self.index, syncs.get(self.transition, self.index) , Activity.delayedGet, self.allocInfo)
 
 class ESModuleTransitionParser(object):
     def __init__(self, payload, moduleInfos, esModuleInfos, recordNames):