[process-stats-dir] Support --merge-timers to sum across modules.

Author: graydon

utils/jobstats/jobstats.py

@@ -170,25 +170,56 @@ def to_lnt_test_obj(self, args):
         }
 
 
+AUXPATSTR = (r"(?P<module>[^-]+)-(?P<input>[^-]+)-(?P<triple>[^-]+)" +
+             r"-(?P<out>[^-]*)-(?P<opt>[^-]+)")
+AUXPAT = re.compile(AUXPATSTR)
+
+TIMERPATSTR = (r"time\.swift-(?P<jobkind>\w+)\." + AUXPATSTR +
+               "\.(?P<timerkind>\w+)$")
+TIMERPAT = re.compile(TIMERPATSTR)
+
+FILEPATSTR = (r"^stats-(?P<start>\d+)-swift-(?P<kind>\w+)-" +
+              AUXPATSTR +
+              r"-(?P<pid>\d+)(-.*)?.json$")
+FILEPAT = re.compile(FILEPATSTR)
+
+
+def match_auxpat(s):
+    m = AUXPAT.match(s)
+    if m is not None:
+        return m.groupdict()
+    else:
+        return None
+
+
+def match_timerpat(s):
+    m = TIMERPAT.match(s)
+    if m is not None:
+        return m.groupdict()
+    else:
+        return None
+
+
+def match_filepat(s):
+    m = FILEPAT.match(s)
+    if m is not None:
+        return m.groupdict()
+    else:
+        return None
+
+
 def load_stats_dir(path, select_module=[], select_stat=[],
-                   exclude_timers=False, **kwargs):
+                   exclude_timers=False, merge_timers=False, **kwargs):
     """Loads all stats-files found in path into a list of JobStats objects"""
     jobstats = []
-    auxpat = (r"(?P<module>[^-]+)-(?P<input>[^-]+)-(?P<triple>[^-]+)" +
-              r"-(?P<out>[^-]*)-(?P<opt>[^-]+)")
-    fpat = (r"^stats-(?P<start>\d+)-swift-(?P<kind>\w+)-" +
-            auxpat +
-            r"-(?P<pid>\d+)(-.*)?.json$")
-    fre = re.compile(fpat)
     sre = re.compile('.*' if len(select_stat) == 0 else
                      '|'.join(select_stat))
     for root, dirs, files in os.walk(path):
         for f in files:
-            m = fre.match(f)
-            if not m:
+            mg = match_filepat(f)
+            if not mg:
                 continue
             # NB: "pid" in fpat is a random number, not unix pid.
-            mg = m.groupdict()
             jobkind = mg['kind']
             jobid = int(mg['pid'])
             start_usec = int(mg['start'])
@@ -200,21 +231,22 @@ def load_stats_dir(path, select_module=[], select_stat=[],
             with open(os.path.join(root, f)) as fp:
                 j = json.load(fp)
             dur_usec = 1
-            patstr = (r"time\.swift-" + jobkind + r"\." + auxpat +
-                      r"\.wall$")
-            pat = re.compile(patstr)
             stats = dict()
             for (k, v) in j.items():
                 if sre.search(k) is None:
                     continue
-                if k.startswith("time."):
+                if k.startswith('time.') and exclude_timers:
+                    continue
+                tm = match_timerpat(k)
+                if tm:
                     v = int(1000000.0 * float(v))
-                    if exclude_timers:
-                        continue
+                    if tm['jobkind'] == jobkind and \
+                       tm['timerkind'] == 'wall':
+                        dur_usec = v
+                    if merge_timers:
+                        k = "time.swift-%s.%s" % (tm['jobkind'],
+                                                  tm['timerkind'])
                 stats[k] = v
-                tm = re.match(pat, k)
-                if tm:
-                    dur_usec = v
 
             e = JobStats(jobkind=jobkind, jobid=jobid,
                          module=module, start_usec=start_usec,

utils/process-stats-dir.py

@@ -528,6 +528,10 @@ def main():
                         default="sum",
                         type=str,
                         help="Merge identical metrics by (sum|min|max)")
+    parser.add_argument("--merge-timers",
+                        default=False,
+                        action="store_true",
+                        help="Merge timers across modules/targets/etc.")
     parser.add_argument("--markdown",
                         default=False,
                         action="store_true",