Merge pull request swiftlang#12962 from graydon/misc-process-stats-dir-fixes

Author: swift-ci

utils/jobstats/jobstats.py

@@ -98,6 +98,13 @@ def prefixed_by(self, prefix):
                         self.module, self.start_usec, self.dur_usec,
                         self.jobargs, prefixed_stats)
 
+    def divided_by(self, n):
+        divided_stats = dict([(k, v / n)
+                              for (k, v) in self.stats.items()])
+        return JobStats(self.jobkind, random.randint(0, 1000000000),
+                        self.module, self.start_usec, self.dur_usec,
+                        self.jobargs, divided_stats)
+
     def incrementality_percentage(self):
         """Assuming the job is a driver job, return the amount of
         jobs that actually ran, as a percentage of the total number."""
@@ -170,25 +177,56 @@ def to_lnt_test_obj(self, args):
         }
 
 
+AUXPATSTR = (r"(?P<module>[^-]+)-(?P<input>[^-]+)-(?P<triple>[^-]+)" +
+             r"-(?P<out>[^-]*)-(?P<opt>[^-]+)")
+AUXPAT = re.compile(AUXPATSTR)
+
+TIMERPATSTR = (r"time\.swift-(?P<jobkind>\w+)\." + AUXPATSTR +
+               "\.(?P<timerkind>\w+)$")
+TIMERPAT = re.compile(TIMERPATSTR)
+
+FILEPATSTR = (r"^stats-(?P<start>\d+)-swift-(?P<kind>\w+)-" +
+              AUXPATSTR +
+              r"-(?P<pid>\d+)(-.*)?.json$")
+FILEPAT = re.compile(FILEPATSTR)
+
+
+def match_auxpat(s):
+    m = AUXPAT.match(s)
+    if m is not None:
+        return m.groupdict()
+    else:
+        return None
+
+
+def match_timerpat(s):
+    m = TIMERPAT.match(s)
+    if m is not None:
+        return m.groupdict()
+    else:
+        return None
+
+
+def match_filepat(s):
+    m = FILEPAT.match(s)
+    if m is not None:
+        return m.groupdict()
+    else:
+        return None
+
+
 def load_stats_dir(path, select_module=[], select_stat=[],
-                   exclude_timers=False, **kwargs):
+                   exclude_timers=False, merge_timers=False, **kwargs):
     """Loads all stats-files found in path into a list of JobStats objects"""
     jobstats = []
-    auxpat = (r"(?P<module>[^-]+)-(?P<input>[^-]+)-(?P<triple>[^-]+)" +
-              r"-(?P<out>[^-]*)-(?P<opt>[^-]+)")
-    fpat = (r"^stats-(?P<start>\d+)-swift-(?P<kind>\w+)-" +
-            auxpat +
-            r"-(?P<pid>\d+)(-.*)?.json$")
-    fre = re.compile(fpat)
     sre = re.compile('.*' if len(select_stat) == 0 else
                      '|'.join(select_stat))
     for root, dirs, files in os.walk(path):
         for f in files:
-            m = fre.match(f)
-            if not m:
+            mg = match_filepat(f)
+            if not mg:
                 continue
             # NB: "pid" in fpat is a random number, not unix pid.
-            mg = m.groupdict()
             jobkind = mg['kind']
             jobid = int(mg['pid'])
             start_usec = int(mg['start'])
@@ -200,21 +238,22 @@ def load_stats_dir(path, select_module=[], select_stat=[],
             with open(os.path.join(root, f)) as fp:
                 j = json.load(fp)
             dur_usec = 1
-            patstr = (r"time\.swift-" + jobkind + r"\." + auxpat +
-                      r"\.wall$")
-            pat = re.compile(patstr)
             stats = dict()
             for (k, v) in j.items():
                 if sre.search(k) is None:
                     continue
-                if k.startswith("time."):
+                if k.startswith('time.') and exclude_timers:
+                    continue
+                tm = match_timerpat(k)
+                if tm:
                     v = int(1000000.0 * float(v))
-                    if exclude_timers:
-                        continue
+                    if tm['jobkind'] == jobkind and \
+                       tm['timerkind'] == 'wall':
+                        dur_usec = v
+                    if merge_timers:
+                        k = "time.swift-%s.%s" % (tm['jobkind'],
+                                                  tm['timerkind'])
                 stats[k] = v
-                tm = re.match(pat, k)
-                if tm:
-                    dur_usec = v
 
             e = JobStats(jobkind=jobkind, jobid=jobid,
                          module=module, start_usec=start_usec,
@@ -225,7 +264,7 @@ def load_stats_dir(path, select_module=[], select_stat=[],
 
 
 def merge_all_jobstats(jobstats, select_module=[], group_by_module=False,
-                       merge_by="sum", **kwargs):
+                       merge_by="sum", divide_by=1, **kwargs):
     """Does a pairwise merge of the elements of list of jobs"""
     m = None
     if len(select_module) > 0:
@@ -237,12 +276,15 @@ def keyfunc(j):
         jobstats.sort(key=keyfunc)
         prefixed = []
         for mod, group in itertools.groupby(jobstats, keyfunc):
-            groupmerge = merge_all_jobstats(group, merge_by=merge_by)
+            groupmerge = merge_all_jobstats(group, merge_by=merge_by,
+                                            divide_by=divide_by)
             prefixed.append(groupmerge.prefixed_by(mod))
         jobstats = prefixed
     for j in jobstats:
         if m is None:
             m = j
         else:
             m = m.merged_with(j, merge_by=merge_by)
-    return m
+    if m is None:
+        return m
+    return m.divided_by(divide_by)

utils/process-stats-dir.py

@@ -48,7 +48,10 @@ def vars_of_args(args):
     vargs = vars(args)
     if args.select_stats_from_csv_baseline is not None:
         b = read_stats_dict_from_csv(args.select_stats_from_csv_baseline)
-        if args.group_by_module:
+        # Sniff baseline stat-names to figure out if they're module-qualified
+        # even when the user isn't asking us to _output_ module-grouped data.
+        all_triples = all(len(k.split('.')) == 3 for k in b.keys())
+        if args.group_by_module or all_triples:
             vargs['select_stat'] = set(stat_name_minus_module(k)
                                        for k in b.keys())
         else:
@@ -308,9 +311,20 @@ def write_comparison(args, old_stats, new_stats):
 
     if args.markdown:
 
+        def format_time(v):
+            if abs(v) > 1000000:
+                return "{:.1f}s".format(v / 1000000.0)
+            elif abs(v) > 1000:
+                return "{:.1f}ms".format(v / 1000.0)
+            else:
+                return "{:.1f}us".format(v)
+
         def format_field(field, row):
-            if field == 'name' and args.group_by_module:
-                return stat_name_minus_module(row.name)
+            if field == 'name':
+                if args.group_by_module:
+                    return stat_name_minus_module(row.name)
+                else:
+                    return row.name
             elif field == 'delta_pct':
                 s = str(row.delta_pct) + "%"
                 if args.github_emoji:
@@ -320,7 +334,11 @@ def format_field(field, row):
                         s += " :white_check_mark:"
                 return s
             else:
-                return str(vars(row)[field])
+                v = int(vars(row)[field])
+                if row.name.startswith('time.'):
+                    return format_time(v)
+                else:
+                    return "{:,d}".format(v)
 
         def format_table(elts):
             out = args.output
@@ -357,10 +375,12 @@ def keyfunc(e):
                 format_table(elts)
             out.write('</details>\n')
 
-        format_details('Regressed', regressed, args.close_regressions)
+        closed_regressions = (args.close_regressions or len(regressed) == 0)
+        format_details('Regressed', regressed, closed_regressions)
         format_details('Improved', improved, True)
-        format_details('Unchanged (abs(delta) < %s%% or %susec)' %
-                       (args.delta_pct_thresh, args.delta_usec_thresh),
+        format_details('Unchanged (delta < %s%% or delta < %s)' %
+                       (args.delta_pct_thresh,
+                        format_time(args.delta_usec_thresh)),
                        unchanged, True)
 
     else:
@@ -528,6 +548,15 @@ def main():
                         default="sum",
                         type=str,
                         help="Merge identical metrics by (sum|min|max)")
+    parser.add_argument("--merge-timers",
+                        default=False,
+                        action="store_true",
+                        help="Merge timers across modules/targets/etc.")
+    parser.add_argument("--divide-by",
+                        default=1,
+                        metavar="D",
+                        type=int,
+                        help="Divide stats by D (to take an average)")
     parser.add_argument("--markdown",
                         default=False,
                         action="store_true",