Merge pull request swiftlang#10084 from graydon/process-stats-dir-mangling

graydon · web-flow · commit 8c32c0dd8c0f · 2017-06-02T20:23:14.000-07:00
[Stats] Mangle a little more information into the file &amp; timer names.
diff --git a/include/swift/Basic/Statistic.h b/include/swift/Basic/Statistic.h
@@ -140,9 +140,16 @@ class UnifiedStatsReporter {
   void publishAlwaysOnStatsToLLVM();
   void printAlwaysOnStatsAndTimers(llvm::raw_ostream &OS);
 
+  UnifiedStatsReporter(StringRef ProgramName,
+                       StringRef AuxName,
+                       StringRef Directory);
 public:
   UnifiedStatsReporter(StringRef ProgramName,
-                       StringRef TargetName,
+                       StringRef ModuleName,
+                       StringRef InputName,
+                       StringRef TripleName,
+                       StringRef OutputType,
+                       StringRef OptType,
                        StringRef Directory);
   ~UnifiedStatsReporter();
 
diff --git a/lib/Basic/Statistic.cpp b/lib/Basic/Statistic.cpp
@@ -43,13 +43,15 @@ getChildrenMaxResidentSetSize() {
 }
 
 static std::string
-makeFileName(StringRef ProcessName) {
+makeFileName(StringRef ProgramName,
+             StringRef AuxName) {
   std::string tmp;
   raw_string_ostream stream(tmp);
   auto now = std::chrono::system_clock::now();
   stream << "stats"
          << "-" << now.time_since_epoch().count()
-         << "-" << ProcessName
+         << "-" << ProgramName
+         << "-" << AuxName
          << "-" << Process::GetRandomNumber()
          << ".json";
   return stream.str();
@@ -58,31 +60,63 @@ makeFileName(StringRef ProcessName) {
 // LLVM's statistics-reporting machinery is sensitive to filenames containing
 // YAML-quote-requiring characters, which occur surprisingly often in the wild;
 // we only need a recognizable and likely-unique name for a target here, not an
-// exact filename, so we go with a crude approximation.
+// exact filename, so we go with a crude approximation. Furthermore, to avoid
+// parse ambiguities when "demangling" counters and filenames we exclude hyphens
+// and slashes.
 static std::string
-cleanTargetName(StringRef TargetName) {
+cleanName(StringRef n) {
   std::string tmp;
-  for (auto c : TargetName) {
+  for (auto c : n) {
     if (('a' <= c && c <= 'z') ||
         ('A' <= c && c <= 'Z') ||
         ('0' <= c && c <= '9') ||
-        (c == '-') || (c == '.') || (c == '/'))
+        (c == '.'))
       tmp += c;
     else
       tmp += '_';
   }
   return tmp;
 }
 
+static std::string
+auxName(StringRef ModuleName,
+        StringRef InputName,
+        StringRef TripleName,
+        StringRef OutputType,
+        StringRef OptType) {
+  return (cleanName(ModuleName)
+          + "-" + cleanName(InputName)
+          + "-" + cleanName(TripleName)
+          + "-" + cleanName(OutputType)
+          + "-" + cleanName(OptType));
+}
+
+UnifiedStatsReporter::UnifiedStatsReporter(StringRef ProgramName,
+                                           StringRef ModuleName,
+                                           StringRef InputName,
+                                           StringRef TripleName,
+                                           StringRef OutputType,
+                                           StringRef OptType,
+                                           StringRef Directory)
+  : UnifiedStatsReporter(ProgramName,
+                         auxName(ModuleName,
+                                 InputName,
+                                 TripleName,
+                                 OutputType,
+                                 OptType),
+                         Directory)
+{
+}
+
 UnifiedStatsReporter::UnifiedStatsReporter(StringRef ProgramName,
-                                           StringRef TargetName,
+                                           StringRef AuxName,
                                            StringRef Directory)
   : Filename(Directory),
-    Timer(make_unique<NamedRegionTimer>(cleanTargetName(TargetName),
+    Timer(make_unique<NamedRegionTimer>(AuxName,
                                         "Building Target",
                                         ProgramName, "Running Program"))
 {
-  path::append(Filename, makeFileName(ProgramName));
+  path::append(Filename, makeFileName(ProgramName, AuxName));
   EnableStatistics(/*PrintOnExit=*/false);
   SharedTimer::enableCompilationTimers();
 }
diff --git a/lib/Driver/Driver.cpp b/lib/Driver/Driver.cpp
@@ -558,8 +558,21 @@ std::unique_ptr<Compilation> Driver::buildCompilation(
   std::unique_ptr<UnifiedStatsReporter> StatsReporter;
   if (const Arg *A =
       ArgList->getLastArgNoClaim(options::OPT_stats_output_dir)) {
+    StringRef OptType = "Onone";
+    if (const Arg *OptA = ArgList->getLastArgNoClaim(options::OPT_O_Group)) {
+      OptType = OptA->getSpelling();
+    }
+    StringRef InputName = "all";
+    if (Inputs.size() == 1) {
+      InputName = Inputs[0].second->getSpelling();
+    }
+    StringRef OutputType = types::getTypeTempSuffix(OI.CompilerOutputType);
     StatsReporter = llvm::make_unique<UnifiedStatsReporter>("swift-driver",
                                                             OI.ModuleName,
+                                                            InputName,
+                                                            DefaultTargetTriple,
+                                                            OutputType,
+                                                            OptType,
                                                             A->getValue());
   }
 
diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp
@@ -1032,6 +1032,18 @@ static bool dumpAPI(ModuleDecl *Mod, StringRef OutDir) {
   return false;
 }
 
+static StringRef
+silOptModeArgStr(SILOptions::SILOptMode mode) {
+  switch (mode) {
+ case SILOptions::SILOptMode::Optimize:
+   return "O";
+ case SILOptions::SILOptMode::OptimizeUnchecked:
+   return "Ounchecked";
+ default:
+   return "Onone";
+  }
+}
+
 int swift::performFrontend(ArrayRef<const char *> Args,
                            const char *Argv0, void *MainAddr,
                            FrontendObserver *observer) {
@@ -1183,16 +1195,29 @@ int swift::performFrontend(ArrayRef<const char *> Args,
       Invocation.getFrontendOptions().StatsOutputDir;
   std::unique_ptr<UnifiedStatsReporter> StatsReporter;
   if (!StatsOutputDir.empty()) {
-    auto &opts = Invocation.getFrontendOptions();
-    std::string TargetName = opts.ModuleName;
-    if (opts.PrimaryInput.hasValue() &&
-        opts.PrimaryInput.getValue().isFilename()) {
-      auto Index = opts.PrimaryInput.getValue().Index;
-      TargetName += ".";
-      TargetName += llvm::sys::path::filename(opts.InputFilenames[Index]);
+    auto &FEOpts = Invocation.getFrontendOptions();
+    auto &LangOpts = Invocation.getLangOptions();
+    auto &SILOpts = Invocation.getSILOptions();
+    StringRef InputName;
+    std::string TargetName = FEOpts.ModuleName;
+    if (FEOpts.PrimaryInput.hasValue() &&
+        FEOpts.PrimaryInput.getValue().isFilename()) {
+      auto Index = FEOpts.PrimaryInput.getValue().Index;
+      InputName = FEOpts.InputFilenames[Index];
+    }
+    StringRef OptType = silOptModeArgStr(SILOpts.Optimization);
+    StringRef OutFile = FEOpts.getSingleOutputFilename();
+    StringRef OutputType = llvm::sys::path::extension(OutFile);
+    if (OutputType.size() > 0 && OutputType.front() == '.') {
+      OutputType = OutputType.substr(1);
     }
+    std::string TripleName = LangOpts.Target.normalize();
     StatsReporter = llvm::make_unique<UnifiedStatsReporter>("swift-frontend",
-                                                            TargetName,
+                                                            FEOpts.ModuleName,
+                                                            InputName,
+                                                            TripleName,
+                                                            OutputType,
+                                                            OptType,
                                                             StatsOutputDir);
   }
 
diff --git a/utils/process-stats-dir.py b/utils/process-stats-dir.py
@@ -148,7 +148,11 @@ def to_lnt_test_obj(self, args):
 # Return an array of JobStats objects
 def load_stats_dir(path):
     jobstats = []
-    fpat = r"^stats-(?P<start>\d+)-swift-(?P<kind>\w+)-(?P<pid>\d+).json$"
+    auxpat = (r"(?P<module>[^-]+)-(?P<input>[^-]+)-(?P<triple>[^-]+)" +
+              r"-(?P<out>[^-]+)-(?P<opt>[^-]+)")
+    fpat = (r"^stats-(?P<start>\d+)-swift-(?P<kind>\w+)-" +
+            auxpat +
+            r"-(?P<pid>\d+)(-.*)?.json$")
     for root, dirs, files in os.walk(path):
         for f in files:
             m = re.match(fpat, f)
@@ -158,13 +162,13 @@ def load_stats_dir(path):
                 jobkind = mg['kind']
                 jobid = int(mg['pid'])
                 start_usec = int(mg['start'])
+                module = mg["module"]
+                jobargs = [mg["input"], mg["triple"], mg["out"], mg["opt"]]
 
                 j = json.load(open(os.path.join(root, f)))
                 dur_usec = 1
-                jobargs = None
-                module = "module"
-                patstr = (r"time\.swift-" + jobkind +
-                          r"\.(?P<module>[^\.]+)(?P<filename>.*)\.wall$")
+                patstr = (r"time\.swift-" + jobkind + r"\." + auxpat +
+                          r"\.wall$")
                 pat = re.compile(patstr)
                 stats = dict()
                 for (k, v) in j.items():
@@ -173,14 +177,7 @@ def load_stats_dir(path):
                     stats[k] = v
                     tm = re.match(pat, k)
                     if tm:
-                        tmg = tm.groupdict()
                         dur_usec = v
-                        module = tmg['module']
-                        if 'filename' in tmg:
-                            ff = tmg['filename']
-                            if ff.startswith('.'):
-                                ff = ff[1:]
-                            jobargs = [ff]
 
                 e = JobStats(jobkind=jobkind, jobid=jobid,
                              module=module, start_usec=start_usec,
