Merge pull request swiftlang#12071 from graydon/trace-stats-deltas

Trace stats deltas

Author: graydon

include/swift/Basic/Statistic.h

@@ -15,6 +15,7 @@
 
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/Statistic.h"
+#include "swift/Basic/SourceLoc.h"
 #include "swift/Basic/Timer.h"
 
 #define SWIFT_FUNC_STAT                                                 \
@@ -53,112 +54,84 @@ class UnifiedStatsReporter {
 public:
   struct AlwaysOnDriverCounters
   {
-    size_t NumDriverJobsRun;
-    size_t NumDriverJobsSkipped;
-
-    size_t DriverDepCascadingTopLevel;
-    size_t DriverDepCascadingDynamic;
-    size_t DriverDepCascadingNominal;
-    size_t DriverDepCascadingMember;
-    size_t DriverDepCascadingExternal;
-
-    size_t DriverDepTopLevel;
-    size_t DriverDepDynamic;
-    size_t DriverDepNominal;
-    size_t DriverDepMember;
-    size_t DriverDepExternal;
-
-    size_t ChildrenMaxRSS;
+#define DRIVER_STATISTIC(ID) size_t ID;
+#include "Statistics.def"
+#undef DRIVER_STATISTIC
   };
 
   struct AlwaysOnFrontendCounters
   {
-    size_t NumSourceBuffers;
-    size_t NumSourceLines;
-    size_t NumSourceLinesPerSecond;
-    size_t NumLinkLibraries;
-    size_t NumLoadedModules;
-    size_t NumImportedExternalDefinitions;
-    size_t NumTotalClangImportedEntities;
-    size_t NumASTBytesAllocated;
-    size_t NumDependencies;
-    size_t NumReferencedTopLevelNames;
-    size_t NumReferencedDynamicNames;
-    size_t NumReferencedMemberNames;
-    size_t NumDecls;
-    size_t NumLocalTypeDecls;
-    size_t NumObjCMethods;
-    size_t NumInfixOperators;
-    size_t NumPostfixOperators;
-    size_t NumPrefixOperators;
-    size_t NumPrecedenceGroups;
-    size_t NumUsedConformances;
-
-    size_t NumConformancesDeserialized;
-    size_t NumConstraintScopes;
-    size_t NumDeclsDeserialized;
-    size_t NumDeclsValidated;
-    size_t NumFunctionsTypechecked;
-    size_t NumGenericSignatureBuilders;
-    size_t NumLazyGenericEnvironments;
-    size_t NumLazyGenericEnvironmentsLoaded;
-    size_t NumLazyIterableDeclContexts;
-    size_t NominalTypeLookupDirectCount;
-    size_t NumTypesDeserialized;
-    size_t NumTypesValidated;
-    size_t NumUnloadedLazyIterableDeclContexts;
-
-    size_t NumSILGenFunctions;
-    size_t NumSILGenVtables;
-    size_t NumSILGenWitnessTables;
-    size_t NumSILGenDefaultWitnessTables;
-    size_t NumSILGenGlobalVariables;
-
-    size_t NumSILOptFunctions;
-    size_t NumSILOptVtables;
-    size_t NumSILOptWitnessTables;
-    size_t NumSILOptDefaultWitnessTables;
-    size_t NumSILOptGlobalVariables;
+#define FRONTEND_STATISTIC(NAME, ID) size_t ID;
+#include "Statistics.def"
+#undef FRONTEND_STATISTIC
+  };
 
-    size_t NumIRGlobals;
-    size_t NumIRFunctions;
-    size_t NumIRAliases;
-    size_t NumIRIFuncs;
-    size_t NumIRNamedMetaData;
-    size_t NumIRValueSymbols;
-    size_t NumIRComdatSymbols;
-    size_t NumIRBasicBlocks;
-    size_t NumIRInsts;
+  struct FrontendStatsTracer
+  {
+    UnifiedStatsReporter *Reporter;
+    llvm::TimeRecord SavedTime;
+    StringRef Name;
+    SourceRange Range;
+    FrontendStatsTracer(StringRef Name,
+                        SourceRange const &Range,
+                        UnifiedStatsReporter *Reporter);
+    FrontendStatsTracer();
+    FrontendStatsTracer(FrontendStatsTracer&& other);
+    FrontendStatsTracer& operator=(FrontendStatsTracer&&);
+    ~FrontendStatsTracer();
+    FrontendStatsTracer(const FrontendStatsTracer&) = delete;
+    FrontendStatsTracer& operator=(const FrontendStatsTracer&) = delete;
+  };
 
-    size_t NumLLVMBytesOutput;
+  struct FrontendStatsEvent
+  {
+    uint64_t TimeUSec;
+    uint64_t LiveUSec;
+    bool IsEntry;
+    StringRef EventName;
+    StringRef CounterName;
+    size_t CounterDelta;
+    size_t CounterValue;
+    SourceRange SourceRange;
   };
 
 private:
-  SmallString<128> Filename;
+  SmallString<128> StatsFilename;
+  SmallString<128> TraceFilename;
   llvm::TimeRecord StartedTime;
   std::unique_ptr<llvm::NamedRegionTimer> Timer;
-
+  SourceManager *SourceMgr;
   std::unique_ptr<AlwaysOnDriverCounters> DriverCounters;
   std::unique_ptr<AlwaysOnFrontendCounters> FrontendCounters;
+  std::unique_ptr<AlwaysOnFrontendCounters> LastTracedFrontendCounters;
+  std::vector<FrontendStatsEvent> FrontendStatsEvents;
 
   void publishAlwaysOnStatsToLLVM();
   void printAlwaysOnStatsAndTimers(llvm::raw_ostream &OS);
 
   UnifiedStatsReporter(StringRef ProgramName,
                        StringRef AuxName,
-                       StringRef Directory);
+                       StringRef Directory,
+                       SourceManager *SM,
+                       bool TraceEvents);
 public:
   UnifiedStatsReporter(StringRef ProgramName,
                        StringRef ModuleName,
                        StringRef InputName,
                        StringRef TripleName,
                        StringRef OutputType,
                        StringRef OptType,
-                       StringRef Directory);
+                       StringRef Directory,
+                       SourceManager *SM=nullptr,
+                       bool TraceEvents=false);
   ~UnifiedStatsReporter();
 
   AlwaysOnDriverCounters &getDriverCounters();
   AlwaysOnFrontendCounters &getFrontendCounters();
+  FrontendStatsTracer getStatsTracer(StringRef N,
+                                     SourceRange const &R);
+  void saveAnyFrontendStatsEvents(FrontendStatsTracer const& T,
+                                  bool IsEntry);
 };
 
 }

include/swift/Basic/Statistics.def

@@ -0,0 +1,103 @@
+//===--- Statistics.def - Statistics Macro Metaprogramming Database -*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the database of always-available statistic counters.
+//
+// DRIVER_STATISTIC(Id)
+//   - Id is an identifier suitable for use in C++
+//
+// FRONTEND_STATISTIC(Subsystem, Id)
+//   - Subsystem is a token to be stringified as a name prefix
+//   - Id is an identifier suitable for use in C++
+//===----------------------------------------------------------------------===//
+
+/// Driver statistics are collected for driver processes
+#ifdef DRIVER_STATISTIC
+DRIVER_STATISTIC(NumDriverJobsRun)
+DRIVER_STATISTIC(NumDriverJobsSkipped)
+
+DRIVER_STATISTIC(DriverDepCascadingTopLevel)
+DRIVER_STATISTIC(DriverDepCascadingDynamic)
+DRIVER_STATISTIC(DriverDepCascadingNominal)
+DRIVER_STATISTIC(DriverDepCascadingMember)
+DRIVER_STATISTIC(DriverDepCascadingExternal)
+
+DRIVER_STATISTIC(DriverDepTopLevel)
+DRIVER_STATISTIC(DriverDepDynamic)
+DRIVER_STATISTIC(DriverDepNominal)
+DRIVER_STATISTIC(DriverDepMember)
+DRIVER_STATISTIC(DriverDepExternal)
+
+DRIVER_STATISTIC(ChildrenMaxRSS)
+#endif
+
+/// Driver statistics are collected for frontend processes
+#ifdef FRONTEND_STATISTIC
+FRONTEND_STATISTIC(AST, NumSourceBuffers)
+FRONTEND_STATISTIC(AST, NumSourceLines)
+FRONTEND_STATISTIC(AST, NumSourceLinesPerSecond)
+FRONTEND_STATISTIC(AST, NumLinkLibraries)
+FRONTEND_STATISTIC(AST, NumLoadedModules)
+FRONTEND_STATISTIC(AST, NumImportedExternalDefinitions)
+FRONTEND_STATISTIC(AST, NumTotalClangImportedEntities)
+FRONTEND_STATISTIC(AST, NumASTBytesAllocated)
+FRONTEND_STATISTIC(AST, NumDependencies)
+FRONTEND_STATISTIC(AST, NumReferencedTopLevelNames)
+FRONTEND_STATISTIC(AST, NumReferencedDynamicNames)
+FRONTEND_STATISTIC(AST, NumReferencedMemberNames)
+FRONTEND_STATISTIC(AST, NumDecls)
+FRONTEND_STATISTIC(AST, NumLocalTypeDecls)
+FRONTEND_STATISTIC(AST, NumObjCMethods)
+FRONTEND_STATISTIC(AST, NumInfixOperators)
+FRONTEND_STATISTIC(AST, NumPostfixOperators)
+FRONTEND_STATISTIC(AST, NumPrefixOperators)
+FRONTEND_STATISTIC(AST, NumPrecedenceGroups)
+FRONTEND_STATISTIC(AST, NumUsedConformances)
+
+FRONTEND_STATISTIC(Sema, NumConformancesDeserialized)
+FRONTEND_STATISTIC(Sema, NumConstraintScopes)
+FRONTEND_STATISTIC(Sema, NumDeclsDeserialized)
+FRONTEND_STATISTIC(Sema, NumDeclsValidated)
+FRONTEND_STATISTIC(Sema, NumFunctionsTypechecked)
+FRONTEND_STATISTIC(Sema, NumGenericSignatureBuilders)
+FRONTEND_STATISTIC(Sema, NumLazyGenericEnvironments)
+FRONTEND_STATISTIC(Sema, NumLazyGenericEnvironmentsLoaded)
+FRONTEND_STATISTIC(Sema, NumLazyIterableDeclContexts)
+FRONTEND_STATISTIC(Sema, NominalTypeLookupDirectCount)
+FRONTEND_STATISTIC(Sema, NumTypesDeserialized)
+FRONTEND_STATISTIC(Sema, NumTypesValidated)
+FRONTEND_STATISTIC(Sema, NumUnloadedLazyIterableDeclContexts)
+
+FRONTEND_STATISTIC(SILModule, NumSILGenFunctions)
+FRONTEND_STATISTIC(SILModule, NumSILGenVtables)
+FRONTEND_STATISTIC(SILModule, NumSILGenWitnessTables)
+FRONTEND_STATISTIC(SILModule, NumSILGenDefaultWitnessTables)
+FRONTEND_STATISTIC(SILModule, NumSILGenGlobalVariables)
+
+FRONTEND_STATISTIC(SILModule, NumSILOptFunctions)
+FRONTEND_STATISTIC(SILModule, NumSILOptVtables)
+FRONTEND_STATISTIC(SILModule, NumSILOptWitnessTables)
+FRONTEND_STATISTIC(SILModule, NumSILOptDefaultWitnessTables)
+FRONTEND_STATISTIC(SILModule, NumSILOptGlobalVariables)
+
+FRONTEND_STATISTIC(IRModule, NumIRGlobals)
+FRONTEND_STATISTIC(IRModule, NumIRFunctions)
+FRONTEND_STATISTIC(IRModule, NumIRAliases)
+FRONTEND_STATISTIC(IRModule, NumIRIFuncs)
+FRONTEND_STATISTIC(IRModule, NumIRNamedMetaData)
+FRONTEND_STATISTIC(IRModule, NumIRValueSymbols)
+FRONTEND_STATISTIC(IRModule, NumIRComdatSymbols)
+FRONTEND_STATISTIC(IRModule, NumIRBasicBlocks)
+FRONTEND_STATISTIC(IRModule, NumIRInsts)
+
+FRONTEND_STATISTIC(LLVM, NumLLVMBytesOutput)
+#endif

include/swift/Frontend/FrontendOptions.h

@@ -222,6 +222,9 @@ class FrontendOptions {
   /// The path to which we should output statistics files.
   std::string StatsOutputDir;
 
+  /// Trace changes to stats to files in StatsOutputDir.
+  bool TraceStats = false;
+
   /// Indicates whether function body parsing should be delayed
   /// until the end of all files.
   bool DelayedFunctionBodyParsing = false;

include/swift/Option/Options.td

@@ -194,6 +194,9 @@ def driver_time_compilation : Flag<["-"], "driver-time-compilation">,
 def stats_output_dir: Separate<["-"], "stats-output-dir">,
   Flags<[FrontendOption, HelpHidden]>,
   HelpText<"Directory to write unified compilation-statistics files to">;
+def trace_stats_events: Flag<["-"], "trace-stats-events">,
+  Flags<[FrontendOption, HelpHidden]>,
+  HelpText<"Trace changes to stats in -stats-output-dir">;
 
 def emit_dependencies : Flag<["-"], "emit-dependencies">,
   Flags<[FrontendOption, NoInteractiveOption, DoesNotAffectIncrementalBuild]>,