Add support for TimingManager (#219)

Introduces support for the TimingManager which was a part of LLVM 17.
This enables time tracing of (parallel) compilation across target
systems. For example this is a report from the mock target.

```
===-------------------------------------------------------------------------===
                         ... Execution time report ...
===-------------------------------------------------------------------------===
  Total Execution Time: 0.0089 seconds

  ----User Time----  ----Wall Time----  ----Name----
    0.0000 (  0.1%)    0.0000 (  0.2%)  build-config
    0.0001 (  0.7%)    0.0001 (  1.3%)  build-target
    0.0004 (  2.7%)    0.0004 (  4.7%)  parse-mlir
    0.0000 (  0.1%)    0.0000 (  0.1%)  command-line-passes
    0.0143 ( 91.6%)    0.0076 ( 85.3%)  build-qem
    0.0142 ( 91.0%)    0.0075 ( 84.2%)    compile-payload
    0.0011 (  7.2%)    0.0011 ( 12.7%)      build-target-pass-managers
    0.0130 ( 83.5%)    0.0063 ( 71.0%)      compile-system
    0.0130 ( 83.5%)    0.0063 ( 71.0%)        MockSystem
    0.0019 ( 12.2%)    0.0010 ( 10.7%)          passes
    0.0001 (  0.4%)    0.0001 (  0.6%)            Canonicalizer
    0.0000 (  0.2%)    0.0000 (  0.3%)            mlir::quir::BreakResetPass
    0.0000 (  0.1%)    0.0000 (  0.1%)            mlir::quir::SubroutineCloningPass
    0.0000 (  0.0%)    0.0000 (  0.0%)            mlir::quir::RemoveQubitOperandsPass
    0.0000 (  0.0%)    0.0000 (  0.1%)            mlir::quir::ClassicalOnlyDetectionPass
    0.0001 (  0.5%)    0.0001 (  0.9%)            qssc::targets::systems::mock::MockQubitLocalizationPass
    0.0000 (  0.0%)    0.0000 (  0.0%)            qssc::targets::systems::mock::SymbolTableBuildPass
    0.0009 (  6.1%)    0.0005 (  5.4%)            'builtin.module' Pipeline
    0.0007 (  4.2%)    0.0004 (  4.6%)              qssc::targets::systems::mock::MockFunctionLocalizationPass
    0.0000 (  0.1%)    0.0000 (  0.1%)              mlir::quir::FunctionArgumentSpecializationPass
    0.0000 (  0.0%)    0.0000 (  0.0%)          emit-to-payload
    0.0110 ( 70.7%)    0.0053 ( 59.3%)          children
    0.0052 ( 33.1%)    0.0052 ( 58.2%)            MockController
    0.0004 (  2.5%)    0.0004 (  4.5%)              passes
    0.0002 (  1.3%)    0.0002 (  2.3%)                qssc::targets::systems::mock::conversion::MockQUIRToStdPass
    0.0000 (  0.1%)    0.0000 (  0.2%)                Canonicalizer
    0.0000 (  0.0%)    0.0000 (  0.0%)                LLVMLegalizeForExport
    0.0048 ( 30.5%)    0.0048 ( 53.6%)              emit-to-payload
    0.0000 (  0.0%)    0.0000 (  0.0%)              emit-to-payload-post-children
    0.0002 (  1.5%)    0.0002 (  2.6%)            MockDrive_0
    0.0000 (  0.0%)    0.0000 (  0.1%)              passes
    0.0002 (  1.4%)    0.0002 (  2.4%)              emit-to-payload
    0.0000 (  0.0%)    0.0000 (  0.0%)              emit-to-payload-post-children
    0.0002 (  1.4%)    0.0002 (  2.5%)            MockAcquire_0
    0.0000 (  0.0%)    0.0000 (  0.0%)              passes
    0.0002 (  1.3%)    0.0002 (  2.3%)              emit-to-payload
    0.0000 (  0.0%)    0.0000 (  0.0%)              emit-to-payload-post-children
    0.0001 (  1.0%)    0.0001 (  1.7%)            MockDrive_1
    0.0000 (  0.1%)    0.0000 (  0.3%)              passes
    0.0001 (  0.6%)    0.0001 (  1.1%)              emit-to-payload
    0.0000 (  0.0%)    0.0000 (  0.0%)              emit-to-payload-post-children
    0.0000 (  0.0%)    0.0000 (  0.0%)          emit-to-payload-post-children
    0.0001 (  0.6%)    0.0001 (  1.0%)    write-payload
    0.0007 (  4.8%)    0.0007 (  8.4%)  Rest
    0.0156 (100.0%)    0.0089 (100.0%)  Total
```

Author: taalexander

include/Frontend/OpenQASM3/OpenQASM3Frontend.h

@@ -25,6 +25,7 @@
 #include "API/errors.h"
 
 #include "mlir/IR/BuiltinOps.h"
+#include "mlir/Support/Timing.h"
 
 #include "llvm/Support/Error.h"
 
@@ -47,7 +48,8 @@ namespace qssc::frontend::openqasm3 {
 llvm::Error parse(std::string const &source, bool sourceIsFilename,
                   bool emitRawAST, bool emitPrettyAST, bool emitMLIR,
                   mlir::ModuleOp newModule,
-                  std::optional<DiagnosticCallback> diagnosticCb);
+                  std::optional<DiagnosticCallback> diagnosticCb,
+                  mlir::TimingScope &timing);
 
 }; // namespace qssc::frontend::openqasm3
 

include/HAL/Compile/TargetCompilationManager.h

@@ -43,15 +43,18 @@ class TargetCompilationManager {
   TargetCompilationManager(hal::TargetSystem &target,
                            mlir::MLIRContext *context);
 
-  using WalkTargetModulesFunction =
-      std::function<llvm::Error(hal::Target *, mlir::ModuleOp)>;
-  using WalkTargetFunction = std::function<llvm::Error(hal::Target *)>;
+  using WalkTargetModulesFunction = std::function<llvm::Error(
+      hal::Target *, mlir::ModuleOp, mlir::TimingScope &timing)>;
+  using WalkTargetFunction =
+      std::function<llvm::Error(hal::Target *, mlir::TimingScope &timing)>;
 
   // Depth first walker for a target system
-  llvm::Error walkTarget(Target *target, const WalkTargetFunction &walkFunc);
+  llvm::Error walkTarget(Target *target, mlir::TimingScope &timing,
+                         const WalkTargetFunction &walkFunc);
   // Depth first walker for a target system modules
   llvm::Error
   walkTargetModules(Target *target, mlir::ModuleOp targetModuleOp,
+                    mlir::TimingScope &timing,
                     const WalkTargetModulesFunction &walkFunc,
                     const WalkTargetModulesFunction &postChildrenCallbackFunc);
 
@@ -68,7 +71,8 @@ class TargetCompilationManager {
   /// @brief Compile only at the MLIR level for the full target
   /// system.
   /// @param moduleOp The root module operation to compile for.
-  /// This must not be specialized to a system already.
+  /// @param timing Root timing scope for tracking timing of payload
+  /// compilation. This must not be specialized to a system already.
   virtual llvm::Error compileMLIR(mlir::ModuleOp moduleOp) = 0;
 
   /// @brief Generate the full configured compilation pipeline
@@ -77,6 +81,8 @@ class TargetCompilationManager {
   /// @param moduleOp The root module operation to compile for.
   /// This must not be specialized to a system already.
   /// @param payload The payload to populate.
+  /// @param timing Root timing scope for tracking timing of payload
+  /// compilation.
   /// @param doCompileMLIR Whether to call compileMLIR prior to compiling the
   /// payload. Defaults to true.
   virtual llvm::Error compilePayload(mlir::ModuleOp moduleOp,
@@ -88,6 +94,9 @@ class TargetCompilationManager {
                         bool printBeforeAllTargetPayload,
                         bool printAfterTargetCompileFailure);
 
+  void enableTiming(mlir::TimingScope &timingScope);
+  void disableTiming();
+
 protected:
   bool getPrintBeforeAllTargetPasses() { return printBeforeAllTargetPasses; }
   bool getPrintAfterAllTargetPasses() { return printAfterAllTargetPasses; }
@@ -97,9 +106,13 @@ class TargetCompilationManager {
   }
 
   /// Thread-safe implementation
-  virtual void printIR(llvm::StringRef msg, mlir::Operation *op,
+  virtual void printIR(llvm::Twine msg, mlir::Operation *op,
                        llvm::raw_ostream &out);
 
+  /// @brief Get a nested timer instance from the root timer
+  /// @param name The name of the timing span
+  mlir::TimingScope getTimer(llvm::StringRef name);
+
 private:
   hal::TargetSystem &target;
   mlir::MLIRContext *context;
@@ -109,6 +122,8 @@ class TargetCompilationManager {
   bool printBeforeAllTargetPayload = false;
   bool printAfterTargetCompileFailure = false;
 
+  mlir::TimingScope rootTimer;
+
 }; // class TargetCompilationManager
 
 /// Register a set of useful command-line options that can be used to configure

include/HAL/Compile/ThreadedCompilationManager.h

@@ -46,17 +46,17 @@ class ThreadedCompilationManager : public TargetCompilationManager {
   /// Threaded depth first walker for a target system using the current
   /// MLIRContext's threadpool.
   llvm::Error walkTargetThreaded(
-      Target *target,
+      Target *target, mlir::TimingScope &timing,
       const TargetCompilationManager::WalkTargetFunction &walkFunc);
   /// Threaded depth first walker for a target system modules using the current
   /// MLIRContext's threadpool.
   llvm::Error walkTargetModulesThreaded(
-      Target *target, mlir::ModuleOp targetModuleOp,
+      Target *target, mlir::ModuleOp targetModuleOp, mlir::TimingScope &timing,
       const TargetCompilationManager::WalkTargetModulesFunction &walkFunc,
       const TargetCompilationManager::WalkTargetModulesFunction
           &postChildrenCallbackFunc);
 
-  virtual void printIR(llvm::StringRef msg, mlir::Operation *op,
+  virtual void printIR(llvm::Twine msg, mlir::Operation *op,
                        llvm::raw_ostream &out) override;
 
 public:
@@ -93,7 +93,8 @@ class ThreadedCompilationManager : public TargetCompilationManager {
 
   /// Prepare pass managers in a threaded way
   /// initializing them with the mlir context safely.
-  llvm::Error buildTargetPassManagers_(Target &target);
+  llvm::Error buildTargetPassManagers_(Target &target,
+                                       mlir::TimingScope &timing);
   /// Threadsafe initialization of PM to work around
   /// non-threadsafe registration of dependent dialects.
   /// I (Thomas) believe this is related to the conversation here
@@ -106,11 +107,13 @@ class ThreadedCompilationManager : public TargetCompilationManager {
   mlir::PassManager &createTargetPassManager_(Target *target);
 
   /// Compiles the input module for a single target.
-  llvm::Error compileMLIRTarget_(Target &target, mlir::ModuleOp targetModuleOp);
+  llvm::Error compileMLIRTarget_(Target &target, mlir::ModuleOp targetModuleOp,
+                                 mlir::TimingScope &timing);
   /// Compiles the input payload for a single target.
   llvm::Error compilePayloadTarget_(Target &target,
                                     mlir::ModuleOp targetModuleOp,
                                     qssc::payload::Payload &payload,
+                                    mlir::TimingScope &timing,
                                     bool doCompileMLIR);
 
   PMBuilder pmBuilder;

include/HAL/TargetSystem.h

@@ -51,8 +51,12 @@ class Target {
   };
 
 public:
-  virtual const std::string &getName() const { return name; }
+  virtual llvm::StringRef getName() const { return name; }
   virtual llvm::StringRef getDescription() const { return ""; }
+  /// @brief Get the Target resource directory sub-path for this target
+  /// which will be used for resolving external static resources at runtime
+  /// that are configured through the build system.
+  virtual llvm::StringRef getResourcePath() const { return getName(); }
   Target *getParent() { return parent; }
   const Target *getParent() const { return parent; }
 
@@ -129,7 +133,17 @@ class Target {
 
   virtual ~Target() = default;
 
+  /// @brief Enable timing from this point for the target and its methods
+  /// @param timingScope the root timer to nest timers from.
+  void enableTiming(mlir::TimingScope &timingScope);
+  /// @brief Disable(stop) ongoing timers
+  void disableTiming();
+
 protected:
+  /// @brief Get a nested timer instance from the root timer
+  /// @param name The name of the timing span
+  mlir::TimingScope getTimer(llvm::StringRef name);
+
   std::string name;
 
   // parent is already owned by unique_ptr
@@ -139,6 +153,9 @@ class Target {
 
   /// @brief Children targets storage.
   std::vector<std::unique_ptr<Target>> children_;
+
+private:
+  mlir::TimingScope rootTimer;
 };
 
 class TargetSystem : public Target {

lib/API/api.cpp

@@ -146,7 +146,11 @@ void showPayloads_() {
 /// @param config The configuration defining the context to build.
 /// @return The constructed TargetSystem.
 llvm::Expected<qssc::hal::TargetSystem &>
-buildTarget_(MLIRContext *context, const qssc::config::QSSConfig &config) {
+buildTarget_(MLIRContext *context, const qssc::config::QSSConfig &config,
+             mlir::TimingScope &timing) {
+
+  mlir::TimingScope const buildTargetTiming = timing.nest("build-target");
+
   const auto &targetName = config.getTargetName();
   const auto &targetConfigPath = config.getTargetConfigPath();
 
@@ -193,13 +197,18 @@ llvm::Error generateQEM_(
     const QSSConfig &config,
     qssc::hal::compile::TargetCompilationManager *targetCompilationManager,
     std::unique_ptr<qssc::payload::Payload> payload, mlir::ModuleOp moduleOp,
-    llvm::raw_ostream *ostream) {
+    llvm::raw_ostream *ostream, mlir::TimingScope &timing) {
 
+  mlir::TimingScope buildQEMTiming = timing.nest("build-qem");
+  targetCompilationManager->enableTiming(buildQEMTiming);
   if (auto err = targetCompilationManager->compilePayload(
           moduleOp, *payload,
           /* doCompileMLIR=*/!config.shouldBypassPayloadTargetCompilation()))
     return err;
+  targetCompilationManager->disableTiming();
 
+  mlir::TimingScope const writePayloadTiming =
+      buildQEMTiming.nest("write-payload");
   if (config.shouldEmitPlaintextPayload())
     payload->writePlain(*ostream);
   else
@@ -223,7 +232,6 @@ llvm::Error buildPassManager_(mlir::PassManager &pm, bool verifyPasses) {
     return llvm::createStringError(
         llvm::inconvertibleErrorCode(),
         "Unable to apply pass manager command line options");
-  mlir::applyDefaultTimingPassManagerCLOptions(pm);
 
   // Configure verifier
   pm.enableVerifier(verifyPasses);
@@ -232,9 +240,13 @@ llvm::Error buildPassManager_(mlir::PassManager &pm, bool verifyPasses) {
 }
 
 llvm::Error buildPassManager(const QSSConfig &config, mlir::PassManager &pm,
-                             ErrorHandler errorHandler, bool verifyPasses) {
+                             ErrorHandler errorHandler, bool verifyPasses,
+                             mlir::TimingScope &timing) {
   if (auto err = buildPassManager_(pm, verifyPasses))
     return err;
+
+  pm.enableTiming(timing);
+
   // Build the provided pipeline.
   if (failed(config.setupPassPipeline(pm)))
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
@@ -254,20 +266,26 @@ llvm::Error emitMLIR_(
     llvm::raw_ostream *ostream, mlir::MLIRContext &context,
     mlir::ModuleOp moduleOp, const QSSConfig &config,
     qssc::hal::compile::ThreadedCompilationManager &targetCompilationManager,
-    ErrorHandler errorHandler) {
+    ErrorHandler errorHandler, mlir::TimingScope &timing) {
+
+  mlir::TimingScope emitMlirTiming = timing.nest("emit-mlir");
+
   if (config.shouldCompileTargetIR()) {
     // Check if we can run the target compilation scheduler.
     if (config.shouldAddTargetPasses()) {
+      targetCompilationManager.enableTiming(emitMlirTiming);
       if (auto err = targetCompilationManager.compileMLIR(moduleOp))
         return llvm::joinErrors(
             llvm::createStringError(llvm::inconvertibleErrorCode(),
                                     "Failure while preparing target passes"),
             std::move(err));
+      targetCompilationManager.disableTiming();
     }
   }
 
   // Print the output.
   dumpMLIR_(ostream, moduleOp);
+
   return llvm::Error::success();
 }
 
@@ -280,7 +298,8 @@ llvm::Error emitMLIR_(
 llvm::Error emitQEM_(
     const QSSConfig &config, llvm::raw_ostream *ostream,
     std::unique_ptr<qssc::payload::Payload> payload, mlir::ModuleOp moduleOp,
-    qssc::hal::compile::ThreadedCompilationManager &targetCompilationManager) {
+    qssc::hal::compile::ThreadedCompilationManager &targetCompilationManager,
+    mlir::TimingScope &timing) {
   if (config.shouldIncludeSource()) {
     if (config.isDirectInput()) {
       if (config.getInputType() == InputType::QASM)
@@ -308,7 +327,7 @@ llvm::Error emitQEM_(
   }
 
   if (auto err = generateQEM_(config, &targetCompilationManager,
-                              std::move(payload), moduleOp, ostream))
+                              std::move(payload), moduleOp, ostream, timing))
     return err;
 
   return llvm::Error::success();
@@ -400,11 +419,13 @@ llvm::Error compile_(int argc, char const **argv, std::string *outputString,
   // Instantiate after parsing command line options.
   MLIRContext context{};
 
+  mlir::TimingScope buildConfigTiming = timing.nest("build-config");
   auto configResult = qssc::config::buildToolConfig();
   if (auto err = configResult.takeError())
     return err;
   qssc::config::QSSConfig const config = configResult.get();
   qssc::config::setContextConfig(&context, config);
+  buildConfigTiming.stop();
 
   // Populate the context
   context.appendDialectRegistry(registry);
@@ -432,7 +453,7 @@ llvm::Error compile_(int argc, char const **argv, std::string *outputString,
   }
 
   // Build the target for compilation
-  auto targetResult = buildTarget_(&context, config);
+  auto targetResult = buildTarget_(&context, config, timing);
   if (auto err = targetResult.takeError())
     return err;
   auto &target = targetResult.get();
@@ -508,6 +529,9 @@ llvm::Error compile_(int argc, char const **argv, std::string *outputString,
   mlir::ModuleOp moduleOp;
 
   if (config.getInputType() == InputType::QASM) {
+
+    mlir::TimingScope loadQASM3Timing = timing.nest("load-qasm3");
+
     if (config.getEmitAction() >= EmitAction::MLIR) {
       moduleOp = mlir::ModuleOp::create(FileLineColLoc::get(
           &context,
@@ -519,7 +543,8 @@ llvm::Error compile_(int argc, char const **argv, std::string *outputString,
             config.getInputSource().str(), !config.isDirectInput(),
             config.getEmitAction() == EmitAction::AST,
             config.getEmitAction() == EmitAction::ASTPretty,
-            config.getEmitAction() >= EmitAction::MLIR, moduleOp, diagnosticCb))
+            config.getEmitAction() >= EmitAction::MLIR, moduleOp, diagnosticCb,
+            loadQASM3Timing))
       return frontendError;
 
     if (config.getEmitAction() < EmitAction::MLIR)
@@ -528,6 +553,8 @@ llvm::Error compile_(int argc, char const **argv, std::string *outputString,
 
   if (config.getInputType() == InputType::MLIR) {
 
+    mlir::TimingScope mlirParserTiming = timing.nest("parse-mlir");
+
     // Tell sourceMgr about this buffer, which is what the parser will pick up.
     auto sourceMgr = std::make_shared<llvm::SourceMgr>();
     sourceMgr->AddNewSourceBuffer(std::move(file), llvm::SMLoc());
@@ -552,11 +579,10 @@ llvm::Error compile_(int argc, char const **argv, std::string *outputString,
       reproOptions.attachResourceParser(parseConfig);
 
     // Parse the input file and reset the context threading state.
-    mlir::TimingScope parserTiming = timing.nest("Parser");
     mlir::OwningOpRef<Operation *> op = mlir::parseSourceFileForTool(
         sourceMgr, parseConfig, !config.shouldUseExplicitModule());
 
-    parserTiming.stop();
+    mlirParserTiming.stop();
 
     if (!op)
       return llvm::createStringError(llvm::inconvertibleErrorCode(),
@@ -606,25 +632,30 @@ llvm::Error compile_(int argc, char const **argv, std::string *outputString,
         "Unable to apply target compilation options.");
 
   // Run additional passes specified on the command line
+
+  mlir::TimingScope commandLinePassesTiming =
+      timing.nest("command-line-passes");
   mlir::PassManager pm(&context);
-  if (auto err = buildPassManager(config, pm, errorHandler, verifyPasses))
+  if (auto err = buildPassManager(config, pm, errorHandler, verifyPasses,
+                                  commandLinePassesTiming))
     return err;
 
   if (pm.size() && failed(pm.run(moduleOp)))
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "Problems running the compiler pipeline!");
+  commandLinePassesTiming.stop();
 
   // Prepare outputs
   if (config.getEmitAction() == EmitAction::MLIR) {
     if (auto err = emitMLIR_(ostream, context, moduleOp, config,
-                             targetCompilationManager, errorHandler))
+                             targetCompilationManager, errorHandler, timing))
       return err;
   }
 
   if (config.getEmitAction() == EmitAction::QEM ||
       config.getEmitAction() == EmitAction::QEQEM) {
     if (auto err = emitQEM_(config, ostream, std::move(payload), moduleOp,
-                            targetCompilationManager))
+                            targetCompilationManager, timing))
       return err;
   }