[Remarks] Add a specialized RemarkStreamer for SIL remarks

This allows the usage of the whole remark infrastructure developed in
LLVM, which includes a new binary format, metadata in object files, etc.

This gets rid of the YAMLTraits-based remark serialization and does the
plumbing for hooking to LLVM's main remark streamer.

For more about the idea behind LLVM's main remark streamer, see the
docs/Remarks.rst changes in https://reviews.llvm.org/D73676.

The flags are now:

* -save-optimization-record: enable remarks, defaults to YAML
* -save-optimization-record=<format>: enable remarks, use <format> for
serialization
* -save-optimization-record-passes <regex>: only serialize passes that
match <regex>.

The YAMLTraits in swift had a different `flow` setting for the debug
location, resulting in some test changes.

Author: francisvm

include/swift/AST/DiagnosticsFrontend.def

@@ -372,6 +372,8 @@ NOTE(compiled_module_invalid_reason,none,
 ERROR(unknown_forced_module_loading_mode,none,
       "unknown value for SWIFT_FORCE_MODULE_LOADING variable: '%0'",
       (StringRef))
+ERROR(error_creating_remark_serializer,none,
+      "error while creating remark serializer: '%0'", (StringRef))
 
 REMARK(interface_file_lock_failure,none,
       "could not acquire lock file for module interface '%0'", (StringRef))

include/swift/AST/SILOptions.h

@@ -23,6 +23,7 @@
 #include "swift/Basic/OptimizationMode.h"
 #include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Remarks/RemarkFormat.h"
 #include <string>
 #include <climits>
 
@@ -161,10 +162,17 @@ class SILOptions {
   /// pipeline or after serialization.
   bool StripOwnershipAfterSerialization = true;
 
-  /// The name of the file to which the backend should save YAML optimization
+  /// The name of the file to which the backend should save optimization
   /// records.
   std::string OptRecordFile;
 
+  /// The regex that filters the passes that should be saved to the optimization
+  /// records.
+  std::string OptRecordPasses;
+
+  /// The format used for serializing remarks (default: YAML)
+  llvm::remarks::Format OptRecordFormat = llvm::remarks::Format::YAML;
+
   SILOptions() {}
 
   /// Return a hash code of any components from these options that should

include/swift/Basic/FileTypes.def

@@ -72,7 +72,8 @@ TYPE("tbd",                 TBD,                       "tbd",             "")
 TYPE("module-trace",        ModuleTrace,               "trace.json",      "")
 
 TYPE("index-data",          IndexData,                 "",                "")
-TYPE("opt-record",          OptRecord,                 "opt.yaml",        "")
+TYPE("yaml-opt-record",     YAMLOptRecord,             "opt.yaml",        "")
+TYPE("bitstream-opt-record",BitstreamOptRecord,        "opt.bitstream",   "")
 
 // Overlay files declare wrapper modules, called "separately-imported overlays",
 // that should be automatically imported when a particular module is imported.

include/swift/Driver/ToolChain.h

@@ -317,6 +317,9 @@ class ToolChain {
   virtual void validateArguments(DiagnosticEngine &diags,
                                  const llvm::opt::ArgList &args,
                                  StringRef defaultTarget) const {}
+
+  llvm::Expected<file_types::ID>
+  remarkFileTypeFromArgs(const llvm::opt::ArgList &Args) const;
 };
 } // end namespace driver
 } // end namespace swift

include/swift/Option/Options.td

@@ -548,10 +548,20 @@ def Rpass_missed_EQ : Joined<["-"], "Rpass-missed=">,
 
 def save_optimization_record : Flag<["-"], "save-optimization-record">,
   Flags<[FrontendOption]>, HelpText<"Generate a YAML optimization record file">;
+def save_optimization_record_EQ : Joined<["-"], "save-optimization-record=">,
+  Flags<[FrontendOption]>,
+  HelpText<"Generate an optimization record file in a specific format "
+           "(default: YAML)">, MetaVarName<"<format>">;
 def save_optimization_record_path :
   Separate<["-"], "save-optimization-record-path">,
   Flags<[FrontendOption, ArgumentIsPath]>,
-  HelpText<"Specify the file name of any generated YAML optimization record">;
+  HelpText<"Specify the file name of any generated optimization record">;
+def save_optimization_record_passes :
+  Separate<["-"], "save-optimization-record-passes">,
+  Flags<[FrontendOption]>,
+  HelpText<"Only include passes which match a specified regular expression in"
+           "the generated optimization record "
+           "(by default, include all passes)">, MetaVarName<"<regex>">;
 
 // Platform options.
 def enable_app_extension : Flag<["-"], "application-extension">,

include/swift/SIL/OptimizationRemark.h

@@ -20,6 +20,7 @@
 #define SWIFT_SIL_OPTIMIZATIONREMARKEMITTER_H
 
 #include "swift/Basic/SourceLoc.h"
+#include "swift/Demangling/Demangler.h"
 #include "swift/SIL/SILBasicBlock.h"
 #include "swift/SIL/SILInstruction.h"
 #include "swift/SIL/SILModule.h"
@@ -75,23 +76,30 @@ template <typename DerivedT> class Remark {
   StringRef passName;
 
   /// Textual identifier for the remark (single-word, camel-case). Can be used
-  /// by external tools reading the YAML output file for optimization remarks to
+  /// by external tools reading the output file for optimization remarks to
   /// identify the remark.
-  StringRef identifier;
+  SmallString<32> identifier;
 
   /// Source location for the diagnostics.
   SourceLoc location;
 
   /// The function for the diagnostics.
   SILFunction *function;
 
+  /// The demangled name of \p Function.
+  SmallString<64> demangledFunctionName;
+
   /// Indentation used if this remarks is printed as a debug message.
   unsigned indentDebugWidth = 0;
 
 protected:
   Remark(StringRef identifier, SILInstruction &i)
-      : identifier(identifier), location(i.getLoc().getSourceLoc()),
-        function(i.getParent()->getParent()) {}
+      : identifier((Twine("sil.") + identifier).str()),
+        location(i.getLoc().getSourceLoc()),
+        function(i.getParent()->getParent()),
+        demangledFunctionName(Demangle::demangleSymbolAsString(
+            function->getName(),
+            Demangle::DemangleOptions::SimplifiedUIDemangleOptions())) {}
 
 public:
   DerivedT &operator<<(StringRef s) {
@@ -112,11 +120,13 @@ template <typename DerivedT> class Remark {
   StringRef getPassName() const { return passName; }
   StringRef getIdentifier() const { return identifier; }
   SILFunction *getFunction() const { return function; }
+  StringRef getDemangledFunctionName() const { return demangledFunctionName; }
   SourceLoc getLocation() const { return location; }
   std::string getMsg() const;
   std::string getDebugMsg() const;
   Remark<DerivedT> &getRemark() { return *this; }
   SmallVector<Argument, 4> &getArgs() { return args; }
+  ArrayRef<Argument> getArgs() const { return args; }
 
   void setPassName(StringRef name) { passName = name; }
 };
@@ -156,7 +166,7 @@ class Emitter {
   void emit(T remarkBuilder, decltype(remarkBuilder()) * = nullptr) {
     using RemarkT = decltype(remarkBuilder());
     // Avoid building the remark unless remarks are enabled.
-    if (isEnabled<RemarkT>() || module.getOptRecordStream()) {
+    if (isEnabled<RemarkT>() || module.getSILRemarkStreamer()) {
       auto rb = remarkBuilder();
       rb.setPassName(passName);
       emit(rb);
@@ -171,7 +181,7 @@ class Emitter {
     using RemarkT = decltype(remarkBuilder());
     // Avoid building the remark unless remarks are enabled.
     bool emitRemark = emitter && (emitter->isEnabled<RemarkT>() ||
-                                  emitter->module.getOptRecordStream());
+                                  emitter->module.getSILRemarkStreamer());
     // Same for DEBUG.
     bool shouldEmitDebug = false;
 #ifndef NDEBUG

include/swift/SIL/SILModule.h

@@ -67,6 +67,7 @@ class SILUndef;
 class SourceFile;
 class SerializedSILLoader;
 class SILFunctionBuilder;
+class SILRemarkStreamer;
 
 namespace Lowering {
 class SILGenModule;
@@ -232,13 +233,11 @@ class SILModule {
   // The list of SILProperties in the module.
   PropertyListType properties;
 
-  /// This is the underlying raw stream of OptRecordStream.
-  ///
-  /// It is also owned by SILModule in order to keep their lifetime in sync.
-  std::unique_ptr<llvm::raw_ostream> OptRecordRawStream;
+  /// The remark output stream used to record SIL remarks to a file.
+  std::unique_ptr<llvm::raw_fd_ostream> silRemarkStream;
 
-  /// If non-null, the YAML file where remarks should be recorded.
-  std::unique_ptr<llvm::yaml::Output> OptRecordStream;
+  /// The remark streamer used to serialize SIL remarks to a file.
+  std::unique_ptr<swift::SILRemarkStreamer> silRemarkStreamer;
 
   /// This is a cache of intrinsic Function declarations to numeric ID mappings.
   llvm::DenseMap<Identifier, IntrinsicInfo> IntrinsicIDCache;
@@ -524,9 +523,12 @@ class SILModule {
     return coverageMaps;
   }
 
-  llvm::yaml::Output *getOptRecordStream() { return OptRecordStream.get(); }
-  void setOptRecordStream(std::unique_ptr<llvm::yaml::Output> &&Stream,
-                          std::unique_ptr<llvm::raw_ostream> &&RawStream);
+  swift::SILRemarkStreamer *getSILRemarkStreamer() {
+    return silRemarkStreamer.get();
+  }
+  void setSILRemarkStreamer(
+      std::unique_ptr<llvm::raw_fd_ostream> &&remarkStream,
+      std::unique_ptr<swift::SILRemarkStreamer> &&remarkStreamer);
 
   // This is currently limited to VarDecl because the visibility of global
   // variables and class properties is straightforward, while the visibility of

include/swift/SIL/SILRemarkStreamer.h

@@ -0,0 +1,104 @@
+//===--- SILRemarkStreamer.h - Interface for streaming remarks --*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// This file defines the interface used to stream SIL optimization diagnostics
+/// through LLVM's RemarkStreamer interface.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_SIL_SILREMARKSTREAMER_H
+#define SWIFT_SIL_SILREMARKSTREAMER_H
+
+#include "swift/SIL/OptimizationRemark.h"
+#include "llvm/Remarks/RemarkStreamer.h"
+
+namespace swift {
+
+class SILRemarkStreamer {
+  llvm::remarks::RemarkStreamer &llvmStreamer;
+  // Source manager for resolving source locations.
+  const SourceManager &srcMgr;
+  /// Convert diagnostics into LLVM remark objects.
+  /// The lifetime of the members of the result is bound to the lifetime of
+  /// the SIL remarks.
+  template <typename RemarkT>
+  llvm::remarks::Remark
+  toLLVMRemark(const OptRemark::Remark<RemarkT> &remark) const;
+
+public:
+  SILRemarkStreamer(llvm::remarks::RemarkStreamer &llvmStreamer,
+                    const SourceManager &srcMgr)
+      : llvmStreamer(llvmStreamer), srcMgr(srcMgr) {}
+  /// Emit a remark through the streamer.
+  template <typename RemarkT>
+  void emit(const OptRemark::Remark<RemarkT> &remark);
+};
+
+std::pair<std::unique_ptr<llvm::raw_fd_ostream>,
+          std::unique_ptr<SILRemarkStreamer>>
+createSILRemarkStreamer(SILModule &srcMgr, StringRef filename, StringRef passes,
+                        llvm::remarks::Format format,
+                        DiagnosticEngine &diagEngine, SourceManager &sourceMgr);
+
+// Implementation for template member functions.
+
+// OptRemark type -> llvm::remarks::Type
+template <typename RemarkT> static llvm::remarks::Type toRemarkType() {
+  if (std::is_same<RemarkT, OptRemark::RemarkPassed>::value)
+    return llvm::remarks::Type::Passed;
+  if (std::is_same<RemarkT, OptRemark::RemarkMissed>::value)
+    return llvm::remarks::Type::Missed;
+  llvm_unreachable("Unknown remark type");
+}
+
+static inline Optional<llvm::remarks::RemarkLocation>
+toRemarkLocation(const SourceLoc &loc, const SourceManager &srcMgr) {
+  if (!loc.isValid())
+    return None;
+
+  StringRef file = srcMgr.getDisplayNameForLoc(loc);
+  unsigned line, col;
+  std::tie(line, col) = srcMgr.getLineAndColumn(loc);
+  return llvm::remarks::RemarkLocation{file, line, col};
+}
+
+template <typename RemarkT>
+llvm::remarks::Remark SILRemarkStreamer::toLLVMRemark(
+    const OptRemark::Remark<RemarkT> &optRemark) const {
+  llvm::remarks::Remark llvmRemark; // The result.
+  llvmRemark.RemarkType = toRemarkType<RemarkT>();
+  llvmRemark.PassName = optRemark.getPassName();
+  llvmRemark.RemarkName = optRemark.getIdentifier();
+  llvmRemark.FunctionName = optRemark.getDemangledFunctionName();
+  llvmRemark.Loc = toRemarkLocation(optRemark.getLocation(), srcMgr);
+
+  for (const OptRemark::Argument &arg : optRemark.getArgs()) {
+    llvmRemark.Args.emplace_back();
+    llvmRemark.Args.back().Key = arg.key;
+    llvmRemark.Args.back().Val = arg.val;
+    llvmRemark.Args.back().Loc = toRemarkLocation(arg.loc, srcMgr);
+  }
+
+  return llvmRemark;
+}
+
+template <typename RemarkT>
+void SILRemarkStreamer::emit(const OptRemark::Remark<RemarkT> &optRemark) {
+  if (!llvmStreamer.matchesFilter(optRemark.getPassName()))
+    return;
+
+  return llvmStreamer.getSerializer().emit(toLLVMRemark(optRemark));
+}
+
+} // namespace swift
+#endif

lib/Basic/FileTypes.cpp

@@ -78,7 +78,7 @@ bool file_types::isTextual(ID Id) {
   case file_types::TY_ImportedModules:
   case file_types::TY_TBD:
   case file_types::TY_ModuleTrace:
-  case file_types::TY_OptRecord:
+  case file_types::TY_YAMLOptRecord:
   case file_types::TY_SwiftModuleInterfaceFile:
   case file_types::TY_PrivateSwiftModuleInterfaceFile:
   case file_types::TY_SwiftOverlayFile:
@@ -102,6 +102,7 @@ bool file_types::isTextual(ID Id) {
   case file_types::TY_Nothing:
   case file_types::TY_Remapping:
   case file_types::TY_IndexData:
+  case file_types::TY_BitstreamOptRecord:
     return false;
   case file_types::TY_INVALID:
     llvm_unreachable("Invalid type ID.");
@@ -146,7 +147,8 @@ bool file_types::isAfterLLVM(ID Id) {
   case file_types::TY_Remapping:
   case file_types::TY_IndexData:
   case file_types::TY_ModuleTrace:
-  case file_types::TY_OptRecord:
+  case file_types::TY_YAMLOptRecord:
+  case file_types::TY_BitstreamOptRecord:
   case file_types::TY_SwiftModuleInterfaceFile:
   case file_types::TY_PrivateSwiftModuleInterfaceFile:
     return false;
@@ -195,7 +197,8 @@ bool file_types::isPartOfSwiftCompilation(ID Id) {
   case file_types::TY_Remapping:
   case file_types::TY_IndexData:
   case file_types::TY_ModuleTrace:
-  case file_types::TY_OptRecord:
+  case file_types::TY_YAMLOptRecord:
+  case file_types::TY_BitstreamOptRecord:
     return false;
   case file_types::TY_INVALID:
     llvm_unreachable("Invalid type ID.");

lib/Driver/Driver.cpp

@@ -47,6 +47,7 @@
 #include "llvm/Option/Arg.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/Option/OptTable.h"
+#include "llvm/Remarks/RemarkFormat.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FileSystem.h"
@@ -1941,7 +1942,8 @@ void Driver::buildActions(SmallVectorImpl<const Action *> &TopLevelActions,
       case file_types::TY_ImportedModules:
       case file_types::TY_TBD:
       case file_types::TY_ModuleTrace:
-      case file_types::TY_OptRecord:
+      case file_types::TY_YAMLOptRecord:
+      case file_types::TY_BitstreamOptRecord:
       case file_types::TY_SwiftModuleInterfaceFile:
       case file_types::TY_PrivateSwiftModuleInterfaceFile:
       case file_types::TY_SwiftCrossImportDir:
@@ -2766,6 +2768,7 @@ Job *Driver::buildJobsForAction(Compilation &C, const JobAction *JA,
                                   Output.get());
 
   if (C.getArgs().hasArg(options::OPT_save_optimization_record,
+                         options::OPT_save_optimization_record_EQ,
                          options::OPT_save_optimization_record_path))
     chooseOptimizationRecordPath(C, workingDirectory, Buf, Output.get());
 
@@ -3202,12 +3205,18 @@ void Driver::chooseOptimizationRecordPath(Compilation &C,
                                           CommandOutput *Output) const {
   const OutputInfo &OI = C.getOutputInfo();
   if (OI.CompilerMode == OutputInfo::Mode::SingleCompile) {
+    llvm::Expected<file_types::ID> FileType =
+        C.getToolChain().remarkFileTypeFromArgs(C.getArgs());
+    if (!FileType) {
+      Diags.diagnose({}, diag::error_creating_remark_serializer,
+                     llvm::toString(FileType.takeError()));
+      return;
+    }
     auto filename = *getOutputFilenameFromPathArgOrAsTopLevel(
-        OI, C.getArgs(), options::OPT_save_optimization_record_path,
-        file_types::TY_OptRecord, /*TreatAsTopLevelOutput=*/true,
-        workingDirectory, Buf);
+        OI, C.getArgs(), options::OPT_save_optimization_record_path, *FileType,
+        /*TreatAsTopLevelOutput=*/true, workingDirectory, Buf);
 
-    Output->setAdditionalOutputForType(file_types::TY_OptRecord, filename);
+    Output->setAdditionalOutputForType(*FileType, filename);
   } else
     // FIXME: We should use the OutputMap in this case.
     Diags.diagnose({}, diag::warn_opt_remark_disabled);