Merge pull request swiftlang#12427 from swiftix/sil-serialization-before-optimizations5

Add support for the early serialization of SIL modules using "high-level" SIL and make it the default

Author: swiftix

include/swift/SIL/SILModule.h

@@ -100,6 +100,7 @@ class SILModule {
   using DefaultWitnessTableListType = llvm::ilist<SILDefaultWitnessTable>;
   using CoverageMapListType = llvm::ilist<SILCoverageMap>;
   using LinkingMode = SILOptions::LinkingMode;
+  using ActionCallback = std::function<void()>;
 
 private:
   friend KeyPathPattern;
@@ -213,6 +214,13 @@ class SILModule {
   /// The options passed into this SILModule.
   SILOptions &Options;
 
+  /// Set if the SILModule was serialized already. It is used
+  /// to ensure that the module is serialized only once.
+  bool serialized;
+
+  /// Action to be executed for serializing the SILModule.
+  ActionCallback SerializeSILAction;
+
   /// A list of clients that need to be notified when an instruction
   /// invalidation message is sent.
   llvm::SetVector<DeleteNotificationHandler*> NotificationHandlers;
@@ -251,6 +259,17 @@ class SILModule {
   /// registered handlers. The order of handlers is deterministic but arbitrary.
   void notifyDeleteHandlers(SILNode *node);
 
+  /// Set a serialization action.
+  void setSerializeSILAction(ActionCallback SerializeSILAction);
+  ActionCallback getSerializeSILAction() const;
+
+  /// Set a flag indicating that this module is serialized already.
+  void setSerialized() { serialized = true; }
+  bool isSerialized() const { return serialized; }
+
+  /// Serialize a SIL module using the configured SerializeSILAction.
+  void serialize();
+
   /// \brief This converts Swift types to SILTypes.
   mutable Lowering::TypeConverter Types;
 

include/swift/SILOptimizer/PassManager/Passes.def

@@ -269,6 +269,8 @@ PASS(MarkUninitializedFixup, "mark-uninitialized-fixup",
      "Temporary pass for staging in mark_uninitialized changes.")
 PASS(SimplifyUnreachableContainingBlocks, "simplify-unreachable-containing-blocks",
      "Utility pass. Removes all non-term insts from blocks with unreachable terms")
+PASS(SerializeSILPass, "serialize-sil",
+     "Utility pass. Serializes the current SILModule")
 PASS(BugReducerTester, "bug-reducer-tester",
      "sil-bug-reducer Tool Testing by Asserting on a Sentinel Function")
 PASS_RANGE(AllPasses, AADumper, BugReducerTester)

lib/FrontendTool/FrontendTool.cpp

@@ -854,6 +854,46 @@ static bool performCompile(CompilerInstance &Instance,
     SM->verify();
   }
 
+  // This is the action to be used to serialize SILModule.
+  // It may be invoked multiple times, but it will perform
+  // serialization only once. The serialization may either happen
+  // after high-level optimizations or after all optimizations are
+  // done, depending on the compiler setting.
+
+  auto SerializeSILModuleAction = [&]() {
+    if (!opts.ModuleOutputPath.empty() || !opts.ModuleDocOutputPath.empty()) {
+      auto DC = PrimarySourceFile ? ModuleOrSourceFile(PrimarySourceFile)
+                                  : Instance.getMainModule();
+      if (!opts.ModuleOutputPath.empty()) {
+        SerializationOptions serializationOpts;
+        serializationOpts.OutputPath = opts.ModuleOutputPath.c_str();
+        serializationOpts.DocOutputPath = opts.ModuleDocOutputPath.c_str();
+        serializationOpts.GroupInfoPath = opts.GroupInfoPath.c_str();
+        if (opts.SerializeBridgingHeader)
+          serializationOpts.ImportedHeader = opts.ImplicitObjCHeaderPath;
+        serializationOpts.ModuleLinkName = opts.ModuleLinkName;
+        serializationOpts.ExtraClangOptions =
+            Invocation.getClangImporterOptions().ExtraArgs;
+        serializationOpts.EnableNestedTypeLookupTable =
+            opts.EnableSerializationNestedTypeLookupTable;
+        if (!IRGenOpts.ForceLoadSymbolName.empty())
+          serializationOpts.AutolinkForceLoad = true;
+
+        // Options contain information about the developer's computer,
+        // so only serialize them if the module isn't going to be shipped to
+        // the public.
+        serializationOpts.SerializeOptionsForDebugging =
+            !moduleIsPublic || opts.AlwaysSerializeDebuggingOptions;
+
+        serialize(DC, serializationOpts, SM.get());
+      }
+    }
+  };
+
+  // Set the serialization action, so that the SIL module
+  // can be serialized at any moment, e.g. during the optimization pipeline.
+  SM->setSerializeSILAction(SerializeSILModuleAction);
+
   // Perform SIL optimization passes if optimizations haven't been disabled.
   // These may change across compiler versions.
   {
@@ -921,32 +961,9 @@ static bool performCompile(CompilerInstance &Instance,
   }
 
   if (!opts.ModuleOutputPath.empty() || !opts.ModuleDocOutputPath.empty()) {
-    auto DC = PrimarySourceFile ? ModuleOrSourceFile(PrimarySourceFile) :
-                                  Instance.getMainModule();
-    if (!opts.ModuleOutputPath.empty()) {
-      SerializationOptions serializationOpts;
-      serializationOpts.OutputPath = opts.ModuleOutputPath.c_str();
-      serializationOpts.DocOutputPath = opts.ModuleDocOutputPath.c_str();
-      serializationOpts.GroupInfoPath = opts.GroupInfoPath.c_str();
-      if (opts.SerializeBridgingHeader)
-        serializationOpts.ImportedHeader = opts.ImplicitObjCHeaderPath;
-      serializationOpts.ModuleLinkName = opts.ModuleLinkName;
-      serializationOpts.ExtraClangOptions =
-          Invocation.getClangImporterOptions().ExtraArgs;
-      serializationOpts.EnableNestedTypeLookupTable =
-          opts.EnableSerializationNestedTypeLookupTable;
-      if (!IRGenOpts.ForceLoadSymbolName.empty())
-        serializationOpts.AutolinkForceLoad = true;
-
-      // Options contain information about the developer's computer,
-      // so only serialize them if the module isn't going to be shipped to
-      // the public.
-      serializationOpts.SerializeOptionsForDebugging =
-          !moduleIsPublic || opts.AlwaysSerializeDebuggingOptions;
-
-      serialize(DC, serializationOpts, SM.get());
-    }
-
+    // Serialize the SILModule if it was not serialized yet.
+    if (!SM.get()->isSerialized())
+      SM.get()->serialize();
     if (Action == FrontendOptions::MergeModules ||
         Action == FrontendOptions::EmitModuleOnly) {
       if (shouldIndex) {

lib/SIL/SILModule.cpp

@@ -87,7 +87,8 @@ SILModule::SILModule(ModuleDecl *SwiftModule, SILOptions &Options,
                      const DeclContext *associatedDC, bool wholeModule)
     : TheSwiftModule(SwiftModule), AssociatedDeclContext(associatedDC),
       Stage(SILStage::Raw), Callback(new SILModule::SerializationCallback()),
-      wholeModule(wholeModule), Options(Options), Types(*this) {}
+      wholeModule(wholeModule), Options(Options), serialized(false),
+      SerializeSILAction(), Types(*this) {}
 
 SILModule::~SILModule() {
   // Decrement ref count for each SILGlobalVariable with static initializers.
@@ -762,3 +763,20 @@ bool SILModule::isOptimizedOnoneSupportModule() const {
   return getOptions().Optimization >= SILOptions::SILOptMode::Optimize &&
          isOnoneSupportModule();
 }
+
+void SILModule::setSerializeSILAction(SILModule::ActionCallback Action) {
+  assert(!SerializeSILAction && "Serialization action can be set only once");
+  SerializeSILAction = Action;
+}
+
+SILModule::ActionCallback SILModule::getSerializeSILAction() const {
+  return SerializeSILAction;
+}
+
+void SILModule::serialize() {
+  assert(SerializeSILAction && "Serialization action should be set");
+  assert(!isSerialized() && "The module was serialized already");
+  SerializeSILAction();
+  setSerialized();
+}
+

lib/SILOptimizer/IPO/DeadFunctionElimination.cpp

@@ -30,6 +30,20 @@ STATISTIC(NumDeadFunc, "Number of dead functions eliminated");
 
 namespace {
 
+/// Returns true if a function should be SIL serialized or emitted by IRGen.
+static bool shouldBeSerializedOrEmitted(SILFunction *F) {
+  // public_external functions are never SIL serialized or emitted by IRGen.
+  if (F->isAvailableExternally() && hasPublicVisibility(F->getLinkage()) &&
+      !F->isTransparent())
+    return false;
+
+  // [serialized] functions should always be SIL serialized.
+  if (F->isSerialized())
+    return true;
+
+  return false;
+}
+
 /// This is a base class for passes that are based on function liveness
 /// computations like e.g. dead function elimination.
 /// It provides a common logic for computing live (i.e. reachable) functions.
@@ -117,6 +131,11 @@ class FunctionLivenessComputation {
       return true;
     }
 
+    // Do not consider public_external functions that do not need to be emitted
+    // into the client as anchors.
+    if (shouldBeSerializedOrEmitted(F))
+      return true;
+
     return false;
   }
 

lib/SILOptimizer/PassManager/PassPipeline.cpp

@@ -243,10 +243,16 @@ void addSSAPasses(SILPassPipelinePlan &P, OptimizationLevelKind OpLevel) {
     P.addEarlyInliner();
     break;
   case OptimizationLevelKind::MidLevel:
-    // Does inline semantics-functions (except "availability"), but not
-    // global-init functions.
     P.addGlobalOpt();
     P.addLetPropertiesOpt();
+    // It is important to serialize before any of the @_semantics
+    // functions are inlined, because otherwise the information about
+    // uses of such functions inside the module is lost,
+    // which reduces the ability of the compiler to optimize clients
+    // importing this module.
+    P.addSerializeSILPass();
+    // Does inline semantics-functions (except "availability"), but not
+    // global-init functions.
     P.addPerfInliner();
     break;
   case OptimizationLevelKind::LowLevel:

lib/SILOptimizer/UtilityPasses/CMakeLists.txt

@@ -21,6 +21,7 @@ set(UTILITYPASSES_SOURCES
   UtilityPasses/LoopRegionPrinter.cpp
   UtilityPasses/MemBehaviorDumper.cpp
   UtilityPasses/RCIdentityDumper.cpp
+  UtilityPasses/SerializeSILPass.cpp
   UtilityPasses/SILDebugInfoGenerator.cpp
   UtilityPasses/SideEffectsDumper.cpp
   UtilityPasses/SimplifyUnreachableContainingBlocks.cpp

lib/SILOptimizer/UtilityPasses/SerializeSILPass.cpp

@@ -0,0 +1,53 @@
+//===--- SerializeSILPass.cpp ---------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "serialize-sil"
+#include "swift/Strings.h"
+#include "swift/SILOptimizer/PassManager/Passes.h"
+#include "swift/SILOptimizer/PassManager/Transforms.h"
+
+using namespace swift;
+
+/// A utility pass to serialize a SILModule at any place inside the optimization
+/// pipeline.
+class SerializeSILPass : public SILModuleTransform {
+  /// Removes [serialized] from all functions. This allows for more
+  /// optimizations and for a better dead function elimination.
+  void removeSerializedFlagFromAllFunctions(SILModule &M) {
+    for (auto &F : M) {
+      F.setSerialized(IsSerialized_t::IsNotSerialized);
+    }
+  }
+
+public:
+  SerializeSILPass() {}
+  void run() override {
+    auto &M = *getModule();
+    // Nothing to do if the module was serialized already.
+    if (M.isSerialized())
+      return;
+
+    // Mark all reachable functions as "anchors" so that they are not
+    // removed later by the dead function elimination pass. This
+    // is required, because clients may reference any of the
+    // serialized functions or anything referenced from them. Therefore,
+    // to avoid linker errors, the object file of the current module should
+    // contain all the symbols which were alive at the time of serialization.
+    DEBUG(llvm::dbgs() << "Serializing SILModule in SerializeSILPass\n");
+    getModule()->serialize();
+    removeSerializedFlagFromAllFunctions(M);
+  }
+};
+
+SILTransform *swift::createSerializeSILPass() {
+  return new SerializeSILPass();
+}

test/SILOptimizer/closure_specialize_fragile.sil

@@ -0,0 +1,60 @@
+// RUN: %target-sil-opt  %s -verify -closure-specialize -assume-parsing-unqualified-ownership-sil -o -  | %FileCheck %s
+
+// Make sure we do not specialize resilientCallee.
+
+sil_stage canonical
+
+import Builtin
+import Swift
+import SwiftShims
+
+@_semantics("optimize.sil.never") public func action()
+
+@inline(__always) public func fragileCaller()
+
+public func resilientCallee(fn: () -> ())
+
+// action()
+sil [_semantics "optimize.sil.never"] @_T026closure_specialize_fragile6actionyyF : $@convention(thin) () -> () {
+bb0:
+  %0 = tuple ()
+  return %0 : $()
+} // end sil function '_T026closure_specialize_fragile6actionyyF'
+
+// CHECK-LABEL: sil [serialized] [always_inline] @_T026closure_specialize_fragile0C6CalleryyF : $@convention(thin) () -> ()
+// CHECK: function_ref @_T026closure_specialize_fragile15resilientCalleeyyyc2fn_tF : $@convention(thin) (@owned @callee_owned () -> ()) -> ()
+// CHECK: return
+// fragileCaller()
+sil [serialized] [always_inline] @_T026closure_specialize_fragile0C6CalleryyF : $@convention(thin) () -> () {
+bb0:
+  // function_ref resilientCallee(fn:)
+  %0 = function_ref @_T026closure_specialize_fragile15resilientCalleeyyyc2fn_tF : $@convention(thin) (@owned @callee_owned () -> ()) -> ()
+  // function_ref closure #1 in fragileCaller()
+  %1 = function_ref @_T026closure_specialize_fragile0C6CalleryyFyycfU_ : $@convention(thin) () -> () 
+  %2 = thin_to_thick_function %1 : $@convention(thin) () -> () to $@callee_owned () -> () 
+  %3 = apply %0(%2) : $@convention(thin) (@owned @callee_owned () -> ()) -> ()
+  %4 = tuple ()
+  return %4 : $()
+} // end sil function '_T026closure_specialize_fragile0C6CalleryyF'
+
+// CHECK-LABEL: sil @_T026closure_specialize_fragile15resilientCalleeyyyc2fn_tF : $@convention(thin) (@owned @callee_owned () -> ()) -> ()
+
+// resilientCallee(fn:)
+sil @_T026closure_specialize_fragile15resilientCalleeyyyc2fn_tF : $@convention(thin) (@owned @callee_owned () -> ()) -> () {
+bb0(%0 : $@callee_owned () -> ()):
+  strong_retain %0 : $@callee_owned () -> ()
+  %3 = apply %0() : $@callee_owned () -> ()
+  strong_release %0 : $@callee_owned () -> ()
+  %5 = tuple ()
+  return %5 : $()
+} // end sil function '_T026closure_specialize_fragile15resilientCalleeyyyc2fn_tF'
+
+// closure #1 in fragileCaller()
+sil shared [serialized] @_T026closure_specialize_fragile0C6CalleryyFyycfU_ : $@convention(thin) () -> () {
+bb0:
+  // function_ref action()
+  %0 = function_ref @_T026closure_specialize_fragile6actionyyF : $@convention(thin) () -> () 
+  %1 = apply %0() : $@convention(thin) () -> ()
+  %2 = tuple ()
+  return %2 : $()
+} // end sil function '_T026closure_specialize_fragile0C6CalleryyFyycfU_'