Merge pull request swiftlang#68295 from kubamracek/embedded-modules

swift-ci · web-flow · commit 3b212ce81f1c · 2023-09-07T01:02:41.000-07:00
[embedded] Add basics of module serialization, importing and validation in embedded Swift.
diff --git a/include/swift/AST/Decl.h b/include/swift/AST/Decl.h
@@ -663,7 +663,7 @@ class alignas(1 << DeclAlignInBits) Decl : public ASTAllocated<Decl> {
     HasAnyUnavailableDuringLoweringValues : 1
   );
 
-  SWIFT_INLINE_BITFIELD(ModuleDecl, TypeDecl, 1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1,
+  SWIFT_INLINE_BITFIELD(ModuleDecl, TypeDecl, 1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1,
     /// If the module is compiled as static library.
     StaticLibrary : 1,
 
@@ -707,6 +707,9 @@ class alignas(1 << DeclAlignInBits) Decl : public ASTAllocated<Decl> {
     /// Whether this module was built with -experimental-hermetic-seal-at-link.
     HasHermeticSealAtLink : 1,
 
+    /// Whether this module was built with embedded Swift.
+    IsEmbeddedSwiftModule : 1,
+
     /// Whether this module has been compiled with comprehensive checking for
     /// concurrency, e.g., Sendable checking.
     IsConcurrencyChecked : 1,
diff --git a/include/swift/AST/DiagnosticsSema.def b/include/swift/AST/DiagnosticsSema.def
@@ -872,6 +872,12 @@ ERROR(need_hermetic_seal_to_import_module,none,
       "module %0 was built with -experimental-hermetic-seal-at-link, but "
       "current compilation does not have -experimental-hermetic-seal-at-link",
       (Identifier))
+ERROR(cannot_import_embedded_module,none,
+      "module %0 cannot be imported because it was built with embedded Swift",
+      (Identifier))
+ERROR(cannot_import_non_embedded_module,none,
+      "module %0 cannot be imported in embedded Swift mode",
+      (Identifier))
 ERROR(need_cxx_interop_to_import_module,none,
       "module %0 was built with C++ interoperability enabled, but "
       "current compilation does not enable C++ interoperability",
diff --git a/include/swift/AST/Module.h b/include/swift/AST/Module.h
@@ -656,6 +656,14 @@ class ModuleDecl
     Bits.ModuleDecl.HasHermeticSealAtLink = enabled;
   }
 
+  /// Returns true if this module was built with embedded Swift
+  bool isEmbeddedSwiftModule() const {
+    return Bits.ModuleDecl.IsEmbeddedSwiftModule;
+  }
+  void setIsEmbeddedSwiftModule(bool enabled = true) {
+    Bits.ModuleDecl.IsEmbeddedSwiftModule = enabled;
+  }
+
   /// Returns true if this module was built with C++ interoperability enabled.
   bool hasCxxInteroperability() const {
     return Bits.ModuleDecl.HasCxxInteroperability;
diff --git a/include/swift/Serialization/SerializationOptions.h b/include/swift/Serialization/SerializationOptions.h
@@ -155,6 +155,7 @@ namespace swift {
     bool DisableCrossModuleIncrementalInfo = false;
     bool StaticLibrary = false;
     bool HermeticSealAtLink = false;
+    bool EmbeddedSwiftModule = false;
     bool IsOSSA = false;
     bool SerializeExternalDeclsOnly = false;
   };
diff --git a/include/swift/Serialization/Validation.h b/include/swift/Serialization/Validation.h
@@ -128,6 +128,7 @@ class ExtendedValidationInfo {
     unsigned IsSIB : 1;
     unsigned IsStaticLibrary : 1;
     unsigned HasHermeticSealAtLink : 1;
+    unsigned IsEmbeddedSwiftModule : 1;
     unsigned IsTestable : 1;
     unsigned ResilienceStrategy : 2;
     unsigned IsImplicitDynamicEnabled : 1;
@@ -181,6 +182,10 @@ class ExtendedValidationInfo {
   void setHasHermeticSealAtLink(bool val) {
     Bits.HasHermeticSealAtLink = val;
   }
+  bool isEmbeddedSwiftModule() const { return Bits.IsEmbeddedSwiftModule; }
+  void setIsEmbeddedSwiftModule(bool val) {
+    Bits.IsEmbeddedSwiftModule = val;
+  }
   bool isTestable() const { return Bits.IsTestable; }
   void setIsTestable(bool val) {
     Bits.IsTestable = val;
diff --git a/lib/AST/Module.cpp b/lib/AST/Module.cpp
@@ -718,6 +718,7 @@ ModuleDecl::ModuleDecl(Identifier name, ASTContext &ctx,
   Bits.ModuleDecl.IsMainModule = 0;
   Bits.ModuleDecl.HasIncrementalInfo = 0;
   Bits.ModuleDecl.HasHermeticSealAtLink = 0;
+  Bits.ModuleDecl.IsEmbeddedSwiftModule = 0;
   Bits.ModuleDecl.IsConcurrencyChecked = 0;
   Bits.ModuleDecl.ObjCNameLookupCachePopulated = 0;
   Bits.ModuleDecl.HasCxxInteroperability = 0;
diff --git a/lib/Frontend/Frontend.cpp b/lib/Frontend/Frontend.cpp
@@ -223,6 +223,9 @@ SerializationOptions CompilerInvocation::computeSerializationOptions(
 
   serializationOpts.HermeticSealAtLink = opts.HermeticSealAtLink;
 
+  serializationOpts.EmbeddedSwiftModule =
+      LangOpts.hasFeature(Feature::Embedded);
+
   serializationOpts.IsOSSA = getSILOptions().EnableOSSAModules;
 
   serializationOpts.SerializeExternalDeclsOnly =
diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp
@@ -1557,6 +1557,11 @@ static bool validateTBDIfNeeded(const CompilerInvocation &Invocation,
       return false;
     }
 
+    // Embedded Swift does not support TBD.
+    if (Invocation.getLangOptions().hasFeature(Feature::Embedded)) {
+      return false;
+    }
+
     // Cross-module optimization does not support TBD.
     if (Invocation.getSILOptions().CMOMode == CrossModuleOptimizationMode::Aggressive) {
       return false;
diff --git a/lib/IRGen/GenDecl.cpp b/lib/IRGen/GenDecl.cpp
@@ -536,7 +536,7 @@ void IRGenModule::emitSourceFile(SourceFile &SF) {
   // libraries. This may however cause the library to get pulled in in
   // situations where it isn't useful, such as for dylibs, though this is
   // harmless aside from code size.
-  if (!IRGen.Opts.UseJIT) {
+  if (!IRGen.Opts.UseJIT && !Context.LangOpts.hasFeature(Feature::Embedded)) {
     auto addBackDeployLib = [&](llvm::VersionTuple version,
                                 StringRef libraryName, bool forceLoad) {
       llvm::Optional<llvm::VersionTuple> compatibilityVersion;
@@ -1416,7 +1416,6 @@ void IRGenerator::emitLazyDefinitions() {
     LazySpecializedTypeMetadataRecords.clear();
     LazyTypeContextDescriptors.clear();
     LazyOpaqueTypeDescriptors.clear();
-    LazyFunctionDefinitions.clear();
     LazyCanonicalSpecializedMetadataAccessors.clear();
     LazyMetadataAccessors.clear();
     LazyWitnessTables.clear();
diff --git a/lib/IRGen/MetadataRequest.cpp b/lib/IRGen/MetadataRequest.cpp
@@ -3303,7 +3303,11 @@ llvm::Value *IRGenFunction::emitTypeMetadataRef(CanType type) {
 MetadataResponse
 IRGenFunction::emitTypeMetadataRef(CanType type,
                                    DynamicMetadataRequest request) {
-  assert(!type->getASTContext().LangOpts.hasFeature(Feature::Embedded));
+  if (type->getASTContext().LangOpts.hasFeature(Feature::Embedded)) {
+    llvm::errs() << "Metadata pointer requested in embedded Swift for type "
+                 << type << "\n";
+    assert(0 && "metadata used in embedded mode");
+  }
 
   type = IGM.getRuntimeReifiedType(type);
   // Look through any opaque types we're allowed to.
diff --git a/lib/SIL/IR/Linker.cpp b/lib/SIL/IR/Linker.cpp
@@ -120,7 +120,8 @@ void SILLinkerVisitor::maybeAddFunctionToWorklist(SILFunction *F,
     return;
   }
 
-  // In the performance pipeline, we deserialize all reachable functions.
+  // In the performance pipeline or embedded mode, we deserialize all reachable
+  // functions.
   if (isLinkAll())
     return deserializeAndPushToWorklist(F);
 
@@ -435,6 +436,11 @@ void SILLinkerVisitor::process() {
       Fn->setSerialized(IsSerialized_t::IsNotSerialized);
     }
 
+    // TODO: This should probably be done as a separate SIL pass ("internalize")
+    if (Fn->getModule().getASTContext().LangOpts.hasFeature(Feature::Embedded)) {
+      Fn->setLinkage(stripExternalFromLinkage(Fn->getLinkage()));
+    }
+
     LLVM_DEBUG(llvm::dbgs() << "Process imports in function: "
                             << Fn->getName() << "\n");
 
diff --git a/lib/SIL/IR/Linker.h b/lib/SIL/IR/Linker.h
@@ -142,7 +142,10 @@ class SILLinkerVisitor : public SILInstructionVisitor<SILLinkerVisitor, void> {
 
   /// Is the current mode link all? Link all implies we should try and link
   /// everything, not just transparent/shared functions.
-  bool isLinkAll() const { return Mode == LinkingMode::LinkAll; }
+  bool isLinkAll() const {
+    return Mode == LinkingMode::LinkAll ||
+           Mod.getASTContext().LangOpts.hasFeature(Feature::Embedded);
+  }
 
   void linkInVTable(ClassDecl *D);
 
diff --git a/lib/SIL/IR/SILModule.cpp b/lib/SIL/IR/SILModule.cpp
@@ -898,6 +898,10 @@ void SILModule::performOnceForPrespecializedImportedExtensions(
   if (prespecializedFunctionDeclsImported)
     return;
 
+  // No prespecitalizations in embedded Swift
+  if (getASTContext().LangOpts.hasFeature(Feature::Embedded))
+    return;
+
   SmallVector<ModuleDecl *, 8> importedModules;
   // Add the Swift module.
   if (!isStdlibModule()) {
diff --git a/lib/SILOptimizer/IPO/CrossModuleOptimization.cpp b/lib/SILOptimizer/IPO/CrossModuleOptimization.cpp
@@ -442,7 +442,9 @@ bool CrossModuleOptimization::shouldSerialize(SILFunction *function) {
   if (function->hasSemanticsAttr("optimize.no.crossmodule"))
     return false;
 
-  if (SerializeEverything)
+  // In embedded Swift we serialize everything.
+  if (SerializeEverything ||
+      function->getASTContext().LangOpts.hasFeature(Feature::Embedded))
     return true;
 
   if (!conservative) {
@@ -652,15 +654,18 @@ class CrossModuleOptimizationPass: public SILModuleTransform {
       return;
       
     bool conservative = false;
-    switch (M.getOptions().CMOMode) {
-      case swift::CrossModuleOptimizationMode::Off:
-        return;
-      case swift::CrossModuleOptimizationMode::Default:
-        conservative = true;
-        break;
-      case swift::CrossModuleOptimizationMode::Aggressive:
-        conservative = false;
-        break;
+    // In embedded Swift we serialize everything.
+    if (!M.getASTContext().LangOpts.hasFeature(Feature::Embedded)) {
+      switch (M.getOptions().CMOMode) {
+        case swift::CrossModuleOptimizationMode::Off:
+          return;
+        case swift::CrossModuleOptimizationMode::Default:
+          conservative = true;
+          break;
+        case swift::CrossModuleOptimizationMode::Aggressive:
+          conservative = false;
+          break;
+      }
     }
 
     CrossModuleOptimization CMO(M, conservative);
diff --git a/lib/SILOptimizer/PassManager/PassPipeline.cpp b/lib/SILOptimizer/PassManager/PassPipeline.cpp
@@ -1004,6 +1004,9 @@ SILPassPipelinePlan::getOnonePassPipeline(const SILOptions &Options) {
   // inlinable functions from imported ones.
   P.addOnonePrespecializations();
 
+  // For embedded Swift: CMO is used to serialize libraries.
+  P.addCrossModuleOptimization();
+
   // First serialize the SIL if we are asked to.
   P.startPipeline("Serialization");
   P.addSerializeSILPass();
diff --git a/lib/Serialization/ModuleFile.h b/lib/Serialization/ModuleFile.h
@@ -621,6 +621,11 @@ class ModuleFile
     return Core->Bits.HasHermeticSealAtLink;
   }
 
+  /// Whether this module was built using embedded Swift.
+  bool isEmbeddedSwiftModule() const {
+    return Core->Bits.IsEmbeddedSwiftModule;
+  }
+
   /// Whether this module was built with C++ interoperability enabled.
   bool hasCxxInteroperability() const {
     return Core->Bits.HasCxxInteroperability;
diff --git a/lib/Serialization/ModuleFileSharedCore.cpp b/lib/Serialization/ModuleFileSharedCore.cpp
@@ -158,6 +158,9 @@ static bool readOptionsBlock(llvm::BitstreamCursor &cursor,
     case options_block::HAS_HERMETIC_SEAL_AT_LINK:
       extendedInfo.setHasHermeticSealAtLink(true);
       break;
+    case options_block::IS_EMBEDDED_SWIFT_MODULE:
+      extendedInfo.setIsEmbeddedSwiftModule(true);
+      break;
     case options_block::IS_TESTABLE:
       extendedInfo.setIsTestable(true);
       break;
@@ -1405,6 +1408,7 @@ ModuleFileSharedCore::ModuleFileSharedCore(
       Bits.IsSIB = extInfo.isSIB();
       Bits.IsStaticLibrary = extInfo.isStaticLibrary();
       Bits.HasHermeticSealAtLink = extInfo.hasHermeticSealAtLink();
+      Bits.IsEmbeddedSwiftModule = extInfo.isEmbeddedSwiftModule();
       Bits.IsTestable = extInfo.isTestable();
       Bits.ResilienceStrategy = unsigned(extInfo.getResilienceStrategy());
       Bits.IsImplicitDynamicEnabled = extInfo.isImplicitDynamicEnabled();
diff --git a/lib/Serialization/ModuleFileSharedCore.h b/lib/Serialization/ModuleFileSharedCore.h
@@ -367,6 +367,9 @@ class ModuleFileSharedCore {
     /// Whether this module was built with -experimental-hermetic-seal-at-link.
     unsigned HasHermeticSealAtLink : 1;
 
+    /// Whether this module was built with embedded Swift.
+    unsigned IsEmbeddedSwiftModule : 1;
+
     /// Whether this module file is compiled with '-enable-testing'.
     unsigned IsTestable : 1;
 
diff --git a/lib/Serialization/ModuleFormat.h b/lib/Serialization/ModuleFormat.h
@@ -58,7 +58,7 @@ const uint16_t SWIFTMODULE_VERSION_MAJOR = 0;
 /// describe what change you made. The content of this comment isn't important;
 /// it just ensures a conflict if two people change the module format.
 /// Don't worry about adhering to the 80-column limit for this line.
-const uint16_t SWIFTMODULE_VERSION_MINOR = 803; // removed initializes and accesses attributes
+const uint16_t SWIFTMODULE_VERSION_MINOR = 804; // embedded swift modules
 
 /// A standard hash seed used for all string hashes in a serialized module.
 ///
@@ -885,6 +885,7 @@ namespace options_block {
     IS_SIB,
     IS_STATIC_LIBRARY,
     HAS_HERMETIC_SEAL_AT_LINK,
+    IS_EMBEDDED_SWIFT_MODULE,
     IS_TESTABLE,
     RESILIENCE_STRATEGY,
     ARE_PRIVATE_IMPORTS_ENABLED,
@@ -928,6 +929,10 @@ namespace options_block {
     HAS_HERMETIC_SEAL_AT_LINK
   >;
 
+  using IsEmbeddedSwiftModuleLayout = BCRecordLayout<
+    IS_EMBEDDED_SWIFT_MODULE
+  >;
+
   using IsTestableLayout = BCRecordLayout<
     IS_TESTABLE
   >;
diff --git a/lib/Serialization/Serialization.cpp b/lib/Serialization/Serialization.cpp
@@ -1038,6 +1038,11 @@ void Serializer::writeHeader() {
         HasHermeticSealAtLink.emit(ScratchRecord);
       }
 
+      if (Options.EmbeddedSwiftModule) {
+        options_block::IsEmbeddedSwiftModuleLayout IsEmbeddedSwiftModule(Out);
+        IsEmbeddedSwiftModule.emit(ScratchRecord);
+      }
+
       if (M->isTestingEnabled()) {
         options_block::IsTestableLayout IsTestable(Out);
         IsTestable.emit(ScratchRecord);
diff --git a/lib/Serialization/SerializedModuleLoader.cpp b/lib/Serialization/SerializedModuleLoader.cpp
@@ -826,6 +826,7 @@ LoadedFile *SerializedModuleLoaderBase::loadAST(
     fileUnit = new (Ctx) SerializedASTFile(M, *loadedModuleFile);
     M.setStaticLibrary(loadedModuleFile->isStaticLibrary());
     M.setHasHermeticSealAtLink(loadedModuleFile->hasHermeticSealAtLink());
+    M.setIsEmbeddedSwiftModule(loadedModuleFile->isEmbeddedSwiftModule());
     if (loadedModuleFile->isTestable())
       M.setTestingEnabled();
     if (loadedModuleFile->arePrivateImportsEnabled())
@@ -916,6 +917,18 @@ LoadedFile *SerializedModuleLoaderBase::loadAST(
                        diag::need_hermetic_seal_to_import_module, M.getName());
   }
 
+  if (M.isEmbeddedSwiftModule() &&
+      !Ctx.LangOpts.hasFeature(Feature::Embedded)) {
+    Ctx.Diags.diagnose(diagLoc.value_or(SourceLoc()),
+                       diag::cannot_import_embedded_module, M.getName());
+  }
+
+  if (!M.isEmbeddedSwiftModule() &&
+      Ctx.LangOpts.hasFeature(Feature::Embedded)) {
+    Ctx.Diags.diagnose(diagLoc.value_or(SourceLoc()),
+                       diag::cannot_import_non_embedded_module, M.getName());
+  }
+
   // Non-resilient modules built with C++ interoperability enabled
   // are typically incompatible with clients that do not enable
   // C++ interoperability.
diff --git a/test/embedded/basic-irgen-no-stdlib.swift b/test/embedded/basic-irgen-no-stdlib.swift
@@ -23,11 +23,11 @@ public func main() {
   start(p: Concrete())
 }
 
-// CHECK-LABEL: declare hidden swiftcc void @"$s4main8ConcreteVACycfC"()
+// CHECK-LABEL: define {{.*}}void @"$s4main8ConcreteVACycfC"()
 
-// CHECK-LABEL: declare hidden swiftcc void @"$s4main5start1pyx_tAA6PlayerRzlFAA8ConcreteV_Tg5"()
+// CHECK-LABEL: define {{.*}}void @"$s4main5start1pyx_tAA6PlayerRzlFAA8ConcreteV_Tg5"()
 
-// CHECK-LABEL: define protected swiftcc void @"$s4mainAAyyF"()
+// CHECK-LABEL: define {{.*}}void @"$s4mainAAyyF"()
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    call swiftcc void @"$s4main8ConcreteVACycfC"()
 // CHECK-NEXT:    call swiftcc void @"$s4main5start1pyx_tAA6PlayerRzlFAA8ConcreteV_Tg5"()
diff --git a/test/embedded/basic-modules-no-stdlib.swift b/test/embedded/basic-modules-no-stdlib.swift
@@ -0,0 +1,46 @@
+// RUN: %empty-directory(%t)
+// RUN: %{python} %utils/split_file.py -o %t %s
+
+// RUN: %target-swift-frontend -swift-version 5 -emit-module -o %t/MyModule.swiftmodule %t/MyModule.swift -parse-stdlib -enable-experimental-feature Embedded
+// RUN: %target-swift-frontend -swift-version 5 -emit-ir     -I %t                      %t/Main.swift     -parse-stdlib -enable-experimental-feature Embedded | %FileCheck %s
+
+// TODO: investigate why windows is generating more metadata.
+// XFAIL: OS=windows-msvc
+
+// BEGIN MyModule.swift
+
+public struct Bool {}
+
+public protocol Player {
+  func play()
+  var canPlay: Bool { get }
+}
+
+public struct Concrete : Player {
+  public init() { }
+  public func play() { }
+  public var canPlay: Bool { Bool() }
+}
+
+public func start(p: some Player) {
+  p.play()
+}
+
+public func moduleMain() {
+  start(p: Concrete())
+}
+
+// BEGIN Main.swift
+
+import MyModule
+
+public func main() {
+  moduleMain()
+}
+
+// CHECK: define {{.*}}@main{{.*}} {
+// CHECK: define {{.*}}void @"$s4Main4mainyyF"{{.*}} {
+// CHECK: define {{.*}}void @"$s8MyModule10moduleMainyyF"{{.*}} {
+// CHECK: define {{.*}}void @"$s8MyModule8ConcreteVACycfC"{{.*}} {
+// CHECK: define {{.*}}void @"$s8MyModule5start1pyx_tAA6PlayerRzlFAA8ConcreteV_Tg5"{{.*}} {
+// CHECK: define {{.*}}void @"$s8MyModule8ConcreteV4playyyF"{{.*}} {
diff --git a/test/embedded/basic-modules-validation-no-stdlib.swift b/test/embedded/basic-modules-validation-no-stdlib.swift
diff --git a/test/sil-passpipeline-dump/basic.test-sh b/test/sil-passpipeline-dump/basic.test-sh
