Merge pull request #11904 from adrian-prantl/cherry-pick-swift-release-6.3-LLDB-Set-up-an-explicit-Swift-module-loader

[Cherry-pick into swift/release/6.3] [LLDB] Set up an explicit Swift module loader

Author: adrian-prantl

lldb/include/lldb/Core/ModuleList.h

@@ -83,6 +83,8 @@ class ModuleListProperties : public Properties {
   bool GetUseSwiftClangImporter() const;
   bool GetUseSwiftDWARFImporter() const;
   bool SetUseSwiftDWARFImporter(bool new_value);
+  bool GetUseSwiftExplicitModuleLoader() const;
+  bool SetUseSwiftExplicitModuleLoader(bool new_value);
   bool GetSwiftValidateTypeSystem() const;
   bool GetSwiftTypeSystemFallback() const;
   bool GetSwiftLoadConformances() const;

lldb/source/Core/CoreProperties.td

@@ -25,6 +25,10 @@ let Definition = "modulelist" in {
   def UseSwiftDWARFImporter: Property<"use-swift-dwarfimporter", "Boolean">,
     DefaultTrue,
     Desc<"Reconstruct Clang module dependencies from DWARF when debugging Swift code">;
+  def UseSwiftExplicitModuleLoader
+      : Property<"use-swift-explicit-module-loader", "Boolean">,
+        DefaultTrue,
+        Desc<"Prefer explicitly specified modules over ones found in dSYMs">;
   def SwiftValidateTypeSystem: Property<"swift-validate-typesystem", "Boolean">,
     DefaultFalse,
     Desc<"Validate all Swift typesystem queries. Used for testing an asserts-enabled LLDB only.">;

lldb/source/Core/ModuleList.cpp

@@ -199,6 +199,17 @@ bool ModuleListProperties::SetUseSwiftDWARFImporter(bool new_value) {
   return SetPropertyAtIndex(idx, new_value);
 }
 
+bool ModuleListProperties::GetUseSwiftExplicitModuleLoader() const {
+  const uint32_t idx = ePropertyUseSwiftExplicitModuleLoader;
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_modulelist_properties[idx].default_uint_value != 0);
+}
+
+bool ModuleListProperties::SetUseSwiftExplicitModuleLoader(bool new_value) {
+  const uint32_t idx = ePropertyUseSwiftExplicitModuleLoader;
+  return SetPropertyAtIndex(idx, new_value);
+}
+
 bool ModuleListProperties::GetSwiftValidateTypeSystem() const {
   const uint32_t idx = ePropertySwiftValidateTypeSystem;
   return GetPropertyAtIndexAs<bool>(

lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.cpp

@@ -1896,6 +1896,22 @@ void SwiftASTContext::AddExtraClangArgs(
     RemoveExplicitModules(importer_options.ExtraArgs);
 }
 
+bool SwiftASTContext::IsModuleAvailableInCAS(const std::string &key) {
+  auto id = m_cas->parseID(key);
+  if (!id) {
+    HEALTH_LOG_PRINTF("failed to parse CASID when loading module: %s",
+                      toString(id.takeError()).c_str());
+    return false;
+  }
+  auto lookup = m_action_cache->get(*id);
+  if (!lookup) {
+    HEALTH_LOG_PRINTF("module lookup failure through action cache: %s",
+                      toString(lookup.takeError()).c_str());
+    return false;
+  }
+  return (bool)*lookup;
+};
+
 void SwiftASTContext::AddExtraClangCC1Args(
     const std::vector<std::string> &source,
     const std::vector<std::pair<std::string, bool>> module_search_paths,
@@ -1967,26 +1983,9 @@ void SwiftASTContext::AddExtraClangCC1Args(
     invocation.getCASOpts().PluginOptions =
         GetCASOptions().Config.PluginOptions;
 
-    // Check the module availability in CAS, if not, fallback to regular load.
-    auto CheckModuleInCAS = [&](const std::string &key) {
-      auto id = m_cas->parseID(key);
-      if (!id) {
-        HEALTH_LOG_PRINTF("failed to parse CASID when loading module: %s",
-                          toString(id.takeError()).c_str());
-        return false;
-      }
-      auto lookup = m_action_cache->get(*id);
-      if (!lookup) {
-        HEALTH_LOG_PRINTF("module lookup failure through action cache: %s",
-                          toString(lookup.takeError()).c_str());
-        return false;
-      }
-      return (bool)*lookup;
-    };
-
     use_cas_module = llvm::all_of(
         invocation.getFrontendOpts().ModuleCacheKeys, [&](const auto &entry) {
-          auto exist = CheckModuleInCAS(entry.second);
+          bool exist = IsModuleAvailableInCAS(entry.second);
           if (!exist)
             HEALTH_LOG_PRINTF("module '%s' cannot be load "
                               "from CAS using key: %s, fallback to "
@@ -3745,7 +3744,6 @@ ThreadSafeASTContext SwiftASTContext::GetASTContext() {
   std::string moduleCachePath =
       GetCompilerInvocation().getClangModuleCachePath().str();
   std::unique_ptr<swift::ClangImporter> clang_importer_up;
-  auto &clang_importer_options = GetClangImporterOptions();
   if (!m_ast_context_up->SearchPathOpts.getSDKPath().empty() ||
       TargetHasNoSDK()) {
     // Create the DWARFImporterDelegate.
@@ -3845,15 +3843,34 @@ ThreadSafeASTContext SwiftASTContext::GetASTContext() {
     m_ast_context_up->addModuleLoader(std::move(memory_buffer_loader_up));
   }
 
+  // 2. Create the explicit swift module loader.
+  if (props.GetUseSwiftExplicitModuleLoader()) {
+    auto &search_path_opts = GetCompilerInvocation().getSearchPathOptions();
+    std::unique_ptr<swift::ModuleLoader> esml_up =
+        swift::ExplicitSwiftModuleLoader::create(
+            *m_ast_context_up, m_dependency_tracker.get(), loading_mode,
+            search_path_opts.ExplicitSwiftModuleMapPath,
+            search_path_opts.ExplicitSwiftModuleInputs,
+            /*IgnoreSwiftSourceInfo*/ false);
+    if (esml_up) {
+      m_explicit_swift_module_loader =
+          static_cast<swift::ExplicitSwiftModuleLoader *>(esml_up.get());
+      m_ast_context_up->addModuleLoader(std::move(esml_up), /*isClang=*/false,
+                                        /*isDwarf=*/false,
+                                        /*isInterface=*/false,
+                                        /*isExplicit=*/true);
+    }
+  }
+
   // Add a module interface checker.
   m_ast_context_up->addModuleInterfaceChecker(
-    std::make_unique<swift::ModuleInterfaceCheckerImpl>(*m_ast_context_up,
-      moduleCachePath, prebuiltModuleCachePath,
-      swift::ModuleInterfaceLoaderOptions()));
+      std::make_unique<swift::ModuleInterfaceCheckerImpl>(
+          *m_ast_context_up, moduleCachePath, prebuiltModuleCachePath,
+          swift::ModuleInterfaceLoaderOptions()));
 
-  // 2. Create and install the module interface loader.
+  // 3. Create and install the module interface loader.
   //
-  // The ordering of 2-4 is the same as the Swift compiler's 1-3,
+  // The ordering of 2-4 is the same as the Swift compiler's 2-4,
   // where unintuitively the serialized module loader comes before the
   // module interface loader. The reason for this is that the module
   // interface loader is actually 2-in-1 and secretly attempts to load
@@ -3865,21 +3882,22 @@ ThreadSafeASTContext SwiftASTContext::GetASTContext() {
   if (loading_mode != swift::ModuleLoadingMode::OnlySerialized) {
     std::unique_ptr<swift::ModuleLoader> module_interface_loader_up(
         swift::ModuleInterfaceLoader::create(
-          *m_ast_context_up, *static_cast<swift::ModuleInterfaceCheckerImpl*>(
-            m_ast_context_up->getModuleInterfaceChecker()), m_dependency_tracker.get(),
-          loading_mode));
+            *m_ast_context_up,
+            *static_cast<swift::ModuleInterfaceCheckerImpl *>(
+                m_ast_context_up->getModuleInterfaceChecker()),
+            m_dependency_tracker.get(), loading_mode));
     if (module_interface_loader_up)
       m_ast_context_up->addModuleLoader(std::move(module_interface_loader_up));
   }
 
-  // 3. Create and install the serialized module loader.
+  // 4. Create and install the serialized module loader.
   std::unique_ptr<swift::ModuleLoader> serialized_module_loader_up(
       swift::ImplicitSerializedModuleLoader::create(
           *m_ast_context_up, m_dependency_tracker.get(), loading_mode));
   if (serialized_module_loader_up)
     m_ast_context_up->addModuleLoader(std::move(serialized_module_loader_up));
 
-  // 4. Install the clang importer.
+  // 5. Install the clang importer.
   if (clang_importer_up) {
     m_clangimporter = (swift::ClangImporter *)clang_importer_up.get();
     m_ast_context_up->addModuleLoader(std::move(clang_importer_up),
@@ -4077,6 +4095,23 @@ SwiftASTContext::GetModule(const SourceModule &module, bool *cached) {
   // Create a diagnostic consumer for the diagnostics produced by the import.
   auto import_diags = getScopedDiagnosticConsumer();
 
+  // Is this an explicitly specified explicit Swift module?
+  StringRef module_path = module.search_path.GetStringRef();
+  bool is_esml_module = (module_path.ends_with(".swiftmodule") &&
+                         llvm::sys::fs::exists(module_path)) ||
+                        (m_cas && IsModuleAvailableInCAS(module_path.str()));
+  if (is_esml_module) {
+    std::string path = module_path.str();
+    bool unloaded = false;
+    if (m_explicit_swift_module_loader) {
+      ast->addExplicitModulePath(module_name, path);
+      if (auto *memory_loader = GetMemoryBufferModuleLoader())
+        unloaded = memory_loader->unregisterMemoryBuffer(module_name);
+    }
+    HEALTH_LOG_PRINTF("found explicit module \"%s\"%s", path.c_str(),
+                      unloaded ? "; replacing AST section module" : "");
+  }
+
   swift::ModuleDecl *module_decl = ast->getModuleByName(module_name);
 
   // Error handling.
@@ -4099,6 +4134,16 @@ SwiftASTContext::GetModule(const SourceModule &module, bool *cached) {
   LOG_PRINTF(GetLog(LLDBLog::Types), "(\"%s\") -- found %s",
              module_name.c_str(), module_decl->getName().str().str().c_str());
 
+  if (is_esml_module) {
+    // Simulate the effect of the BypassResilience flag in the
+    // MemoryBufferSerializedModuleLoader.  Explicitly specified
+    // modules are not typically produced from textual interfaces. By
+    // disabling resilience, the debugger can directly access private
+    // members.
+    //if (!module_decl->isBuiltFromInterface())
+    //  module_decl->setBypassResilience();
+  }
+
   m_swift_module_cache.insert({module_name, *module_decl});
   return *module_decl;
 }

lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.h

@@ -278,6 +278,9 @@ class SwiftASTContext : public TypeSystemSwift {
 
   void ConfigureModuleValidation(std::vector<std::string> &extra_args);
 
+  /// Check whether a module with key \c key is available in CAS.
+  bool IsModuleAvailableInCAS(const std::string &key);
+
   /// Add a list of Clang arguments to the ClangImporter options and
   /// apply the working directory to any relative paths.
   void AddExtraClangArgs(
@@ -972,6 +975,7 @@ class SwiftASTContext : public TypeSystemSwift {
   /// Owned by the AST.
   swift::MemoryBufferSerializedModuleLoader *m_memory_buffer_module_loader =
       nullptr;
+  swift::ModuleLoader *m_explicit_swift_module_loader = nullptr;
   swift::ClangImporter *m_clangimporter = nullptr;
   /// Wraps the clang::ASTContext owned by ClangImporter.
   std::shared_ptr<TypeSystemClang> m_clangimporter_typesystem;

lldb/test/API/lang/swift/explicit_modules/simple/TestSwiftExplicitModules.py

@@ -13,8 +13,30 @@ def test(self):
         target, process, thread, bkpt = lldbutil.run_to_source_breakpoint(
             self, 'Set breakpoint here', lldb.SBFileSpec('main.swift'))
 
+        log = self.getBuildArtifact("types.log")
+        self.expect('log enable lldb types -f "%s"' % log)
         self.expect("expression c", substrs=['hello explicit'])
+        self.filecheck('platform shell cat "%s"' % log, __file__)
+        # CHECK: SwiftASTContextForExpressions(module: "a", cu: "main.swift"){{.*}} found explicit module {{.*}}a.swiftmodule
+        # CHECK: SwiftASTContextForExpressions(module: "a", cu: "main.swift"){{.*}} Module import remark: loaded module 'a'; source: '{{.*}}a.swiftmodule', loaded: '{{.*}}a.swiftmodule'
 
+    @swiftTest
+    def test_disable_esml(self):
+        """Test disabling the explicit Swift module loader"""
+        self.build()
+        self.expect("settings set symbols.use-swift-explicit-module-loader false")
+
+        target, process, thread, bkpt = lldbutil.run_to_source_breakpoint(
+            self, 'Set breakpoint here', lldb.SBFileSpec('main.swift'))
+
+        log = self.getBuildArtifact("types.log")
+        self.expect('log enable lldb types -f "%s"' % log)
+        self.expect("expression c", substrs=['hello explicit'])
+        self.filecheck('platform shell cat "%s"' % log, __file__, '--check-prefix=DISABLED')
+        # DISABLED: SwiftASTContextForExpressions(module: "a", cu: "main.swift"){{.*}} found explicit module {{.*}}a.swiftmodule
+        # DISABLED: SwiftASTContextForExpressions(module: "a", cu: "main.swift"){{.*}} Module import remark: loaded module 'a'; source: 'a', loaded: 'a'
+
+        
     @swiftTest
     @skipUnlessDarwin
     def test_import(self):