[Dependency Scanning] Add ClangImporter's mandatory path remaps to dependency scanning query filesystem

On creation, 'ClangImporter' adds overlay modulemap files for non-modular platform libraries (e.g. glibc, libstdc++), which allows Swift code to import and use those libraries.

This change adds the same filesystem overlay to dependency scanning queries by applying them to the filesystem instantiated for each depndency scanning worker. Without these overlays EBM builds cannot discover and use non-modular system libraries on non-Darwin platforms.

Resolves rdar://151780437

Author: artemcm

include/swift/AST/ModuleDependencies.h

@@ -1097,13 +1097,8 @@ class SwiftDependencyScanningService {
     return SwiftDependencyTracker(CAS, Mapper.get(), CI);
   }
 
-  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> getClangScanningFS() const {
-    if (CAS)
-      return llvm::cas::createCASProvidingFileSystem(
-          CAS, llvm::vfs::createPhysicalFileSystem());
-
-    return llvm::vfs::createPhysicalFileSystem();
-  }
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>
+  getClangScanningFS(ASTContext &ctx) const;
 
   bool hasPathMapping() const {
     return Mapper && !Mapper->getMappings().empty();

include/swift/ClangImporter/ClangImporter.h

@@ -873,10 +873,19 @@ struct ClangInvocationFileMapping {
 /// `suppressDiagnostic` prevents us from emitting warning messages when we
 /// are unable to find headers.
 ClangInvocationFileMapping getClangInvocationFileMapping(
-    ASTContext &ctx,
+    const ASTContext &ctx,
     llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> vfs = nullptr,
     bool suppressDiagnostic = false);
 
+/// Apply the given file mapping to the specified 'fileSystem', used
+/// primarily to inject modulemaps on platforms with non-modularized
+/// platform libraries.
+ClangInvocationFileMapping applyClangInvocationMapping(
+    const ASTContext &ctx,
+    llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> baseVFS,
+    llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> &fileSystem,
+    bool suppressDiagnostics = false);
+
 /// Information used to compute the access level of inherited C++ members.
 class ClangInheritanceInfo {
   /// The cumulative inheritance access specifier, that is used to compute the

lib/AST/ModuleDependencies.cpp

@@ -499,6 +499,18 @@ SwiftDependencyScanningService::SwiftDependencyScanningService() {
   SharedFilesystemCache.emplace();
 }
 
+llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>
+SwiftDependencyScanningService::getClangScanningFS(ASTContext &ctx) const {
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> baseFileSystem =
+      llvm::vfs::createPhysicalFileSystem();
+  ClangInvocationFileMapping fileMapping =
+    applyClangInvocationMapping(ctx, nullptr, baseFileSystem, false);
+
+  if (CAS)
+    return llvm::cas::createCASProvidingFileSystem(CAS, baseFileSystem);
+  return baseFileSystem;
+}
+
 bool
 swift::dependencies::checkImportNotTautological(const ImportPath::Module modulePath,
                                                 const SourceLoc importLoc,

lib/ClangImporter/ClangImporter.cpp

@@ -1300,49 +1300,11 @@ ClangImporter::create(ASTContext &ctx,
       ctx.SourceMgr.getFileSystem();
 
   ClangInvocationFileMapping fileMapping =
-    getClangInvocationFileMapping(ctx, nullptr, ignoreFileMapping);
+    applyClangInvocationMapping(ctx, nullptr, VFS, ignoreFileMapping);
 
   importer->requiresBuiltinHeadersInSystemModules =
       fileMapping.requiresBuiltinHeadersInSystemModules;
 
-  // Avoid creating indirect file system when using include tree.
-  if (!ctx.CASOpts.HasImmutableFileSystem) {
-    // Wrap Swift's FS to allow Clang to override the working directory
-    VFS = llvm::vfs::RedirectingFileSystem::create(
-        fileMapping.redirectedFiles, true, *ctx.SourceMgr.getFileSystem());
-    if (importerOpts.DumpClangDiagnostics) {
-      llvm::errs() << "clang importer redirected file mappings:\n";
-      for (const auto &mapping : fileMapping.redirectedFiles) {
-        llvm::errs() << "   mapping real file '" << mapping.second
-                     << "' to virtual file '" << mapping.first << "'\n";
-      }
-      llvm::errs() << "\n";
-    }
-
-    if (!fileMapping.overridenFiles.empty()) {
-      llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> overridenVFS =
-          new llvm::vfs::InMemoryFileSystem();
-      for (const auto &file : fileMapping.overridenFiles) {
-        if (importerOpts.DumpClangDiagnostics) {
-          llvm::errs() << "clang importer overriding file '" << file.first
-                       << "' with the following contents:\n";
-          llvm::errs() << file.second << "\n";
-        }
-        auto contents = ctx.Allocate<char>(file.second.size() + 1);
-        std::copy(file.second.begin(), file.second.end(), contents.begin());
-        // null terminate the buffer.
-        contents[contents.size() - 1] = '\0';
-        overridenVFS->addFile(file.first, 0,
-                              llvm::MemoryBuffer::getMemBuffer(StringRef(
-                                  contents.begin(), contents.size() - 1)));
-      }
-      llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> overlayVFS =
-          new llvm::vfs::OverlayFileSystem(VFS);
-      VFS = overlayVFS;
-      overlayVFS->pushOverlay(overridenVFS);
-    }
-  }
-
   // Create a new Clang compiler invocation.
   {
     if (auto ClangArgs = importer->getClangCC1Arguments(ctx, VFS))

lib/ClangImporter/ClangIncludePaths.cpp

@@ -197,7 +197,7 @@ ClangImporter::createClangArgs(const ClangImporterOptions &ClangImporterOpts,
 }
 
 static SmallVector<std::pair<std::string, std::string>, 2>
-getLibcFileMapping(ASTContext &ctx, StringRef modulemapFileName,
+getLibcFileMapping(const ASTContext &ctx, StringRef modulemapFileName,
                    std::optional<ArrayRef<StringRef>> maybeHeaderFileNames,
                    const llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> &vfs,
                    bool suppressDiagnostic) {
@@ -263,7 +263,7 @@ getLibcFileMapping(ASTContext &ctx, StringRef modulemapFileName,
 }
 
 static void getLibStdCxxFileMapping(
-    ClangInvocationFileMapping &fileMapping, ASTContext &ctx,
+    ClangInvocationFileMapping &fileMapping, const ASTContext &ctx,
     const llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> &vfs,
     bool suppressDiagnostic) {
   assert(ctx.LangOpts.EnableCXXInterop &&
@@ -454,7 +454,7 @@ GetPlatformAuxiliaryFile(StringRef Platform, StringRef File,
 }
 
 void GetWindowsFileMappings(
-    ClangInvocationFileMapping &fileMapping, ASTContext &Context,
+    ClangInvocationFileMapping &fileMapping, const ASTContext &Context,
     const llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> &driverVFS,
     bool &requiresBuiltinHeadersInSystemModules) {
   const llvm::Triple &Triple = Context.LangOpts.Target;
@@ -585,7 +585,7 @@ void GetWindowsFileMappings(
 } // namespace
 
 ClangInvocationFileMapping swift::getClangInvocationFileMapping(
-  ASTContext &ctx, llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> vfs,
+  const ASTContext &ctx, llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> vfs,
   bool suppressDiagnostic) {
   ClangInvocationFileMapping result;
   if (!vfs)
@@ -655,3 +655,52 @@ ClangInvocationFileMapping swift::getClangInvocationFileMapping(
                          result.requiresBuiltinHeadersInSystemModules);
   return result;
 }
+
+ClangInvocationFileMapping swift::applyClangInvocationMapping(const ASTContext &ctx,
+                                        llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> baseVFS,
+                                        llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> &fileSystem,
+                                        bool suppressDiagnostics) {
+  if (ctx.CASOpts.HasImmutableFileSystem)
+    return ClangInvocationFileMapping();
+
+  ClangInvocationFileMapping fileMapping =
+    getClangInvocationFileMapping(ctx, baseVFS, suppressDiagnostics);
+
+  auto importerOpts = ctx.ClangImporterOpts;
+  // Wrap Swift's FS to allow Clang to override the working directory
+  fileSystem = llvm::vfs::RedirectingFileSystem::create(
+      fileMapping.redirectedFiles, true, *fileSystem);
+  if (importerOpts.DumpClangDiagnostics) {
+    llvm::errs() << "clang importer redirected file mappings:\n";
+    for (const auto &mapping : fileMapping.redirectedFiles) {
+      llvm::errs() << "   mapping real file '" << mapping.second
+                   << "' to virtual file '" << mapping.first << "'\n";
+    }
+    llvm::errs() << "\n";
+  }
+
+  if (!fileMapping.overridenFiles.empty()) {
+    llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> overridenVFS =
+        new llvm::vfs::InMemoryFileSystem();
+    for (const auto &file : fileMapping.overridenFiles) {
+      if (importerOpts.DumpClangDiagnostics) {
+        llvm::errs() << "clang importer overriding file '" << file.first
+                     << "' with the following contents:\n";
+        llvm::errs() << file.second << "\n";
+      }
+      auto contents = ctx.Allocate<char>(file.second.size() + 1);
+      std::copy(file.second.begin(), file.second.end(), contents.begin());
+      // null terminate the buffer.
+      contents[contents.size() - 1] = '\0';
+      overridenVFS->addFile(file.first, 0,
+                            llvm::MemoryBuffer::getMemBuffer(StringRef(
+                                contents.begin(), contents.size() - 1)));
+    }
+    llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> overlayVFS =
+        new llvm::vfs::OverlayFileSystem(fileSystem);
+    fileSystem = overlayVFS;
+    overlayVFS->pushOverlay(overridenVFS);
+  }
+
+  return fileMapping;
+}

lib/DependencyScan/ModuleDependencyScanner.cpp

@@ -31,6 +31,7 @@
 #include "swift/Frontend/ModuleInterfaceLoader.h"
 #include "swift/Serialization/SerializedModuleLoader.h"
 #include "swift/Subsystems.h"
+#include "clang/Frontend/CompilerInstance.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/SetOperations.h"
 #include "llvm/CAS/CachingOnDiskFileSystem.h"
@@ -267,7 +268,7 @@ ModuleDependencyScanningWorker::ModuleDependencyScanningWorker(
     const SILOptions &SILOptions, ASTContext &ScanASTContext,
     swift::DependencyTracker &DependencyTracker, DiagnosticEngine &Diagnostics)
     : clangScanningTool(*globalScanningService.ClangScanningService,
-                        globalScanningService.getClangScanningFS()) {
+                        globalScanningService.getClangScanningFS(ScanASTContext)) {
   // Create a scanner-specific Invocation and ASTContext.
   workerCompilerInvocation =
       std::make_unique<CompilerInvocation>(ScanCompilerInvocation);

test/ScanDependencies/cxx-overlay-underlying-module-lookup.swift

@@ -3,9 +3,6 @@
 // RUN: %target-swift-frontend -scan-dependencies -o %t/deps.json %s -cxx-interoperability-mode=default -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import
 // RUN: %validate-json %t/deps.json | %FileCheck %s
 
-// rdar://151780437: libstdc++ VFS modulemap redirects not functioning with EBM enabled
-// REQUIRES: OS=macosx
-
 import CxxStdlib
 
 // CHECK: "mainModuleName": "deps"