Merge pull request #81720 from cachemeifyoucan/eng/PR-release-148752988-151339073-151395300

[6.2][Caching] Swift Caching Dependency Scanning Improvements

Author: cachemeifyoucan

include/swift/AST/DiagnosticsFrontend.def

@@ -520,7 +520,7 @@ REMARK(matching_output_produced,none,
 
 // Caching related diagnostics
 ERROR(error_caching_no_cas_fs, none,
-    "caching is enabled without -cas-fs option, input is not immutable", ())
+    "caching is enabled without CAS file-system options, input is not immutable", ())
 ERROR(error_prefix_mapping, none, "cannot create scanner prefix mapping: '%0'", (StringRef))
 
 REMARK(replay_output, none, "replay output file '%0': key '%1'", (StringRef, StringRef))
@@ -534,6 +534,7 @@ ERROR(error_cache_key_creation, none, "cannot create cache key for compilation %
 ERROR(error_cas_file_ref, none, "cannot load file %0 from CAS filesystem", (StringRef))
 ERROR(error_cas_conflict_options, none, "cannot setup CAS due to conflicting '-cas-*' options", ())
 ERROR(error_cas_initialization, none, "CAS cannot be initialized from the specified '-cas-*' options: %0", (StringRef))
+ERROR(error_cas_malformed_input, none, "CAS input '%0' is malformed: %1", (StringRef, StringRef))
 WARNING(cache_replay_failed, none, "cache replay failed: %0", (StringRef))
 
 ERROR(error_failed_cached_diag, none, "failed to serialize cached diagnostics: %0", (StringRef))

include/swift/AST/ModuleDependencies.h

@@ -1005,7 +1005,7 @@ using ModuleDependenciesKindMap =
 /// Track swift dependency
 class SwiftDependencyTracker {
 public:
-  SwiftDependencyTracker(llvm::cas::CachingOnDiskFileSystem &FS,
+  SwiftDependencyTracker(std::shared_ptr<llvm::cas::ObjectStore> CAS,
                          llvm::PrefixMapper *Mapper,
                          const CompilerInvocation &CI);
 
@@ -1014,7 +1014,8 @@ class SwiftDependencyTracker {
   llvm::Expected<llvm::cas::ObjectProxy> createTreeFromDependencies();
 
 private:
-  llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> FS;
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS;
+  std::shared_ptr<llvm::cas::ObjectStore> CAS;
   llvm::PrefixMapper *Mapper;
 
   struct FileEntry {
@@ -1040,10 +1041,7 @@ class SwiftDependencyScanningService {
       ClangScanningService;
 
   /// CachingOnDiskFileSystem for dependency tracking.
-  llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> CacheFS;
-
-  /// If use clang include tree.
-  bool UseClangIncludeTree = false;
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> CacheFS;
 
   /// CAS Instance.
   std::shared_ptr<llvm::cas::ObjectStore> CAS;
@@ -1081,8 +1079,8 @@ class SwiftDependencyScanningService {
     return *SharedFilesystemCache;
   }
 
-  llvm::cas::CachingOnDiskFileSystem &getSharedCachingFS() const {
-    assert(CacheFS && "Expect CachingOnDiskFileSystem");
+  llvm::vfs::FileSystem &getSharedCachingFS() const {
+    assert(CacheFS && "Expect a CASFileSystem");
     return *CacheFS;
   }
 
@@ -1093,20 +1091,17 @@ class SwiftDependencyScanningService {
 
   std::optional<SwiftDependencyTracker>
   createSwiftDependencyTracker(const CompilerInvocation &CI) {
-    if (!CacheFS)
+    if (!CAS)
       return std::nullopt;
 
-    return SwiftDependencyTracker(*CacheFS, Mapper.get(), CI);
+    return SwiftDependencyTracker(CAS, Mapper.get(), CI);
   }
 
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> getClangScanningFS() const {
-    if (UseClangIncludeTree)
+    if (CAS)
       return llvm::cas::createCASProvidingFileSystem(
           CAS, llvm::vfs::createPhysicalFileSystem());
 
-    if (CacheFS)
-      return CacheFS->createProxyFS();
-
     return llvm::vfs::createPhysicalFileSystem();
   }
 

include/swift/Basic/CASOptions.h

@@ -37,14 +37,11 @@ class CASOptions final {
   /// CASOptions
   clang::CASOptions CASOpts;
 
-  /// CASFS Root.
-  std::vector<std::string> CASFSRootIDs;
-
   /// Clang Include Trees.
-  std::vector<std::string> ClangIncludeTrees;
+  std::string ClangIncludeTree;
 
   /// Clang Include Tree FileList.
-  std::vector<std::string> ClangIncludeTreeFileList;
+  std::string ClangIncludeTreeFileList;
 
   /// CacheKey for input file.
   std::string InputFileKey;
@@ -62,9 +59,8 @@ class CASOptions final {
   /// Check to see if a CASFileSystem is required.
   bool requireCASFS() const {
     return EnableCaching &&
-           (!CASFSRootIDs.empty() || !ClangIncludeTrees.empty() ||
-            !ClangIncludeTreeFileList.empty() || !InputFileKey.empty() ||
-            !BridgingHeaderPCHCacheKey.empty());
+           (!ClangIncludeTree.empty() || !ClangIncludeTreeFileList.empty() ||
+            !InputFileKey.empty() || !BridgingHeaderPCHCacheKey.empty());
   }
 
   /// Return a hash code of any components from these options that should

include/swift/Basic/LangOptions.h

@@ -1094,12 +1094,6 @@ namespace swift {
     /// built and provided to the compiler invocation.
     bool DisableImplicitClangModules = false;
 
-    /// Enable ClangIncludeTree for explicit module builds scanning.
-    bool UseClangIncludeTree = false;
-
-    /// Using ClangIncludeTreeRoot for compilation.
-    bool HasClangIncludeTreeRoot = false;
-
     /// Whether the dependency scanner should construct all swift-frontend
     /// invocations directly from clang cc1 args.
     bool ClangImporterDirectCC1Scan = false;

include/swift/ClangImporter/ClangImporter.h

@@ -472,11 +472,6 @@ class ClangImporter final : public ClangModuleLoader {
   /// Reads the original source file name from PCH.
   std::string getOriginalSourceFile(StringRef PCHFilename);
 
-  /// Add clang dependency file names.
-  ///
-  /// \param files The list of file to append dependencies to.
-  void addClangInvovcationDependencies(std::vector<std::string> &files);
-
   /// Makes a temporary replica of the ClangImporter's CompilerInstance, reads a
   /// module map into the replica and emits a PCM file for one of the modules it
   /// declares. Delegates to clang for everything except construction of the

include/swift/Frontend/CachingUtils.h

@@ -70,9 +70,9 @@ std::unique_ptr<llvm::MemoryBuffer> loadCachedCompileResultFromCacheKey(
     llvm::StringRef Filename = "");
 
 llvm::Expected<llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>>
-createCASFileSystem(llvm::cas::ObjectStore &CAS, ArrayRef<std::string> FSRoots,
-                    ArrayRef<std::string> IncludeTreeRoots,
-                    ArrayRef<std::string> IncludeTreeFileList);
+createCASFileSystem(llvm::cas::ObjectStore &CAS,
+                    const std::string &IncludeTreeRoot,
+                    const std::string &IncludeTreeFileList);
 
 std::vector<std::string> remapPathsFromCommandLine(
     ArrayRef<std::string> Args,

include/swift/Option/FrontendOptions.td

@@ -1454,9 +1454,6 @@ def bridging_header_pch_key : Separate<["-"], "bridging-header-pch-key">,
 def no_clang_include_tree: Flag<["-"], "no-clang-include-tree">,
   HelpText<"Do not use clang include tree, fallback to use CAS filesystem to build clang modules">;
 
-def cas_fs: Separate<["-"], "cas-fs">,
-  HelpText<"Root CASID for CAS FileSystem">, MetaVarName<"<cas-id>">;
-
 def clang_include_tree_root: Separate<["-"], "clang-include-tree-root">,
   HelpText<"Clang Include Tree CASID">, MetaVarName<"<cas-id>">;
 def clang_include_tree_filelist: Separate<["-"], "clang-include-tree-filelist">,

lib/AST/ModuleDependencies.cpp

@@ -25,7 +25,6 @@
 #include "swift/Strings.h"
 #include "clang/CAS/IncludeTree.h"
 #include "llvm/CAS/CASProvidingFileSystem.h"
-#include "llvm/CAS/CachingOnDiskFileSystem.h"
 #include "llvm/Config/config.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
@@ -602,13 +601,16 @@ swift::dependencies::registerBackDeployLibraries(
 }
 
 SwiftDependencyTracker::SwiftDependencyTracker(
-    llvm::cas::CachingOnDiskFileSystem &FS, llvm::PrefixMapper *Mapper,
+    std::shared_ptr<llvm::cas::ObjectStore> CAS, llvm::PrefixMapper *Mapper,
     const CompilerInvocation &CI)
-    : FS(FS.createProxyFS()), Mapper(Mapper) {
+    : CAS(CAS), Mapper(Mapper) {
   auto &SearchPathOpts = CI.getSearchPathOptions();
 
+  FS = llvm::cas::createCASProvidingFileSystem(
+      CAS, llvm::vfs::createPhysicalFileSystem());
+
   auto addCommonFile = [&](StringRef path) {
-    auto file = FS.openFileForRead(path);
+    auto file = FS->openFileForRead(path);
     if (!file)
       return;
     auto status = (*file)->status();
@@ -679,7 +681,7 @@ SwiftDependencyTracker::createTreeFromDependencies() {
   llvm::SmallVector<clang::cas::IncludeTree::FileList::FileEntry> Files;
   for (auto &file : TrackedFiles) {
     auto includeTreeFile = clang::cas::IncludeTree::File::create(
-        FS->getCAS(), file.first, file.second.FileRef);
+        *CAS, file.first, file.second.FileRef);
     if (!includeTreeFile) {
       return llvm::createStringError("CASFS createTree failed for " +
                                      file.first + ": " +
@@ -691,7 +693,7 @@ SwiftDependencyTracker::createTreeFromDependencies() {
   }
 
   auto includeTreeList =
-      clang::cas::IncludeTree::FileList::create(FS->getCAS(), Files, {});
+      clang::cas::IncludeTree::FileList::create(*CAS, Files, {});
   if (!includeTreeList)
     return llvm::createStringError("casfs include-tree filelist error: " +
                                    toString(includeTreeList.takeError()));
@@ -719,14 +721,8 @@ bool SwiftDependencyScanningService::setupCachingDependencyScanningService(
   CAS = Instance.getSharedCASInstance();
   ActionCache = Instance.getSharedCacheInstance();
 
-  auto CachingFS =
-      llvm::cas::createCachingOnDiskFileSystem(Instance.getObjectStore());
-  if (!CachingFS) {
-    Instance.getDiags().diagnose(SourceLoc(), diag::error_cas_fs_creation,
-                                 toString(CachingFS.takeError()));
-    return true;
-  }
-  CacheFS = std::move(*CachingFS);
+  CacheFS = llvm::cas::createCASProvidingFileSystem(
+      CAS, llvm::vfs::createPhysicalFileSystem());
 
   // Setup prefix mapping.
   auto &ScannerPrefixMapper =
@@ -744,19 +740,14 @@ bool SwiftDependencyScanningService::setupCachingDependencyScanningService(
     Mapper->sort();
   }
 
-  UseClangIncludeTree =
-      Instance.getInvocation().getClangImporterOptions().UseClangIncludeTree;
   const clang::tooling::dependencies::ScanningOutputFormat ClangScanningFormat =
-      UseClangIncludeTree
-          ? clang::tooling::dependencies::ScanningOutputFormat::FullIncludeTree
-          : clang::tooling::dependencies::ScanningOutputFormat::FullTree;
+      clang::tooling::dependencies::ScanningOutputFormat::FullIncludeTree;
 
   ClangScanningService.emplace(
       clang::tooling::dependencies::ScanningMode::DependencyDirectivesScan,
-      ClangScanningFormat,
-      Instance.getInvocation().getCASOptions().CASOpts,
+      ClangScanningFormat, Instance.getInvocation().getCASOptions().CASOpts,
       Instance.getSharedCASInstance(), Instance.getSharedCacheInstance(),
-      UseClangIncludeTree ? nullptr : CacheFS,
+      /*CachingOnDiskFileSystem=*/nullptr,
       // The current working directory optimization (off by default)
       // should not impact CAS. We set the optization to all to be
       // consistent with the non-CAS case.

lib/AST/PluginLoader.cpp

@@ -64,9 +64,9 @@ static StringRef pluginModuleNameStringFromPath(StringRef path) {
 
 static llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>
 getPluginLoadingFS(ASTContext &Ctx) {
-  // If there is a clang include tree FS, using real file system to load plugin
+  // If there is an immutable file system, using real file system to load plugin
   // as the FS in SourceMgr doesn't support directory iterator.
-  if (Ctx.ClangImporterOpts.HasClangIncludeTreeRoot)
+  if (Ctx.CASOpts.HasImmutableFileSystem)
     return llvm::vfs::getRealFileSystem();
   return Ctx.SourceMgr.getFileSystem();
 }

lib/ClangImporter/ClangImporter.cpp

@@ -90,6 +90,7 @@
 #include "clang/Serialization/ASTReader.h"
 #include "clang/Serialization/ASTWriter.h"
 #include "clang/Serialization/ObjectFilePCHContainerReader.h"
+#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
 #include "clang/Tooling/DependencyScanning/ScanAndUpdateArgs.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/STLExtras.h"
@@ -1062,27 +1063,6 @@ std::string ClangImporter::getOriginalSourceFile(StringRef PCHFilename) {
       Impl.Instance->getPCHContainerReader(), Impl.Instance->getDiagnostics());
 }
 
-void ClangImporter::addClangInvovcationDependencies(
-    std::vector<std::string> &files) {
-  auto addFiles = [&files](const auto &F) {
-    files.insert(files.end(), F.begin(), F.end());
-  };
-  auto &invocation = *Impl.Invocation;
-  // FIXME: Add file dependencies that are not accounted. The long term solution
-  // is to do a dependency scanning for clang importer and use that directly.
-  SmallVector<std::string, 4> HeaderMapFileNames;
-  Impl.Instance->getPreprocessor().getHeaderSearchInfo().getHeaderMapFileNames(
-      HeaderMapFileNames);
-  addFiles(HeaderMapFileNames);
-  addFiles(invocation.getHeaderSearchOpts().VFSOverlayFiles);
-  // FIXME: Should not depend on working directory. Build system/swift driver
-  // should not pass working directory here but if that option is passed,
-  // repect that and add that into CASFS.
-  auto CWD = invocation.getFileSystemOpts().WorkingDir;
-  if (!CWD.empty())
-    files.push_back(CWD);
-}
-
 std::optional<std::string>
 ClangImporter::getPCHFilename(const ClangImporterOptions &ImporterOptions,
                               StringRef SwiftPCHHash, bool &isExplicit) {
@@ -1210,9 +1190,8 @@ std::optional<std::vector<std::string>> ClangImporter::getClangCC1Arguments(
                clang::frontend::ActionKind::GenerateModule ||
            CI->getFrontendOpts().ProgramAction ==
                clang::frontend::ActionKind::GeneratePCH) &&
-          ctx.ClangImporterOpts.HasClangIncludeTreeRoot) {
-        CI->getFrontendOpts().CASIncludeTreeID =
-            ctx.CASOpts.ClangIncludeTrees.back();
+          !ctx.CASOpts.ClangIncludeTree.empty()) {
+        CI->getFrontendOpts().CASIncludeTreeID = ctx.CASOpts.ClangIncludeTree;
         CI->getFrontendOpts().Inputs.clear();
       }
     }
@@ -1271,7 +1250,7 @@ std::optional<std::vector<std::string>> ClangImporter::getClangCC1Arguments(
 
   std::vector<std::string> FilteredModuleMapFiles;
   for (auto ModuleMapFile : CI->getFrontendOpts().ModuleMapFiles) {
-    if (ctx.ClangImporterOpts.HasClangIncludeTreeRoot) {
+    if (ctx.CASOpts.HasImmutableFileSystem) {
       // There is no need to add any module map file here. Issue a warning and
       // drop the option.
       Impl.diagnose(SourceLoc(), diag::module_map_ignored, ModuleMapFile);
@@ -1351,7 +1330,7 @@ ClangImporter::create(ASTContext &ctx,
       fileMapping.requiresBuiltinHeadersInSystemModules;
 
   // Avoid creating indirect file system when using include tree.
-  if (!ctx.ClangImporterOpts.HasClangIncludeTreeRoot) {
+  if (!ctx.CASOpts.HasImmutableFileSystem) {
     // Wrap Swift's FS to allow Clang to override the working directory
     VFS = llvm::vfs::RedirectingFileSystem::create(
         fileMapping.redirectedFiles, true, *ctx.SourceMgr.getFileSystem());
@@ -2756,7 +2735,6 @@ ClangImporter::Implementation::Implementation(
           !ctx.ClangImporterOpts.BridgingHeader.empty()),
       DisableOverlayModules(ctx.ClangImporterOpts.DisableOverlayModules),
       EnableClangSPI(ctx.ClangImporterOpts.EnableClangSPI),
-      UseClangIncludeTree(ctx.ClangImporterOpts.UseClangIncludeTree),
       importSymbolicCXXDecls(
           ctx.LangOpts.hasFeature(Feature::ImportSymbolicCXXDecls)),
       IsReadingBridgingPCH(false),
@@ -4281,15 +4259,14 @@ ClangImporter::getSwiftExplicitModuleDirectCC1Args() const {
   PPOpts.MacroIncludes.clear();
   PPOpts.Includes.clear();
 
-  // CodeGenOptions.
-  auto &CGOpts = instance.getCodeGenOpts();
-  CGOpts.DebugCompilationDir.clear();
+  // Clear benign CodeGenOptions.
+  clang::tooling::dependencies::resetBenignCodeGenOptions(
+      clang::frontend::ActionKind::GenerateModule, instance.getLangOpts(),
+      instance.getCodeGenOpts());
 
-  if (Impl.SwiftContext.ClangImporterOpts.UseClangIncludeTree) {
-    // FileSystemOptions.
-    auto &FSOpts = instance.getFileSystemOpts();
-    FSOpts.WorkingDir.clear();
-  }
+  // FileSystemOptions.
+  auto &FSOpts = instance.getFileSystemOpts();
+  FSOpts.WorkingDir.clear();
 
   if (!Impl.SwiftContext.SearchPathOpts.ScannerPrefixMapper.empty()) {
     // Remap all the paths if requested.