[clang][modules] Introduce new ModuleCache interface (llvm#131193)

This PR adds new `ModuleCache` interface to Clang's implicitly-built
modules machinery. The main motivation for this change is to create a
second implementation that uses a more efficient kind of
`llvm::AdvisoryLock` during dependency scanning.

In addition to the lock abstraction, the `ModuleCache` interface also
manages the existing `InMemoryModuleCache` instance. I found that
compared to keeping these separate/independent, the code is a bit
simpler now, since these are two tightly coupled concepts. I can
envision a more efficient implementation of the `InMemoryModuleCache`
for the single-process case too, which will be much easier to implement
with the current setup.

This is not intended to be a functional change.

(cherry picked from commit c84d8e8)

Author: jansvoboda11

clang/include/clang/Frontend/ASTUnit.h

@@ -70,7 +70,7 @@ class FileManager;
 class FrontendAction;
 class HeaderSearch;
 class InputKind;
-class InMemoryModuleCache;
+class ModuleCache;
 class PCHContainerOperations;
 class PCHContainerReader;
 class Preprocessor;
@@ -110,7 +110,7 @@ class ASTUnit {
   IntrusiveRefCntPtr<DiagnosticsEngine>   Diagnostics;
   IntrusiveRefCntPtr<FileManager>         FileMgr;
   IntrusiveRefCntPtr<SourceManager>       SourceMgr;
-  IntrusiveRefCntPtr<InMemoryModuleCache> ModuleCache;
+  IntrusiveRefCntPtr<ModuleCache> ModCache;
   std::unique_ptr<HeaderSearch>           HeaderInfo;
   IntrusiveRefCntPtr<TargetInfo>          Target;
   std::shared_ptr<Preprocessor>           PP;

clang/include/clang/Frontend/CompilerInstance.h

@@ -54,8 +54,8 @@ class DiagnosticsEngine;
 class DiagnosticConsumer;
 class FileManager;
 class FrontendAction;
-class InMemoryModuleCache;
 class Module;
+class ModuleCache;
 class Preprocessor;
 class Sema;
 class SourceManager;
@@ -115,7 +115,7 @@ class CompilerInstance : public ModuleLoader {
   IntrusiveRefCntPtr<SourceManager> SourceMgr;
 
   /// The cache of PCM files.
-  IntrusiveRefCntPtr<InMemoryModuleCache> ModuleCache;
+  IntrusiveRefCntPtr<ModuleCache> ModCache;
 
   /// The preprocessor.
   std::shared_ptr<Preprocessor> PP;
@@ -223,7 +223,7 @@ class CompilerInstance : public ModuleLoader {
   explicit CompilerInstance(
       std::shared_ptr<PCHContainerOperations> PCHContainerOps =
           std::make_shared<PCHContainerOperations>(),
-      InMemoryModuleCache *SharedModuleCache = nullptr);
+      ModuleCache *ModCache = nullptr);
   ~CompilerInstance() override;
 
   /// @name High-Level Operations
@@ -793,9 +793,8 @@ class CompilerInstance : public ModuleLoader {
   static IntrusiveRefCntPtr<ASTReader> createPCHExternalASTSource(
       StringRef Path, StringRef Sysroot,
       DisableValidationForModuleKind DisableValidation,
-      bool AllowPCHWithCompilerErrors, Preprocessor &PP,
-      InMemoryModuleCache &ModuleCache, ASTContext &Context,
-      const PCHContainerReader &PCHContainerRdr,
+      bool AllowPCHWithCompilerErrors, Preprocessor &PP, ModuleCache &ModCache,
+      ASTContext &Context, const PCHContainerReader &PCHContainerRdr,
       ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions,
       ArrayRef<std::shared_ptr<DependencyCollector>> DependencyCollectors,
       void *DeserializationListener, bool OwnDeserializationListener,
@@ -966,7 +965,7 @@ class CompilerInstance : public ModuleLoader {
   bool addCachedModuleFile(StringRef Path, StringRef CacheKey,
                            StringRef Provider);
 
-  InMemoryModuleCache &getModuleCache() const { return *ModuleCache; }
+  ModuleCache &getModuleCache() const { return *ModCache; }
 };
 
 } // end namespace clang

clang/include/clang/Serialization/ASTReader.h

@@ -88,7 +88,7 @@ struct HeaderFileInfo;
 class HeaderSearchOptions;
 class LangOptions;
 class MacroInfo;
-class InMemoryModuleCache;
+class ModuleCache;
 class NamedDecl;
 class NamespaceDecl;
 class ObjCCategoryDecl;
@@ -1641,8 +1641,8 @@ class ASTReader
   ///
   /// \param ReadTimer If non-null, a timer used to track the time spent
   /// deserializing.
-  ASTReader(Preprocessor &PP, InMemoryModuleCache &ModuleCache,
-            ASTContext *Context, const PCHContainerReader &PCHContainerRdr,
+  ASTReader(Preprocessor &PP, ModuleCache &ModCache, ASTContext *Context,
+            const PCHContainerReader &PCHContainerRdr,
             ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions,
             StringRef isysroot = "",
             DisableValidationForModuleKind DisableValidationKind =
@@ -1853,8 +1853,7 @@ class ASTReader
   ///
   /// \returns true if an error occurred, false otherwise.
   static bool readASTFileControlBlock(
-      StringRef Filename, FileManager &FileMgr,
-      const InMemoryModuleCache &ModuleCache,
+      StringRef Filename, FileManager &FileMgr, const ModuleCache &ModCache,
       const PCHContainerReader &PCHContainerRdr, bool FindModuleFileExtensions,
       ASTReaderListener &Listener, bool ValidateDiagnosticOptions,
       unsigned ClientLoadCapabilities = ARR_ConfigurationMismatch |
@@ -1863,7 +1862,7 @@ class ASTReader
   /// Determine whether the given AST file is acceptable to load into a
   /// translation unit with the given language and target options.
   static bool isAcceptableASTFile(StringRef Filename, FileManager &FileMgr,
-                                  const InMemoryModuleCache &ModuleCache,
+                                  const ModuleCache &ModCache,
                                   const PCHContainerReader &PCHContainerRdr,
                                   const LangOptions &LangOpts,
                                   const TargetOptions &TargetOpts,

clang/include/clang/Serialization/ASTWriter.h

@@ -60,7 +60,7 @@ class LangOptions;
 class MacroDefinitionRecord;
 class MacroInfo;
 class Module;
-class InMemoryModuleCache;
+class ModuleCache;
 class ModuleFileExtension;
 class ModuleFileExtensionWriter;
 class NamedDecl;
@@ -117,7 +117,7 @@ class ASTWriter : public ASTDeserializationListener,
   const SmallVectorImpl<char> &Buffer;
 
   /// The PCM manager which manages memory buffers for pcm files.
-  InMemoryModuleCache &ModuleCache;
+  ModuleCache &ModCache;
 
   /// The preprocessor we're writing.
   Preprocessor *PP = nullptr;
@@ -638,7 +638,7 @@ class ASTWriter : public ASTDeserializationListener,
   /// Create a new precompiled header writer that outputs to
   /// the given bitstream.
   ASTWriter(llvm::BitstreamWriter &Stream, SmallVectorImpl<char> &Buffer,
-            InMemoryModuleCache &ModuleCache,
+            ModuleCache &ModCache,
             ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions,
             bool IncludeTimestamps = true, bool BuildingImplicitModule = false,
             bool GeneratingReducedBMI = false);
@@ -948,9 +948,8 @@ class PCHGenerator : public SemaConsumer {
   virtual Module *getEmittingModule(ASTContext &Ctx);
 
 public:
-  PCHGenerator(Preprocessor &PP, InMemoryModuleCache &ModuleCache,
-               StringRef OutputFile, StringRef isysroot,
-               std::shared_ptr<PCHBuffer> Buffer,
+  PCHGenerator(Preprocessor &PP, ModuleCache &ModCache, StringRef OutputFile,
+               StringRef isysroot, std::shared_ptr<PCHBuffer> Buffer,
                ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions,
                bool AllowASTWithErrors = false, bool IncludeTimestamps = true,
                bool BuildingImplicitModule = false,
@@ -972,13 +971,13 @@ class CXX20ModulesGenerator : public PCHGenerator {
 protected:
   virtual Module *getEmittingModule(ASTContext &Ctx) override;
 
-  CXX20ModulesGenerator(Preprocessor &PP, InMemoryModuleCache &ModuleCache,
+  CXX20ModulesGenerator(Preprocessor &PP, ModuleCache &ModCache,
                         StringRef OutputFile, bool GeneratingReducedBMI);
 
 public:
-  CXX20ModulesGenerator(Preprocessor &PP, InMemoryModuleCache &ModuleCache,
+  CXX20ModulesGenerator(Preprocessor &PP, ModuleCache &ModCache,
                         StringRef OutputFile)
-      : CXX20ModulesGenerator(PP, ModuleCache, OutputFile,
+      : CXX20ModulesGenerator(PP, ModCache, OutputFile,
                               /*GeneratingReducedBMI=*/false) {}
 
   void HandleTranslationUnit(ASTContext &Ctx) override;
@@ -988,9 +987,9 @@ class ReducedBMIGenerator : public CXX20ModulesGenerator {
   void anchor() override;
 
 public:
-  ReducedBMIGenerator(Preprocessor &PP, InMemoryModuleCache &ModuleCache,
+  ReducedBMIGenerator(Preprocessor &PP, ModuleCache &ModCache,
                       StringRef OutputFile)
-      : CXX20ModulesGenerator(PP, ModuleCache, OutputFile,
+      : CXX20ModulesGenerator(PP, ModCache, OutputFile,
                               /*GeneratingReducedBMI=*/true) {}
 };
 

clang/include/clang/Serialization/ModuleCache.h

@@ -0,0 +1,50 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_SERIALIZATION_MODULECACHE_H
+#define LLVM_CLANG_SERIALIZATION_MODULECACHE_H
+
+#include "clang/Basic/LLVM.h"
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+
+namespace llvm {
+class AdvisoryLock;
+} // namespace llvm
+
+namespace clang {
+class InMemoryModuleCache;
+
+/// The module cache used for compiling modules implicitly. This centralizes the
+/// operations the compiler might want to perform on the cache.
+class ModuleCache : public RefCountedBase<ModuleCache> {
+public:
+  /// May perform any work that only needs to be performed once for multiple
+  /// calls \c getLock() with the same module filename.
+  virtual void prepareForGetLock(StringRef ModuleFilename) = 0;
+
+  /// Returns lock for the given module file. The lock is initially unlocked.
+  virtual std::unique_ptr<llvm::AdvisoryLock>
+  getLock(StringRef ModuleFilename) = 0;
+
+  /// Returns this process's view of the module cache.
+  virtual InMemoryModuleCache &getInMemoryModuleCache() = 0;
+  virtual const InMemoryModuleCache &getInMemoryModuleCache() const = 0;
+
+  // TODO: Virtualize writing/reading PCM files, timestamping, pruning, etc.
+
+  virtual ~ModuleCache() = default;
+};
+
+/// Creates new \c ModuleCache backed by a file system directory that may be
+/// operated on by multiple processes. This instance must be used across all
+/// \c CompilerInstance instances participating in building modules for single
+/// translation unit in order to share the same \c InMemoryModuleCache.
+IntrusiveRefCntPtr<ModuleCache> createCrossProcessModuleCache();
+} // namespace clang
+
+#endif

clang/include/clang/Serialization/ModuleManager.h

@@ -37,7 +37,7 @@ class FileEntry;
 class FileManager;
 class GlobalModuleIndex;
 class HeaderSearch;
-class InMemoryModuleCache;
+class ModuleCache;
 class PCHContainerReader;
 
 namespace serialization {
@@ -65,7 +65,7 @@ class ModuleManager {
   FileManager &FileMgr;
 
   /// Cache of PCM files.
-  IntrusiveRefCntPtr<InMemoryModuleCache> ModuleCache;
+  IntrusiveRefCntPtr<ModuleCache> ModCache;
 
   /// Knows how to unwrap module containers.
   const PCHContainerReader &PCHContainerRdr;
@@ -133,9 +133,9 @@ class ModuleManager {
       SmallVectorImpl<std::unique_ptr<ModuleFile>>::reverse_iterator>;
   using ModuleOffset = std::pair<uint32_t, StringRef>;
 
-  explicit ModuleManager(FileManager &FileMgr, InMemoryModuleCache &ModuleCache,
-                         const PCHContainerReader &PCHContainerRdr,
-                         const HeaderSearch &HeaderSearchInfo);
+  ModuleManager(FileManager &FileMgr, ModuleCache &ModCache,
+                const PCHContainerReader &PCHContainerRdr,
+                const HeaderSearch &HeaderSearchInfo);
 
   /// Forward iterator to traverse all loaded modules.
   ModuleIterator begin() { return Chain.begin(); }
@@ -306,7 +306,7 @@ class ModuleManager {
   /// View the graphviz representation of the module graph.
   void viewGraph();
 
-  InMemoryModuleCache &getModuleCache() const { return *ModuleCache; }
+  ModuleCache &getModuleCache() const { return *ModCache; }
 };
 
 } // namespace serialization

clang/lib/Frontend/ASTUnit.cpp

@@ -61,7 +61,7 @@
 #include "clang/Serialization/ASTReader.h"
 #include "clang/Serialization/ASTWriter.h"
 #include "clang/Serialization/ContinuousRangeMap.h"
-#include "clang/Serialization/InMemoryModuleCache.h"
+#include "clang/Serialization/ModuleCache.h"
 #include "clang/Serialization/ModuleFile.h"
 #include "clang/Serialization/PCHContainerOperations.h"
 #include "llvm/ADT/ArrayRef.h"
@@ -219,8 +219,8 @@ struct ASTUnit::ASTWriterData {
   llvm::BitstreamWriter Stream;
   ASTWriter Writer;
 
-  ASTWriterData(InMemoryModuleCache &ModuleCache)
-      : Stream(Buffer), Writer(Stream, Buffer, ModuleCache, {}) {}
+  ASTWriterData(ModuleCache &ModCache)
+      : Stream(Buffer), Writer(Stream, Buffer, ModCache, {}) {}
 };
 
 void ASTUnit::clearFileLevelDecls() {
@@ -825,7 +825,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromASTFile(
   AST->SourceMgr = new SourceManager(AST->getDiagnostics(),
                                      AST->getFileManager(),
                                      UserFilesAreVolatile);
-  AST->ModuleCache = new InMemoryModuleCache;
+  AST->ModCache = createCrossProcessModuleCache();
   AST->HSOpts = HSOpts ? HSOpts : std::make_shared<HeaderSearchOptions>();
   AST->HSOpts->ModuleFormat = std::string(PCHContainerRdr.getFormats().front());
   AST->HeaderInfo.reset(new HeaderSearch(AST->HSOpts,
@@ -857,8 +857,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromASTFile(
   if (::getenv("LIBCLANG_DISABLE_PCH_VALIDATION"))
     disableValid = DisableValidationForModuleKind::All;
   AST->Reader = new ASTReader(
-      PP, *AST->ModuleCache, AST->Ctx.get(), PCHContainerRdr, {},
-      /*isysroot=*/"",
+      PP, *AST->ModCache, AST->Ctx.get(), PCHContainerRdr, {}, /*isysroot=*/"",
       /*DisableValidationKind=*/disableValid, AllowASTWithCompilerErrors);
 
   unsigned Counter = 0;
@@ -1542,7 +1541,7 @@ ASTUnit::create(std::shared_ptr<CompilerInvocation> CI,
   AST->UserFilesAreVolatile = UserFilesAreVolatile;
   AST->SourceMgr = new SourceManager(AST->getDiagnostics(), *AST->FileMgr,
                                      UserFilesAreVolatile);
-  AST->ModuleCache = new InMemoryModuleCache;
+  AST->ModCache = createCrossProcessModuleCache();
 
   return AST;
 }
@@ -1829,7 +1828,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromCommandLine(
   AST->FileMgr = new FileManager(AST->FileSystemOpts, VFS);
   AST->StorePreamblesInMemory = StorePreamblesInMemory;
   AST->PreambleStoragePath = PreambleStoragePath;
-  AST->ModuleCache = new InMemoryModuleCache;
+  AST->ModCache = createCrossProcessModuleCache();
   AST->OnlyLocalDecls = OnlyLocalDecls;
   AST->CaptureDiagnostics = CaptureDiagnostics;
   AST->TUKind = TUKind;
@@ -1840,7 +1839,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromCommandLine(
   AST->Invocation = CI;
   AST->SkipFunctionBodies = SkipFunctionBodies;
   if (ForSerialization)
-    AST->WriterData.reset(new ASTWriterData(*AST->ModuleCache));
+    AST->WriterData.reset(new ASTWriterData(*AST->ModCache));
   // Zero out now to ease cleanup during crash recovery.
   CI = nullptr;
   Diags = nullptr;
@@ -2373,8 +2372,8 @@ bool ASTUnit::serialize(raw_ostream &OS) {
 
   SmallString<128> Buffer;
   llvm::BitstreamWriter Stream(Buffer);
-  InMemoryModuleCache ModuleCache;
-  ASTWriter Writer(Stream, Buffer, ModuleCache, {});
+  IntrusiveRefCntPtr<ModuleCache> ModCache = createCrossProcessModuleCache();
+  ASTWriter Writer(Stream, Buffer, *ModCache, {});
   return serializeUnit(Writer, Buffer, getSema(), OS);
 }