Skip to content

Commit 42e6d99

Browse files
jansvoboda11jansvoboda11
authored
Merge pull request swiftlang#10286 from swiftlang/jan_svoboda/stable-no-file-locks
🍒 [clang][deps] Implement efficient in-process `ModuleCache`
2 parents 305d823 + dcdca84 commit 42e6d99

28 files changed

+554
-330
lines changed

‎clang/include/clang/Frontend/ASTUnit.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ class FileManager;
7070
class FrontendAction;
7171
class HeaderSearch;
7272
class InputKind;
73-
class InMemoryModuleCache;
73+
class ModuleCache;
7474
class PCHContainerOperations;
7575
class PCHContainerReader;
7676
class Preprocessor;
@@ -110,7 +110,7 @@ class ASTUnit {
110110
IntrusiveRefCntPtr<DiagnosticsEngine> Diagnostics;
111111
IntrusiveRefCntPtr<FileManager> FileMgr;
112112
IntrusiveRefCntPtr<SourceManager> SourceMgr;
113-
IntrusiveRefCntPtr<InMemoryModuleCache> ModuleCache;
113+
IntrusiveRefCntPtr<ModuleCache> ModCache;
114114
std::unique_ptr<HeaderSearch> HeaderInfo;
115115
IntrusiveRefCntPtr<TargetInfo> Target;
116116
std::shared_ptr<Preprocessor> PP;

‎clang/include/clang/Frontend/CompilerInstance.h‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,8 @@ class DiagnosticsEngine;
5454
class DiagnosticConsumer;
5555
class FileManager;
5656
class FrontendAction;
57-
class InMemoryModuleCache;
5857
class Module;
58+
class ModuleCache;
5959
class Preprocessor;
6060
class Sema;
6161
class SourceManager;
@@ -115,7 +115,7 @@ class CompilerInstance : public ModuleLoader {
115115
IntrusiveRefCntPtr<SourceManager> SourceMgr;
116116

117117
/// The cache of PCM files.
118-
IntrusiveRefCntPtr<InMemoryModuleCache> ModuleCache;
118+
IntrusiveRefCntPtr<ModuleCache> ModCache;
119119

120120
/// The preprocessor.
121121
std::shared_ptr<Preprocessor> PP;
@@ -223,7 +223,7 @@ class CompilerInstance : public ModuleLoader {
223223
explicit CompilerInstance(
224224
std::shared_ptr<PCHContainerOperations> PCHContainerOps =
225225
std::make_shared<PCHContainerOperations>(),
226-
InMemoryModuleCache *SharedModuleCache = nullptr);
226+
ModuleCache *ModCache = nullptr);
227227
~CompilerInstance() override;
228228

229229
/// @name High-Level Operations
@@ -793,9 +793,8 @@ class CompilerInstance : public ModuleLoader {
793793
static IntrusiveRefCntPtr<ASTReader> createPCHExternalASTSource(
794794
StringRef Path, StringRef Sysroot,
795795
DisableValidationForModuleKind DisableValidation,
796-
bool AllowPCHWithCompilerErrors, Preprocessor &PP,
797-
InMemoryModuleCache &ModuleCache, ASTContext &Context,
798-
const PCHContainerReader &PCHContainerRdr,
796+
bool AllowPCHWithCompilerErrors, Preprocessor &PP, ModuleCache &ModCache,
797+
ASTContext &Context, const PCHContainerReader &PCHContainerRdr,
799798
ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions,
800799
ArrayRef<std::shared_ptr<DependencyCollector>> DependencyCollectors,
801800
void *DeserializationListener, bool OwnDeserializationListener,
@@ -966,7 +965,7 @@ class CompilerInstance : public ModuleLoader {
966965
bool addCachedModuleFile(StringRef Path, StringRef CacheKey,
967966
StringRef Provider);
968967

969-
InMemoryModuleCache &getModuleCache() const { return *ModuleCache; }
968+
ModuleCache &getModuleCache() const { return *ModCache; }
970969
};
971970

972971
} // end namespace clang

‎clang/include/clang/Serialization/ASTReader.h‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ struct HeaderFileInfo;
8888
class HeaderSearchOptions;
8989
class LangOptions;
9090
class MacroInfo;
91-
class InMemoryModuleCache;
91+
class ModuleCache;
9292
class NamedDecl;
9393
class NamespaceDecl;
9494
class ObjCCategoryDecl;
@@ -1641,8 +1641,8 @@ class ASTReader
16411641
///
16421642
/// \param ReadTimer If non-null, a timer used to track the time spent
16431643
/// deserializing.
1644-
ASTReader(Preprocessor &PP, InMemoryModuleCache &ModuleCache,
1645-
ASTContext *Context, const PCHContainerReader &PCHContainerRdr,
1644+
ASTReader(Preprocessor &PP, ModuleCache &ModCache, ASTContext *Context,
1645+
const PCHContainerReader &PCHContainerRdr,
16461646
ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions,
16471647
StringRef isysroot = "",
16481648
DisableValidationForModuleKind DisableValidationKind =
@@ -1853,8 +1853,7 @@ class ASTReader
18531853
///
18541854
/// \returns true if an error occurred, false otherwise.
18551855
static bool readASTFileControlBlock(
1856-
StringRef Filename, FileManager &FileMgr,
1857-
const InMemoryModuleCache &ModuleCache,
1856+
StringRef Filename, FileManager &FileMgr, const ModuleCache &ModCache,
18581857
const PCHContainerReader &PCHContainerRdr, bool FindModuleFileExtensions,
18591858
ASTReaderListener &Listener, bool ValidateDiagnosticOptions,
18601859
unsigned ClientLoadCapabilities = ARR_ConfigurationMismatch |
@@ -1863,7 +1862,7 @@ class ASTReader
18631862
/// Determine whether the given AST file is acceptable to load into a
18641863
/// translation unit with the given language and target options.
18651864
static bool isAcceptableASTFile(StringRef Filename, FileManager &FileMgr,
1866-
const InMemoryModuleCache &ModuleCache,
1865+
const ModuleCache &ModCache,
18671866
const PCHContainerReader &PCHContainerRdr,
18681867
const LangOptions &LangOpts,
18691868
const TargetOptions &TargetOpts,

‎clang/include/clang/Serialization/ASTWriter.h‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ class LangOptions;
6060
class MacroDefinitionRecord;
6161
class MacroInfo;
6262
class Module;
63-
class InMemoryModuleCache;
63+
class ModuleCache;
6464
class ModuleFileExtension;
6565
class ModuleFileExtensionWriter;
6666
class NamedDecl;
@@ -117,7 +117,7 @@ class ASTWriter : public ASTDeserializationListener,
117117
const SmallVectorImpl<char> &Buffer;
118118

119119
/// The PCM manager which manages memory buffers for pcm files.
120-
InMemoryModuleCache &ModuleCache;
120+
ModuleCache &ModCache;
121121

122122
/// The preprocessor we're writing.
123123
Preprocessor *PP = nullptr;
@@ -638,7 +638,7 @@ class ASTWriter : public ASTDeserializationListener,
638638
/// Create a new precompiled header writer that outputs to
639639
/// the given bitstream.
640640
ASTWriter(llvm::BitstreamWriter &Stream, SmallVectorImpl<char> &Buffer,
641-
InMemoryModuleCache &ModuleCache,
641+
ModuleCache &ModCache,
642642
ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions,
643643
bool IncludeTimestamps = true, bool BuildingImplicitModule = false,
644644
bool GeneratingReducedBMI = false);
@@ -948,9 +948,8 @@ class PCHGenerator : public SemaConsumer {
948948
virtual Module *getEmittingModule(ASTContext &Ctx);
949949

950950
public:
951-
PCHGenerator(Preprocessor &PP, InMemoryModuleCache &ModuleCache,
952-
StringRef OutputFile, StringRef isysroot,
953-
std::shared_ptr<PCHBuffer> Buffer,
951+
PCHGenerator(Preprocessor &PP, ModuleCache &ModCache, StringRef OutputFile,
952+
StringRef isysroot, std::shared_ptr<PCHBuffer> Buffer,
954953
ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions,
955954
bool AllowASTWithErrors = false, bool IncludeTimestamps = true,
956955
bool BuildingImplicitModule = false,
@@ -972,13 +971,13 @@ class CXX20ModulesGenerator : public PCHGenerator {
972971
protected:
973972
virtual Module *getEmittingModule(ASTContext &Ctx) override;
974973

975-
CXX20ModulesGenerator(Preprocessor &PP, InMemoryModuleCache &ModuleCache,
974+
CXX20ModulesGenerator(Preprocessor &PP, ModuleCache &ModCache,
976975
StringRef OutputFile, bool GeneratingReducedBMI);
977976

978977
public:
979-
CXX20ModulesGenerator(Preprocessor &PP, InMemoryModuleCache &ModuleCache,
978+
CXX20ModulesGenerator(Preprocessor &PP, ModuleCache &ModCache,
980979
StringRef OutputFile)
981-
: CXX20ModulesGenerator(PP, ModuleCache, OutputFile,
980+
: CXX20ModulesGenerator(PP, ModCache, OutputFile,
982981
/*GeneratingReducedBMI=*/false) {}
983982

984983
void HandleTranslationUnit(ASTContext &Ctx) override;
@@ -988,9 +987,9 @@ class ReducedBMIGenerator : public CXX20ModulesGenerator {
988987
void anchor() override;
989988

990989
public:
991-
ReducedBMIGenerator(Preprocessor &PP, InMemoryModuleCache &ModuleCache,
990+
ReducedBMIGenerator(Preprocessor &PP, ModuleCache &ModCache,
992991
StringRef OutputFile)
993-
: CXX20ModulesGenerator(PP, ModuleCache, OutputFile,
992+
: CXX20ModulesGenerator(PP, ModCache, OutputFile,
994993
/*GeneratingReducedBMI=*/true) {}
995994
};
996995

‎clang/include/clang/Serialization/ModuleCache.h‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#ifndef LLVM_CLANG_SERIALIZATION_MODULECACHE_H
10+
#define LLVM_CLANG_SERIALIZATION_MODULECACHE_H
11+
12+
#include "clang/Basic/LLVM.h"
13+
#include "llvm/ADT/IntrusiveRefCntPtr.h"
14+
15+
namespace llvm {
16+
class AdvisoryLock;
17+
} // namespace llvm
18+
19+
namespace clang {
20+
class InMemoryModuleCache;
21+
22+
/// The module cache used for compiling modules implicitly. This centralizes the
23+
/// operations the compiler might want to perform on the cache.
24+
class ModuleCache : public RefCountedBase<ModuleCache> {
25+
public:
26+
/// May perform any work that only needs to be performed once for multiple
27+
/// calls \c getLock() with the same module filename.
28+
virtual void prepareForGetLock(StringRef ModuleFilename) = 0;
29+
30+
/// Returns lock for the given module file. The lock is initially unlocked.
31+
virtual std::unique_ptr<llvm::AdvisoryLock>
32+
getLock(StringRef ModuleFilename) = 0;
33+
34+
/// Returns this process's view of the module cache.
35+
virtual InMemoryModuleCache &getInMemoryModuleCache() = 0;
36+
virtual const InMemoryModuleCache &getInMemoryModuleCache() const = 0;
37+
38+
// TODO: Virtualize writing/reading PCM files, timestamping, pruning, etc.
39+
40+
virtual ~ModuleCache() = default;
41+
};
42+
43+
/// Creates new \c ModuleCache backed by a file system directory that may be
44+
/// operated on by multiple processes. This instance must be used across all
45+
/// \c CompilerInstance instances participating in building modules for single
46+
/// translation unit in order to share the same \c InMemoryModuleCache.
47+
IntrusiveRefCntPtr<ModuleCache> createCrossProcessModuleCache();
48+
} // namespace clang
49+
50+
#endif

‎clang/include/clang/Serialization/ModuleManager.h‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ class FileEntry;
3737
class FileManager;
3838
class GlobalModuleIndex;
3939
class HeaderSearch;
40-
class InMemoryModuleCache;
40+
class ModuleCache;
4141
class PCHContainerReader;
4242

4343
namespace serialization {
@@ -65,7 +65,7 @@ class ModuleManager {
6565
FileManager &FileMgr;
6666

6767
/// Cache of PCM files.
68-
IntrusiveRefCntPtr<InMemoryModuleCache> ModuleCache;
68+
IntrusiveRefCntPtr<ModuleCache> ModCache;
6969

7070
/// Knows how to unwrap module containers.
7171
const PCHContainerReader &PCHContainerRdr;
@@ -133,9 +133,9 @@ class ModuleManager {
133133
SmallVectorImpl<std::unique_ptr<ModuleFile>>::reverse_iterator>;
134134
using ModuleOffset = std::pair<uint32_t, StringRef>;
135135

136-
explicit ModuleManager(FileManager &FileMgr, InMemoryModuleCache &ModuleCache,
137-
const PCHContainerReader &PCHContainerRdr,
138-
const HeaderSearch &HeaderSearchInfo);
136+
ModuleManager(FileManager &FileMgr, ModuleCache &ModCache,
137+
const PCHContainerReader &PCHContainerRdr,
138+
const HeaderSearch &HeaderSearchInfo);
139139

140140
/// Forward iterator to traverse all loaded modules.
141141
ModuleIterator begin() { return Chain.begin(); }
@@ -306,7 +306,7 @@ class ModuleManager {
306306
/// View the graphviz representation of the module graph.
307307
void viewGraph();
308308

309-
InMemoryModuleCache &getModuleCache() const { return *ModuleCache; }
309+
ModuleCache &getModuleCache() const { return *ModCache; }
310310
};
311311

312312
} // namespace serialization

‎clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
#include "clang/CAS/CASOptions.h"
1313
#include "clang/Tooling/DependencyScanning/DependencyScanningCASFilesystem.h"
1414
#include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h"
15+
#include "clang/Tooling/DependencyScanning/InProcessModuleCache.h"
1516
#include "llvm/ADT/BitmaskEnum.h"
1617
#include "llvm/CAS/ActionCache.h"
1718

@@ -130,6 +131,8 @@ class DependencyScanningService {
130131

131132
bool useCASFS() const { return (bool)SharedFS; }
132133

134+
ModuleCacheMutexes &getModuleCacheMutexes() { return ModCacheMutexes; }
135+
133136
private:
134137
const ScanningMode Mode;
135138
const ScanningOutputFormat Format;
@@ -147,6 +150,8 @@ class DependencyScanningService {
147150
IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> SharedFS;
148151
/// The global file system cache.
149152
std::optional<DependencyScanningFilesystemSharedCache> SharedCache;
153+
/// The global module cache mutexes.
154+
ModuleCacheMutexes ModCacheMutexes;
150155
};
151156

152157
} // end namespace dependencies

‎clang/include/clang/Tooling/DependencyScanning/InProcessModuleCache.h‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#ifndef LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_INPROCESSMODULECACHE_H
10+
#define LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_INPROCESSMODULECACHE_H
11+
12+
#include "clang/Serialization/ModuleCache.h"
13+
#include "llvm/ADT/StringMap.h"
14+
15+
#include <mutex>
16+
#include <shared_mutex>
17+
18+
namespace clang {
19+
namespace tooling {
20+
namespace dependencies {
21+
struct ModuleCacheMutexes {
22+
std::mutex Mutex;
23+
llvm::StringMap<std::unique_ptr<std::shared_mutex>> Map;
24+
};
25+
26+
IntrusiveRefCntPtr<ModuleCache>
27+
makeInProcessModuleCache(ModuleCacheMutexes &Mutexes);
28+
} // namespace dependencies
29+
} // namespace tooling
30+
} // namespace clang
31+
32+
#endif

0 commit comments

Comments
 (0)