Skip to content

Commit 8f6ed3a

Browse files
jansvoboda11jansvoboda11
committed
[clang][modules] Virtualize module cache pruning (llvm#149113)
This PR virtualizes module cache pruning via the new `ModuleCache` interface. Currently this is an NFC, but I left a FIXME in `InProcessModuleCache` to make this more efficient for the dependency scanner.
1 parent 18e6773 commit 8f6ed3a

File tree

4 files changed

+105
-90
lines changed

4 files changed

+105
-90
lines changed

‎clang/include/clang/Serialization/ModuleCache.h‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,11 +45,15 @@ class ModuleCache : public RefCountedBase<ModuleCache> {
4545
/// were validated.
4646
virtual void updateModuleTimestamp(StringRef ModuleFilename) = 0;
4747

48+
/// Prune module files that haven't been accessed in a long time.
49+
virtual void maybePrune(StringRef Path, time_t PruneInterval,
50+
time_t PruneAfter) = 0;
51+
4852
/// Returns this process's view of the module cache.
4953
virtual InMemoryModuleCache &getInMemoryModuleCache() = 0;
5054
virtual const InMemoryModuleCache &getInMemoryModuleCache() const = 0;
5155

52-
// TODO: Virtualize writing/reading PCM files, pruning, etc.
56+
// TODO: Virtualize writing/reading PCM files, etc.
5357

5458
virtual ~ModuleCache() = default;
5559
};
@@ -59,6 +63,9 @@ class ModuleCache : public RefCountedBase<ModuleCache> {
5963
/// \c CompilerInstance instances participating in building modules for single
6064
/// translation unit in order to share the same \c InMemoryModuleCache.
6165
IntrusiveRefCntPtr<ModuleCache> createCrossProcessModuleCache();
66+
67+
/// Shared implementation of `ModuleCache::maybePrune()`.
68+
void maybePruneImpl(StringRef Path, time_t PruneInterval, time_t PruneAfter);
6269
} // namespace clang
6370

6471
#endif

‎clang/lib/Frontend/CompilerInstance.cpp‎

Lines changed: 4 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -1848,90 +1848,6 @@ static void checkConfigMacros(Preprocessor &PP, Module *M,
18481848
}
18491849
}
18501850

1851-
/// Write a new timestamp file with the given path.
1852-
static void writeTimestampFile(StringRef TimestampFile) {
1853-
std::error_code EC;
1854-
llvm::raw_fd_ostream Out(TimestampFile.str(), EC, llvm::sys::fs::OF_None);
1855-
}
1856-
1857-
/// Prune the module cache of modules that haven't been accessed in
1858-
/// a long time.
1859-
static void pruneModuleCache(const HeaderSearchOptions &HSOpts) {
1860-
llvm::sys::fs::file_status StatBuf;
1861-
llvm::SmallString<128> TimestampFile;
1862-
TimestampFile = HSOpts.ModuleCachePath;
1863-
assert(!TimestampFile.empty());
1864-
llvm::sys::path::append(TimestampFile, "modules.timestamp");
1865-
1866-
// Try to stat() the timestamp file.
1867-
if (std::error_code EC = llvm::sys::fs::status(TimestampFile, StatBuf)) {
1868-
// If the timestamp file wasn't there, create one now.
1869-
if (EC == std::errc::no_such_file_or_directory) {
1870-
writeTimestampFile(TimestampFile);
1871-
}
1872-
return;
1873-
}
1874-
1875-
// Check whether the time stamp is older than our pruning interval.
1876-
// If not, do nothing.
1877-
time_t TimeStampModTime =
1878-
llvm::sys::toTimeT(StatBuf.getLastModificationTime());
1879-
time_t CurrentTime = time(nullptr);
1880-
if (CurrentTime - TimeStampModTime <= time_t(HSOpts.ModuleCachePruneInterval))
1881-
return;
1882-
1883-
// Write a new timestamp file so that nobody else attempts to prune.
1884-
// There is a benign race condition here, if two Clang instances happen to
1885-
// notice at the same time that the timestamp is out-of-date.
1886-
writeTimestampFile(TimestampFile);
1887-
1888-
// Walk the entire module cache, looking for unused module files and module
1889-
// indices.
1890-
std::error_code EC;
1891-
for (llvm::sys::fs::directory_iterator Dir(HSOpts.ModuleCachePath, EC),
1892-
DirEnd;
1893-
Dir != DirEnd && !EC; Dir.increment(EC)) {
1894-
// If we don't have a directory, there's nothing to look into.
1895-
if (!llvm::sys::fs::is_directory(Dir->path()))
1896-
continue;
1897-
1898-
// Walk all of the files within this directory.
1899-
for (llvm::sys::fs::directory_iterator File(Dir->path(), EC), FileEnd;
1900-
File != FileEnd && !EC; File.increment(EC)) {
1901-
// We only care about module and global module index files.
1902-
StringRef Extension = llvm::sys::path::extension(File->path());
1903-
if (Extension != ".pcm" && Extension != ".timestamp" &&
1904-
llvm::sys::path::filename(File->path()) != "modules.idx")
1905-
continue;
1906-
1907-
// Look at this file. If we can't stat it, there's nothing interesting
1908-
// there.
1909-
if (llvm::sys::fs::status(File->path(), StatBuf))
1910-
continue;
1911-
1912-
// If the file has been used recently enough, leave it there.
1913-
time_t FileAccessTime = llvm::sys::toTimeT(StatBuf.getLastAccessedTime());
1914-
if (CurrentTime - FileAccessTime <=
1915-
time_t(HSOpts.ModuleCachePruneAfter)) {
1916-
continue;
1917-
}
1918-
1919-
// Remove the file.
1920-
llvm::sys::fs::remove(File->path());
1921-
1922-
// Remove the timestamp file.
1923-
std::string TimpestampFilename = File->path() + ".timestamp";
1924-
llvm::sys::fs::remove(TimpestampFilename);
1925-
}
1926-
1927-
// If we removed all of the files in the directory, remove the directory
1928-
// itself.
1929-
if (llvm::sys::fs::directory_iterator(Dir->path(), EC) ==
1930-
llvm::sys::fs::directory_iterator() && !EC)
1931-
llvm::sys::fs::remove(Dir->path());
1932-
}
1933-
}
1934-
19351851
void CompilerInstance::createASTReader() {
19361852
if (TheASTReader)
19371853
return;
@@ -1942,11 +1858,10 @@ void CompilerInstance::createASTReader() {
19421858
// If we're implicitly building modules but not currently recursively
19431859
// building a module, check whether we need to prune the module cache.
19441860
if (getSourceManager().getModuleBuildStack().empty() &&
1945-
!getPreprocessor().getHeaderSearchInfo().getModuleCachePath().empty() &&
1946-
getHeaderSearchOpts().ModuleCachePruneInterval > 0 &&
1947-
getHeaderSearchOpts().ModuleCachePruneAfter > 0) {
1948-
pruneModuleCache(getHeaderSearchOpts());
1949-
}
1861+
!getPreprocessor().getHeaderSearchInfo().getModuleCachePath().empty())
1862+
ModCache->maybePrune(getHeaderSearchOpts().ModuleCachePath,
1863+
getHeaderSearchOpts().ModuleCachePruneInterval,
1864+
getHeaderSearchOpts().ModuleCachePruneAfter);
19501865

19511866
HeaderSearchOptions &HSOpts = getHeaderSearchOpts();
19521867
std::string Sysroot = HSOpts.Sysroot;

‎clang/lib/Serialization/ModuleCache.cpp‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,87 @@
1616

1717
using namespace clang;
1818

19+
/// Write a new timestamp file with the given path.
20+
static void writeTimestampFile(StringRef TimestampFile) {
21+
std::error_code EC;
22+
llvm::raw_fd_ostream Out(TimestampFile.str(), EC, llvm::sys::fs::OF_None);
23+
}
24+
25+
void clang::maybePruneImpl(StringRef Path, time_t PruneInterval,
26+
time_t PruneAfter) {
27+
if (PruneInterval <= 0 || PruneAfter <= 0)
28+
return;
29+
30+
llvm::SmallString<128> TimestampFile(Path);
31+
llvm::sys::path::append(TimestampFile, "modules.timestamp");
32+
33+
// Try to stat() the timestamp file.
34+
llvm::sys::fs::file_status StatBuf;
35+
if (std::error_code EC = llvm::sys::fs::status(TimestampFile, StatBuf)) {
36+
// If the timestamp file wasn't there, create one now.
37+
if (EC == std::errc::no_such_file_or_directory)
38+
writeTimestampFile(TimestampFile);
39+
return;
40+
}
41+
42+
// Check whether the time stamp is older than our pruning interval.
43+
// If not, do nothing.
44+
time_t TimestampModTime =
45+
llvm::sys::toTimeT(StatBuf.getLastModificationTime());
46+
time_t CurrentTime = time(nullptr);
47+
if (CurrentTime - TimestampModTime <= PruneInterval)
48+
return;
49+
50+
// Write a new timestamp file so that nobody else attempts to prune.
51+
// There is a benign race condition here, if two Clang instances happen to
52+
// notice at the same time that the timestamp is out-of-date.
53+
writeTimestampFile(TimestampFile);
54+
55+
// Walk the entire module cache, looking for unused module files and module
56+
// indices.
57+
std::error_code EC;
58+
for (llvm::sys::fs::directory_iterator Dir(Path, EC), DirEnd;
59+
Dir != DirEnd && !EC; Dir.increment(EC)) {
60+
// If we don't have a directory, there's nothing to look into.
61+
if (!llvm::sys::fs::is_directory(Dir->path()))
62+
continue;
63+
64+
// Walk all the files within this directory.
65+
for (llvm::sys::fs::directory_iterator File(Dir->path(), EC), FileEnd;
66+
File != FileEnd && !EC; File.increment(EC)) {
67+
// We only care about module and global module index files.
68+
StringRef Extension = llvm::sys::path::extension(File->path());
69+
if (Extension != ".pcm" && Extension != ".timestamp" &&
70+
llvm::sys::path::filename(File->path()) != "modules.idx")
71+
continue;
72+
73+
// Look at this file. If we can't stat it, there's nothing interesting
74+
// there.
75+
if (llvm::sys::fs::status(File->path(), StatBuf))
76+
continue;
77+
78+
// If the file has been used recently enough, leave it there.
79+
time_t FileAccessTime = llvm::sys::toTimeT(StatBuf.getLastAccessedTime());
80+
if (CurrentTime - FileAccessTime <= PruneAfter)
81+
continue;
82+
83+
// Remove the file.
84+
llvm::sys::fs::remove(File->path());
85+
86+
// Remove the timestamp file.
87+
std::string TimpestampFilename = File->path() + ".timestamp";
88+
llvm::sys::fs::remove(TimpestampFilename);
89+
}
90+
91+
// If we removed all the files in the directory, remove the directory
92+
// itself.
93+
if (llvm::sys::fs::directory_iterator(Dir->path(), EC) ==
94+
llvm::sys::fs::directory_iterator() &&
95+
!EC)
96+
llvm::sys::fs::remove(Dir->path());
97+
}
98+
}
99+
19100
namespace {
20101
class CrossProcessModuleCache : public ModuleCache {
21102
InMemoryModuleCache InMemory;
@@ -55,6 +136,11 @@ class CrossProcessModuleCache : public ModuleCache {
55136
OS.clear_error(); // Avoid triggering a fatal error.
56137
}
57138

139+
void maybePrune(StringRef Path, time_t PruneInterval,
140+
time_t PruneAfter) override {
141+
maybePruneImpl(Path, PruneInterval, PruneAfter);
142+
}
143+
58144
InMemoryModuleCache &getInMemoryModuleCache() override { return InMemory; }
59145
const InMemoryModuleCache &getInMemoryModuleCache() const override {
60146
return InMemory;

‎clang/lib/Tooling/DependencyScanning/InProcessModuleCache.cpp‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,13 @@ class InProcessModuleCache : public ModuleCache {
100100
Timestamp.store(llvm::sys::toTimeT(std::chrono::system_clock::now()));
101101
}
102102

103+
void maybePrune(StringRef Path, time_t PruneInterval,
104+
time_t PruneAfter) override {
105+
// FIXME: This only needs to be ran once per build, not in every
106+
// compilation. Call it once per service.
107+
maybePruneImpl(Path, PruneInterval, PruneAfter);
108+
}
109+
103110
InMemoryModuleCache &getInMemoryModuleCache() override { return InMemory; }
104111
const InMemoryModuleCache &getInMemoryModuleCache() const override {
105112
return InMemory;

0 commit comments

Comments
 (0)