Skip to content

Commit b411e1a

Browse files
jansvoboda11jansvoboda11
committed
[clang][modules] Virtualize module cache pruning
1 parent 76058c0 commit b411e1a

File tree

4 files changed

+105
-90
lines changed

4 files changed

+105
-90
lines changed

‎clang/include/clang/Serialization/ModuleCache.h‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,11 +45,15 @@ class ModuleCache : public RefCountedBase<ModuleCache> {
4545
/// were validated.
4646
virtual void updateModuleTimestamp(StringRef ModuleFilename) = 0;
4747

48+
/// Prune module files that haven't been accessed in a long time.
49+
virtual void maybePrune(StringRef Path, time_t PruneInterval,
50+
time_t PruneAfter) = 0;
51+
4852
/// Returns this process's view of the module cache.
4953
virtual InMemoryModuleCache &getInMemoryModuleCache() = 0;
5054
virtual const InMemoryModuleCache &getInMemoryModuleCache() const = 0;
5155

52-
// TODO: Virtualize writing/reading PCM files, pruning, etc.
56+
// TODO: Virtualize writing/reading PCM files, etc.
5357

5458
virtual ~ModuleCache() = default;
5559
};
@@ -59,6 +63,9 @@ class ModuleCache : public RefCountedBase<ModuleCache> {
5963
/// \c CompilerInstance instances participating in building modules for single
6064
/// translation unit in order to share the same \c InMemoryModuleCache.
6165
IntrusiveRefCntPtr<ModuleCache> createCrossProcessModuleCache();
66+
67+
/// Shared implementation of `ModuleCache::maybePrune()`.
68+
void maybePruneImpl(StringRef Path, time_t PruneInterval, time_t PruneAfter);
6269
} // namespace clang
6370

6471
#endif

‎clang/lib/Frontend/CompilerInstance.cpp‎

Lines changed: 4 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -1636,90 +1636,6 @@ static void checkConfigMacros(Preprocessor &PP, Module *M,
16361636
}
16371637
}
16381638

1639-
/// Write a new timestamp file with the given path.
1640-
static void writeTimestampFile(StringRef TimestampFile) {
1641-
std::error_code EC;
1642-
llvm::raw_fd_ostream Out(TimestampFile.str(), EC, llvm::sys::fs::OF_None);
1643-
}
1644-
1645-
/// Prune the module cache of modules that haven't been accessed in
1646-
/// a long time.
1647-
static void pruneModuleCache(const HeaderSearchOptions &HSOpts) {
1648-
llvm::sys::fs::file_status StatBuf;
1649-
llvm::SmallString<128> TimestampFile;
1650-
TimestampFile = HSOpts.ModuleCachePath;
1651-
assert(!TimestampFile.empty());
1652-
llvm::sys::path::append(TimestampFile, "modules.timestamp");
1653-
1654-
// Try to stat() the timestamp file.
1655-
if (std::error_code EC = llvm::sys::fs::status(TimestampFile, StatBuf)) {
1656-
// If the timestamp file wasn't there, create one now.
1657-
if (EC == std::errc::no_such_file_or_directory) {
1658-
writeTimestampFile(TimestampFile);
1659-
}
1660-
return;
1661-
}
1662-
1663-
// Check whether the time stamp is older than our pruning interval.
1664-
// If not, do nothing.
1665-
time_t TimeStampModTime =
1666-
llvm::sys::toTimeT(StatBuf.getLastModificationTime());
1667-
time_t CurrentTime = time(nullptr);
1668-
if (CurrentTime - TimeStampModTime <= time_t(HSOpts.ModuleCachePruneInterval))
1669-
return;
1670-
1671-
// Write a new timestamp file so that nobody else attempts to prune.
1672-
// There is a benign race condition here, if two Clang instances happen to
1673-
// notice at the same time that the timestamp is out-of-date.
1674-
writeTimestampFile(TimestampFile);
1675-
1676-
// Walk the entire module cache, looking for unused module files and module
1677-
// indices.
1678-
std::error_code EC;
1679-
for (llvm::sys::fs::directory_iterator Dir(HSOpts.ModuleCachePath, EC),
1680-
DirEnd;
1681-
Dir != DirEnd && !EC; Dir.increment(EC)) {
1682-
// If we don't have a directory, there's nothing to look into.
1683-
if (!llvm::sys::fs::is_directory(Dir->path()))
1684-
continue;
1685-
1686-
// Walk all of the files within this directory.
1687-
for (llvm::sys::fs::directory_iterator File(Dir->path(), EC), FileEnd;
1688-
File != FileEnd && !EC; File.increment(EC)) {
1689-
// We only care about module and global module index files.
1690-
StringRef Extension = llvm::sys::path::extension(File->path());
1691-
if (Extension != ".pcm" && Extension != ".timestamp" &&
1692-
llvm::sys::path::filename(File->path()) != "modules.idx")
1693-
continue;
1694-
1695-
// Look at this file. If we can't stat it, there's nothing interesting
1696-
// there.
1697-
if (llvm::sys::fs::status(File->path(), StatBuf))
1698-
continue;
1699-
1700-
// If the file has been used recently enough, leave it there.
1701-
time_t FileAccessTime = llvm::sys::toTimeT(StatBuf.getLastAccessedTime());
1702-
if (CurrentTime - FileAccessTime <=
1703-
time_t(HSOpts.ModuleCachePruneAfter)) {
1704-
continue;
1705-
}
1706-
1707-
// Remove the file.
1708-
llvm::sys::fs::remove(File->path());
1709-
1710-
// Remove the timestamp file.
1711-
std::string TimpestampFilename = File->path() + ".timestamp";
1712-
llvm::sys::fs::remove(TimpestampFilename);
1713-
}
1714-
1715-
// If we removed all of the files in the directory, remove the directory
1716-
// itself.
1717-
if (llvm::sys::fs::directory_iterator(Dir->path(), EC) ==
1718-
llvm::sys::fs::directory_iterator() && !EC)
1719-
llvm::sys::fs::remove(Dir->path());
1720-
}
1721-
}
1722-
17231639
void CompilerInstance::createASTReader() {
17241640
if (TheASTReader)
17251641
return;
@@ -1730,11 +1646,10 @@ void CompilerInstance::createASTReader() {
17301646
// If we're implicitly building modules but not currently recursively
17311647
// building a module, check whether we need to prune the module cache.
17321648
if (getSourceManager().getModuleBuildStack().empty() &&
1733-
!getPreprocessor().getHeaderSearchInfo().getModuleCachePath().empty() &&
1734-
getHeaderSearchOpts().ModuleCachePruneInterval > 0 &&
1735-
getHeaderSearchOpts().ModuleCachePruneAfter > 0) {
1736-
pruneModuleCache(getHeaderSearchOpts());
1737-
}
1649+
!getPreprocessor().getHeaderSearchInfo().getModuleCachePath().empty())
1650+
ModCache->maybePrune(getHeaderSearchOpts().ModuleCachePath,
1651+
getHeaderSearchOpts().ModuleCachePruneInterval,
1652+
getHeaderSearchOpts().ModuleCachePruneAfter);
17381653

17391654
HeaderSearchOptions &HSOpts = getHeaderSearchOpts();
17401655
std::string Sysroot = HSOpts.Sysroot;

‎clang/lib/Serialization/ModuleCache.cpp‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,87 @@
1616

1717
using namespace clang;
1818

19+
/// Write a new timestamp file with the given path.
20+
static void writeTimestampFile(StringRef TimestampFile) {
21+
std::error_code EC;
22+
llvm::raw_fd_ostream Out(TimestampFile.str(), EC, llvm::sys::fs::OF_None);
23+
}
24+
25+
void clang::maybePruneImpl(StringRef Path, time_t PruneInterval,
26+
time_t PruneAfter) {
27+
if (PruneInterval <= 0 || PruneAfter <= 0)
28+
return;
29+
30+
llvm::SmallString<128> TimestampFile(Path);
31+
llvm::sys::path::append(TimestampFile, "modules.timestamp");
32+
33+
// Try to stat() the timestamp file.
34+
llvm::sys::fs::file_status StatBuf;
35+
if (std::error_code EC = llvm::sys::fs::status(TimestampFile, StatBuf)) {
36+
// If the timestamp file wasn't there, create one now.
37+
if (EC == std::errc::no_such_file_or_directory)
38+
writeTimestampFile(TimestampFile);
39+
return;
40+
}
41+
42+
// Check whether the time stamp is older than our pruning interval.
43+
// If not, do nothing.
44+
time_t TimestampModTime =
45+
llvm::sys::toTimeT(StatBuf.getLastModificationTime());
46+
time_t CurrentTime = time(nullptr);
47+
if (CurrentTime - TimestampModTime <= PruneInterval)
48+
return;
49+
50+
// Write a new timestamp file so that nobody else attempts to prune.
51+
// There is a benign race condition here, if two Clang instances happen to
52+
// notice at the same time that the timestamp is out-of-date.
53+
writeTimestampFile(TimestampFile);
54+
55+
// Walk the entire module cache, looking for unused module files and module
56+
// indices.
57+
std::error_code EC;
58+
for (llvm::sys::fs::directory_iterator Dir(Path, EC), DirEnd;
59+
Dir != DirEnd && !EC; Dir.increment(EC)) {
60+
// If we don't have a directory, there's nothing to look into.
61+
if (!llvm::sys::fs::is_directory(Dir->path()))
62+
continue;
63+
64+
// Walk all the files within this directory.
65+
for (llvm::sys::fs::directory_iterator File(Dir->path(), EC), FileEnd;
66+
File != FileEnd && !EC; File.increment(EC)) {
67+
// We only care about module and global module index files.
68+
StringRef Extension = llvm::sys::path::extension(File->path());
69+
if (Extension != ".pcm" && Extension != ".timestamp" &&
70+
llvm::sys::path::filename(File->path()) != "modules.idx")
71+
continue;
72+
73+
// Look at this file. If we can't stat it, there's nothing interesting
74+
// there.
75+
if (llvm::sys::fs::status(File->path(), StatBuf))
76+
continue;
77+
78+
// If the file has been used recently enough, leave it there.
79+
time_t FileAccessTime = llvm::sys::toTimeT(StatBuf.getLastAccessedTime());
80+
if (CurrentTime - FileAccessTime <= PruneAfter)
81+
continue;
82+
83+
// Remove the file.
84+
llvm::sys::fs::remove(File->path());
85+
86+
// Remove the timestamp file.
87+
std::string TimpestampFilename = File->path() + ".timestamp";
88+
llvm::sys::fs::remove(TimpestampFilename);
89+
}
90+
91+
// If we removed all the files in the directory, remove the directory
92+
// itself.
93+
if (llvm::sys::fs::directory_iterator(Dir->path(), EC) ==
94+
llvm::sys::fs::directory_iterator() &&
95+
!EC)
96+
llvm::sys::fs::remove(Dir->path());
97+
}
98+
}
99+
19100
namespace {
20101
class CrossProcessModuleCache : public ModuleCache {
21102
InMemoryModuleCache InMemory;
@@ -53,6 +134,11 @@ class CrossProcessModuleCache : public ModuleCache {
53134
OS.clear_error(); // Avoid triggering a fatal error.
54135
}
55136

137+
void maybePrune(StringRef Path, time_t PruneInterval,
138+
time_t PruneAfter) override {
139+
maybePruneImpl(Path, PruneInterval, PruneAfter);
140+
}
141+
56142
InMemoryModuleCache &getInMemoryModuleCache() override { return InMemory; }
57143
const InMemoryModuleCache &getInMemoryModuleCache() const override {
58144
return InMemory;

‎clang/lib/Tooling/DependencyScanning/InProcessModuleCache.cpp‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,13 @@ class InProcessModuleCache : public ModuleCache {
100100
Timestamp.store(llvm::sys::toTimeT(std::chrono::system_clock::now()));
101101
}
102102

103+
void maybePrune(StringRef Path, time_t PruneInterval,
104+
time_t PruneAfter) override {
105+
// FIXME: This only needs to be ran once per build, not in every
106+
// compilation. Call it once per service.
107+
maybePruneImpl(Path, PruneInterval, PruneAfter);
108+
}
109+
103110
InMemoryModuleCache &getInMemoryModuleCache() override { return InMemory; }
104111
const InMemoryModuleCache &getInMemoryModuleCache() const override {
105112
return InMemory;

0 commit comments

Comments
 (0)