Skip to content

Commit 6110dea

Browse files
BigcheeseBigcheese
authored
[clang][scan-deps] Add option to disable caching stat failures (#144000)
While the source code isn't supposed to change during a build, in some environments it does. This adds an option that disables caching of stat failures, meaning that source files can be added to the build during scanning. This adds a `-no-cache-negative-stats` option to clang-scan-deps to enable this behavior. There are no tests for clang-scan-deps as there's no reliable way to do so from it. A unit test has been added that modifies the filesystem between scans to test it.
1 parent 491b82a commit 6110dea

File tree

9 files changed

+132
-42
lines changed

9 files changed

+132
-42
lines changed

‎clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h‎

Lines changed: 9 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ namespace clang {
2323
namespace tooling {
2424
namespace dependencies {
2525

26+
class DependencyScanningService;
27+
2628
using DependencyDirectivesTy =
2729
SmallVector<dependency_directives_scan::Directive, 20>;
2830

@@ -349,7 +351,7 @@ class DependencyScanningWorkerFilesystem
349351
static const char ID;
350352

351353
DependencyScanningWorkerFilesystem(
352-
DependencyScanningFilesystemSharedCache &SharedCache,
354+
DependencyScanningService &Service,
353355
IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS);
354356

355357
llvm::ErrorOr<llvm::vfs::Status> status(const Twine &Path) override;
@@ -435,10 +437,7 @@ class DependencyScanningWorkerFilesystem
435437
/// Returns entry associated with the unique ID in the shared cache or nullptr
436438
/// if none is found.
437439
const CachedFileSystemEntry *
438-
findSharedEntryByUID(llvm::vfs::Status Stat) const {
439-
return SharedCache.getShardForUID(Stat.getUniqueID())
440-
.findEntryByUID(Stat.getUniqueID());
441-
}
440+
findSharedEntryByUID(llvm::vfs::Status Stat) const;
442441

443442
/// Associates the given entry with the filename in the local cache and
444443
/// returns it.
@@ -452,20 +451,14 @@ class DependencyScanningWorkerFilesystem
452451
/// some. Otherwise, constructs new one with the given error code, associates
453452
/// it with the filename and returns the result.
454453
const CachedFileSystemEntry &
455-
getOrEmplaceSharedEntryForFilename(StringRef Filename, std::error_code EC) {
456-
return SharedCache.getShardForFilename(Filename)
457-
.getOrEmplaceEntryForFilename(Filename, EC);
458-
}
454+
getOrEmplaceSharedEntryForFilename(StringRef Filename, std::error_code EC);
459455

460456
/// Returns entry associated with the filename in the shared cache if there is
461457
/// some. Otherwise, associates the given entry with the filename and returns
462458
/// it.
463459
const CachedFileSystemEntry &
464460
getOrInsertSharedEntryForFilename(StringRef Filename,
465-
const CachedFileSystemEntry &Entry) {
466-
return SharedCache.getShardForFilename(Filename)
467-
.getOrInsertEntryForFilename(Filename, Entry);
468-
}
461+
const CachedFileSystemEntry &Entry);
469462

470463
void printImpl(raw_ostream &OS, PrintType Type,
471464
unsigned IndentLevel) const override {
@@ -478,8 +471,9 @@ class DependencyScanningWorkerFilesystem
478471
/// VFS.
479472
bool shouldBypass(StringRef Path) const;
480473

481-
/// The global cache shared between worker threads.
482-
DependencyScanningFilesystemSharedCache &SharedCache;
474+
/// The service associated with this VFS.
475+
DependencyScanningService &Service;
476+
483477
/// The local cache is used by the worker thread to cache file system queries
484478
/// locally instead of querying the global cache every time.
485479
DependencyScanningFilesystemLocalCache LocalCache;

‎clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,8 @@ class DependencyScanningService {
8787
ScanningOptimizations OptimizeArgs = ScanningOptimizations::Default,
8888
bool EagerLoadModules = false, bool TraceVFS = false,
8989
std::time_t BuildSessionTimestamp =
90-
llvm::sys::toTimeT(std::chrono::system_clock::now()));
90+
llvm::sys::toTimeT(std::chrono::system_clock::now()),
91+
bool CacheNegativeStats = true);
9192

9293
ScanningMode getMode() const { return Mode; }
9394

@@ -99,6 +100,8 @@ class DependencyScanningService {
99100

100101
bool shouldTraceVFS() const { return TraceVFS; }
101102

103+
bool shouldCacheNegativeStats() const { return CacheNegativeStats; }
104+
102105
DependencyScanningFilesystemSharedCache &getSharedCache() {
103106
return SharedCache;
104107
}
@@ -116,6 +119,7 @@ class DependencyScanningService {
116119
const bool EagerLoadModules;
117120
/// Whether to trace VFS accesses.
118121
const bool TraceVFS;
122+
const bool CacheNegativeStats;
119123
/// The global file system cache.
120124
DependencyScanningFilesystemSharedCache SharedCache;
121125
/// The global module cache entries.

‎clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp‎

Lines changed: 34 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
//===----------------------------------------------------------------------===//
88

99
#include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h"
10+
#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
1011
#include "llvm/Support/MemoryBuffer.h"
1112
#include "llvm/Support/Threading.h"
1213
#include <optional>
@@ -232,19 +233,19 @@ bool DependencyScanningWorkerFilesystem::shouldBypass(StringRef Path) const {
232233
}
233234

234235
DependencyScanningWorkerFilesystem::DependencyScanningWorkerFilesystem(
235-
DependencyScanningFilesystemSharedCache &SharedCache,
236+
DependencyScanningService &Service,
236237
IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)
237238
: llvm::RTTIExtends<DependencyScanningWorkerFilesystem,
238239
llvm::vfs::ProxyFileSystem>(std::move(FS)),
239-
SharedCache(SharedCache),
240-
WorkingDirForCacheLookup(llvm::errc::invalid_argument) {
240+
Service(Service), WorkingDirForCacheLookup(llvm::errc::invalid_argument) {
241241
updateWorkingDirForCacheLookup();
242242
}
243243

244244
const CachedFileSystemEntry &
245245
DependencyScanningWorkerFilesystem::getOrEmplaceSharedEntryForUID(
246246
TentativeEntry TEntry) {
247-
auto &Shard = SharedCache.getShardForUID(TEntry.Status.getUniqueID());
247+
auto &Shard =
248+
Service.getSharedCache().getShardForUID(TEntry.Status.getUniqueID());
248249
return Shard.getOrEmplaceEntryForUID(TEntry.Status.getUniqueID(),
249250
std::move(TEntry.Status),
250251
std::move(TEntry.Contents));
@@ -255,18 +256,44 @@ DependencyScanningWorkerFilesystem::findEntryByFilenameWithWriteThrough(
255256
StringRef Filename) {
256257
if (const auto *Entry = LocalCache.findEntryByFilename(Filename))
257258
return Entry;
258-
auto &Shard = SharedCache.getShardForFilename(Filename);
259+
auto &Shard = Service.getSharedCache().getShardForFilename(Filename);
259260
if (const auto *Entry = Shard.findEntryByFilename(Filename))
260261
return &LocalCache.insertEntryForFilename(Filename, *Entry);
261262
return nullptr;
262263
}
263264

265+
const CachedFileSystemEntry *
266+
DependencyScanningWorkerFilesystem::findSharedEntryByUID(
267+
llvm::vfs::Status Stat) const {
268+
return Service.getSharedCache()
269+
.getShardForUID(Stat.getUniqueID())
270+
.findEntryByUID(Stat.getUniqueID());
271+
}
272+
273+
const CachedFileSystemEntry &
274+
DependencyScanningWorkerFilesystem::getOrEmplaceSharedEntryForFilename(
275+
StringRef Filename, std::error_code EC) {
276+
return Service.getSharedCache()
277+
.getShardForFilename(Filename)
278+
.getOrEmplaceEntryForFilename(Filename, EC);
279+
}
280+
281+
const CachedFileSystemEntry &
282+
DependencyScanningWorkerFilesystem::getOrInsertSharedEntryForFilename(
283+
StringRef Filename, const CachedFileSystemEntry &Entry) {
284+
return Service.getSharedCache()
285+
.getShardForFilename(Filename)
286+
.getOrInsertEntryForFilename(Filename, Entry);
287+
}
288+
264289
llvm::ErrorOr<const CachedFileSystemEntry &>
265290
DependencyScanningWorkerFilesystem::computeAndStoreResult(
266291
StringRef OriginalFilename, StringRef FilenameForLookup) {
267292
llvm::ErrorOr<llvm::vfs::Status> Stat =
268293
getUnderlyingFS().status(OriginalFilename);
269294
if (!Stat) {
295+
if (!Service.shouldCacheNegativeStats())
296+
return Stat.getError();
270297
const auto &Entry =
271298
getOrEmplaceSharedEntryForFilename(FilenameForLookup, Stat.getError());
272299
return insertLocalEntryForFilename(FilenameForLookup, Entry);
@@ -420,7 +447,8 @@ DependencyScanningWorkerFilesystem::getRealPath(const Twine &Path,
420447
return HandleCachedRealPath(*RealPath);
421448

422449
// If we have the result in the shared cache, cache it locally.
423-
auto &Shard = SharedCache.getShardForFilename(*FilenameForLookup);
450+
auto &Shard =
451+
Service.getSharedCache().getShardForFilename(*FilenameForLookup);
424452
if (const auto *ShardRealPath =
425453
Shard.findRealPathByFilename(*FilenameForLookup)) {
426454
const auto &RealPath = LocalCache.insertRealPathForFilename(

‎clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@ using namespace dependencies;
1515
DependencyScanningService::DependencyScanningService(
1616
ScanningMode Mode, ScanningOutputFormat Format,
1717
ScanningOptimizations OptimizeArgs, bool EagerLoadModules, bool TraceVFS,
18-
std::time_t BuildSessionTimestamp)
18+
std::time_t BuildSessionTimestamp, bool CacheNegativeStats)
1919
: Mode(Mode), Format(Format), OptimizeArgs(OptimizeArgs),
2020
EagerLoadModules(EagerLoadModules), TraceVFS(TraceVFS),
21+
CacheNegativeStats(CacheNegativeStats),
2122
BuildSessionTimestamp(BuildSessionTimestamp) {}

‎clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -611,8 +611,7 @@ DependencyScanningWorker::DependencyScanningWorker(
611611

612612
switch (Service.getMode()) {
613613
case ScanningMode::DependencyDirectivesScan:
614-
DepFS =
615-
new DependencyScanningWorkerFilesystem(Service.getSharedCache(), FS);
614+
DepFS = new DependencyScanningWorkerFilesystem(Service, FS);
616615
BaseFS = DepFS;
617616
break;
618617
case ScanningMode::CanonicalPreprocessing:

‎clang/tools/clang-scan-deps/ClangScanDeps.cpp‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ static ScanningOutputFormat Format = ScanningOutputFormat::Make;
8585
static ScanningOptimizations OptimizeArgs;
8686
static std::string ModuleFilesDir;
8787
static bool EagerLoadModules;
88+
static bool CacheNegativeStats = true;
8889
static unsigned NumThreads = 0;
8990
static std::string CompilationDB;
9091
static std::optional<std::string> ModuleName;
@@ -191,6 +192,8 @@ static void ParseArgs(int argc, char **argv) {
191192

192193
EagerLoadModules = Args.hasArg(OPT_eager_load_pcm);
193194

195+
CacheNegativeStats = !Args.hasArg(OPT_no_cache_negative_stats);
196+
194197
if (const llvm::opt::Arg *A = Args.getLastArg(OPT_j)) {
195198
StringRef S{A->getValue()};
196199
if (!llvm::to_integer(S, NumThreads, 0)) {
@@ -1080,8 +1083,9 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) {
10801083
});
10811084
};
10821085

1083-
DependencyScanningService Service(ScanMode, Format, OptimizeArgs,
1084-
EagerLoadModules, /*TraceVFS=*/Verbose);
1086+
DependencyScanningService Service(
1087+
ScanMode, Format, OptimizeArgs, EagerLoadModules, /*TraceVFS=*/Verbose,
1088+
llvm::sys::toTimeT(std::chrono::system_clock::now()), CacheNegativeStats);
10851089

10861090
llvm::Timer T;
10871091
T.startTimer();

‎clang/tools/clang-scan-deps/Opts.td‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ defm module_files_dir : Eq<"module-files-dir",
2222

2323
def optimize_args_EQ : CommaJoined<["-", "--"], "optimize-args=">, HelpText<"Which command-line arguments of modules to optimize">;
2424
def eager_load_pcm : F<"eager-load-pcm", "Load PCM files eagerly (instead of lazily on import)">;
25+
def no_cache_negative_stats : F<"no-cache-negative-stats", "Don't cache stat failures">;
2526

2627
def j : Arg<"j", "Number of worker threads to use (default: use all concurrent threads)">;
2728

‎clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -384,3 +384,55 @@ TEST(DependencyScanner, ScanDepsWithDiagConsumer) {
384384
EXPECT_TRUE(DiagConsumer.Finished);
385385
}
386386
}
387+
388+
TEST(DependencyScanner, NoNegativeCache) {
389+
StringRef CWD = "/root";
390+
391+
auto VFS = new llvm::vfs::InMemoryFileSystem();
392+
VFS->setCurrentWorkingDirectory(CWD);
393+
auto Sept = llvm::sys::path::get_separator();
394+
std::string HeaderPath =
395+
std::string(llvm::formatv("{0}root{0}header.h", Sept));
396+
std::string Test0Path =
397+
std::string(llvm::formatv("{0}root{0}test0.cpp", Sept));
398+
std::string Test1Path =
399+
std::string(llvm::formatv("{0}root{0}test1.cpp", Sept));
400+
401+
VFS->addFile(Test0Path, 0,
402+
llvm::MemoryBuffer::getMemBuffer(
403+
"#if __has_include(\"header.h\")\n#endif"));
404+
VFS->addFile(Test1Path, 0,
405+
llvm::MemoryBuffer::getMemBuffer("#include \"header.h\""));
406+
407+
DependencyScanningService Service(
408+
ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Make,
409+
ScanningOptimizations::All, false, false,
410+
llvm::sys::toTimeT(std::chrono::system_clock::now()), false);
411+
DependencyScanningTool ScanTool(Service, VFS);
412+
413+
std::vector<std::string> CommandLine0 = {"clang",
414+
"-target",
415+
"x86_64-apple-macosx10.7",
416+
"-c",
417+
"test0.cpp",
418+
"-o"
419+
"test0.cpp.o"};
420+
std::vector<std::string> CommandLine1 = {"clang",
421+
"-target",
422+
"x86_64-apple-macosx10.7",
423+
"-c",
424+
"test1.cpp",
425+
"-o"
426+
"test1.cpp.o"};
427+
428+
std::string Result;
429+
ASSERT_THAT_ERROR(
430+
ScanTool.getDependencyFile(CommandLine0, CWD).moveInto(Result),
431+
llvm::Succeeded());
432+
433+
VFS->addFile(HeaderPath, 0, llvm::MemoryBuffer::getMemBuffer(""));
434+
435+
ASSERT_THAT_ERROR(
436+
ScanTool.getDependencyFile(CommandLine1, CWD).moveInto(Result),
437+
llvm::Succeeded());
438+
}

‎clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp‎

Lines changed: 22 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
//===----------------------------------------------------------------------===//
88

99
#include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h"
10+
#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
1011
#include "llvm/ADT/SmallString.h"
1112
#include "llvm/Support/VirtualFileSystem.h"
1213
#include "gtest/gtest.h"
@@ -19,9 +20,10 @@ TEST(DependencyScanningWorkerFilesystem, CacheStatusFailures) {
1920
auto InstrumentingFS =
2021
llvm::makeIntrusiveRefCnt<llvm::vfs::TracingFileSystem>(InMemoryFS);
2122

22-
DependencyScanningFilesystemSharedCache SharedCache;
23-
DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS);
24-
DependencyScanningWorkerFilesystem DepFS2(SharedCache, InstrumentingFS);
23+
DependencyScanningService Service(ScanningMode::DependencyDirectivesScan,
24+
ScanningOutputFormat::Make);
25+
DependencyScanningWorkerFilesystem DepFS(Service, InstrumentingFS);
26+
DependencyScanningWorkerFilesystem DepFS2(Service, InstrumentingFS);
2527

2628
DepFS.status("/foo.c");
2729
EXPECT_EQ(InstrumentingFS->NumStatusCalls, 1u);
@@ -45,9 +47,10 @@ TEST(DependencyScanningFilesystem, CacheGetRealPath) {
4547
auto InstrumentingFS =
4648
llvm::makeIntrusiveRefCnt<llvm::vfs::TracingFileSystem>(InMemoryFS);
4749

48-
DependencyScanningFilesystemSharedCache SharedCache;
49-
DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS);
50-
DependencyScanningWorkerFilesystem DepFS2(SharedCache, InstrumentingFS);
50+
DependencyScanningService Service(ScanningMode::DependencyDirectivesScan,
51+
ScanningOutputFormat::Make);
52+
DependencyScanningWorkerFilesystem DepFS(Service, InstrumentingFS);
53+
DependencyScanningWorkerFilesystem DepFS2(Service, InstrumentingFS);
5154

5255
{
5356
llvm::SmallString<128> Result;
@@ -80,8 +83,9 @@ TEST(DependencyScanningFilesystem, RealPathAndStatusInvariants) {
8083
InMemoryFS->addFile("/foo.c", 0, llvm::MemoryBuffer::getMemBuffer(""));
8184
InMemoryFS->addFile("/bar.c", 0, llvm::MemoryBuffer::getMemBuffer(""));
8285

83-
DependencyScanningFilesystemSharedCache SharedCache;
84-
DependencyScanningWorkerFilesystem DepFS(SharedCache, InMemoryFS);
86+
DependencyScanningService Service(ScanningMode::DependencyDirectivesScan,
87+
ScanningOutputFormat::Make);
88+
DependencyScanningWorkerFilesystem DepFS(Service, InMemoryFS);
8589

8690
// Success.
8791
{
@@ -133,8 +137,9 @@ TEST(DependencyScanningFilesystem, CacheStatOnExists) {
133137
InMemoryFS->setCurrentWorkingDirectory("/");
134138
InMemoryFS->addFile("/foo", 0, llvm::MemoryBuffer::getMemBuffer(""));
135139
InMemoryFS->addFile("/bar", 0, llvm::MemoryBuffer::getMemBuffer(""));
136-
DependencyScanningFilesystemSharedCache SharedCache;
137-
DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS);
140+
DependencyScanningService Service(ScanningMode::DependencyDirectivesScan,
141+
ScanningOutputFormat::Make);
142+
DependencyScanningWorkerFilesystem DepFS(Service, InstrumentingFS);
138143

139144
DepFS.status("/foo");
140145
DepFS.status("/foo");
@@ -156,8 +161,9 @@ TEST(DependencyScanningFilesystem, CacheStatFailures) {
156161
auto InstrumentingFS =
157162
llvm::makeIntrusiveRefCnt<llvm::vfs::TracingFileSystem>(InMemoryFS);
158163

159-
DependencyScanningFilesystemSharedCache SharedCache;
160-
DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS);
164+
DependencyScanningService Service(ScanningMode::DependencyDirectivesScan,
165+
ScanningOutputFormat::Make);
166+
DependencyScanningWorkerFilesystem DepFS(Service, InstrumentingFS);
161167

162168
DepFS.status("/dir");
163169
DepFS.status("/dir");
@@ -183,8 +189,9 @@ TEST(DependencyScanningFilesystem, DiagnoseStaleStatFailures) {
183189
auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
184190
InMemoryFS->setCurrentWorkingDirectory("/");
185191

186-
DependencyScanningFilesystemSharedCache SharedCache;
187-
DependencyScanningWorkerFilesystem DepFS(SharedCache, InMemoryFS);
192+
DependencyScanningService Service(ScanningMode::DependencyDirectivesScan,
193+
ScanningOutputFormat::Make);
194+
DependencyScanningWorkerFilesystem DepFS(Service, InMemoryFS);
188195

189196
bool Path1Exists = DepFS.exists("/path1.suffix");
190197
EXPECT_EQ(Path1Exists, false);
@@ -197,7 +204,7 @@ TEST(DependencyScanningFilesystem, DiagnoseStaleStatFailures) {
197204
EXPECT_EQ(Path1Exists, false);
198205

199206
std::vector<llvm::StringRef> InvalidPaths =
200-
SharedCache.getInvalidNegativeStatCachedPaths(*InMemoryFS);
207+
Service.getSharedCache().getInvalidNegativeStatCachedPaths(*InMemoryFS);
201208

202209
EXPECT_EQ(InvalidPaths.size(), 1u);
203210
ASSERT_STREQ("/path1.suffix", InvalidPaths[0].str().c_str());

0 commit comments

Comments
 (0)