Skip to content

Commit 20a4da9

Browse files
benlangmuirbenlangmuir
committed
[clang][cas] Initial support for prefix-mapping with modules and PCH
Add initial support to compilation caching to allow prefix-mapping paths when working with modules and PCH in a cas-fs/caching-on-disk-filesystem. (cherry picked from commit a334080)
1 parent f62450e commit 20a4da9

File tree

7 files changed

+703
-13
lines changed

7 files changed

+703
-13
lines changed

‎clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp

Lines changed: 25 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,22 @@ class IncludeTreePPConsumer : public PPIncludeActionsConsumer {
249249
};
250250
} // namespace
251251

252+
/// The PCH recorded file paths with canonical paths, create a VFS that
253+
/// allows remapping back to the non-canonical source paths so that they are
254+
/// found during dep-scanning.
255+
static void
256+
addReversePrefixMappingFileSystem(const llvm::PrefixMapper &PrefixMapper,
257+
CompilerInstance &ScanInstance) {
258+
llvm::PrefixMapper ReverseMapper;
259+
ReverseMapper.addInverseRange(PrefixMapper.getMappings());
260+
ReverseMapper.sort();
261+
std::unique_ptr<llvm::vfs::FileSystem> FS =
262+
llvm::vfs::createPrefixMappingFileSystem(
263+
std::move(ReverseMapper), &ScanInstance.getVirtualFileSystem());
264+
265+
ScanInstance.getFileManager().setVirtualFileSystem(std::move(FS));
266+
}
267+
252268
Error IncludeTreePPConsumer::initialize(CompilerInstance &ScanInstance,
253269
CompilerInvocation &NewInvocation) {
254270
if (Error E =
@@ -263,17 +279,7 @@ Error IncludeTreePPConsumer::initialize(CompilerInstance &ScanInstance,
263279
if (PPOpts.Includes.empty() && PPOpts.ImplicitPCHInclude.empty())
264280
return;
265281

266-
// The PCH recorded file paths with canonical paths, create a VFS that
267-
// allows remapping back to the non-canonical source paths so that they are
268-
// found during dep-scanning.
269-
llvm::PrefixMapper ReverseMapper;
270-
ReverseMapper.addInverseRange(PrefixMapper.getMappings());
271-
ReverseMapper.sort();
272-
std::unique_ptr<llvm::vfs::FileSystem> FS =
273-
llvm::vfs::createPrefixMappingFileSystem(
274-
std::move(ReverseMapper), &ScanInstance.getVirtualFileSystem());
275-
276-
ScanInstance.getFileManager().setVirtualFileSystem(std::move(FS));
282+
addReversePrefixMappingFileSystem(PrefixMapper, ScanInstance);
277283

278284
// These are written in the predefines buffer, so we need to remap them.
279285
for (std::string &Include : PPOpts.Includes)
@@ -594,6 +600,10 @@ Error FullDependencyConsumer::initialize(CompilerInstance &ScanInstance,
594600
if (Error E = PrefixMapping.configurePrefixMapper(NewInvocation, *Mapper))
595601
return E;
596602

603+
const PreprocessorOptions &PPOpts = ScanInstance.getPreprocessorOpts();
604+
if (!PPOpts.Includes.empty() || !PPOpts.ImplicitPCHInclude.empty())
605+
addReversePrefixMappingFileSystem(*Mapper, ScanInstance);
606+
597607
CacheFS->trackNewAccesses();
598608
if (auto CWD =
599609
ScanInstance.getVirtualFileSystem().getCurrentWorkingDirectory())
@@ -716,6 +726,10 @@ Error FullDependencyConsumer::finalizeModuleInvocation(CompilerInvocation &CI,
716726
CacheFS->getCurrentWorkingDirectory().get(),
717727
/*ProduceIncludeTree=*/false);
718728
}
729+
730+
if (Mapper)
731+
DepscanPrefixMapping::remapInvocationPaths(CI, *Mapper);
732+
719733
return llvm::Error::success();
720734
}
721735

‎clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -289,10 +289,12 @@ static std::string getModuleContextHash(const ModuleDeps &MD,
289289
HashBuilder.add(serialization::VERSION_MAJOR, serialization::VERSION_MINOR);
290290

291291
// Save and restore options that should not affect the hash, e.g. the exact
292-
// contents of input files.
292+
// contents of input files, or prefix mappings.
293293
auto &MutableCI = const_cast<CompilerInvocation &>(CI);
294294
llvm::SaveAndRestore<std::string> RestoreCASFSRootID(
295-
MutableCI.getFileSystemOpts().CASFileSystemRootID);
295+
MutableCI.getFileSystemOpts().CASFileSystemRootID, "");
296+
llvm::SaveAndRestore<std::vector<std::string>> RestorePrefixMappings(
297+
MutableCI.getFrontendOpts().PathPrefixMappings, {});
296298

297299
// Hash the BuildInvocation without any input files.
298300
SmallVector<const char *, 32> DummyArgs;

‎clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,8 @@ void DepscanPrefixMapping::remapInvocationPaths(CompilerInvocation &Invocation,
146146
mapInPlaceAll(FrontendOpts.ASTMergeFiles);
147147
Mapper.mapInPlace(FrontendOpts.OverrideRecordLayoutsFile);
148148
Mapper.mapInPlace(FrontendOpts.StatsFile);
149+
for (auto &[Path, _] : FrontendOpts.ModuleCacheKeys)
150+
Mapper.mapInPlace(Path);
149151

150152
// Filesystem options.
151153
Mapper.mapInPlace(FileSystemOpts.WorkingDir);

‎clang/test/ClangScanDeps/modules-cas-fs-prefix-mapping-caching.c

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
// Test that we get cache hits across directories with modules.
2+
3+
// REQUIRES: ondisk_cas
4+
5+
// RUN: rm -rf %t
6+
// RUN: split-file %s %t
7+
// RUN: cp -r %t/dir1 %t/dir2
8+
// RUN: sed -e "s|DIR|%t/dir1|g" -e "s|CLANG|%clang|g" -e "s|SDK|%S/Inputs/SDK|g" %t/cdb.json.template > %t/dir1/cdb.json
9+
// RUN: sed -e "s|DIR|%t/dir2|g" -e "s|CLANG|%clang|g" -e "s|SDK|%S/Inputs/SDK|g" %t/cdb.json.template > %t/dir2/cdb.json
10+
11+
// RUN: clang-scan-deps -compilation-database %t/dir1/cdb.json -format experimental-full \
12+
// RUN: -cas-path %t/cas -action-cache-path %t/cache -module-files-dir %t/dir1/modules \
13+
// RUN: -prefix-map-sdk=/^sdk -prefix-map-toolchain=/^tc \
14+
// RUN: -prefix-map=%t/dir1/modules=/^modules -prefix-map=%t/dir1=/^src \
15+
// RUN: > %t/dir1.txt
16+
17+
// RUN: clang-scan-deps -compilation-database %t/dir2/cdb.json -format experimental-full \
18+
// RUN: -cas-path %t/cas -action-cache-path %t/cache -module-files-dir %t/dir2/modules \
19+
// RUN: -prefix-map-sdk=/^sdk -prefix-map-toolchain=/^tc \
20+
// RUN: -prefix-map=%t/dir2/modules=/^modules -prefix-map=%t/dir2=/^src \
21+
// RUN: > %t/dir2.txt
22+
23+
// Extract individual commands.
24+
// RUN: %deps-to-rsp %t/dir1.txt --module-name=B > %t/dir1/B.cc1.rsp
25+
// RUN: %deps-to-rsp %t/dir1.txt --module-name=A > %t/dir1/A.cc1.rsp
26+
// RUN: %deps-to-rsp %t/dir1.txt --tu-index 0 > %t/dir1/tu.cc1.rsp
27+
28+
// RUN: %deps-to-rsp %t/dir2.txt --module-name=B > %t/dir2/B.cc1.rsp
29+
// RUN: %deps-to-rsp %t/dir2.txt --module-name=A > %t/dir2/A.cc1.rsp
30+
// RUN: %deps-to-rsp %t/dir2.txt --tu-index 0 > %t/dir2/tu.cc1.rsp
31+
32+
// RUN: (cd %t/dir1; %clang @B.cc1.rsp) 2>&1 | FileCheck %s -check-prefix=CACHE-MISS
33+
// RUN: (cd %t/dir1; %clang @A.cc1.rsp) 2>&1 | FileCheck %s -check-prefix=CACHE-MISS
34+
// RUN: (cd %t/dir1; %clang @tu.cc1.rsp) 2>&1 | FileCheck %s -check-prefix=CACHE-MISS
35+
36+
// CACHE-MISS: compile job cache miss
37+
38+
// RUN: (cd %t/dir2; %clang @B.cc1.rsp) 2>&1 | FileCheck %s -check-prefix=CACHE-HIT
39+
// RUN: (cd %t/dir2; %clang @A.cc1.rsp) 2>&1 | FileCheck %s -check-prefix=CACHE-HIT
40+
// RUN: (cd %t/dir2; %clang @tu.cc1.rsp) 2>&1 | FileCheck %s -check-prefix=CACHE-HIT
41+
42+
// CACHE-HIT: compile job cache hit
43+
44+
// RUN: diff -r -u %t/dir1/modules %t/dir2/modules
45+
46+
//--- cdb.json.template
47+
[
48+
{
49+
"directory": "DIR",
50+
"command": "CLANG -fsyntax-only DIR/t.c -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/mcp -target x86_64-apple-macos11 -isysroot SDK -Rcompile-job-cache",
51+
"file": "DIR/t.c"
52+
}
53+
]
54+
55+
//--- dir1/t.c
56+
#include "a.h"
57+
58+
//--- dir1/module.modulemap
59+
module A { header "a.h" }
60+
module B { header "b.h" }
61+
62+
//--- dir1/a.h
63+
#include "b.h"
64+
65+
//--- dir1/b.h
66+
#include <stdarg.h>
67+
#include <stdlib.h>

‎clang/test/ClangScanDeps/modules-cas-fs-prefix-mapping.c

Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
// Test path prefix-mapping when using a cas-fs with clang-scan-deps in
2+
// modules.
3+
4+
// REQUIRES: ondisk_cas
5+
6+
// RUN: rm -rf %t
7+
// RUN: split-file %s %t
8+
// RUN: sed -e "s|DIR|%t|g" -e "s|CLANG|%clang|g" -e "s|SDK|%S/Inputs/SDK|g" %t/cdb.json.template > %t/cdb.json
9+
10+
// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full \
11+
// RUN: -cas-path %t/cas -action-cache-path %t/cache -module-files-dir %t/modules \
12+
// RUN: -prefix-map=%t/modules=/^modules -prefix-map=%t=/^src -prefix-map-sdk=/^sdk -prefix-map-toolchain=/^tc \
13+
// RUN: > %t/full_result.txt
14+
15+
// Check the command-lines.
16+
// RUN: FileCheck %s -input-file %t/full_result.txt -DPREFIX=%t -DSDK_PREFIX=%S/Inputs/SDK
17+
18+
// Extract individual commands.
19+
// RUN: %deps-to-rsp %t/full_result.txt --module-name=B > %t/B.cc1.rsp
20+
// RUN: %deps-to-rsp %t/full_result.txt --module-name=A > %t/A.cc1.rsp
21+
// RUN: %deps-to-rsp %t/full_result.txt --tu-index 0 > %t/tu.cc1.rsp
22+
23+
// Check the casfs.
24+
// RUN: cat %t/B.cc1.rsp | sed -E 's/.* "-fcas-fs" "([^ ]+)" .*/\1/' > %t/B_id.txt
25+
// RUN: cat %t/A.cc1.rsp | sed -E 's/.* "-fcas-fs" "([^ ]+)" .*/\1/' > %t/A_id.txt
26+
// RUN: cat %t/tu.cc1.rsp | sed -E 's/.* "-fcas-fs" "([^ ]+)" .*/\1/' > %t/tu_id.txt
27+
// RUN: llvm-cas -cas %t/cas -ls-tree-recursive @%t/B_id.txt > %t/B_fs.txt
28+
// RUN: llvm-cas -cas %t/cas -ls-tree-recursive @%t/A_id.txt > %t/A_fs.txt
29+
// RUN: llvm-cas -cas %t/cas -ls-tree-recursive @%t/tu_id.txt > %t/tu_fs.txt
30+
// RUN: FileCheck %s -input-file %t/A_fs.txt -DPREFIX=%t -DSDK_PREFIX=%S/Inputs/SDK -check-prefixes=FS_NEG,FS
31+
// RUN: FileCheck %s -input-file %t/B_fs.txt -DPREFIX=%t -DSDK_PREFIX=%S/Inputs/SDK -check-prefixes=FS_NEG,FS
32+
// RUN: FileCheck %s -input-file %t/tu_fs.txt -DPREFIX=%t -DSDK_PREFIX=%S/Inputs/SDK -check-prefixes=FS_NEG,FS
33+
34+
// FS_NEG-NOT: [[PREFIX]]
35+
// FS_NEG-NOT: [[SDK_PREFIX]]
36+
// FS_NEG-NOT: .pcm{{$}}
37+
// FS: file llvmcas://{{.*}} /^sdk/usr/include/stdlib.h
38+
// FS: file llvmcas://{{.*}} /^src/a.h
39+
// FS: file llvmcas://{{.*}} /^src/b.h
40+
// FS: file llvmcas://{{.*}} /^src/module.modulemap
41+
// FS: file llvmcas://{{.*}} /^tc/lib/clang/{{.*}}/include/stdarg.h
42+
43+
// Check that it builds.
44+
// RUN: %clang @%t/B.cc1.rsp
45+
// RUN: %clang @%t/A.cc1.rsp
46+
// RUN: %clang @%t/tu.cc1.rsp
47+
48+
// CHECK: {
49+
// CHECK: "modules": [
50+
// CHECK: {
51+
// CHECK: "casfs-root-id": "[[A_ROOT_ID:llvmcas://[[:xdigit:]]+]]"
52+
// CHECK: "clang-module-deps": [
53+
// CHECK: {
54+
// CHECK: "module-name": "B"
55+
// CHECK: }
56+
// CHECK: ]
57+
// CHECK: "clang-modulemap-file": "[[PREFIX]]/module.modulemap"
58+
// CHECK: "command-line": [
59+
// CHECK: "-fcas-path"
60+
// CHECK: "[[PREFIX]]/cas"
61+
// CHECK: "-faction-cache-path"
62+
// CHECK: "[[PREFIX]]/cache"
63+
// CHECK: "-fcas-fs"
64+
// CHECK: "[[A_ROOT_ID]]"
65+
// CHECK: "-fcas-fs-working-directory"
66+
// CHECK: "/^src"
67+
// CHECK: "-fmodule-map-file=/^src/module.modulemap"
68+
// CHECK: "-o"
69+
// CHECK: "[[PREFIX]]/modules/{{.*}}/A-{{.*}}.pcm"
70+
// CHECK: "-fmodule-file-cache-key=/^modules/{{.*}}/B-[[B_CONTEXT_HASH:[^.]+]].pcm=llvmcas://{{.*}}"
71+
// CHECK: "-x"
72+
// CHECK: "c"
73+
// CHECK: "/^src/module.modulemap"
74+
// CHECK: "-isysroot"
75+
// CHECK: "/^sdk"
76+
// CHECK: "-resource-dir"
77+
// CHECK: "/^tc/lib/clang/{{.*}}"
78+
// CHECK: "-fmodule-file=B=/^modules/{{.*}}/B-[[B_CONTEXT_HASH]].pcm"
79+
// CHECK: "-isystem"
80+
// CHECK: "/^tc/lib/clang/{{.*}}/include"
81+
// CHECK: "-internal-externc-isystem"
82+
// CHECK: "/^sdk/usr/include"
83+
// CHECK: ]
84+
// CHECK: "file-deps": [
85+
// CHECK: "[[PREFIX]]/a.h"
86+
// CHECK: "[[PREFIX]]/module.modulemap"
87+
// CHECK: ]
88+
// CHECK: "name": "A"
89+
// CHECK: }
90+
// CHECK: {
91+
// CHECK: "casfs-root-id": "[[B_ROOT_ID:llvmcas://[[:xdigit:]]+]]"
92+
// CHECK: "clang-module-deps": [],
93+
// CHECK: "clang-modulemap-file": "[[PREFIX]]/module.modulemap"
94+
// CHECK: "command-line": [
95+
// CHECK: "-fcas-path"
96+
// CHECK: "[[PREFIX]]/cas"
97+
// CHECK: "-faction-cache-path"
98+
// CHECK: "[[PREFIX]]/cache"
99+
// CHECK: "-fcas-fs"
100+
// CHECK: "[[B_ROOT_ID]]"
101+
// CHECK: "-fcas-fs-working-directory"
102+
// CHECK: "/^src"
103+
// CHECK: "-o"
104+
// CHECK: "[[PREFIX]]/modules/{{.*}}/B-[[B_CONTEXT_HASH]].pcm"
105+
// CHECK: "-x"
106+
// CHECK: "c"
107+
// CHECK: "/^src/module.modulemap"
108+
// CHECK: "-isysroot"
109+
// CHECK: "/^sdk"
110+
// CHECK: "-resource-dir"
111+
// CHECK: "/^tc/lib/clang/{{.*}}"
112+
// CHECK: "-isystem"
113+
// CHECK: "/^tc/lib/clang/{{.*}}/include"
114+
// CHECK: "-internal-externc-isystem"
115+
// CHECK: "/^sdk/usr/include"
116+
// CHECK: ]
117+
// CHECK: "context-hash": "[[B_CONTEXT_HASH]]"
118+
// CHECK: "file-deps": [
119+
// CHECK: "{{.*}}/include/stdarg.h"
120+
// CHECK: "[[PREFIX]]/b.h"
121+
// CHECK: "[[PREFIX]]/module.modulemap"
122+
// CHECK: "[[SDK_PREFIX]]/usr/include/stdlib.h"
123+
// CHECK: ]
124+
// CHECK: "name": "B"
125+
// CHECK: }
126+
// CHECK: ]
127+
// CHECK: "translation-units": [
128+
// CHECK: {
129+
// CHECK: "commands": [
130+
// CHECK: {
131+
// CHECK: "casfs-root-id": "[[TU_ROOT_ID:llvmcas://[[:xdigit:]]+]]"
132+
// CHECK: "clang-module-deps": [
133+
// CHECK: {
134+
// CHECK: "module-name": "A"
135+
// CHECK: }
136+
// CHECK: ]
137+
// CHECK: "command-line": [
138+
// CHECK: "-fcas-path"
139+
// CHECK: "[[PREFIX]]/cas"
140+
// CHECK: "-faction-cache-path"
141+
// CHECK: "[[PREFIX]]/cache"
142+
// CHECK: "-fcas-fs"
143+
// CHECK: "[[TU_ROOT_ID]]"
144+
// CHECK: "-fcas-fs-working-directory"
145+
// CHECK: "/^src"
146+
// CHECK: "-fmodule-map-file=/^src/module.modulemap"
147+
// CHECK: "-fmodule-file-cache-key=/^modules/{{.*}}A-{{.*}}.pcm=llvmcas://{{.*}}"
148+
// CHECK: "-x"
149+
// CHECK: "c"
150+
// CHECK: "/^src/t.c"
151+
// CHECK: "-isysroot"
152+
// CHECK: "/^sdk"
153+
// CHECK: "-resource-dir"
154+
// CHECK: "/^tc/lib/clang/{{.*}}"
155+
// CHECK: "-fmodule-file=A=/^modules/{{.*}}/A-{{.*}}.pcm"
156+
// CHECK: "-isystem"
157+
// CHECK: "/^tc/lib/clang/{{.*}}/include"
158+
// CHECK: "-internal-externc-isystem"
159+
// CHECK: "/^sdk/usr/include"
160+
// CHECK: ],
161+
// CHECK: "file-deps": [
162+
// CHECK: "[[PREFIX]]/t.c"
163+
// CHECK: ]
164+
// CHECK: "input-file": "[[PREFIX]]/t.c"
165+
// CHECK: }
166+
167+
168+
//--- cdb.json.template
169+
[
170+
{
171+
"directory": "DIR",
172+
"command": "CLANG -fsyntax-only DIR/t.c -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/mcp -target x86_64-apple-macos11 -isysroot SDK",
173+
"file": "DIR/t.c"
174+
}
175+
]
176+
177+
//--- t.c
178+
#include "a.h"
179+
180+
//--- module.modulemap
181+
module A { header "a.h" }
182+
module B { header "b.h" }
183+
184+
//--- a.h
185+
#include "b.h"
186+
187+
//--- b.h
188+
#include <stdarg.h>
189+
#include <stdlib.h>

0 commit comments

Comments
 (0)