Skip to content

Commit bd2b3c7

Browse files
cachemeifyoucancachemeifyoucan
committed
[Caching] Allow prefix mapping on legacy layout file
To support distributed caching for targets that need to use legacy layout file, the path of legacy layout needs to be remapped. Current handling of the legacy layout file is to ingest all layout files to the CAS FileSystem as part of the compiler resource files. But it cannot convey remapped legacy layout file path to the swift-frontend when distributed caching is enabled. In order to properly support path remapping, the legacy layout file is ingested on demand (thus it doesn't need to be ingested for module compilation), and the remapped path is communicated to swift-front via frontend flag. rdar://162793678
1 parent 9120a53 commit bd2b3c7

File tree

5 files changed

+110
-25
lines changed

5 files changed

+110
-25
lines changed

‎include/swift/DependencyScan/ModuleDependencyScanner.h‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,11 @@ class SwiftDependencyTracker {
145145
const CompilerInvocation &CI);
146146

147147
void startTracking(bool includeCommonDeps = true);
148-
void trackFile(const Twine &path);
148+
149+
/// Track a file with path.
150+
/// \returns true if the file is tracked, false if the file doesn't exist.
151+
bool trackFile(const Twine &path);
152+
149153
llvm::Expected<llvm::cas::ObjectProxy> createTreeFromDependencies();
150154

151155
private:

‎lib/DependencyScan/ModuleDependencyScanner.cpp‎

Lines changed: 5 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -452,20 +452,6 @@ SwiftDependencyTracker::SwiftDependencyTracker(
452452
"SDKSettings.json");
453453
addCommonFile(SDKSettingPath);
454454

455-
// Add Legacy layout file.
456-
const std::vector<std::string> AllSupportedArches = {
457-
"arm64", "arm64e", "x86_64", "i386",
458-
"armv7", "armv7s", "armv7k", "arm64_32"};
459-
460-
for (auto RuntimeLibPath : SearchPathOpts.RuntimeLibraryPaths) {
461-
std::error_code EC;
462-
for (auto &Arch : AllSupportedArches) {
463-
SmallString<256> LayoutFile(RuntimeLibPath);
464-
llvm::sys::path::append(LayoutFile, "layouts-" + Arch + ".yaml");
465-
addCommonFile(LayoutFile);
466-
}
467-
}
468-
469455
// Add VFSOverlay file.
470456
for (auto &Overlay: SearchPathOpts.VFSOverlayFiles)
471457
addCommonFile(Overlay);
@@ -488,20 +474,21 @@ void SwiftDependencyTracker::startTracking(bool includeCommonDeps) {
488474
}
489475
}
490476

491-
void SwiftDependencyTracker::trackFile(const Twine &path) {
477+
bool SwiftDependencyTracker::trackFile(const Twine &path) {
492478
auto file = FS->openFileForRead(path);
493479
if (!file)
494-
return;
480+
return false;
495481
auto status = (*file)->status();
496482
if (!status)
497-
return;
483+
return false;
498484
auto CASFile = dyn_cast<llvm::cas::CASBackedFile>(*file);
499485
if (!CASFile)
500-
return;
486+
return false;
501487
auto fileRef = CASFile->getObjectRefForContent();
502488
std::string realPath =
503489
Mapper ? Mapper->mapToString(path.str()) : path.str();
504490
TrackedFiles.try_emplace(realPath, fileRef, (size_t)status->getSize());
491+
return true;
505492
}
506493

507494
llvm::Expected<llvm::cas::ObjectProxy>

‎lib/DependencyScan/ScanDependencies.cpp‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -471,6 +471,27 @@ class ExplicitModuleDependencyResolver {
471471
[this](const auto &entry) {
472472
tracker->trackFile(entry.second.LibraryPath);
473473
});
474+
StringRef readLegacyTypeInfoPath =
475+
instance.getInvocation().getIRGenOptions().ReadLegacyTypeInfoPath;
476+
if (!readLegacyTypeInfoPath.empty()) {
477+
// If legacy layout is specifed, just need to track that file.
478+
if (tracker->trackFile(readLegacyTypeInfoPath))
479+
commandline.push_back("-read-legacy-type-info-path=" +
480+
scanner.remapPath(readLegacyTypeInfoPath));
481+
} else {
482+
// Otherwise, search RuntimeLibrary Path for implicit legacy layout.
483+
// Search logic need to match IRGen/GenType.cpp.
484+
const auto &triple = instance.getInvocation().getLangOptions().Target;
485+
auto archName = swift::getMajorArchitectureName(triple);
486+
SmallString<256> legacyLayoutPath(instance.getInvocation()
487+
.getSearchPathOptions()
488+
.RuntimeLibraryPaths[0]);
489+
llvm::sys::path::append(legacyLayoutPath,
490+
"layouts-" + archName + ".yaml");
491+
if (tracker->trackFile(legacyLayoutPath))
492+
commandline.push_back("-read-legacy-type-info-path=" +
493+
scanner.remapPath(legacyLayoutPath));
494+
}
474495
auto root = tracker->createTreeFromDependencies();
475496
if (!root)
476497
return diagnoseCASFSCreationError(root.takeError());

‎test/CAS/deps_cas_fs.swift‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
// RUN: mkdir -p %t/cas
66

77
// RUN: mkdir -p %t/resource/macosx
8-
// RUN: cp %S/../IRGen/Inputs/legacy_type_info/a.yaml %t/resource/macosx/layouts-x86_64.yaml
8+
// RUN: cp %S/../IRGen/Inputs/legacy_type_info/a.yaml %t/resource/macosx/layouts-%target-arch.yaml
99

1010
// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/../ScanDependencies/Inputs/CHeaders -I %S/../ScanDependencies/Inputs/Swift -emit-dependencies -emit-dependencies-path %t/deps.d -import-objc-header %S/../ScanDependencies/Inputs/CHeaders/Bridging.h -swift-version 4 -cache-compile-job -cas-path %t/cas -module-name Test -resource-dir %t/resource -scanner-output-dir %t
1111
// Check the contents of the JSON output
@@ -19,13 +19,13 @@
1919
// RUN: %{python} %S/Inputs/SwiftDepsExtractor.py %t/deps.json Test casFSRootID > %t/Test_fs.casid
2020
// RUN: %cache-tool -cas-path %t/cas -cache-tool-action print-include-tree-list @%t/Test_fs.casid | %FileCheck %s -check-prefix FS_ROOT_TEST
2121

22-
// FS_ROOT_E-DAG: layouts-x86_64.yaml
23-
// FS_ROOT_E-DAG: E.swiftinterface
22+
// FS_ROOT_E-NOT: layouts-{{.*}}.yaml
23+
// FS_ROOT_E: E.swiftinterface
2424

25-
// FS_ROOT_F-DAG: layouts-x86_64.yaml
26-
// FS_ROOT_F-DAG: F.swiftinterface
25+
// FS_ROOT_F-NOT: layouts-{{.*}}.yaml
26+
// FS_ROOT_F: F.swiftinterface
2727

28-
// FS_ROOT_TEST-DAG: layouts-x86_64.yaml
28+
// FS_ROOT_TEST-DAG: layouts-{{.*}}.yaml
2929
// FS_ROOT_TEST-DAG: deps_cas_fs.swift
3030

3131
import E

‎test/CAS/legacy_layout_remap.swift‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
// REQUIRES: objc_interop
2+
// UNSUPPORTED: swift_only_stable_abi
3+
4+
// RUN: %empty-directory(%t)
5+
// RUN: split-file %s %t
6+
7+
/// Build legacy module.
8+
// RUN: %target-swift-frontend -target %target-pre-stable-abi-triple -emit-module -enable-library-evolution \
9+
// RUN: -emit-module-path=%t/resilient_struct.swiftmodule -module-name=resilient_struct -O \
10+
// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
11+
// RUN: %S/../Inputs/resilient_struct.swift
12+
13+
/// Scan with legacy layout.
14+
// RUN: %target-swift-frontend -target %target-pre-stable-abi-triple -I %t -c -enable-library-evolution -read-legacy-type-info-path=%t/layout.yaml \
15+
// RUN: -scan-dependencies -module-name Test -module-cache-path %t/clang-module-cache -O -module-load-mode prefer-serialized \
16+
// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
17+
// RUN: %t/main.swift -o %t/deps.json -swift-version 4 -cache-compile-job -cas-path %t/cas \
18+
// RUN: -scanner-prefix-map-paths %t /^tmp
19+
20+
// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json clang:SwiftShims > %t/shim.cmd
21+
// RUN: %swift_frontend_plain @%t/shim.cmd
22+
23+
// RUN: %{python} %S/Inputs/GenerateExplicitModuleMap.py %t/deps.json > %t/map.json
24+
// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map.json > %t/map.casid
25+
26+
// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json Test > %t/MyApp.cmd
27+
28+
// RUN: %target-swift-frontend -target %target-pre-stable-abi-triple -enable-library-evolution \
29+
// RUN: -cache-compile-job -cas-path %t/cas -O \
30+
// RUN: -swift-version 4 -disable-implicit-swift-modules \
31+
// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
32+
// RUN: -module-name Test -explicit-swift-module-map-file @%t/map.casid \
33+
// RUN: /^tmp/main.swift @%t/MyApp.cmd -c -o %t/main.o
34+
35+
/// Now do implicit search.
36+
// RUN: mkdir -p %t/resource/macosx
37+
// RUN: cp %t/layout.yaml %t/resource/macosx/layouts-%target-arch.yaml
38+
39+
// RUN: %target-swift-frontend -target %target-pre-stable-abi-triple -I %t -c -enable-library-evolution \
40+
// RUN: -scan-dependencies -module-name Test -module-cache-path %t/clang-module-cache -O -module-load-mode prefer-serialized \
41+
// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
42+
// RUN: %t/main.swift -o %t/deps2.json -swift-version 4 -cache-compile-job -cas-path %t/cas \
43+
// RUN: -scanner-prefix-map-paths %t /^tmp -resource-dir %t/resource -I %platform-dylib-dir
44+
45+
// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps2.json clang:SwiftShims > %t/shim2.cmd
46+
// RUN: %swift_frontend_plain @%t/shim2.cmd
47+
48+
// RUN: %{python} %S/Inputs/GenerateExplicitModuleMap.py %t/deps2.json > %t/map2.json
49+
// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map2.json > %t/map2.casid
50+
// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps2.json Test > %t/MyApp2.cmd
51+
52+
// RUN: %target-swift-frontend -target %target-pre-stable-abi-triple -enable-library-evolution \
53+
// RUN: -cache-compile-job -cas-path %t/cas -O \
54+
// RUN: -swift-version 4 -disable-implicit-swift-modules \
55+
// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
56+
// RUN: -module-name Test -explicit-swift-module-map-file @%t/map2.casid \
57+
// RUN: /^tmp/main.swift @%t/MyApp2.cmd -c -o %t/main.o
58+
59+
//--- main.swift
60+
import resilient_struct
61+
62+
public class ClassWithResilientRef {
63+
var first: ResilientRef? = nil
64+
var second: Int = 0
65+
}
66+
67+
//--- layout.yaml
68+
Name: resilient_struct
69+
Decls:
70+
- Name: 16resilient_struct12ResilientRefV
71+
Size: 8
72+
Alignment: 8
73+
ExtraInhabitants: 4096

0 commit comments

Comments
 (0)