Skip to content

Commit cd07d53

Browse files
cachemeifyoucancachemeifyoucan
committed
[CAS] Use IncludeTreeFileList instead of full CASFS for caching
Use IncludeTreeFileList instead of full feature CASFS for swift dependency filesystem. This allows smaller CAS based VFS that is smaller and faster. This is enabled by the CAS enabled compilation does not need to iterate file system. rdar://136787368
1 parent 353a1c2 commit cd07d53

27 files changed

+323
-98
lines changed

‎include/swift/AST/ModuleDependencies.h

Lines changed: 28 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -935,17 +935,26 @@ using ModuleDependenciesKindRefMap =
935935
class SwiftDependencyTracker {
936936
public:
937937
SwiftDependencyTracker(llvm::cas::CachingOnDiskFileSystem &FS,
938-
llvm::TreePathPrefixMapper *Mapper)
939-
: FS(FS.createProxyFS()), Mapper(Mapper) {}
938+
llvm::PrefixMapper *Mapper,
939+
const CompilerInvocation &CI);
940940

941-
void startTracking();
942-
void addCommonSearchPathDeps(const CompilerInvocation &CI);
943-
void trackFile(const Twine &path) { (void)FS->status(path); }
941+
void startTracking(bool includeCommonDeps = true);
942+
void trackFile(const Twine &path);
944943
llvm::Expected<llvm::cas::ObjectProxy> createTreeFromDependencies();
945944

946945
private:
947946
llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> FS;
948-
llvm::TreePathPrefixMapper *Mapper;
947+
llvm::PrefixMapper *Mapper;
948+
949+
struct FileEntry {
950+
llvm::cas::ObjectRef FileRef;
951+
size_t Size;
952+
953+
FileEntry(llvm::cas::ObjectRef FileRef, size_t Size)
954+
: FileRef(FileRef), Size(Size) {}
955+
};
956+
llvm::StringMap<FileEntry> CommonFiles;
957+
std::map<std::string, FileEntry> TrackedFiles;
949958
};
950959

951960
// MARK: SwiftDependencyScanningService
@@ -984,7 +993,7 @@ class SwiftDependencyScanningService {
984993
std::shared_ptr<llvm::cas::ObjectStore> CAS;
985994

986995
/// File prefix mapper.
987-
std::unique_ptr<llvm::TreePathPrefixMapper> Mapper;
996+
std::unique_ptr<llvm::PrefixMapper> Mapper;
988997

989998
/// The global file system cache.
990999
std::optional<
@@ -1033,39 +1042,39 @@ class SwiftDependencyScanningService {
10331042
return *SharedFilesystemCache;
10341043
}
10351044

1036-
bool usingCachingFS() const { return !UseClangIncludeTree && (bool)CacheFS; }
1037-
llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem>
1038-
getCachingFS() const {
1039-
return CacheFS;
1040-
}
1041-
10421045
llvm::cas::CachingOnDiskFileSystem &getSharedCachingFS() const {
10431046
assert(CacheFS && "Expect CachingOnDiskFileSystem");
10441047
return *CacheFS;
10451048
}
10461049

1047-
std::optional<SwiftDependencyTracker> createSwiftDependencyTracker() {
1050+
llvm::cas::ObjectStore &getCAS() const {
1051+
assert(CAS && "Expect CAS available");
1052+
return *CAS;
1053+
}
1054+
1055+
std::optional<SwiftDependencyTracker>
1056+
createSwiftDependencyTracker(const CompilerInvocation &CI) {
10481057
if (!CacheFS)
10491058
return std::nullopt;
10501059

1051-
return SwiftDependencyTracker(*CacheFS, Mapper.get());
1060+
return SwiftDependencyTracker(*CacheFS, Mapper.get(), CI);
10521061
}
10531062

10541063
llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> getClangScanningFS() const {
1055-
if (usingCachingFS())
1056-
return CacheFS->createProxyFS();
1057-
10581064
if (UseClangIncludeTree)
10591065
return llvm::cas::createCASProvidingFileSystem(
10601066
CAS, llvm::vfs::createPhysicalFileSystem());
10611067

1068+
if (CacheFS)
1069+
return CacheFS->createProxyFS();
1070+
10621071
return llvm::vfs::createPhysicalFileSystem();
10631072
}
10641073

10651074
bool hasPathMapping() const {
10661075
return Mapper && !Mapper->getMappings().empty();
10671076
}
1068-
llvm::TreePathPrefixMapper *getPrefixMapper() const { return Mapper.get(); }
1077+
llvm::PrefixMapper *getPrefixMapper() const { return Mapper.get(); }
10691078
std::string remapPath(StringRef Path) const {
10701079
if (!Mapper)
10711080
return Path.str();

‎include/swift/AST/ModuleLoader.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@
3535

3636
namespace llvm {
3737
class FileCollectorBase;
38-
class TreePathPrefixMapper;
38+
class PrefixMapper;
3939
namespace vfs {
4040
class OutputBackend;
4141
}
@@ -376,7 +376,7 @@ class ModuleLoader {
376376
const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
377377
clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
378378
InterfaceSubContextDelegate &delegate,
379-
llvm::TreePathPrefixMapper *mapper = nullptr,
379+
llvm::PrefixMapper *mapper = nullptr,
380380
bool isTestableImport = false) = 0;
381381
};
382382

‎include/swift/Basic/CASOptions.h

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,9 @@ class CASOptions final {
4343
/// Clang Include Trees.
4444
std::vector<std::string> ClangIncludeTrees;
4545

46+
/// Clang Include Tree FileList.
47+
std::vector<std::string> ClangIncludeTreeFileList;
48+
4649
/// CacheKey for input file.
4750
std::string InputFileKey;
4851

@@ -60,7 +63,8 @@ class CASOptions final {
6063
bool requireCASFS() const {
6164
return EnableCaching &&
6265
(!CASFSRootIDs.empty() || !ClangIncludeTrees.empty() ||
63-
!InputFileKey.empty() || !BridgingHeaderPCHCacheKey.empty());
66+
!ClangIncludeTreeFileList.empty() || !InputFileKey.empty() ||
67+
!BridgingHeaderPCHCacheKey.empty());
6468
}
6569

6670
/// Return a hash code of any components from these options that should

‎include/swift/ClangImporter/ClangImporter.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -484,7 +484,7 @@ class ClangImporter final : public ClangModuleLoader {
484484
const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
485485
clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
486486
InterfaceSubContextDelegate &delegate,
487-
llvm::TreePathPrefixMapper *mapper,
487+
llvm::PrefixMapper *mapper,
488488
bool isTestableImport = false) override;
489489

490490
void recordBridgingHeaderOptions(

‎include/swift/Frontend/CachingUtils.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,8 @@ std::unique_ptr<llvm::MemoryBuffer> loadCachedCompileResultFromCacheKey(
5454

5555
llvm::Expected<llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>>
5656
createCASFileSystem(llvm::cas::ObjectStore &CAS, ArrayRef<std::string> FSRoots,
57-
ArrayRef<std::string> IncludeTreeRoots);
57+
ArrayRef<std::string> IncludeTreeRoots,
58+
ArrayRef<std::string> IncludeTreeFileList);
5859

5960
std::vector<std::string> remapPathsFromCommandLine(
6061
ArrayRef<std::string> Args,

‎include/swift/Option/FrontendOptions.td

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1386,7 +1386,8 @@ def cas_fs: Separate<["-"], "cas-fs">,
13861386

13871387
def clang_include_tree_root: Separate<["-"], "clang-include-tree-root">,
13881388
HelpText<"Clang Include Tree CASID">, MetaVarName<"<cas-id>">;
1389-
1389+
def clang_include_tree_filelist: Separate<["-"], "clang-include-tree-filelist">,
1390+
HelpText<"Clang Include Tree FileList CASID">, MetaVarName<"<cas-id>">;
13901391

13911392
def experimental_spi_only_imports :
13921393
Flag<["-"], "experimental-spi-only-imports">,

‎include/swift/Sema/SourceLoader.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ class SourceLoader : public ModuleLoader {
103103
const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
104104
clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
105105
InterfaceSubContextDelegate &delegate,
106-
llvm::TreePathPrefixMapper *mapper,
106+
llvm::PrefixMapper *mapper,
107107
bool isTestableImport) override;
108108
};
109109
}

‎include/swift/Serialization/SerializedModuleLoader.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -265,7 +265,7 @@ class SerializedModuleLoaderBase : public ModuleLoader {
265265
const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
266266
clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
267267
InterfaceSubContextDelegate &delegate,
268-
llvm::TreePathPrefixMapper *mapper,
268+
llvm::PrefixMapper *mapper,
269269
bool isTestableImport) override;
270270
};
271271

‎lib/AST/ModuleDependencies.cpp

Lines changed: 76 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
#include "swift/AST/PluginLoader.h"
2323
#include "swift/AST/SourceFile.h"
2424
#include "swift/Frontend/Frontend.h"
25+
#include "clang/CAS/IncludeTree.h"
2526
#include "llvm/CAS/CASProvidingFileSystem.h"
2627
#include "llvm/CAS/CachingOnDiskFileSystem.h"
2728
#include "llvm/Config/config.h"
@@ -597,14 +598,32 @@ swift::dependencies::registerBackDeployLibraries(
597598
#include "swift/Frontend/BackDeploymentLibs.def"
598599
}
599600

600-
void SwiftDependencyTracker::addCommonSearchPathDeps(
601-
const CompilerInvocation &CI) {
601+
SwiftDependencyTracker::SwiftDependencyTracker(
602+
llvm::cas::CachingOnDiskFileSystem &FS, llvm::PrefixMapper *Mapper,
603+
const CompilerInvocation &CI)
604+
: FS(FS.createProxyFS()), Mapper(Mapper) {
602605
auto &SearchPathOpts = CI.getSearchPathOptions();
606+
607+
auto addCommonFile = [&](StringRef path) {
608+
auto file = FS.openFileForRead(path);
609+
if (!file)
610+
return;
611+
auto status = (*file)->status();
612+
if (!status)
613+
return;
614+
auto fileRef = (*file)->getObjectRefForContent();
615+
if (!fileRef)
616+
return;
617+
618+
std::string realPath = Mapper ? Mapper->mapToString(path) : path.str();
619+
CommonFiles.try_emplace(realPath, **fileRef, (size_t)status->getSize());
620+
};
621+
603622
// Add SDKSetting file.
604623
SmallString<256> SDKSettingPath;
605624
llvm::sys::path::append(SDKSettingPath, SearchPathOpts.getSDKPath(),
606625
"SDKSettings.json");
607-
FS->status(SDKSettingPath);
626+
addCommonFile(SDKSettingPath);
608627

609628
// Add Legacy layout file.
610629
const std::vector<std::string> AllSupportedArches = {
@@ -616,32 +635,61 @@ void SwiftDependencyTracker::addCommonSearchPathDeps(
616635
for (auto &Arch : AllSupportedArches) {
617636
SmallString<256> LayoutFile(RuntimeLibPath);
618637
llvm::sys::path::append(LayoutFile, "layouts-" + Arch + ".yaml");
619-
FS->status(LayoutFile);
638+
addCommonFile(LayoutFile);
620639
}
621640
}
622641

623642
// Add VFSOverlay file.
624643
for (auto &Overlay: SearchPathOpts.VFSOverlayFiles)
625-
FS->status(Overlay);
644+
addCommonFile(Overlay);
626645

627646
// Add blocklist file.
628647
for (auto &File: CI.getFrontendOptions().BlocklistConfigFilePaths)
629-
FS->status(File);
648+
addCommonFile(File);
630649
}
631650

632-
void SwiftDependencyTracker::startTracking() {
633-
FS->trackNewAccesses();
651+
void SwiftDependencyTracker::startTracking(bool includeCommonDeps) {
652+
TrackedFiles.clear();
653+
if (includeCommonDeps) {
654+
for (auto &entry : CommonFiles)
655+
TrackedFiles.emplace(entry.first(), entry.second);
656+
}
657+
}
658+
659+
void SwiftDependencyTracker::trackFile(const Twine &path) {
660+
auto file = FS->openFileForRead(path);
661+
if (!file)
662+
return;
663+
auto status = (*file)->status();
664+
if (!status)
665+
return;
666+
auto fileRef = (*file)->getObjectRefForContent();
667+
if (!fileRef)
668+
return;
669+
std::string realPath =
670+
Mapper ? Mapper->mapToString(path.str()) : path.str();
671+
TrackedFiles.try_emplace(realPath, **fileRef, (size_t)status->getSize());
634672
}
635673

636674
llvm::Expected<llvm::cas::ObjectProxy>
637675
SwiftDependencyTracker::createTreeFromDependencies() {
638-
return FS->createTreeFromNewAccesses(
639-
[&](const llvm::vfs::CachedDirectoryEntry &Entry,
640-
llvm::SmallVectorImpl<char> &Storage) {
641-
if (Mapper)
642-
return Mapper->mapDirEntry(Entry, Storage);
643-
return Entry.getTreePath();
644-
});
676+
llvm::SmallVector<clang::cas::IncludeTree::FileList::FileEntry> Files;
677+
for (auto &file : TrackedFiles) {
678+
auto includeTreeFile = clang::cas::IncludeTree::File::create(
679+
FS->getCAS(), file.first, file.second.FileRef);
680+
if (!includeTreeFile)
681+
return includeTreeFile.takeError();
682+
Files.push_back(
683+
{includeTreeFile->getRef(),
684+
(clang::cas::IncludeTree::FileList::FileSizeTy)file.second.Size});
685+
}
686+
687+
auto includeTreeList =
688+
clang::cas::IncludeTree::FileList::create(FS->getCAS(), Files, {});
689+
if (!includeTreeList)
690+
return includeTreeList.takeError();
691+
692+
return *includeTreeList;
645693
}
646694

647695
bool SwiftDependencyScanningService::setupCachingDependencyScanningService(
@@ -675,17 +723,20 @@ bool SwiftDependencyScanningService::setupCachingDependencyScanningService(
675723
CacheFS = std::move(*CachingFS);
676724

677725
// Setup prefix mapping.
678-
Mapper = std::make_unique<llvm::TreePathPrefixMapper>(CacheFS);
679-
SmallVector<llvm::MappedPrefix, 4> Prefixes;
680-
if (auto E = llvm::MappedPrefix::transformJoined(
681-
Instance.getInvocation().getSearchPathOptions().ScannerPrefixMapper,
682-
Prefixes)) {
683-
Instance.getDiags().diagnose(SourceLoc(), diag::error_prefix_mapping,
684-
toString(std::move(E)));
685-
return true;
726+
auto &ScannerPrefixMapper =
727+
Instance.getInvocation().getSearchPathOptions().ScannerPrefixMapper;
728+
if (!ScannerPrefixMapper.empty()) {
729+
Mapper = std::make_unique<llvm::PrefixMapper>();
730+
SmallVector<llvm::MappedPrefix, 4> Prefixes;
731+
if (auto E = llvm::MappedPrefix::transformJoined(ScannerPrefixMapper,
732+
Prefixes)) {
733+
Instance.getDiags().diagnose(SourceLoc(), diag::error_prefix_mapping,
734+
toString(std::move(E)));
735+
return true;
736+
}
737+
Mapper->addRange(Prefixes);
738+
Mapper->sort();
686739
}
687-
Mapper->addRange(Prefixes);
688-
Mapper->sort();
689740

690741
UseClangIncludeTree =
691742
Instance.getInvocation().getClangImporterOptions().UseClangIncludeTree;

‎lib/ClangImporter/ClangModuleDependencyScanner.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -424,7 +424,7 @@ ClangImporter::getModuleDependencies(Identifier moduleName,
424424
const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
425425
clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
426426
InterfaceSubContextDelegate &delegate,
427-
llvm::TreePathPrefixMapper *mapper,
427+
llvm::PrefixMapper *mapper,
428428
bool isTestableImport) {
429429
auto &ctx = Impl.SwiftContext;
430430
// Determine the command-line arguments for dependency scanning.

0 commit comments

Comments
 (0)