Skip to content

Commit 5fa12d3

Browse files
cachemeifyoucancachemeifyoucan
authored
Merge pull request #76738 from cachemeifyoucan/eng/PR-136787368
[CAS] Use IncludeTreeFileList instead of full CASFS for caching
2 parents d7d385d + cd07d53 commit 5fa12d3

27 files changed

+323
-98
lines changed

‎include/swift/AST/ModuleDependencies.h

Lines changed: 28 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -935,17 +935,26 @@ using ModuleDependenciesKindRefMap =
935935
class SwiftDependencyTracker {
936936
public:
937937
SwiftDependencyTracker(llvm::cas::CachingOnDiskFileSystem &FS,
938-
llvm::TreePathPrefixMapper *Mapper)
939-
: FS(FS.createProxyFS()), Mapper(Mapper) {}
938+
llvm::PrefixMapper *Mapper,
939+
const CompilerInvocation &CI);
940940

941-
void startTracking();
942-
void addCommonSearchPathDeps(const CompilerInvocation &CI);
943-
void trackFile(const Twine &path) { (void)FS->status(path); }
941+
void startTracking(bool includeCommonDeps = true);
942+
void trackFile(const Twine &path);
944943
llvm::Expected<llvm::cas::ObjectProxy> createTreeFromDependencies();
945944

946945
private:
947946
llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> FS;
948-
llvm::TreePathPrefixMapper *Mapper;
947+
llvm::PrefixMapper *Mapper;
948+
949+
struct FileEntry {
950+
llvm::cas::ObjectRef FileRef;
951+
size_t Size;
952+
953+
FileEntry(llvm::cas::ObjectRef FileRef, size_t Size)
954+
: FileRef(FileRef), Size(Size) {}
955+
};
956+
llvm::StringMap<FileEntry> CommonFiles;
957+
std::map<std::string, FileEntry> TrackedFiles;
949958
};
950959

951960
// MARK: SwiftDependencyScanningService
@@ -984,7 +993,7 @@ class SwiftDependencyScanningService {
984993
std::shared_ptr<llvm::cas::ObjectStore> CAS;
985994

986995
/// File prefix mapper.
987-
std::unique_ptr<llvm::TreePathPrefixMapper> Mapper;
996+
std::unique_ptr<llvm::PrefixMapper> Mapper;
988997

989998
/// The global file system cache.
990999
std::optional<
@@ -1033,39 +1042,39 @@ class SwiftDependencyScanningService {
10331042
return *SharedFilesystemCache;
10341043
}
10351044

1036-
bool usingCachingFS() const { return !UseClangIncludeTree && (bool)CacheFS; }
1037-
llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem>
1038-
getCachingFS() const {
1039-
return CacheFS;
1040-
}
1041-
10421045
llvm::cas::CachingOnDiskFileSystem &getSharedCachingFS() const {
10431046
assert(CacheFS && "Expect CachingOnDiskFileSystem");
10441047
return *CacheFS;
10451048
}
10461049

1047-
std::optional<SwiftDependencyTracker> createSwiftDependencyTracker() {
1050+
llvm::cas::ObjectStore &getCAS() const {
1051+
assert(CAS && "Expect CAS available");
1052+
return *CAS;
1053+
}
1054+
1055+
std::optional<SwiftDependencyTracker>
1056+
createSwiftDependencyTracker(const CompilerInvocation &CI) {
10481057
if (!CacheFS)
10491058
return std::nullopt;
10501059

1051-
return SwiftDependencyTracker(*CacheFS, Mapper.get());
1060+
return SwiftDependencyTracker(*CacheFS, Mapper.get(), CI);
10521061
}
10531062

10541063
llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> getClangScanningFS() const {
1055-
if (usingCachingFS())
1056-
return CacheFS->createProxyFS();
1057-
10581064
if (UseClangIncludeTree)
10591065
return llvm::cas::createCASProvidingFileSystem(
10601066
CAS, llvm::vfs::createPhysicalFileSystem());
10611067

1068+
if (CacheFS)
1069+
return CacheFS->createProxyFS();
1070+
10621071
return llvm::vfs::createPhysicalFileSystem();
10631072
}
10641073

10651074
bool hasPathMapping() const {
10661075
return Mapper && !Mapper->getMappings().empty();
10671076
}
1068-
llvm::TreePathPrefixMapper *getPrefixMapper() const { return Mapper.get(); }
1077+
llvm::PrefixMapper *getPrefixMapper() const { return Mapper.get(); }
10691078
std::string remapPath(StringRef Path) const {
10701079
if (!Mapper)
10711080
return Path.str();

‎include/swift/AST/ModuleLoader.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@
3535

3636
namespace llvm {
3737
class FileCollectorBase;
38-
class TreePathPrefixMapper;
38+
class PrefixMapper;
3939
namespace vfs {
4040
class OutputBackend;
4141
}
@@ -376,7 +376,7 @@ class ModuleLoader {
376376
const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
377377
clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
378378
InterfaceSubContextDelegate &delegate,
379-
llvm::TreePathPrefixMapper *mapper = nullptr,
379+
llvm::PrefixMapper *mapper = nullptr,
380380
bool isTestableImport = false) = 0;
381381
};
382382

‎include/swift/Basic/CASOptions.h

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,9 @@ class CASOptions final {
4343
/// Clang Include Trees.
4444
std::vector<std::string> ClangIncludeTrees;
4545

46+
/// Clang Include Tree FileList.
47+
std::vector<std::string> ClangIncludeTreeFileList;
48+
4649
/// CacheKey for input file.
4750
std::string InputFileKey;
4851

@@ -60,7 +63,8 @@ class CASOptions final {
6063
bool requireCASFS() const {
6164
return EnableCaching &&
6265
(!CASFSRootIDs.empty() || !ClangIncludeTrees.empty() ||
63-
!InputFileKey.empty() || !BridgingHeaderPCHCacheKey.empty());
66+
!ClangIncludeTreeFileList.empty() || !InputFileKey.empty() ||
67+
!BridgingHeaderPCHCacheKey.empty());
6468
}
6569

6670
/// Return a hash code of any components from these options that should

‎include/swift/ClangImporter/ClangImporter.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -485,7 +485,7 @@ class ClangImporter final : public ClangModuleLoader {
485485
const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
486486
clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
487487
InterfaceSubContextDelegate &delegate,
488-
llvm::TreePathPrefixMapper *mapper,
488+
llvm::PrefixMapper *mapper,
489489
bool isTestableImport = false) override;
490490

491491
void recordBridgingHeaderOptions(

‎include/swift/Frontend/CachingUtils.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,8 @@ std::unique_ptr<llvm::MemoryBuffer> loadCachedCompileResultFromCacheKey(
5656

5757
llvm::Expected<llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>>
5858
createCASFileSystem(llvm::cas::ObjectStore &CAS, ArrayRef<std::string> FSRoots,
59-
ArrayRef<std::string> IncludeTreeRoots);
59+
ArrayRef<std::string> IncludeTreeRoots,
60+
ArrayRef<std::string> IncludeTreeFileList);
6061

6162
std::vector<std::string> remapPathsFromCommandLine(
6263
ArrayRef<std::string> Args,

‎include/swift/Option/FrontendOptions.td

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1386,7 +1386,8 @@ def cas_fs: Separate<["-"], "cas-fs">,
13861386

13871387
def clang_include_tree_root: Separate<["-"], "clang-include-tree-root">,
13881388
HelpText<"Clang Include Tree CASID">, MetaVarName<"<cas-id>">;
1389-
1389+
def clang_include_tree_filelist: Separate<["-"], "clang-include-tree-filelist">,
1390+
HelpText<"Clang Include Tree FileList CASID">, MetaVarName<"<cas-id>">;
13901391

13911392
def experimental_spi_only_imports :
13921393
Flag<["-"], "experimental-spi-only-imports">,

‎include/swift/Sema/SourceLoader.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ class SourceLoader : public ModuleLoader {
103103
const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
104104
clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
105105
InterfaceSubContextDelegate &delegate,
106-
llvm::TreePathPrefixMapper *mapper,
106+
llvm::PrefixMapper *mapper,
107107
bool isTestableImport) override;
108108
};
109109
}

‎include/swift/Serialization/SerializedModuleLoader.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -265,7 +265,7 @@ class SerializedModuleLoaderBase : public ModuleLoader {
265265
const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
266266
clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
267267
InterfaceSubContextDelegate &delegate,
268-
llvm::TreePathPrefixMapper *mapper,
268+
llvm::PrefixMapper *mapper,
269269
bool isTestableImport) override;
270270
};
271271

‎lib/AST/ModuleDependencies.cpp

Lines changed: 76 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
#include "swift/AST/PluginLoader.h"
2323
#include "swift/AST/SourceFile.h"
2424
#include "swift/Frontend/Frontend.h"
25+
#include "clang/CAS/IncludeTree.h"
2526
#include "llvm/CAS/CASProvidingFileSystem.h"
2627
#include "llvm/CAS/CachingOnDiskFileSystem.h"
2728
#include "llvm/Config/config.h"
@@ -597,14 +598,32 @@ swift::dependencies::registerBackDeployLibraries(
597598
#include "swift/Frontend/BackDeploymentLibs.def"
598599
}
599600

600-
void SwiftDependencyTracker::addCommonSearchPathDeps(
601-
const CompilerInvocation &CI) {
601+
SwiftDependencyTracker::SwiftDependencyTracker(
602+
llvm::cas::CachingOnDiskFileSystem &FS, llvm::PrefixMapper *Mapper,
603+
const CompilerInvocation &CI)
604+
: FS(FS.createProxyFS()), Mapper(Mapper) {
602605
auto &SearchPathOpts = CI.getSearchPathOptions();
606+
607+
auto addCommonFile = [&](StringRef path) {
608+
auto file = FS.openFileForRead(path);
609+
if (!file)
610+
return;
611+
auto status = (*file)->status();
612+
if (!status)
613+
return;
614+
auto fileRef = (*file)->getObjectRefForContent();
615+
if (!fileRef)
616+
return;
617+
618+
std::string realPath = Mapper ? Mapper->mapToString(path) : path.str();
619+
CommonFiles.try_emplace(realPath, **fileRef, (size_t)status->getSize());
620+
};
621+
603622
// Add SDKSetting file.
604623
SmallString<256> SDKSettingPath;
605624
llvm::sys::path::append(SDKSettingPath, SearchPathOpts.getSDKPath(),
606625
"SDKSettings.json");
607-
FS->status(SDKSettingPath);
626+
addCommonFile(SDKSettingPath);
608627

609628
// Add Legacy layout file.
610629
const std::vector<std::string> AllSupportedArches = {
@@ -616,32 +635,61 @@ void SwiftDependencyTracker::addCommonSearchPathDeps(
616635
for (auto &Arch : AllSupportedArches) {
617636
SmallString<256> LayoutFile(RuntimeLibPath);
618637
llvm::sys::path::append(LayoutFile, "layouts-" + Arch + ".yaml");
619-
FS->status(LayoutFile);
638+
addCommonFile(LayoutFile);
620639
}
621640
}
622641

623642
// Add VFSOverlay file.
624643
for (auto &Overlay: SearchPathOpts.VFSOverlayFiles)
625-
FS->status(Overlay);
644+
addCommonFile(Overlay);
626645

627646
// Add blocklist file.
628647
for (auto &File: CI.getFrontendOptions().BlocklistConfigFilePaths)
629-
FS->status(File);
648+
addCommonFile(File);
630649
}
631650

632-
void SwiftDependencyTracker::startTracking() {
633-
FS->trackNewAccesses();
651+
void SwiftDependencyTracker::startTracking(bool includeCommonDeps) {
652+
TrackedFiles.clear();
653+
if (includeCommonDeps) {
654+
for (auto &entry : CommonFiles)
655+
TrackedFiles.emplace(entry.first(), entry.second);
656+
}
657+
}
658+
659+
void SwiftDependencyTracker::trackFile(const Twine &path) {
660+
auto file = FS->openFileForRead(path);
661+
if (!file)
662+
return;
663+
auto status = (*file)->status();
664+
if (!status)
665+
return;
666+
auto fileRef = (*file)->getObjectRefForContent();
667+
if (!fileRef)
668+
return;
669+
std::string realPath =
670+
Mapper ? Mapper->mapToString(path.str()) : path.str();
671+
TrackedFiles.try_emplace(realPath, **fileRef, (size_t)status->getSize());
634672
}
635673

636674
llvm::Expected<llvm::cas::ObjectProxy>
637675
SwiftDependencyTracker::createTreeFromDependencies() {
638-
return FS->createTreeFromNewAccesses(
639-
[&](const llvm::vfs::CachedDirectoryEntry &Entry,
640-
llvm::SmallVectorImpl<char> &Storage) {
641-
if (Mapper)
642-
return Mapper->mapDirEntry(Entry, Storage);
643-
return Entry.getTreePath();
644-
});
676+
llvm::SmallVector<clang::cas::IncludeTree::FileList::FileEntry> Files;
677+
for (auto &file : TrackedFiles) {
678+
auto includeTreeFile = clang::cas::IncludeTree::File::create(
679+
FS->getCAS(), file.first, file.second.FileRef);
680+
if (!includeTreeFile)
681+
return includeTreeFile.takeError();
682+
Files.push_back(
683+
{includeTreeFile->getRef(),
684+
(clang::cas::IncludeTree::FileList::FileSizeTy)file.second.Size});
685+
}
686+
687+
auto includeTreeList =
688+
clang::cas::IncludeTree::FileList::create(FS->getCAS(), Files, {});
689+
if (!includeTreeList)
690+
return includeTreeList.takeError();
691+
692+
return *includeTreeList;
645693
}
646694

647695
bool SwiftDependencyScanningService::setupCachingDependencyScanningService(
@@ -675,17 +723,20 @@ bool SwiftDependencyScanningService::setupCachingDependencyScanningService(
675723
CacheFS = std::move(*CachingFS);
676724

677725
// Setup prefix mapping.
678-
Mapper = std::make_unique<llvm::TreePathPrefixMapper>(CacheFS);
679-
SmallVector<llvm::MappedPrefix, 4> Prefixes;
680-
if (auto E = llvm::MappedPrefix::transformJoined(
681-
Instance.getInvocation().getSearchPathOptions().ScannerPrefixMapper,
682-
Prefixes)) {
683-
Instance.getDiags().diagnose(SourceLoc(), diag::error_prefix_mapping,
684-
toString(std::move(E)));
685-
return true;
726+
auto &ScannerPrefixMapper =
727+
Instance.getInvocation().getSearchPathOptions().ScannerPrefixMapper;
728+
if (!ScannerPrefixMapper.empty()) {
729+
Mapper = std::make_unique<llvm::PrefixMapper>();
730+
SmallVector<llvm::MappedPrefix, 4> Prefixes;
731+
if (auto E = llvm::MappedPrefix::transformJoined(ScannerPrefixMapper,
732+
Prefixes)) {
733+
Instance.getDiags().diagnose(SourceLoc(), diag::error_prefix_mapping,
734+
toString(std::move(E)));
735+
return true;
736+
}
737+
Mapper->addRange(Prefixes);
738+
Mapper->sort();
686739
}
687-
Mapper->addRange(Prefixes);
688-
Mapper->sort();
689740

690741
UseClangIncludeTree =
691742
Instance.getInvocation().getClangImporterOptions().UseClangIncludeTree;

‎lib/ClangImporter/ClangModuleDependencyScanner.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -424,7 +424,7 @@ ClangImporter::getModuleDependencies(Identifier moduleName,
424424
const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
425425
clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
426426
InterfaceSubContextDelegate &delegate,
427-
llvm::TreePathPrefixMapper *mapper,
427+
llvm::PrefixMapper *mapper,
428428
bool isTestableImport) {
429429
auto &ctx = Impl.SwiftContext;
430430
// Determine the command-line arguments for dependency scanning.

0 commit comments

Comments
 (0)