Skip to content

Commit ef78b80

Browse files
bnbarhambnbarham
committed
[Index] Prevent re-indexing system modules repeatedly
If a module was first read using the adjacent swiftmodule and then reloaded using the swiftinterface, we would do an up to date check on the adjacent module but write out the unit using the swiftinterface. This would cause the same modules to be indexed repeatedly for the first invocation using a new SDK. On the next run we would instead raad the swiftmodule from the cache and thus the out of date check would match up. The impact of this varies depending on the size of the module graph in the initial compilation and the number of jobs started at the same time. Each SDK dependency is re-indexed *and* reloaded, which is a drain on both CPU and memory. Thus, if many jobs are initially started and they're all going down this path, it can cause the system to run out of memory very quickly. Resolves rdar://103119964. (cherry picked from commit e5b25a6)
1 parent b0980d9 commit ef78b80

18 files changed

+373
-232
lines changed

‎include/swift/AST/FileUnit.h

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -409,6 +409,14 @@ class LoadedFile : public FileUnit {
409409
/// applicable.
410410
virtual StringRef getLoadedFilename() const { return StringRef(); }
411411

412+
/// Returns the path of the file for the module represented by this
413+
/// \c FileUnit, or an empty string if there is none. For modules either
414+
/// built by or adjacent to a module interface, returns the module
415+
/// interface instead.
416+
virtual StringRef getSourceFilename() const {
417+
return getModuleDefiningPath();
418+
}
419+
412420
virtual StringRef getFilenameForPrivateDecl(const ValueDecl *decl) const {
413421
return StringRef();
414422
}

‎include/swift/Frontend/ModuleInterfaceLoader.h

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -137,20 +137,21 @@ class ExplicitSwiftModuleLoader: public SerializedModuleLoaderBase {
137137

138138
bool findModule(ImportPath::Element moduleID,
139139
SmallVectorImpl<char> *moduleInterfacePath,
140+
SmallVectorImpl<char> *moduleInterfaceSourcePath,
140141
std::unique_ptr<llvm::MemoryBuffer> *moduleBuffer,
141142
std::unique_ptr<llvm::MemoryBuffer> *moduleDocBuffer,
142143
std::unique_ptr<llvm::MemoryBuffer> *moduleSourceInfoBuffer,
143144
bool skipBuildingInterface, bool &isFramework,
144145
bool &isSystemModule) override;
145146

146147
std::error_code findModuleFilesInDirectory(
147-
ImportPath::Element ModuleID,
148-
const SerializedModuleBaseName &BaseName,
149-
SmallVectorImpl<char> *ModuleInterfacePath,
150-
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
151-
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
152-
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
153-
bool skipBuildingInterface, bool IsFramework) override;
148+
ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName,
149+
SmallVectorImpl<char> *ModuleInterfacePath,
150+
SmallVectorImpl<char> *ModuleInterfaceSourcePath,
151+
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
152+
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
153+
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
154+
bool skipBuildingInterface, bool IsFramework) override;
154155

155156
bool canImportModule(ImportPath::Module named,
156157
ModuleVersionInfo *versionInfo) override;
@@ -403,13 +404,13 @@ class ModuleInterfaceLoader : public SerializedModuleLoaderBase {
403404
ArrayRef<std::string> PreferInterfaceForModules;
404405

405406
std::error_code findModuleFilesInDirectory(
406-
ImportPath::Element ModuleID,
407-
const SerializedModuleBaseName &BaseName,
408-
SmallVectorImpl<char> *ModuleInterfacePath,
409-
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
410-
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
411-
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
412-
bool skipBuildingInterface, bool IsFramework) override;
407+
ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName,
408+
SmallVectorImpl<char> *ModuleInterfacePath,
409+
SmallVectorImpl<char> *ModuleInterfaceSourcePath,
410+
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
411+
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
412+
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
413+
bool skipBuildingInterface, bool IsFramework) override;
413414

414415
bool isCached(StringRef DepPath) override;
415416
public:

‎include/swift/Serialization/ModuleDependencyScanner.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ namespace swift {
5757
ImportPath::Element ModuleID,
5858
const SerializedModuleBaseName &BaseName,
5959
SmallVectorImpl<char> *ModuleInterfacePath,
60+
SmallVectorImpl<char> *ModuleInterfaceSourcePath,
6061
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
6162
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
6263
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
@@ -118,6 +119,7 @@ namespace swift {
118119
virtual bool
119120
findModule(ImportPath::Element moduleID,
120121
SmallVectorImpl<char> *moduleInterfacePath,
122+
SmallVectorImpl<char> *moduleInterfaceSourcePath,
121123
std::unique_ptr<llvm::MemoryBuffer> *moduleBuffer,
122124
std::unique_ptr<llvm::MemoryBuffer> *moduleDocBuffer,
123125
std::unique_ptr<llvm::MemoryBuffer> *moduleSourceInfoBuffer,

‎include/swift/Serialization/SerializedModuleLoader.h

Lines changed: 22 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,12 @@ struct SerializedModuleBaseName {
4949

5050
/// Gets the filename with a particular extension appended to it.
5151
std::string getName(file_types::ID fileTy) const;
52+
53+
/// If the interface with \p baseName exists, returns its path (which may be
54+
/// the private interface if there is one). Return an empty optional
55+
/// otherwise.
56+
llvm::Optional<std::string>
57+
findInterfacePath(llvm::vfs::FileSystem &fs) const;
5258
};
5359

5460
/// Common functionality shared between \c ImplicitSerializedModuleLoader,
@@ -72,12 +78,13 @@ class SerializedModuleLoaderBase : public ModuleLoader {
7278
void collectVisibleTopLevelModuleNamesImpl(SmallVectorImpl<Identifier> &names,
7379
StringRef extension) const;
7480

75-
virtual bool findModule(ImportPath::Element moduleID,
76-
SmallVectorImpl<char> *moduleInterfacePath,
77-
std::unique_ptr<llvm::MemoryBuffer> *moduleBuffer,
78-
std::unique_ptr<llvm::MemoryBuffer> *moduleDocBuffer,
79-
std::unique_ptr<llvm::MemoryBuffer> *moduleSourceInfoBuffer,
80-
bool skipBuildingInterface, bool &isFramework, bool &isSystemModule);
81+
virtual bool findModule(
82+
ImportPath::Element moduleID, SmallVectorImpl<char> *moduleInterfacePath,
83+
SmallVectorImpl<char> *moduleInterfaceSourcePath,
84+
std::unique_ptr<llvm::MemoryBuffer> *moduleBuffer,
85+
std::unique_ptr<llvm::MemoryBuffer> *moduleDocBuffer,
86+
std::unique_ptr<llvm::MemoryBuffer> *moduleSourceInfoBuffer,
87+
bool skipBuildingInterface, bool &isFramework, bool &isSystemModule);
8188

8289
/// Attempts to search the provided directory for a loadable serialized
8390
/// .swiftmodule with the provided `ModuleFilename`. Subclasses must
@@ -92,9 +99,9 @@ class SerializedModuleLoaderBase : public ModuleLoader {
9299
/// modules and will defer to the remaining module loaders to look up this
93100
/// module.
94101
virtual std::error_code findModuleFilesInDirectory(
95-
ImportPath::Element ModuleID,
96-
const SerializedModuleBaseName &BaseName,
102+
ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName,
97103
SmallVectorImpl<char> *ModuleInterfacePath,
104+
SmallVectorImpl<char> *ModuleInterfaceSourcePath,
98105
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
99106
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
100107
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
@@ -155,7 +162,7 @@ class SerializedModuleLoaderBase : public ModuleLoader {
155162
/// printed. (Note that \p diagLoc is allowed to be invalid.)
156163
LoadedFile *
157164
loadAST(ModuleDecl &M, Optional<SourceLoc> diagLoc,
158-
StringRef moduleInterfacePath,
165+
StringRef moduleInterfacePath, StringRef moduleInterfaceSourcePath,
159166
std::unique_ptr<llvm::MemoryBuffer> moduleInputBuffer,
160167
std::unique_ptr<llvm::MemoryBuffer> moduleDocInputBuffer,
161168
std::unique_ptr<llvm::MemoryBuffer> moduleSourceInfoInputBuffer,
@@ -219,9 +226,9 @@ class ImplicitSerializedModuleLoader : public SerializedModuleLoaderBase {
219226
{}
220227

221228
std::error_code findModuleFilesInDirectory(
222-
ImportPath::Element ModuleID,
223-
const SerializedModuleBaseName &BaseName,
229+
ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName,
224230
SmallVectorImpl<char> *ModuleInterfacePath,
231+
SmallVectorImpl<char> *ModuleInterfaceSourcePath,
225232
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
226233
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
227234
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
@@ -273,9 +280,9 @@ class MemoryBufferSerializedModuleLoader : public SerializedModuleLoaderBase {
273280
BypassResilience(BypassResilience) {}
274281

275282
std::error_code findModuleFilesInDirectory(
276-
ImportPath::Element ModuleID,
277-
const SerializedModuleBaseName &BaseName,
283+
ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName,
278284
SmallVectorImpl<char> *ModuleInterfacePath,
285+
SmallVectorImpl<char> *ModuleInterfaceSourcePath,
279286
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
280287
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
281288
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
@@ -453,6 +460,8 @@ class SerializedASTFile final : public LoadedFile {
453460

454461
virtual StringRef getLoadedFilename() const override;
455462

463+
virtual StringRef getSourceFilename() const override;
464+
456465
virtual StringRef getModuleDefiningPath() const override;
457466

458467
ValueDecl *getMainDecl() const override;

‎lib/AST/Module.cpp

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1989,11 +1989,9 @@ StringRef ModuleDecl::getModuleFilename() const {
19891989

19901990
StringRef ModuleDecl::getModuleSourceFilename() const {
19911991
for (auto F : getFiles()) {
1992-
if (auto *SFU = dyn_cast<SynthesizedFileUnit>(F))
1993-
continue;
1994-
return F->getModuleDefiningPath();
1992+
if (auto LF = dyn_cast<LoadedFile>(F))
1993+
return LF->getSourceFilename();
19951994
}
1996-
19971995
return StringRef();
19981996
}
19991997

‎lib/Frontend/Frontend.cpp

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1196,11 +1196,11 @@ bool CompilerInstance::loadPartialModulesAndImplicitImports(
11961196
bool hadLoadError = false;
11971197
for (auto &PM : PartialModules) {
11981198
assert(PM.ModuleBuffer);
1199-
auto *file =
1200-
DefaultSerializedLoader->loadAST(*mod, /*diagLoc*/ SourceLoc(), /*moduleInterfacePath*/ "",
1201-
std::move(PM.ModuleBuffer), std::move(PM.ModuleDocBuffer),
1202-
std::move(PM.ModuleSourceInfoBuffer),
1203-
/*isFramework*/ false);
1199+
auto *file = DefaultSerializedLoader->loadAST(
1200+
*mod, /*diagLoc=*/SourceLoc(), /*moduleInterfacePath*/ "",
1201+
/*moduleInterfaceSourcePath=*/"", std::move(PM.ModuleBuffer),
1202+
std::move(PM.ModuleDocBuffer), std::move(PM.ModuleSourceInfoBuffer),
1203+
/*isFramework*/ false);
12041204
if (file) {
12051205
partialModules.push_back(file);
12061206
} else {

‎lib/Frontend/ModuleInterfaceLoader.cpp

Lines changed: 31 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
#include "swift/Frontend/ModuleInterfaceSupport.h"
2525
#include "swift/Parse/ParseVersion.h"
2626
#include "swift/Serialization/SerializationOptions.h"
27+
#include "swift/Serialization/SerializedModuleLoader.h"
2728
#include "swift/Serialization/Validation.h"
2829
#include "swift/Strings.h"
2930
#include "clang/Basic/Module.h"
@@ -1106,47 +1107,43 @@ bool ModuleInterfaceLoader::isCached(StringRef DepPath) {
11061107
/// cache or by converting it in a subordinate \c CompilerInstance, caching
11071108
/// the results.
11081109
std::error_code ModuleInterfaceLoader::findModuleFilesInDirectory(
1109-
ImportPath::Element ModuleID,
1110-
const SerializedModuleBaseName &BaseName,
1111-
SmallVectorImpl<char> *ModuleInterfacePath,
1112-
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
1113-
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
1114-
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
1115-
bool skipBuildingInterface, bool IsFramework) {
1110+
ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName,
1111+
SmallVectorImpl<char> *ModuleInterfacePath,
1112+
SmallVectorImpl<char> *ModuleInterfaceSourcePath,
1113+
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
1114+
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
1115+
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
1116+
bool skipBuildingInterface, bool IsFramework) {
11161117

11171118
// If running in OnlySerialized mode, ModuleInterfaceLoader
11181119
// should not have been constructed at all.
11191120
assert(LoadMode != ModuleLoadingMode::OnlySerialized);
11201121

1121-
llvm::SmallString<256>
1122-
ModPath{ BaseName.getName(file_types::TY_SwiftModuleFile) },
1123-
InPath{ BaseName.getName(file_types::TY_SwiftModuleInterfaceFile) },
1124-
PrivateInPath{BaseName.getName(file_types::TY_PrivateSwiftModuleInterfaceFile)};
1122+
std::string ModPath{BaseName.getName(file_types::TY_SwiftModuleFile)};
11251123

11261124
// First check to see if the .swiftinterface exists at all. Bail if not.
11271125
auto &fs = *Ctx.SourceMgr.getFileSystem();
1128-
if (!fs.exists(InPath)) {
1126+
std::string InPath = BaseName.findInterfacePath(fs).getValueOr("");
1127+
if (InPath.empty()) {
11291128
if (fs.exists(ModPath)) {
11301129
LLVM_DEBUG(llvm::dbgs()
1131-
<< "No .swiftinterface file found adjacent to module file "
1132-
<< ModPath.str() << "\n");
1130+
<< "No .swiftinterface file found adjacent to module file "
1131+
<< ModPath << "\n");
11331132
return std::make_error_code(std::errc::not_supported);
11341133
}
11351134
return std::make_error_code(std::errc::no_such_file_or_directory);
11361135
}
11371136

1138-
// If present, use the private interface instead of the public one.
1139-
if (fs.exists(PrivateInPath)) {
1140-
InPath = PrivateInPath;
1141-
}
1137+
if (ModuleInterfaceSourcePath)
1138+
ModuleInterfaceSourcePath->assign(InPath.begin(), InPath.end());
11421139

11431140
// If we've been told to skip building interfaces, we are done here and do
11441141
// not need to have the module actually built. For example, if we are
11451142
// currently answering a `canImport` query, it is enough to have found
11461143
// the interface.
11471144
if (skipBuildingInterface) {
11481145
if (ModuleInterfacePath)
1149-
*ModuleInterfacePath = InPath;
1146+
ModuleInterfacePath->assign(InPath.begin(), InPath.end());
11501147
return std::error_code();
11511148
}
11521149

@@ -1170,7 +1167,7 @@ std::error_code ModuleInterfaceLoader::findModuleFilesInDirectory(
11701167
if (ModuleBuffer) {
11711168
*ModuleBuffer = std::move(*ModuleBufferOrErr);
11721169
if (ModuleInterfacePath)
1173-
*ModuleInterfacePath = InPath;
1170+
ModuleInterfacePath->assign(InPath.begin(), InPath.end());
11741171
}
11751172

11761173
// Open .swiftsourceinfo file if it's present.
@@ -1888,12 +1885,13 @@ ExplicitSwiftModuleLoader::ExplicitSwiftModuleLoader(
18881885

18891886
ExplicitSwiftModuleLoader::~ExplicitSwiftModuleLoader() { delete &Impl; }
18901887

1891-
bool ExplicitSwiftModuleLoader::findModule(ImportPath::Element ModuleID,
1892-
SmallVectorImpl<char> *ModuleInterfacePath,
1893-
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
1894-
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
1895-
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
1896-
bool skipBuildingInterface, bool &IsFramework, bool &IsSystemModule) {
1888+
bool ExplicitSwiftModuleLoader::findModule(
1889+
ImportPath::Element ModuleID, SmallVectorImpl<char> *ModuleInterfacePath,
1890+
SmallVectorImpl<char> *ModuleInterfaceSourcePath,
1891+
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
1892+
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
1893+
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
1894+
bool skipBuildingInterface, bool &IsFramework, bool &IsSystemModule) {
18971895
// Find a module with an actual, physical name on disk, in case
18981896
// -module-alias is used (otherwise same).
18991897
//
@@ -1965,13 +1963,13 @@ bool ExplicitSwiftModuleLoader::findModule(ImportPath::Element ModuleID,
19651963
}
19661964

19671965
std::error_code ExplicitSwiftModuleLoader::findModuleFilesInDirectory(
1968-
ImportPath::Element ModuleID,
1969-
const SerializedModuleBaseName &BaseName,
1970-
SmallVectorImpl<char> *ModuleInterfacePath,
1971-
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
1972-
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
1973-
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
1974-
bool skipBuildingInterface, bool IsFramework) {
1966+
ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName,
1967+
SmallVectorImpl<char> *ModuleInterfacePath,
1968+
SmallVectorImpl<char> *ModuleInterfaceSourcePath,
1969+
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
1970+
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
1971+
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
1972+
bool skipBuildingInterface, bool IsFramework) {
19751973
llvm_unreachable("Not supported in the Explicit Swift Module Loader.");
19761974
return std::make_error_code(std::errc::not_supported);
19771975
}

0 commit comments

Comments
 (0)