Skip to content

Commit 6269643

Browse files
bnbarhambnbarham
committed
[Index] Prevent re-indexing system modules repeatedly
If a module was first read using the adjacent swiftmodule and then reloaded using the swiftinterface, we would do an up to date check on the adjacent module but write out the unit using the swiftinterface. This would cause the same modules to be indexed repeatedly for the first invocation using a new SDK. On the next run we would instead raad the swiftmodule from the cache and thus the out of date check would match up. The impact of this varies depending on the size of the module graph in the initial compilation and the number of jobs started at the same time. Each SDK dependency is re-indexed *and* reloaded, which is a drain on both CPU and memory. Thus, if many jobs are initially started and they're all going down this path, it can cause the system to run out of memory very quickly. Resolves rdar://103119964.
1 parent 9401926 commit 6269643

18 files changed

+389
-247
lines changed

‎include/swift/AST/FileUnit.h

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -412,6 +412,14 @@ class LoadedFile : public FileUnit {
412412
/// applicable.
413413
virtual StringRef getLoadedFilename() const { return StringRef(); }
414414

415+
/// Returns the path of the file for the module represented by this
416+
/// \c FileUnit, or an empty string if there is none. For modules either
417+
/// built by or adjacent to a module interface, returns the module
418+
/// interface instead.
419+
virtual StringRef getSourceFilename() const {
420+
return getModuleDefiningPath();
421+
}
422+
415423
virtual StringRef getFilenameForPrivateDecl(const ValueDecl *decl) const {
416424
return StringRef();
417425
}

‎include/swift/Frontend/ModuleInterfaceLoader.h

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -138,20 +138,21 @@ class ExplicitSwiftModuleLoader: public SerializedModuleLoaderBase {
138138

139139
bool findModule(ImportPath::Element moduleID,
140140
SmallVectorImpl<char> *moduleInterfacePath,
141+
SmallVectorImpl<char> *moduleInterfaceSourcePath,
141142
std::unique_ptr<llvm::MemoryBuffer> *moduleBuffer,
142143
std::unique_ptr<llvm::MemoryBuffer> *moduleDocBuffer,
143144
std::unique_ptr<llvm::MemoryBuffer> *moduleSourceInfoBuffer,
144145
bool skipBuildingInterface, bool &isFramework,
145146
bool &isSystemModule) override;
146147

147148
std::error_code findModuleFilesInDirectory(
148-
ImportPath::Element ModuleID,
149-
const SerializedModuleBaseName &BaseName,
150-
SmallVectorImpl<char> *ModuleInterfacePath,
151-
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
152-
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
153-
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
154-
bool skipBuildingInterface, bool IsFramework) override;
149+
ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName,
150+
SmallVectorImpl<char> *ModuleInterfacePath,
151+
SmallVectorImpl<char> *ModuleInterfaceSourcePath,
152+
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
153+
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
154+
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
155+
bool skipBuildingInterface, bool IsFramework) override;
155156

156157
bool canImportModule(ImportPath::Module named,
157158
ModuleVersionInfo *versionInfo) override;
@@ -418,13 +419,13 @@ class ModuleInterfaceLoader : public SerializedModuleLoaderBase {
418419
ArrayRef<std::string> PreferInterfaceForModules;
419420

420421
std::error_code findModuleFilesInDirectory(
421-
ImportPath::Element ModuleID,
422-
const SerializedModuleBaseName &BaseName,
423-
SmallVectorImpl<char> *ModuleInterfacePath,
424-
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
425-
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
426-
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
427-
bool skipBuildingInterface, bool IsFramework) override;
422+
ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName,
423+
SmallVectorImpl<char> *ModuleInterfacePath,
424+
SmallVectorImpl<char> *ModuleInterfaceSourcePath,
425+
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
426+
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
427+
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
428+
bool skipBuildingInterface, bool IsFramework) override;
428429

429430
bool isCached(StringRef DepPath) override;
430431
public:

‎include/swift/Serialization/ModuleDependencyScanner.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ namespace swift {
5757
ImportPath::Element ModuleID,
5858
const SerializedModuleBaseName &BaseName,
5959
SmallVectorImpl<char> *ModuleInterfacePath,
60+
SmallVectorImpl<char> *ModuleInterfaceSourcePath,
6061
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
6162
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
6263
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
@@ -118,6 +119,7 @@ namespace swift {
118119
virtual bool
119120
findModule(ImportPath::Element moduleID,
120121
SmallVectorImpl<char> *moduleInterfacePath,
122+
SmallVectorImpl<char> *moduleInterfaceSourcePath,
121123
std::unique_ptr<llvm::MemoryBuffer> *moduleBuffer,
122124
std::unique_ptr<llvm::MemoryBuffer> *moduleDocBuffer,
123125
std::unique_ptr<llvm::MemoryBuffer> *moduleSourceInfoBuffer,

‎include/swift/Serialization/SerializedModuleLoader.h

Lines changed: 22 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,12 @@ struct SerializedModuleBaseName {
4949

5050
/// Gets the filename with a particular extension appended to it.
5151
std::string getName(file_types::ID fileTy) const;
52+
53+
/// If the interface with \p baseName exists, returns its path (which may be
54+
/// the private interface if there is one). Return an empty optional
55+
/// otherwise.
56+
llvm::Optional<std::string>
57+
findInterfacePath(llvm::vfs::FileSystem &fs) const;
5258
};
5359

5460
/// Common functionality shared between \c ImplicitSerializedModuleLoader,
@@ -72,12 +78,13 @@ class SerializedModuleLoaderBase : public ModuleLoader {
7278
void collectVisibleTopLevelModuleNamesImpl(SmallVectorImpl<Identifier> &names,
7379
StringRef extension) const;
7480

75-
virtual bool findModule(ImportPath::Element moduleID,
76-
SmallVectorImpl<char> *moduleInterfacePath,
77-
std::unique_ptr<llvm::MemoryBuffer> *moduleBuffer,
78-
std::unique_ptr<llvm::MemoryBuffer> *moduleDocBuffer,
79-
std::unique_ptr<llvm::MemoryBuffer> *moduleSourceInfoBuffer,
80-
bool skipBuildingInterface, bool &isFramework, bool &isSystemModule);
81+
virtual bool findModule(
82+
ImportPath::Element moduleID, SmallVectorImpl<char> *moduleInterfacePath,
83+
SmallVectorImpl<char> *moduleInterfaceSourcePath,
84+
std::unique_ptr<llvm::MemoryBuffer> *moduleBuffer,
85+
std::unique_ptr<llvm::MemoryBuffer> *moduleDocBuffer,
86+
std::unique_ptr<llvm::MemoryBuffer> *moduleSourceInfoBuffer,
87+
bool skipBuildingInterface, bool &isFramework, bool &isSystemModule);
8188

8289
/// Attempts to search the provided directory for a loadable serialized
8390
/// .swiftmodule with the provided `ModuleFilename`. Subclasses must
@@ -92,9 +99,9 @@ class SerializedModuleLoaderBase : public ModuleLoader {
9299
/// modules and will defer to the remaining module loaders to look up this
93100
/// module.
94101
virtual std::error_code findModuleFilesInDirectory(
95-
ImportPath::Element ModuleID,
96-
const SerializedModuleBaseName &BaseName,
102+
ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName,
97103
SmallVectorImpl<char> *ModuleInterfacePath,
104+
SmallVectorImpl<char> *ModuleInterfaceSourcePath,
98105
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
99106
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
100107
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
@@ -155,7 +162,7 @@ class SerializedModuleLoaderBase : public ModuleLoader {
155162
/// printed. (Note that \p diagLoc is allowed to be invalid.)
156163
LoadedFile *
157164
loadAST(ModuleDecl &M, Optional<SourceLoc> diagLoc,
158-
StringRef moduleInterfacePath,
165+
StringRef moduleInterfacePath, StringRef moduleInterfaceSourcePath,
159166
std::unique_ptr<llvm::MemoryBuffer> moduleInputBuffer,
160167
std::unique_ptr<llvm::MemoryBuffer> moduleDocInputBuffer,
161168
std::unique_ptr<llvm::MemoryBuffer> moduleSourceInfoInputBuffer,
@@ -219,9 +226,9 @@ class ImplicitSerializedModuleLoader : public SerializedModuleLoaderBase {
219226
{}
220227

221228
std::error_code findModuleFilesInDirectory(
222-
ImportPath::Element ModuleID,
223-
const SerializedModuleBaseName &BaseName,
229+
ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName,
224230
SmallVectorImpl<char> *ModuleInterfacePath,
231+
SmallVectorImpl<char> *ModuleInterfaceSourcePath,
225232
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
226233
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
227234
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
@@ -273,9 +280,9 @@ class MemoryBufferSerializedModuleLoader : public SerializedModuleLoaderBase {
273280
BypassResilience(BypassResilience) {}
274281

275282
std::error_code findModuleFilesInDirectory(
276-
ImportPath::Element ModuleID,
277-
const SerializedModuleBaseName &BaseName,
283+
ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName,
278284
SmallVectorImpl<char> *ModuleInterfacePath,
285+
SmallVectorImpl<char> *ModuleInterfaceSourcePath,
279286
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
280287
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
281288
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
@@ -453,6 +460,8 @@ class SerializedASTFile final : public LoadedFile {
453460

454461
virtual StringRef getLoadedFilename() const override;
455462

463+
virtual StringRef getSourceFilename() const override;
464+
456465
virtual StringRef getModuleDefiningPath() const override;
457466

458467
virtual StringRef getExportedModuleName() const override;

‎lib/AST/Module.cpp

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2064,11 +2064,9 @@ StringRef ModuleDecl::getModuleFilename() const {
20642064

20652065
StringRef ModuleDecl::getModuleSourceFilename() const {
20662066
for (auto F : getFiles()) {
2067-
if (auto *SFU = dyn_cast<SynthesizedFileUnit>(F))
2068-
continue;
2069-
return F->getModuleDefiningPath();
2067+
if (auto LF = dyn_cast<LoadedFile>(F))
2068+
return LF->getSourceFilename();
20702069
}
2071-
20722070
return StringRef();
20732071
}
20742072

‎lib/Frontend/Frontend.cpp

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1229,11 +1229,11 @@ bool CompilerInstance::loadPartialModulesAndImplicitImports(
12291229
bool hadLoadError = false;
12301230
for (auto &PM : PartialModules) {
12311231
assert(PM.ModuleBuffer);
1232-
auto *file =
1233-
DefaultSerializedLoader->loadAST(*mod, /*diagLoc*/ SourceLoc(), /*moduleInterfacePath*/ "",
1234-
std::move(PM.ModuleBuffer), std::move(PM.ModuleDocBuffer),
1235-
std::move(PM.ModuleSourceInfoBuffer),
1236-
/*isFramework*/ false);
1232+
auto *file = DefaultSerializedLoader->loadAST(
1233+
*mod, /*diagLoc=*/SourceLoc(), /*moduleInterfacePath*/ "",
1234+
/*moduleInterfaceSourcePath=*/"", std::move(PM.ModuleBuffer),
1235+
std::move(PM.ModuleDocBuffer), std::move(PM.ModuleSourceInfoBuffer),
1236+
/*isFramework*/ false);
12371237
if (file) {
12381238
partialModules.push_back(file);
12391239
} else {

‎lib/Frontend/ModuleInterfaceLoader.cpp

Lines changed: 31 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
#include "swift/Frontend/ModuleInterfaceSupport.h"
2525
#include "swift/Parse/ParseVersion.h"
2626
#include "swift/Serialization/SerializationOptions.h"
27+
#include "swift/Serialization/SerializedModuleLoader.h"
2728
#include "swift/Serialization/Validation.h"
2829
#include "swift/Strings.h"
2930
#include "clang/Basic/Module.h"
@@ -1106,47 +1107,43 @@ bool ModuleInterfaceLoader::isCached(StringRef DepPath) {
11061107
/// cache or by converting it in a subordinate \c CompilerInstance, caching
11071108
/// the results.
11081109
std::error_code ModuleInterfaceLoader::findModuleFilesInDirectory(
1109-
ImportPath::Element ModuleID,
1110-
const SerializedModuleBaseName &BaseName,
1111-
SmallVectorImpl<char> *ModuleInterfacePath,
1112-
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
1113-
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
1114-
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
1115-
bool skipBuildingInterface, bool IsFramework) {
1110+
ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName,
1111+
SmallVectorImpl<char> *ModuleInterfacePath,
1112+
SmallVectorImpl<char> *ModuleInterfaceSourcePath,
1113+
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
1114+
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
1115+
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
1116+
bool skipBuildingInterface, bool IsFramework) {
11161117

11171118
// If running in OnlySerialized mode, ModuleInterfaceLoader
11181119
// should not have been constructed at all.
11191120
assert(LoadMode != ModuleLoadingMode::OnlySerialized);
11201121

1121-
llvm::SmallString<256>
1122-
ModPath{ BaseName.getName(file_types::TY_SwiftModuleFile) },
1123-
InPath{ BaseName.getName(file_types::TY_SwiftModuleInterfaceFile) },
1124-
PrivateInPath{BaseName.getName(file_types::TY_PrivateSwiftModuleInterfaceFile)};
1122+
std::string ModPath{BaseName.getName(file_types::TY_SwiftModuleFile)};
11251123

11261124
// First check to see if the .swiftinterface exists at all. Bail if not.
11271125
auto &fs = *Ctx.SourceMgr.getFileSystem();
1128-
if (!fs.exists(InPath)) {
1126+
std::string InPath = BaseName.findInterfacePath(fs).getValueOr("");
1127+
if (InPath.empty()) {
11291128
if (fs.exists(ModPath)) {
11301129
LLVM_DEBUG(llvm::dbgs()
1131-
<< "No .swiftinterface file found adjacent to module file "
1132-
<< ModPath.str() << "\n");
1130+
<< "No .swiftinterface file found adjacent to module file "
1131+
<< ModPath << "\n");
11331132
return std::make_error_code(std::errc::not_supported);
11341133
}
11351134
return std::make_error_code(std::errc::no_such_file_or_directory);
11361135
}
11371136

1138-
// If present, use the private interface instead of the public one.
1139-
if (fs.exists(PrivateInPath)) {
1140-
InPath = PrivateInPath;
1141-
}
1137+
if (ModuleInterfaceSourcePath)
1138+
ModuleInterfaceSourcePath->assign(InPath.begin(), InPath.end());
11421139

11431140
// If we've been told to skip building interfaces, we are done here and do
11441141
// not need to have the module actually built. For example, if we are
11451142
// currently answering a `canImport` query, it is enough to have found
11461143
// the interface.
11471144
if (skipBuildingInterface) {
11481145
if (ModuleInterfacePath)
1149-
*ModuleInterfacePath = InPath;
1146+
ModuleInterfacePath->assign(InPath.begin(), InPath.end());
11501147
return std::error_code();
11511148
}
11521149

@@ -1170,7 +1167,7 @@ std::error_code ModuleInterfaceLoader::findModuleFilesInDirectory(
11701167
if (ModuleBuffer) {
11711168
*ModuleBuffer = std::move(*ModuleBufferOrErr);
11721169
if (ModuleInterfacePath)
1173-
*ModuleInterfacePath = InPath;
1170+
ModuleInterfacePath->assign(InPath.begin(), InPath.end());
11741171
}
11751172

11761173
// Open .swiftsourceinfo file if it's present.
@@ -1920,12 +1917,13 @@ ExplicitSwiftModuleLoader::ExplicitSwiftModuleLoader(
19201917

19211918
ExplicitSwiftModuleLoader::~ExplicitSwiftModuleLoader() { delete &Impl; }
19221919

1923-
bool ExplicitSwiftModuleLoader::findModule(ImportPath::Element ModuleID,
1924-
SmallVectorImpl<char> *ModuleInterfacePath,
1925-
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
1926-
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
1927-
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
1928-
bool skipBuildingInterface, bool &IsFramework, bool &IsSystemModule) {
1920+
bool ExplicitSwiftModuleLoader::findModule(
1921+
ImportPath::Element ModuleID, SmallVectorImpl<char> *ModuleInterfacePath,
1922+
SmallVectorImpl<char> *ModuleInterfaceSourcePath,
1923+
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
1924+
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
1925+
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
1926+
bool skipBuildingInterface, bool &IsFramework, bool &IsSystemModule) {
19291927
// Find a module with an actual, physical name on disk, in case
19301928
// -module-alias is used (otherwise same).
19311929
//
@@ -2007,13 +2005,13 @@ bool ExplicitSwiftModuleLoader::findModule(ImportPath::Element ModuleID,
20072005
}
20082006

20092007
std::error_code ExplicitSwiftModuleLoader::findModuleFilesInDirectory(
2010-
ImportPath::Element ModuleID,
2011-
const SerializedModuleBaseName &BaseName,
2012-
SmallVectorImpl<char> *ModuleInterfacePath,
2013-
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
2014-
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
2015-
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
2016-
bool skipBuildingInterface, bool IsFramework) {
2008+
ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName,
2009+
SmallVectorImpl<char> *ModuleInterfacePath,
2010+
SmallVectorImpl<char> *ModuleInterfaceSourcePath,
2011+
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
2012+
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
2013+
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
2014+
bool skipBuildingInterface, bool IsFramework) {
20172015
llvm_unreachable("Not supported in the Explicit Swift Module Loader.");
20182016
return std::make_error_code(std::errc::not_supported);
20192017
}

0 commit comments

Comments
 (0)