Skip to content

Commit 76c9c35

Browse files
jansvoboda11jansvoboda11
authored
Merge pull request #9507 from swiftlang/jan_svoboda/stable-20240723-scanner-perf
🍒 [clang][deps][modules] Speed up dependency scanning
2 parents fc467b0 + 96b7d1f commit 76c9c35

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+781
-555
lines changed

‎clang-tools-extra/clangd/index/SymbolCollector.cpp

Lines changed: 28 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -290,9 +290,10 @@ class SymbolCollector::HeaderFileURICache {
290290
}
291291

292292
struct FrameworkHeaderPath {
293-
// Path to the framework directory containing the Headers/PrivateHeaders
294-
// directories e.g. /Frameworks/Foundation.framework/
295-
llvm::StringRef HeadersParentDir;
293+
// Path to the frameworks directory containing the .framework directory.
294+
llvm::StringRef FrameworkParentDir;
295+
// Name of the framework.
296+
llvm::StringRef FrameworkName;
296297
// Subpath relative to the Headers or PrivateHeaders dir, e.g. NSObject.h
297298
// Note: This is NOT relative to the `HeadersParentDir`.
298299
llvm::StringRef HeaderSubpath;
@@ -306,19 +307,17 @@ class SymbolCollector::HeaderFileURICache {
306307
path::reverse_iterator I = path::rbegin(Path);
307308
path::reverse_iterator Prev = I;
308309
path::reverse_iterator E = path::rend(Path);
310+
FrameworkHeaderPath HeaderPath;
309311
while (I != E) {
310-
if (*I == "Headers") {
311-
FrameworkHeaderPath HeaderPath;
312-
HeaderPath.HeadersParentDir = Path.substr(0, I - E);
312+
if (*I == "Headers" || *I == "PrivateHeaders") {
313313
HeaderPath.HeaderSubpath = Path.substr(Prev - E);
314-
HeaderPath.IsPrivateHeader = false;
315-
return HeaderPath;
316-
}
317-
if (*I == "PrivateHeaders") {
318-
FrameworkHeaderPath HeaderPath;
319-
HeaderPath.HeadersParentDir = Path.substr(0, I - E);
320-
HeaderPath.HeaderSubpath = Path.substr(Prev - E);
321-
HeaderPath.IsPrivateHeader = true;
314+
HeaderPath.IsPrivateHeader = *I == "PrivateHeaders";
315+
if (++I == E)
316+
break;
317+
HeaderPath.FrameworkName = *I;
318+
if (!HeaderPath.FrameworkName.consume_back(".framework"))
319+
break;
320+
HeaderPath.FrameworkParentDir = Path.substr(0, I - E);
322321
return HeaderPath;
323322
}
324323
Prev = I;
@@ -334,26 +333,27 @@ class SymbolCollector::HeaderFileURICache {
334333
// <Foundation/NSObject_Private.h> which should be used instead of directly
335334
// importing the header.
336335
std::optional<std::string>
337-
getFrameworkUmbrellaSpelling(llvm::StringRef Framework,
338-
const HeaderSearch &HS,
336+
getFrameworkUmbrellaSpelling(const HeaderSearch &HS,
339337
FrameworkHeaderPath &HeaderPath) {
338+
StringRef Framework = HeaderPath.FrameworkName;
340339
auto Res = CacheFrameworkToUmbrellaHeaderSpelling.try_emplace(Framework);
341340
auto *CachedSpelling = &Res.first->second;
342341
if (!Res.second) {
343342
return HeaderPath.IsPrivateHeader ? CachedSpelling->PrivateHeader
344343
: CachedSpelling->PublicHeader;
345344
}
346-
SmallString<256> UmbrellaPath(HeaderPath.HeadersParentDir);
347-
llvm::sys::path::append(UmbrellaPath, "Headers", Framework + ".h");
345+
SmallString<256> UmbrellaPath(HeaderPath.FrameworkParentDir);
346+
llvm::sys::path::append(UmbrellaPath, Framework + ".framework", "Headers",
347+
Framework + ".h");
348348

349349
llvm::vfs::Status Status;
350350
auto StatErr = HS.getFileMgr().getNoncachedStatValue(UmbrellaPath, Status);
351351
if (!StatErr)
352352
CachedSpelling->PublicHeader = llvm::formatv("<{0}/{0}.h>", Framework);
353353

354-
UmbrellaPath = HeaderPath.HeadersParentDir;
355-
llvm::sys::path::append(UmbrellaPath, "PrivateHeaders",
356-
Framework + "_Private.h");
354+
UmbrellaPath = HeaderPath.FrameworkParentDir;
355+
llvm::sys::path::append(UmbrellaPath, Framework + ".framework",
356+
"PrivateHeaders", Framework + "_Private.h");
357357

358358
StatErr = HS.getFileMgr().getNoncachedStatValue(UmbrellaPath, Status);
359359
if (!StatErr)
@@ -369,8 +369,7 @@ class SymbolCollector::HeaderFileURICache {
369369
// give <Foundation/Foundation.h> if the umbrella header exists, otherwise
370370
// <Foundation/NSObject.h>.
371371
std::optional<llvm::StringRef>
372-
getFrameworkHeaderIncludeSpelling(FileEntryRef FE, llvm::StringRef Framework,
373-
HeaderSearch &HS) {
372+
getFrameworkHeaderIncludeSpelling(FileEntryRef FE, HeaderSearch &HS) {
374373
auto Res = CachePathToFrameworkSpelling.try_emplace(FE.getName());
375374
auto *CachedHeaderSpelling = &Res.first->second;
376375
if (!Res.second)
@@ -384,13 +383,15 @@ class SymbolCollector::HeaderFileURICache {
384383
return std::nullopt;
385384
}
386385
if (auto UmbrellaSpelling =
387-
getFrameworkUmbrellaSpelling(Framework, HS, *HeaderPath)) {
386+
getFrameworkUmbrellaSpelling(HS, *HeaderPath)) {
388387
*CachedHeaderSpelling = *UmbrellaSpelling;
389388
return llvm::StringRef(*CachedHeaderSpelling);
390389
}
391390

392391
*CachedHeaderSpelling =
393-
llvm::formatv("<{0}/{1}>", Framework, HeaderPath->HeaderSubpath).str();
392+
llvm::formatv("<{0}/{1}>", HeaderPath->FrameworkName,
393+
HeaderPath->HeaderSubpath)
394+
.str();
394395
return llvm::StringRef(*CachedHeaderSpelling);
395396
}
396397

@@ -409,11 +410,8 @@ class SymbolCollector::HeaderFileURICache {
409410
// Framework headers are spelled as <FrameworkName/Foo.h>, not
410411
// "path/FrameworkName.framework/Headers/Foo.h".
411412
auto &HS = PP->getHeaderSearchInfo();
412-
if (const auto *HFI = HS.getExistingFileInfo(*FE))
413-
if (!HFI->Framework.empty())
414-
if (auto Spelling =
415-
getFrameworkHeaderIncludeSpelling(*FE, HFI->Framework, HS))
416-
return *Spelling;
413+
if (auto Spelling = getFrameworkHeaderIncludeSpelling(*FE, HS))
414+
return *Spelling;
417415

418416
if (!tooling::isSelfContainedHeader(*FE, PP->getSourceManager(),
419417
PP->getHeaderSearchInfo())) {

‎clang-tools-extra/modularize/CoverageChecker.cpp

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -223,10 +223,9 @@ bool CoverageChecker::collectModuleHeaders(const Module &Mod) {
223223
return false;
224224
}
225225

226-
for (auto &HeaderKind : Mod.Headers)
227-
for (auto &Header : HeaderKind)
228-
ModuleMapHeadersSet.insert(
229-
ModularizeUtilities::getCanonicalPath(Header.Entry.getName()));
226+
for (const auto &Header : Mod.getAllHeaders())
227+
ModuleMapHeadersSet.insert(
228+
ModularizeUtilities::getCanonicalPath(Header.Entry.getName()));
230229

231230
for (auto *Submodule : Mod.submodules())
232231
collectModuleHeaders(*Submodule);

‎clang-tools-extra/modularize/ModularizeUtilities.cpp

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -358,7 +358,7 @@ bool ModularizeUtilities::collectModuleHeaders(const clang::Module &Mod) {
358358
} else if (std::optional<clang::Module::DirectoryName> UmbrellaDir =
359359
Mod.getUmbrellaDirAsWritten()) {
360360
// If there normal headers, assume these are umbrellas and skip collection.
361-
if (Mod.Headers->size() == 0) {
361+
if (Mod.getHeaders(Module::HK_Normal).empty()) {
362362
// Collect headers in umbrella directory.
363363
if (!collectUmbrellaHeaders(UmbrellaDir->Entry.getName(),
364364
UmbrellaDependents))
@@ -371,16 +371,8 @@ bool ModularizeUtilities::collectModuleHeaders(const clang::Module &Mod) {
371371
// modules or because they are meant to be included by another header,
372372
// and thus should be ignored by modularize.
373373

374-
int NormalHeaderCount = Mod.Headers[clang::Module::HK_Normal].size();
375-
376-
for (int Index = 0; Index < NormalHeaderCount; ++Index) {
377-
DependentsVector NormalDependents;
378-
// Collect normal header.
379-
const clang::Module::Header &Header(
380-
Mod.Headers[clang::Module::HK_Normal][Index]);
381-
std::string HeaderPath = getCanonicalPath(Header.Entry.getName());
382-
HeaderFileNames.push_back(HeaderPath);
383-
}
374+
for (const auto &Header : Mod.getHeaders(clang::Module::HK_Normal))
375+
HeaderFileNames.push_back(getCanonicalPath(Header.Entry.getName()));
384376

385377
int MissingCountThisModule = Mod.MissingHeaders.size();
386378

‎clang/include/clang/Basic/Module.h

Lines changed: 38 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ namespace clang {
4848

4949
class FileManager;
5050
class LangOptions;
51+
class ModuleMap;
5152
class TargetInfo;
5253

5354
/// Describes the name of a module.
@@ -99,6 +100,15 @@ struct ASTFileSignature : std::array<uint8_t, 20> {
99100
}
100101
};
101102

103+
/// Required to construct a Module.
104+
///
105+
/// This tag type is only constructible by ModuleMap, guaranteeing it ownership
106+
/// of all Module instances.
107+
class ModuleConstructorTag {
108+
explicit ModuleConstructorTag() = default;
109+
friend ModuleMap;
110+
};
111+
102112
/// Describes a module or submodule.
103113
///
104114
/// Aligned to 8 bytes to allow for llvm::PointerIntPair<Module *, 3>.
@@ -223,7 +233,7 @@ class alignas(8) Module {
223233

224234
/// A mapping from the submodule name to the index into the
225235
/// \c SubModules vector at which that submodule resides.
226-
llvm::StringMap<unsigned> SubModuleIndex;
236+
mutable llvm::StringMap<unsigned> SubModuleIndex;
227237

228238
/// The AST file if this is a top-level module which has a
229239
/// corresponding serialized AST file, or null otherwise.
@@ -252,8 +262,6 @@ class alignas(8) Module {
252262
HK_PrivateTextual,
253263
HK_Excluded
254264
};
255-
static const int NumHeaderKinds = HK_Excluded + 1;
256-
257265
/// Information about a header directive as found in the module map
258266
/// file.
259267
struct Header {
@@ -262,17 +270,36 @@ class alignas(8) Module {
262270
FileEntryRef Entry;
263271
};
264272

265-
/// Information about a directory name as found in the module map
266-
/// file.
273+
private:
274+
static const int NumHeaderKinds = HK_Excluded + 1;
275+
// The begin index for a HeaderKind also acts the end index of HeaderKind - 1.
276+
// The extra element at the end acts as the end index of the last HeaderKind.
277+
unsigned HeaderKindBeginIndex[NumHeaderKinds + 1] = {};
278+
SmallVector<Header, 2> HeadersStorage;
279+
280+
public:
281+
ArrayRef<Header> getAllHeaders() const { return HeadersStorage; }
282+
ArrayRef<Header> getHeaders(HeaderKind HK) const {
283+
assert(HK < NumHeaderKinds && "Invalid Module::HeaderKind");
284+
auto BeginIt = HeadersStorage.begin() + HeaderKindBeginIndex[HK];
285+
auto EndIt = HeadersStorage.begin() + HeaderKindBeginIndex[HK + 1];
286+
return {BeginIt, EndIt};
287+
}
288+
void addHeader(HeaderKind HK, Header H) {
289+
assert(HK < NumHeaderKinds && "Invalid Module::HeaderKind");
290+
auto EndIt = HeadersStorage.begin() + HeaderKindBeginIndex[HK + 1];
291+
HeadersStorage.insert(EndIt, std::move(H));
292+
for (unsigned HKI = HK + 1; HKI != NumHeaderKinds + 1; ++HKI)
293+
++HeaderKindBeginIndex[HKI];
294+
}
295+
296+
/// Information about a directory name as found in the module map file.
267297
struct DirectoryName {
268298
std::string NameAsWritten;
269299
std::string PathRelativeToRootModuleDirectory;
270300
DirectoryEntryRef Entry;
271301
};
272302

273-
/// The headers that are part of this module.
274-
SmallVector<Header, 2> Headers[5];
275-
276303
/// Stored information about a header directive that was found in the
277304
/// module map file but has not been resolved to a file.
278305
struct UnresolvedHeaderDirective {
@@ -514,8 +541,9 @@ class alignas(8) Module {
514541
std::vector<Conflict> Conflicts;
515542

516543
/// Construct a new module or submodule.
517-
Module(StringRef Name, SourceLocation DefinitionLoc, Module *Parent,
518-
bool IsFramework, bool IsExplicit, unsigned VisibilityID);
544+
Module(ModuleConstructorTag, StringRef Name, SourceLocation DefinitionLoc,
545+
Module *Parent, bool IsFramework, bool IsExplicit,
546+
unsigned VisibilityID);
519547

520548
~Module();
521549

@@ -601,7 +629,6 @@ class alignas(8) Module {
601629
void setParent(Module *M) {
602630
assert(!Parent);
603631
Parent = M;
604-
Parent->SubModuleIndex[Name] = Parent->SubModules.size();
605632
Parent->SubModules.push_back(this);
606633
}
607634

@@ -775,7 +802,6 @@ class alignas(8) Module {
775802
///
776803
/// \returns The submodule if found, or NULL otherwise.
777804
Module *findSubmodule(StringRef Name) const;
778-
Module *findOrInferSubmodule(StringRef Name);
779805

780806
/// Get the Global Module Fragment (sub-module) for this module, it there is
781807
/// one.

‎clang/include/clang/Driver/Options.td

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4542,9 +4542,6 @@ def ibuiltininc : Flag<["-"], "ibuiltininc">, Group<clang_i_Group>,
45424542
HelpText<"Enable builtin #include directories even when -nostdinc is used "
45434543
"before or after -ibuiltininc. "
45444544
"Using -nobuiltininc after the option disables it">;
4545-
def index_header_map : Flag<["-"], "index-header-map">,
4546-
Visibility<[ClangOption, CC1Option]>,
4547-
HelpText<"Make the next included directory (-I or -F) an indexer header map">;
45484545
def iapinotes_modules : JoinedOrSeparate<["-"], "iapinotes-modules">, Group<clang_i_Group>,
45494546
Visibility<[ClangOption, CC1Option]>,
45504547
HelpText<"Add directory to the API notes search path referenced by module name">, MetaVarName<"<directory>">;

‎clang/include/clang/Lex/DirectoryLookup.h

Lines changed: 3 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -58,10 +58,6 @@ class DirectoryLookup {
5858
LLVM_PREFERRED_TYPE(LookupType_t)
5959
unsigned LookupType : 2;
6060

61-
/// Whether this is a header map used when building a framework.
62-
LLVM_PREFERRED_TYPE(bool)
63-
unsigned IsIndexHeaderMap : 1;
64-
6561
/// Whether we've performed an exhaustive search for module maps
6662
/// within the subdirectories of this directory.
6763
LLVM_PREFERRED_TYPE(bool)
@@ -73,13 +69,12 @@ class DirectoryLookup {
7369
bool isFramework)
7470
: u(Dir), DirCharacteristic(DT),
7571
LookupType(isFramework ? LT_Framework : LT_NormalDir),
76-
IsIndexHeaderMap(false), SearchedAllModuleMaps(false) {}
72+
SearchedAllModuleMaps(false) {}
7773

7874
/// This ctor *does not take ownership* of 'Map'.
79-
DirectoryLookup(const HeaderMap *Map, SrcMgr::CharacteristicKind DT,
80-
bool isIndexHeaderMap)
75+
DirectoryLookup(const HeaderMap *Map, SrcMgr::CharacteristicKind DT)
8176
: u(Map), DirCharacteristic(DT), LookupType(LT_HeaderMap),
82-
IsIndexHeaderMap(isIndexHeaderMap), SearchedAllModuleMaps(false) {}
77+
SearchedAllModuleMaps(false) {}
8378

8479
/// getLookupType - Return the kind of directory lookup that this is: either a
8580
/// normal directory, a framework path, or a HeaderMap.
@@ -146,11 +141,6 @@ class DirectoryLookup {
146141
return getDirCharacteristic() != SrcMgr::C_User;
147142
}
148143

149-
/// Whether this header map is building a framework or not.
150-
bool isIndexHeaderMap() const {
151-
return isHeaderMap() && IsIndexHeaderMap;
152-
}
153-
154144
/// LookupFile - Lookup the specified file in this search path, returning it
155145
/// if it exists or returning null if not.
156146
///

‎clang/include/clang/Lex/HeaderSearch.h

Lines changed: 3 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -108,16 +108,6 @@ struct HeaderFileInfo {
108108
LLVM_PREFERRED_TYPE(bool)
109109
unsigned Resolved : 1;
110110

111-
/// Whether this is a header inside a framework that is currently
112-
/// being built.
113-
///
114-
/// When a framework is being built, the headers have not yet been placed
115-
/// into the appropriate framework subdirectories, and therefore are
116-
/// provided via a header map. This bit indicates when this is one of
117-
/// those framework headers.
118-
LLVM_PREFERRED_TYPE(bool)
119-
unsigned IndexHeaderMapHeader : 1;
120-
121111
/// Whether this file has been looked up as a header.
122112
LLVM_PREFERRED_TYPE(bool)
123113
unsigned IsValid : 1;
@@ -132,15 +122,11 @@ struct HeaderFileInfo {
132122
/// external storage.
133123
LazyIdentifierInfoPtr LazyControllingMacro;
134124

135-
/// If this header came from a framework include, this is the name
136-
/// of the framework.
137-
StringRef Framework;
138-
139125
HeaderFileInfo()
140126
: IsLocallyIncluded(false), isImport(false), isPragmaOnce(false),
141127
DirInfo(SrcMgr::C_User), External(false), isModuleHeader(false),
142128
isTextualModuleHeader(false), isCompilingModuleHeader(false),
143-
Resolved(false), IndexHeaderMapHeader(false), IsValid(false) {}
129+
Resolved(false), IsValid(false) {}
144130

145131
/// Retrieve the controlling macro for this header file, if
146132
/// any.
@@ -154,6 +140,8 @@ struct HeaderFileInfo {
154140
void mergeModuleMembership(ModuleMap::ModuleHeaderRole Role);
155141
};
156142

143+
static_assert(sizeof(HeaderFileInfo) <= 16);
144+
157145
/// An external source of header file information, which may supply
158146
/// information about header files already included.
159147
class ExternalHeaderFileInfoSource {

‎clang/include/clang/Lex/HeaderSearchOptions.h

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,6 @@ enum IncludeDirGroup {
3535
/// Paths for '\#include <>' added by '-I'.
3636
Angled,
3737

38-
/// Like Angled, but marks header maps used when building frameworks.
39-
IndexHeaderMap,
40-
4138
/// Like Angled, but marks system directories.
4239
System,
4340

0 commit comments

Comments
 (0)