Skip to content

Commit 3e338db

Browse files
varungandhi-srcvarungandhi-src
authored
refactor: StableFileIdMap->FileMetadataMap which tracks PackageMetadata (#340)
1 parent 2e3e9d6 commit 3e338db

12 files changed

+221
-168
lines changed

‎indexer/AstConsumer.cc

Lines changed: 21 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -21,17 +21,17 @@ namespace {
2121
class IndexerAstVisitor : public clang::RecursiveASTVisitor<IndexerAstVisitor> {
2222
using Base = RecursiveASTVisitor;
2323

24-
const StableFileIdMap &stableFileIdMap;
24+
const FileMetadataMap &fileMetadataMap;
2525
FileIdsToBeIndexedSet toBeIndexed;
2626
bool deterministic;
2727

2828
TuIndexer &tuIndexer;
2929

3030
public:
31-
IndexerAstVisitor(const StableFileIdMap &pathMap,
31+
IndexerAstVisitor(const FileMetadataMap &fileMetadataMap,
3232
FileIdsToBeIndexedSet &&toBeIndexed, bool deterministic,
3333
TuIndexer &tuIndexer)
34-
: stableFileIdMap(pathMap), toBeIndexed(std::move(toBeIndexed)),
34+
: fileMetadataMap(fileMetadataMap), toBeIndexed(std::move(toBeIndexed)),
3535
deterministic(deterministic), tuIndexer(tuIndexer) {}
3636

3737
// See clang/include/clang/Basic/DeclNodes.td for list of declarations.
@@ -106,7 +106,7 @@ class IndexerAstVisitor : public clang::RecursiveASTVisitor<IndexerAstVisitor> {
106106
indexedProjectFiles;
107107
for (auto wrappedFileId : this->toBeIndexed) {
108108
if (auto optStableFileId =
109-
this->stableFileIdMap.getStableFileId(wrappedFileId.data)) {
109+
this->fileMetadataMap.getStableFileId(wrappedFileId.data)) {
110110
if (optStableFileId->isInProject) {
111111
indexedProjectFiles.emplace_back(optStableFileId->path,
112112
wrappedFileId.data);
@@ -191,26 +191,22 @@ void IndexerAstConsumer::HandleTranslationUnit(clang::ASTContext &astContext) {
191191
return;
192192
}
193193

194-
StableFileIdMap stableFileIdMap{this->options.projectRootPath,
194+
FileMetadataMap fileMetadataMap{this->options.projectRootPath,
195195
this->options.buildRootPath};
196196
FileIdsToBeIndexedSet toBeIndexed{};
197197
this->computeFileIdsToBeIndexed(astContext, emitIndexDetails,
198-
clangIdLookupMap, stableFileIdMap,
198+
clangIdLookupMap, fileMetadataMap,
199199
toBeIndexed);
200200

201-
auto getStableFileId =
202-
[&](clang::FileID fileId) -> std::optional<StableFileId> {
203-
return stableFileIdMap.getStableFileId(fileId);
204-
};
205-
SymbolFormatter symbolFormatter{sourceManager, getStableFileId};
201+
SymbolFormatter symbolFormatter{sourceManager, fileMetadataMap};
206202
TuIndexer tuIndexer{sourceManager, this->sema->getLangOpts(),
207203
this->sema->getASTContext(), symbolFormatter,
208-
getStableFileId};
204+
fileMetadataMap};
209205

210206
this->saveIncludeReferences(toBeIndexed, macroIndexer, clangIdLookupMap,
211-
stableFileIdMap, tuIndexer);
207+
fileMetadataMap, tuIndexer);
212208

213-
IndexerAstVisitor visitor{stableFileIdMap, std::move(toBeIndexed),
209+
IndexerAstVisitor visitor{fileMetadataMap, std::move(toBeIndexed),
214210
this->options.deterministic, tuIndexer};
215211
visitor.TraverseAST(astContext);
216212

@@ -231,16 +227,16 @@ void IndexerAstConsumer::ForgetSema() {
231227
void IndexerAstConsumer::computeFileIdsToBeIndexed(
232228
const clang::ASTContext &astContext,
233229
const EmitIndexJobDetails &emitIndexDetails,
234-
const ClangIdLookupMap &clangIdLookupMap, StableFileIdMap &stableFileIdMap,
230+
const ClangIdLookupMap &clangIdLookupMap, FileMetadataMap &fileMetadataMap,
235231
FileIdsToBeIndexedSet &toBeIndexed) {
236232
auto &sourceManager = astContext.getSourceManager();
237233
auto mainFileId = sourceManager.getMainFileID();
238234

239-
stableFileIdMap.populate(clangIdLookupMap);
235+
fileMetadataMap.populate(clangIdLookupMap);
240236
if (auto *mainFileEntry = sourceManager.getFileEntryForID(mainFileId)) {
241237
if (auto optMainFileAbsPath =
242238
AbsolutePathRef::tryFrom(mainFileEntry->tryGetRealPathName())) {
243-
stableFileIdMap.insert(mainFileId, optMainFileAbsPath.value());
239+
fileMetadataMap.insert(mainFileId, optMainFileAbsPath.value());
244240
toBeIndexed.insert({mainFileId});
245241
} else {
246242
spdlog::debug(
@@ -265,12 +261,11 @@ void IndexerAstConsumer::computeFileIdsToBeIndexed(
265261
void IndexerAstConsumer::saveIncludeReferences(
266262
const FileIdsToBeIndexedSet &toBeIndexed, const MacroIndexer &macroIndexer,
267263
const ClangIdLookupMap &clangIdLookupMap,
268-
const StableFileIdMap &stableFileIdMap, TuIndexer &tuIndexer) {
264+
const FileMetadataMap &fileMetadataMap, TuIndexer &tuIndexer) {
269265
for (auto &wrappedFileId : toBeIndexed) {
270-
if (auto optStableFileId =
271-
stableFileIdMap.getStableFileId(wrappedFileId.data)) {
272-
tuIndexer.saveSyntheticFileDefinition(wrappedFileId.data,
273-
*optStableFileId);
266+
if (auto *fileMetadata =
267+
fileMetadataMap.getFileMetadata(wrappedFileId.data)) {
268+
tuIndexer.saveSyntheticFileDefinition(wrappedFileId.data, *fileMetadata);
274269
}
275270
macroIndexer.forEachIncludeInFile(
276271
wrappedFileId.data,
@@ -281,11 +276,11 @@ void IndexerAstConsumer::saveIncludeReferences(
281276
return;
282277
}
283278
auto refFileId = *optRefFileId;
284-
auto optStableFileId = stableFileIdMap.getStableFileId(*optRefFileId);
285-
ENFORCE(optStableFileId.has_value(),
286-
"missing StableFileId value for path {} (FileID = {})",
279+
auto *fileMetadata = fileMetadataMap.getFileMetadata(*optRefFileId);
280+
ENFORCE(fileMetadata,
281+
"missing FileMetadata value for path {} (FileID = {})",
287282
importedFilePath.asStringView(), refFileId.getHashValue());
288-
tuIndexer.saveInclude(range, *optStableFileId);
283+
tuIndexer.saveInclude(range, *fileMetadata);
289284
});
290285
}
291286
}

‎indexer/AstConsumer.h

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ class CompilerInstance;
2020

2121
namespace scip_clang {
2222
class ClangIdLookupMap;
23-
class StableFileIdMap;
23+
class FileMetadataMap;
2424
class IndexerPreprocessorWrapper;
2525
class MacroIndexer;
2626
class TuIndexer;
@@ -88,13 +88,13 @@ class IndexerAstConsumer : public clang::SemaConsumer {
8888
void computeFileIdsToBeIndexed(const clang::ASTContext &astContext,
8989
const EmitIndexJobDetails &emitIndexDetails,
9090
const ClangIdLookupMap &clangIdLookupMap,
91-
StableFileIdMap &stableFileIdMap,
91+
FileMetadataMap &fileMetadataMap,
9292
FileIdsToBeIndexedSet &toBeIndexed);
9393

9494
void saveIncludeReferences(const FileIdsToBeIndexedSet &toBeIndexed,
9595
const MacroIndexer &macroIndexer,
9696
const ClangIdLookupMap &clangIdLookupMap,
97-
const StableFileIdMap &stableFileIdMap,
97+
const FileMetadataMap &fileMetadataMap,
9898
TuIndexer &tuIndexer);
9999
};
100100

‎indexer/FileMetadata.h

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
#ifndef SCIP_CLANG_STABLE_FILE_ID_H
2+
#define SCIP_CLANG_STABLE_FILE_ID_H
3+
4+
#include <compare>
5+
#include <optional>
6+
#include <string_view>
7+
#include <utility>
8+
9+
#include "absl/functional/function_ref.h"
10+
11+
#include "clang/Basic/SourceLocation.h"
12+
13+
#include "indexer/Comparison.h"
14+
#include "indexer/Derive.h"
15+
#include "indexer/Path.h"
16+
17+
namespace scip_clang {
18+
19+
/// An identifier for a file that is stable across indexing runs,
20+
/// represented as a path.
21+
///
22+
/// There are 4 kinds of files:
23+
/// 1. In-project files.
24+
/// 2. Generated files: These are present in the build root,
25+
/// but not in the project root.
26+
/// 3. External files: From libraries (stdlib, SDKs etc.)
27+
/// 4. Magic files: Corresponding to the builtin header,
28+
/// and command-line arguments.
29+
///
30+
/// For generated files and magic files, we make up fake paths
31+
/// that are likely to be distinct from actual in-project paths.
32+
///
33+
/// For external files, if available, we track package information
34+
/// in the packageId field of \c FileMetadata. In that case,
35+
/// the path represents the "true" in-project path of the external
36+
/// file. In the absence of package information, the path is
37+
/// fake for external files too.
38+
struct StableFileId {
39+
RootRelativePathRef path;
40+
/// Does this file belong to the project being indexed?
41+
bool isInProject;
42+
/// Was this path synthesized by truncating an actual path?
43+
bool isSynthetic;
44+
45+
template <typename H> friend H AbslHashValue(H h, const StableFileId &s) {
46+
return H::combine(std::move(h), s.path, s.isInProject, s.isSynthetic);
47+
}
48+
49+
DERIVE_CMP_ALL(StableFileId)
50+
};
51+
52+
struct PackageId {
53+
std::string_view name;
54+
std::string_view version;
55+
};
56+
57+
/// Summary information about a package.
58+
struct PackageMetadata {
59+
PackageId id;
60+
AbsolutePathRef rootPath;
61+
};
62+
63+
/// Represents important metadata related to a file.
64+
///
65+
/// The possible cases are:
66+
/// 1. stableFileId.isInProject = true
67+
/// - stableFileId.isSynthetic = false
68+
/// - packageInfo MAY be present.
69+
/// 2. isInProject = false, stableFileId.isSynthetic = false
70+
/// - packageInfo MUST be present.
71+
/// - stableFileId.path is a path relative to an actual root directory.
72+
/// 3. stableFileId.isInProject = false, stableFileId.isSynthetic = true
73+
/// - packageInfo MUST be absent.
74+
/// - stableFileId.path is a fake path
75+
struct FileMetadata {
76+
StableFileId stableFileId;
77+
AbsolutePathRef originalPath;
78+
std::optional<PackageMetadata> packageInfo;
79+
80+
PackageId packageId() const {
81+
if (this->packageInfo.has_value()) {
82+
return this->packageInfo->id;
83+
}
84+
return PackageId{};
85+
}
86+
};
87+
88+
} // namespace scip_clang
89+
90+
#endif // SCIP_CLANG_STABLE_FILE_ID_H

‎indexer/IdPathMappings.cc

Lines changed: 43 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
#include "clang/Basic/SourceLocation.h"
99

10+
#include "indexer/FileMetadata.h"
1011
#include "indexer/Hash.h"
1112
#include "indexer/IdPathMappings.h"
1213
#include "indexer/Path.h"
@@ -63,7 +64,7 @@ ClangIdLookupMap::lookupAnyFileId(AbsolutePathRef absPathRef) const {
6364
return {};
6465
}
6566

66-
void StableFileIdMap::populate(const ClangIdLookupMap &clangIdLookupMap) {
67+
void FileMetadataMap::populate(const ClangIdLookupMap &clangIdLookupMap) {
6768
clangIdLookupMap.forEachPathAndHash( // force formatting break
6869
[&](AbsolutePathRef absPathRef,
6970
const absl::flat_hash_map<HashValue, clang::FileID> &map) {
@@ -77,16 +78,23 @@ void StableFileIdMap::populate(const ClangIdLookupMap &clangIdLookupMap) {
7778
});
7879
}
7980

80-
bool StableFileIdMap::insert(clang::FileID fileId, AbsolutePathRef absPathRef) {
81+
bool FileMetadataMap::insert(clang::FileID fileId, AbsolutePathRef absPathRef) {
8182
ENFORCE(fileId.isValid(),
8283
"invalid FileIDs should be filtered out after preprocessing");
8384
ENFORCE(!absPathRef.asStringView().empty(),
8485
"inserting file with empty absolute path");
8586

86-
auto insertRelPath = [&](RootRelativePathRef projectRootRelPath) -> bool {
87-
ENFORCE(!projectRootRelPath.asStringView().empty(),
87+
std::optional<PackageMetadata> optPackageMetadata = std::nullopt;
88+
auto insertRelPath = [&](RootRelativePathRef relPathRef,
89+
bool isInProject) -> bool {
90+
ENFORCE(!relPathRef.asStringView().empty(),
8891
"file path is unexpectedly equal to project root");
89-
return this->map.insert({{fileId}, projectRootRelPath}).second;
92+
auto metadata = FileMetadata{
93+
StableFileId{relPathRef, isInProject, /*isSynthetic*/ false},
94+
absPathRef,
95+
optPackageMetadata,
96+
};
97+
return this->map.insert({{fileId}, std::move(metadata)}).second;
9098
};
9199

92100
// In practice, CMake ends up passing paths to project files as well
@@ -102,11 +110,24 @@ bool StableFileIdMap::insert(clang::FileID fileId, AbsolutePathRef absPathRef) {
102110
if (std::filesystem::exists(originalFileSourcePath.asStringRef(), error)
103111
&& !error) {
104112
return insertRelPath(RootRelativePathRef(buildRootRelPath->asStringView(),
105-
RootKind::Project));
113+
RootKind::Project),
114+
/*isInProject*/ true);
106115
}
107116
} else if (auto optProjectRootRelPath =
108117
this->projectRootPath.tryMakeRelative(absPathRef)) {
109-
return insertRelPath(optProjectRootRelPath.value());
118+
return insertRelPath(optProjectRootRelPath.value(), /*isInProject*/ true);
119+
}
120+
if (optPackageMetadata.has_value()) {
121+
if (auto optStrView =
122+
optPackageMetadata->rootPath.makeRelative(absPathRef)) {
123+
return insertRelPath(RootRelativePathRef(*optStrView, RootKind::External),
124+
/*isInProject*/ false);
125+
} else {
126+
spdlog::warn("package info map determined '{}' as root for path '{}', "
127+
"but prefix check failed",
128+
optPackageMetadata->rootPath.asStringView(),
129+
absPathRef.asStringView());
130+
}
110131
}
111132
auto optFileName = absPathRef.fileName();
112133
ENFORCE(optFileName.has_value(),
@@ -118,38 +139,36 @@ bool StableFileIdMap::insert(clang::FileID fileId, AbsolutePathRef absPathRef) {
118139
RootKind::Build); // fake value to satisfy the RootRelativePathRef API
119140
return this->map
120141
.insert({{fileId},
121-
ExternalFileEntry{absPathRef, this->storage.back().asRef()}})
142+
FileMetadata{StableFileId{this->storage.back().asRef(),
143+
/*isInProject*/ false,
144+
/*isSynthetic*/ true},
145+
absPathRef, optPackageMetadata}})
122146
.second;
123147
}
124148

125-
void StableFileIdMap::forEachFileId(
149+
void FileMetadataMap::forEachFileId(
126150
absl::FunctionRef<void(clang::FileID, StableFileId)> callback) {
127151
for (auto &[wrappedFileId, entry] : this->map) {
128-
callback(wrappedFileId.data, Self::mapValueToStableFileId(entry));
152+
callback(wrappedFileId.data, entry.stableFileId);
129153
}
130154
}
131155

132156
std::optional<StableFileId>
133-
StableFileIdMap::getStableFileId(clang::FileID fileId) const {
157+
FileMetadataMap::getStableFileId(clang::FileID fileId) const {
134158
auto it = this->map.find({fileId});
135159
if (it == this->map.end()) {
136160
return {};
137161
}
138-
return Self::mapValueToStableFileId(it->second);
162+
return it->second.stableFileId;
139163
}
140164

141-
// static
142-
StableFileId
143-
StableFileIdMap::mapValueToStableFileId(const MapValueType &variant) {
144-
if (std::holds_alternative<RootRelativePathRef>(variant)) {
145-
return StableFileId{.path = std::get<RootRelativePathRef>(variant),
146-
.isInProject = true,
147-
.isSynthetic = false};
165+
const FileMetadata *
166+
FileMetadataMap::getFileMetadata(clang::FileID fileId) const {
167+
auto it = this->map.find({fileId});
168+
if (it == this->map.end()) {
169+
return {};
148170
}
149-
return StableFileId{.path =
150-
std::get<ExternalFileEntry>(variant).fakeRelativePath,
151-
.isInProject = false,
152-
.isSynthetic = true};
171+
return &it->second;
153172
}
154173

155-
} // namespace scip_clang
174+
} // namespace scip_clang

0 commit comments

Comments
 (0)