Skip to content

Commit d0e2e2b

Browse files
committed
Swift: introduce SwiftDispatcher
1 parent 8f8ece6 commit d0e2e2b

File tree

6 files changed

+331
-11
lines changed

6 files changed

+331
-11
lines changed

swift/extractor/BUILD.bazel

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ swift_cc_binary(
66
"SwiftExtractor.cpp",
77
"SwiftExtractor.h",
88
"SwiftExtractorConfiguration.h",
9+
"SwiftDispatcher.h",
10+
"SwiftTagTraits.h",
911
"main.cpp",
1012
],
1113
visibility = ["//swift:__pkg__"],

swift/extractor/SwiftDispatcher.h

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
#pragma once
2+
3+
#include "swift/extractor/trap/TrapArena.h"
4+
#include "swift/extractor/trap/TrapLabelStore.h"
5+
// autogenerated file
6+
#include "swift/extractor/trap/TrapClasses.h"
7+
#include "swift/extractor/SwiftTagTraits.h"
8+
#include <swift/AST/SourceFile.h>
9+
#include <swift/Basic/SourceManager.h>
10+
#include <llvm/Support/FileSystem.h>
11+
12+
namespace codeql {
13+
14+
namespace detail {
15+
16+
// The following `getKindName`s are used within "TBD" TRAP entries to visually mark an AST node as
17+
// not properly emitted yet.
18+
// TODO: To be replaced with QL counterpart
19+
template <typename Parent, typename Kind>
20+
inline std::string getKindName(Kind kind) {
21+
return Parent::getKindName(kind).str();
22+
}
23+
24+
template <>
25+
inline std::string getKindName<swift::TypeBase, swift::TypeKind>(swift::TypeKind kind) {
26+
switch (kind) {
27+
#define TYPE(CLASS, PARENT) \
28+
case swift::TypeKind::CLASS: \
29+
return #CLASS;
30+
#include "swift/AST/TypeNodes.def"
31+
default:
32+
return "Unknown";
33+
}
34+
}
35+
36+
template <>
37+
std::string inline getKindName<swift::TypeRepr, swift::TypeReprKind>(swift::TypeReprKind kind) {
38+
switch (kind) {
39+
#define TYPEREPR(CLASS, PARENT) \
40+
case swift::TypeReprKind::CLASS: \
41+
return #CLASS;
42+
#include "swift/AST/TypeReprNodes.def"
43+
default:
44+
return "Unknown";
45+
}
46+
}
47+
48+
} // namespace detail
49+
50+
// The main reponsibilities of the SwiftDispatcher are as follows:
51+
// * redirect specific AST node emission to a corresponding visitor (statements, expressions, etc.)
52+
// * storing TRAP labels for emitted AST nodes (in the TrapLabelStore) to avoid re-emission
53+
// Since SwiftDispatcher sees all the AST nodes, it also attaches a location to every 'locatable'
54+
// node (AST nodes that are not types: declarations, statements, expressions, etc.).
55+
class SwiftDispatcher {
56+
public:
57+
SwiftDispatcher(const swift::SourceManager& sourceManager, TrapArena& arena, TrapOutput& trap)
58+
: sourceManager{sourceManager}, arena{arena}, trap{trap} {}
59+
60+
template <typename T>
61+
void extract(T* entity) {
62+
fetchLabel(entity);
63+
}
64+
65+
private:
66+
// This method gives a TRAP label for already emitted AST node.
67+
// If the AST node was not emitted yet, then the emission is dispatched to a corresponding
68+
// visitor (see `visit(T *)` methods below).
69+
template <typename E>
70+
TrapLabel<ToTag<E>> fetchLabel(E* e) {
71+
// this is required so we avoid any recursive loop: a `fetchLabel` during the visit of `e` might
72+
// end up calling `fetchLabel` on `e` itself, so we want the visit of `e` to call `fetchLabel`
73+
// only after having called `assignNewLabel` on `e`
74+
assert(!waitingForNewLabel && "fetchLabel called before assignNewLabel");
75+
if (auto l = store.get(e)) {
76+
return *l;
77+
}
78+
waitingForNewLabel = getCanonicalPtr(e);
79+
visit(e);
80+
if (auto l = store.get(e)) {
81+
if constexpr (!std::is_base_of_v<swift::TypeBase, E>) {
82+
attachLocation(e, *l);
83+
}
84+
return *l;
85+
}
86+
assert(!"assignNewLabel not called during visit");
87+
return {};
88+
}
89+
90+
// Due to the lazy emission approach, we must assign a label to a corresponding AST node before
91+
// it actually gets emitted to handle recursive cases such as recursive calls, or recursive type
92+
// declarations
93+
template <typename E>
94+
TrapLabel<ToTag<E>> assignNewLabel(E* e) {
95+
assert(waitingForNewLabel == getCanonicalPtr(e) && "assignNewLabel called on wrong entity");
96+
auto label = getLabel<ToTag<E>>();
97+
trap.assignStar(label);
98+
store.insert(e, label);
99+
waitingForNewLabel = nullptr;
100+
return label;
101+
}
102+
103+
template <typename Tag>
104+
TrapLabel<Tag> getLabel() {
105+
return arena.allocateLabel<Tag>();
106+
}
107+
108+
// This is a helper method to emit TRAP entries for AST nodes that we don't fully support yet.
109+
template <typename Parent, typename Child>
110+
void TBD(Child* entity, const std::string& suffix) {
111+
using namespace std::string_literals;
112+
auto label = assignNewLabel(entity);
113+
auto kind = detail::getKindName<Parent>(static_cast<const Parent*>(entity)->getKind());
114+
auto name = "TBD ("s + kind + suffix + ")";
115+
if constexpr (std::is_same_v<Parent, swift::TypeBase>) {
116+
trap.emit(UnknownTypesTrap{label, name});
117+
} else {
118+
trap.emit(UnknownAstNodesTrap{label, name});
119+
}
120+
}
121+
122+
template <typename Locatable>
123+
void attachLocation(Locatable locatable, TrapLabel<LocatableTag> locatableLabel) {
124+
attachLocation(&locatable, locatableLabel);
125+
}
126+
127+
// Emits a Location TRAP entry and attaches it to an AST node
128+
template <typename Locatable>
129+
void attachLocation(Locatable* locatable, TrapLabel<LocatableTag> locatableLabel) {
130+
auto start = locatable->getStartLoc();
131+
auto end = locatable->getEndLoc();
132+
if (!start.isValid() || !end.isValid()) {
133+
// invalid locations seem to come from entities synthesized by the compiler
134+
return;
135+
}
136+
std::string filepath = getFilepath(start);
137+
auto fileLabel = arena.allocateLabel<FileTag>();
138+
trap.assignKey(fileLabel, filepath);
139+
/// TODO: do not emit duplicate trap entries for Files
140+
trap.emit(FilesTrap{fileLabel, filepath});
141+
auto [startLine, startColumn] = sourceManager.getLineAndColumnInBuffer(start);
142+
auto [endLine, endColumn] = sourceManager.getLineAndColumnInBuffer(end);
143+
auto locLabel = arena.allocateLabel<LocationTag>();
144+
trap.assignKey(locLabel, '{', fileLabel, "}:", startLine, ':', startColumn, ':', endLine, ':',
145+
endColumn);
146+
trap.emit(LocationsTrap{locLabel, fileLabel, startLine, startColumn, endLine, endColumn});
147+
trap.emit(LocatablesTrap{locatableLabel, locLabel});
148+
}
149+
150+
std::string getFilepath(swift::SourceLoc loc) {
151+
/// TODO: this needs more testing
152+
std::string displayName = sourceManager.getDisplayNameForLoc(loc).str();
153+
llvm::SmallString<PATH_MAX> filePath(displayName);
154+
if (std::error_code ec = llvm::sys::fs::make_absolute(filePath)) {
155+
std::cerr << "Cannot make absolute path: '" << displayName << "': " << ec.message() << "\n";
156+
return {};
157+
}
158+
llvm::sys::path::remove_dots(filePath);
159+
return filePath.str().str();
160+
}
161+
162+
// TODO: The following methods are supposed to redirect TRAP emission to correpsonding visitors,
163+
// which are to be introduced in follow-up PRs
164+
void visit(swift::Decl* decl) { TBD<swift::Decl>(decl, "Decl"); }
165+
void visit(swift::Stmt* stmt) { TBD<swift::Stmt>(stmt, "Stmt"); }
166+
void visit(swift::Expr* expr) { TBD<swift::Expr>(expr, "Expr"); }
167+
void visit(swift::Pattern* pattern) { TBD<swift::Pattern>(pattern, "Pattern"); }
168+
void visit(swift::TypeRepr* type) { TBD<swift::TypeRepr>(type, "TypeRepr"); }
169+
void visit(swift::TypeBase* type) { TBD<swift::TypeBase>(type, "Type"); }
170+
171+
const swift::SourceManager& sourceManager;
172+
TrapArena& arena;
173+
TrapOutput& trap;
174+
TrapLabelStore store;
175+
const void* waitingForNewLabel{nullptr};
176+
};
177+
178+
} // namespace codeql

swift/extractor/SwiftExtractor.cpp

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,14 @@
1313
#include <llvm/Support/Path.h>
1414

1515
#include "swift/extractor/trap/TrapClasses.h"
16-
#include "swift/extractor/trap/TrapArena.h"
1716
#include "swift/extractor/trap/TrapOutput.h"
17+
#include "swift/extractor/SwiftDispatcher.h"
1818

1919
using namespace codeql;
2020

21-
static void extractFile(const SwiftExtractorConfiguration& config, swift::SourceFile& file) {
21+
static void extractFile(const SwiftExtractorConfiguration& config,
22+
swift::CompilerInstance& compiler,
23+
swift::SourceFile& file) {
2224
if (std::error_code ec = llvm::sys::fs::create_directories(config.trapDir)) {
2325
std::cerr << "Cannot create TRAP directory: " << ec.message() << "\n";
2426
return;
@@ -79,12 +81,17 @@ static void extractFile(const SwiftExtractorConfiguration& config, swift::Source
7981

8082
TrapOutput trap{trapStream};
8183
TrapArena arena{};
82-
auto label = arena.allocateLabel<FileTag>();
83-
trap.assignStar(label);
84-
File f{};
85-
f.id = label;
86-
f.name = srcFilePath.str().str();
87-
trap.emit(f);
84+
85+
// In the case of emtpy files, the dispatcher is not called, but we still want to 'record' the
86+
// fact that the file was extracted
87+
auto fileLabel = arena.allocateLabel<FileTag>();
88+
trap.assignKey(fileLabel, srcFilePath.str().str());
89+
trap.emit(FilesTrap{fileLabel, srcFilePath.str().str()});
90+
91+
SwiftDispatcher dispatcher(compiler.getSourceMgr(), arena, trap);
92+
for (swift::Decl* decl : file.getTopLevelDecls()) {
93+
dispatcher.extract(decl);
94+
}
8895

8996
// TODO: Pick a better name to avoid collisions
9097
std::string trapName = file.getFilename().str() + ".trap";
@@ -108,11 +115,11 @@ void codeql::extractSwiftFiles(const SwiftExtractorConfiguration& config,
108115
module->getFiles().front()->getKind() == swift::FileUnitKind::Source) {
109116
// We can only call getMainSourceFile if the first file is of a Source kind
110117
swift::SourceFile& file = module->getMainSourceFile();
111-
extractFile(config, file);
118+
extractFile(config, compiler, file);
112119
}
113120
} else {
114121
for (auto s : compiler.getPrimarySourceFiles()) {
115-
extractFile(config, *s);
122+
extractFile(config, compiler, *s);
116123
}
117124
}
118125
}

swift/extractor/SwiftTagTraits.h

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
#pragma once
2+
3+
#include <swift/AST/ASTVisitor.h>
4+
5+
// autogenerated header
6+
#include "swift/extractor/trap/TrapTags.h"
7+
8+
namespace codeql {
9+
10+
// codegen goes with QL acronym convention (Sil instead of SIL), we need to remap it to Swift's
11+
// convention
12+
using SILBlockStorageTypeTag = SilBlockStorageTypeTag;
13+
using SILBoxTypeTag = SilBoxTypeTag;
14+
using SILFunctionTypeTag = SilFunctionTypeTag;
15+
using SILTokenTypeTag = SilTokenTypeTag;
16+
17+
#define MAP_TYPE_TO_TAG(TYPE, TAG) \
18+
template <> \
19+
struct ToTagFunctor<swift::TYPE> { \
20+
using type = TAG; \
21+
}
22+
#define MAP_TAG(TYPE) MAP_TYPE_TO_TAG(TYPE, TYPE##Tag)
23+
#define MAP_SUBTAG(TYPE, PARENT) \
24+
MAP_TAG(TYPE); \
25+
static_assert(std::is_base_of_v<PARENT##Tag, TYPE##Tag>, \
26+
#PARENT "Tag must be a base of " #TYPE "Tag");
27+
28+
#define OVERRIDE_TAG(TYPE, TAG) \
29+
template <> \
30+
struct ToTagOverride<swift::TYPE> { \
31+
using type = TAG; \
32+
}; \
33+
static_assert(std::is_base_of_v<TYPE##Tag, TAG>, "override is not a subtag");
34+
35+
MAP_TAG(Stmt);
36+
#define ABSTRACT_STMT(CLASS, PARENT) MAP_SUBTAG(CLASS##Stmt, PARENT)
37+
#define STMT(CLASS, PARENT) ABSTRACT_STMT(CLASS, PARENT)
38+
#include "swift/AST/StmtNodes.def"
39+
40+
MAP_TAG(Expr);
41+
#define ABSTRACT_EXPR(CLASS, PARENT) MAP_SUBTAG(CLASS##Expr, PARENT)
42+
#define EXPR(CLASS, PARENT) ABSTRACT_EXPR(CLASS, PARENT)
43+
#include "swift/AST/ExprNodes.def"
44+
45+
MAP_TAG(Decl);
46+
#define ABSTRACT_DECL(CLASS, PARENT) MAP_SUBTAG(CLASS##Decl, PARENT)
47+
#define DECL(CLASS, PARENT) ABSTRACT_DECL(CLASS, PARENT)
48+
#include "swift/AST/DeclNodes.def"
49+
50+
MAP_TAG(Pattern);
51+
#define ABSTRACT_PATTERN(CLASS, PARENT) MAP_SUBTAG(CLASS##Pattern, PARENT)
52+
#define PATTERN(CLASS, PARENT) ABSTRACT_PATTERN(CLASS, PARENT)
53+
#include "swift/AST/PatternNodes.def"
54+
55+
MAP_TAG(TypeRepr);
56+
MAP_TYPE_TO_TAG(TypeBase, TypeTag);
57+
#define ABSTRACT_TYPE(CLASS, PARENT) MAP_SUBTAG(CLASS##Type, PARENT)
58+
#define TYPE(CLASS, PARENT) ABSTRACT_TYPE(CLASS, PARENT)
59+
#include "swift/AST/TypeNodes.def"
60+
61+
OVERRIDE_TAG(FuncDecl, ConcreteFuncDeclTag);
62+
OVERRIDE_TAG(VarDecl, ConcreteVarDeclTag);
63+
64+
#undef MAP_TAG
65+
#undef MAP_SUBTAG
66+
#undef MAP_TYPE_TO_TAG
67+
#undef OVERRIDE_TAG
68+
69+
// All the other macros defined here are undefined by the .def files
70+
71+
} // namespace codeql

swift/extractor/trap/TrapLabel.h

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ class UntypedTrapLabel {
1414

1515
friend class std::hash<UntypedTrapLabel>;
1616

17+
// we want to have access to the untyped, underlying id
18+
friend class TrapLabelStore;
19+
1720
protected:
1821
UntypedTrapLabel() : id_{0xffffffffffffffff} {}
1922
UntypedTrapLabel(uint64_t id) : id_{id} {}
@@ -33,7 +36,6 @@ class TrapLabel : public UntypedTrapLabel {
3336
friend class TrapLabel;
3437

3538
using UntypedTrapLabel::UntypedTrapLabel;
36-
3739
public:
3840
using Tag = TagParam;
3941

swift/extractor/trap/TrapLabelStore.h

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
#pragma once
2+
3+
#include <cassert>
4+
#include <optional>
5+
#include <unordered_map>
6+
7+
#include <swift/AST/ASTVisitor.h>
8+
9+
#include "swift/extractor/trap/TrapLabel.h"
10+
#include "swift/extractor/trap/TrapTagTraits.h"
11+
// autogenerated file
12+
#include "swift/extractor/trap/TrapTags.h"
13+
14+
namespace codeql {
15+
16+
// the following is needed to avoid the problem of subclass pointers not necessarily coinciding
17+
// with superclass ones in case of multiple inheritance
18+
inline const void* getCanonicalPtr(const swift::Decl* e) {
19+
return e;
20+
}
21+
inline const void* getCanonicalPtr(const swift::Stmt* e) {
22+
return e;
23+
}
24+
inline const void* getCanonicalPtr(const swift::Expr* e) {
25+
return e;
26+
}
27+
inline const void* getCanonicalPtr(const swift::Pattern* e) {
28+
return e;
29+
}
30+
inline const void* getCanonicalPtr(const swift::TypeRepr* e) {
31+
return e;
32+
}
33+
inline const void* getCanonicalPtr(const swift::TypeBase* e) {
34+
return e;
35+
}
36+
37+
// The extraction is done in a lazy/on-demand fashion:
38+
// Each emitted TRAP entry for an AST node gets a TRAP label assigned to it.
39+
// To avoid re-emission, we store the "AST node <> label" entry in the TrapLabelStore.
40+
class TrapLabelStore {
41+
public:
42+
template <typename T>
43+
std::optional<TrapLabel<ToTag<T>>> get(const T* e) {
44+
if (auto found = store_.find(getCanonicalPtr(e)); found != store_.end()) {
45+
return TrapLabel<ToTag<T>>::unsafeCreateFromExplicitId(found->second);
46+
}
47+
return std::nullopt;
48+
}
49+
50+
template <typename T>
51+
void insert(const T* e, TrapLabel<ToTag<T>> l) {
52+
auto [_, inserted] = store_.emplace(getCanonicalPtr(e), l.id_);
53+
assert(inserted && "already inserted");
54+
}
55+
56+
private:
57+
std::unordered_map<const void*, uint64_t> store_;
58+
};
59+
60+
} // namespace codeql

0 commit comments

Comments
 (0)