Skip to content

Commit 132fa46

Browse files
christianparpartchristianparpart
committed
Yul: Adds parsing @src comment in AsmParser to customize the AST's sourcer locations.
1 parent 43cde4e commit 132fa46

File tree

6 files changed

+513
-17
lines changed

6 files changed

+513
-17
lines changed

‎libsolidity/interface/CompilerStack.cpp

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,6 @@
8080

8181
#include <utility>
8282
#include <map>
83-
#include <range/v3/view/concat.hpp>
8483

8584
#include <boost/algorithm/string/replace.hpp>
8685

@@ -927,6 +926,19 @@ map<string, unsigned> CompilerStack::sourceIndices() const
927926
return indices;
928927
}
929928

929+
map<unsigned, shared_ptr<CharStream>> CompilerStack::indicesToCharStreams() const
930+
{
931+
map<unsigned, shared_ptr<CharStream>> result;
932+
unsigned index = 0;
933+
for (auto const& s: m_sources)
934+
result[index++] = s.second.scanner->charStream();
935+
936+
// NB: CompilerContext::yulUtilityFileName() does not have a source,
937+
result[index++] = shared_ptr<CharStream>{};
938+
939+
return result;
940+
}
941+
930942
Json::Value const& CompilerStack::contractABI(string const& _contractName) const
931943
{
932944
if (m_stackState < AnalysisPerformed)

‎libsolidity/interface/CompilerStack.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,10 @@ class CompilerStack
239239
/// by sourceNames().
240240
std::map<std::string, unsigned> sourceIndices() const;
241241

242+
/// @returns the reverse mapping of source indices to their respective
243+
/// CharStream instances.
244+
std::map<unsigned, std::shared_ptr<langutil::CharStream>> indicesToCharStreams() const;
245+
242246
/// @returns the previously used scanner, useful for counting lines during error reporting.
243247
langutil::Scanner const& scanner(std::string const& _sourceName) const;
244248

‎libyul/AsmParser.cpp

Lines changed: 90 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,12 @@
2929
#include <libsolutil/Common.h>
3030
#include <libsolutil/Visitor.h>
3131

32+
#include <range/v3/view/subrange.hpp>
33+
3234
#include <boost/algorithm/string.hpp>
3335

3436
#include <algorithm>
37+
#include <regex>
3538

3639
using namespace std;
3740
using namespace solidity;
@@ -53,6 +56,18 @@ shared_ptr<DebugData const> updateLocationEndFrom(
5356
return make_shared<DebugData const>(updatedLocation);
5457
}
5558

59+
optional<int> toInt(string const& _value)
60+
{
61+
try
62+
{
63+
return stoi(_value);
64+
}
65+
catch (...)
66+
{
67+
return nullopt;
68+
}
69+
}
70+
5671
}
5772

5873
unique_ptr<Block> Parser::parse(std::shared_ptr<Scanner> const& _scanner, bool _reuseScanner)
@@ -65,6 +80,8 @@ unique_ptr<Block> Parser::parse(std::shared_ptr<Scanner> const& _scanner, bool _
6580
try
6681
{
6782
m_scanner = _scanner;
83+
if (m_charStreamMap)
84+
fetchSourceLocationFromComment();
6885
auto block = make_unique<Block>(parseBlock());
6986
if (!_reuseScanner)
7087
expectToken(Token::EOS);
@@ -78,14 +95,65 @@ unique_ptr<Block> Parser::parse(std::shared_ptr<Scanner> const& _scanner, bool _
7895
return nullptr;
7996
}
8097

98+
langutil::Token Parser::advance()
99+
{
100+
auto const token = ParserBase::advance();
101+
if (m_useSourceLocationFrom == UseSourceLocationFrom::Comments)
102+
fetchSourceLocationFromComment();
103+
return token;
104+
}
105+
106+
void Parser::fetchSourceLocationFromComment()
107+
{
108+
solAssert(m_charStreamMap.has_value(), "");
109+
110+
if (m_scanner->currentCommentLiteral().empty())
111+
return;
112+
113+
static regex const lineRE = std::regex(
114+
R"~~~((^|\s*)@src\s+(-1|\d+):(-1|\d+):(-1|\d+)(\s+|$))~~~",
115+
std::regex_constants::ECMAScript | std::regex_constants::optimize
116+
);
117+
118+
string const text = m_scanner->currentCommentLiteral();
119+
auto from = sregex_iterator(text.begin(), text.end(), lineRE);
120+
auto to = sregex_iterator();
121+
122+
for (auto const& matchResult: ranges::make_subrange(from, to))
123+
{
124+
solAssert(matchResult.size() == 6, "");
125+
126+
auto const sourceIndex = toInt(matchResult[2].str());
127+
auto const start = toInt(matchResult[3].str());
128+
auto const end = toInt(matchResult[4].str());
129+
130+
auto const commentLocation = m_scanner->currentCommentLocation();
131+
m_locationOverride = SourceLocation{};
132+
if (!sourceIndex || !start || !end)
133+
m_errorReporter.syntaxError(6367_error, commentLocation, "Invalid value in source location mapping. Could not parse location specification.");
134+
else if (!((start < 0 && end < 0) || (start >= 0 && *start <= *end)))
135+
m_errorReporter.syntaxError(5798_error, commentLocation, "Invalid value in source location mapping. Start offset larger than end offset.");
136+
else if (!(sourceIndex >= 0 && m_charStreamMap->count(static_cast<unsigned>(*sourceIndex))))
137+
m_errorReporter.syntaxError(2674_error, commentLocation, "Invalid source mapping. Source index not defined via @use-src.");
138+
else if (sourceIndex >= 0)
139+
{
140+
shared_ptr<CharStream> charStream = m_charStreamMap->at(static_cast<unsigned>(*sourceIndex));
141+
solAssert(charStream, "");
142+
143+
m_locationOverride = SourceLocation{*start, *end, charStream};
144+
}
145+
}
146+
}
147+
81148
Block Parser::parseBlock()
82149
{
83150
RecursionGuard recursionGuard(*this);
84151
Block block = createWithLocation<Block>();
85152
expectToken(Token::LBrace);
86153
while (currentToken() != Token::RBrace)
87154
block.statements.emplace_back(parseStatement());
88-
block.debugData = updateLocationEndFrom(block.debugData, currentLocation());
155+
if (m_useSourceLocationFrom == UseSourceLocationFrom::Scanner)
156+
block.debugData = updateLocationEndFrom(block.debugData, currentLocation());
89157
advance();
90158
return block;
91159
}
@@ -107,7 +175,8 @@ Statement Parser::parseStatement()
107175
advance();
108176
_if.condition = make_unique<Expression>(parseExpression());
109177
_if.body = parseBlock();
110-
_if.debugData = updateLocationEndFrom(_if.debugData, _if.body.debugData->location);
178+
if (m_useSourceLocationFrom == UseSourceLocationFrom::Scanner)
179+
_if.debugData = updateLocationEndFrom(_if.debugData, _if.body.debugData->location);
111180
return Statement{move(_if)};
112181
}
113182
case Token::Switch:
@@ -125,7 +194,8 @@ Statement Parser::parseStatement()
125194
fatalParserError(4904_error, "Case not allowed after default case.");
126195
if (_switch.cases.empty())
127196
fatalParserError(2418_error, "Switch statement without any cases.");
128-
_switch.debugData = updateLocationEndFrom(_switch.debugData, _switch.cases.back().body.debugData->location);
197+
if (m_useSourceLocationFrom == UseSourceLocationFrom::Scanner)
198+
_switch.debugData = updateLocationEndFrom(_switch.debugData, _switch.cases.back().body.debugData->location);
129199
return Statement{move(_switch)};
130200
}
131201
case Token::For:
@@ -207,7 +277,8 @@ Statement Parser::parseStatement()
207277
expectToken(Token::AssemblyAssign);
208278

209279
assignment.value = make_unique<Expression>(parseExpression());
210-
assignment.debugData = updateLocationEndFrom(assignment.debugData, locationOf(*assignment.value));
280+
if (m_useSourceLocationFrom == UseSourceLocationFrom::Scanner)
281+
assignment.debugData = updateLocationEndFrom(assignment.debugData, locationOf(*assignment.value));
211282

212283
return Statement{move(assignment)};
213284
}
@@ -237,7 +308,8 @@ Case Parser::parseCase()
237308
else
238309
yulAssert(false, "Case or default case expected.");
239310
_case.body = parseBlock();
240-
_case.debugData = updateLocationEndFrom(_case.debugData, _case.body.debugData->location);
311+
if (m_useSourceLocationFrom == UseSourceLocationFrom::Scanner)
312+
_case.debugData = updateLocationEndFrom(_case.debugData, _case.body.debugData->location);
241313
return _case;
242314
}
243315

@@ -257,7 +329,8 @@ ForLoop Parser::parseForLoop()
257329
forLoop.post = parseBlock();
258330
m_currentForLoopComponent = ForLoopComponent::ForLoopBody;
259331
forLoop.body = parseBlock();
260-
forLoop.debugData = updateLocationEndFrom(forLoop.debugData, forLoop.body.debugData->location);
332+
if (m_useSourceLocationFrom == UseSourceLocationFrom::Scanner)
333+
forLoop.debugData = updateLocationEndFrom(forLoop.debugData, forLoop.body.debugData->location);
261334

262335
m_currentForLoopComponent = outerForLoopComponent;
263336

@@ -336,7 +409,8 @@ variant<Literal, Identifier> Parser::parseLiteralOrIdentifier()
336409
if (currentToken() == Token::Colon)
337410
{
338411
expectToken(Token::Colon);
339-
literal.debugData = updateLocationEndFrom(literal.debugData, currentLocation());
412+
if (m_useSourceLocationFrom == UseSourceLocationFrom::Scanner)
413+
literal.debugData = updateLocationEndFrom(literal.debugData, currentLocation());
340414
literal.type = expectAsmIdentifier();
341415
}
342416

@@ -368,9 +442,10 @@ VariableDeclaration Parser::parseVariableDeclaration()
368442
{
369443
expectToken(Token::AssemblyAssign);
370444
varDecl.value = make_unique<Expression>(parseExpression());
371-
varDecl.debugData = updateLocationEndFrom(varDecl.debugData, locationOf(*varDecl.value));
445+
if (m_useSourceLocationFrom == UseSourceLocationFrom::Scanner)
446+
varDecl.debugData = updateLocationEndFrom(varDecl.debugData, locationOf(*varDecl.value));
372447
}
373-
else
448+
else if (m_useSourceLocationFrom == UseSourceLocationFrom::Scanner)
374449
varDecl.debugData = updateLocationEndFrom(varDecl.debugData, varDecl.variables.back().debugData->location);
375450

376451
return varDecl;
@@ -417,7 +492,8 @@ FunctionDefinition Parser::parseFunctionDefinition()
417492
m_insideFunction = true;
418493
funDef.body = parseBlock();
419494
m_insideFunction = preInsideFunction;
420-
funDef.debugData = updateLocationEndFrom(funDef.debugData, funDef.body.debugData->location);
495+
if (m_useSourceLocationFrom == UseSourceLocationFrom::Scanner)
496+
funDef.debugData = updateLocationEndFrom(funDef.debugData, funDef.body.debugData->location);
421497

422498
m_currentForLoopComponent = outerForLoopComponent;
423499
return funDef;
@@ -444,7 +520,8 @@ FunctionCall Parser::parseCall(variant<Literal, Identifier>&& _initialOp)
444520
ret.arguments.emplace_back(parseExpression());
445521
}
446522
}
447-
ret.debugData = updateLocationEndFrom(ret.debugData, currentLocation());
523+
if (m_useSourceLocationFrom == UseSourceLocationFrom::Scanner)
524+
ret.debugData = updateLocationEndFrom(ret.debugData, currentLocation());
448525
expectToken(Token::RParen);
449526
return ret;
450527
}
@@ -457,7 +534,8 @@ TypedName Parser::parseTypedName()
457534
if (currentToken() == Token::Colon)
458535
{
459536
expectToken(Token::Colon);
460-
typedName.debugData = updateLocationEndFrom(typedName.debugData, currentLocation());
537+
if (m_useSourceLocationFrom == UseSourceLocationFrom::Scanner)
538+
typedName.debugData = updateLocationEndFrom(typedName.debugData, currentLocation());
461539
typedName.type = expectAsmIdentifier();
462540
}
463541
else

‎libyul/AsmParser.h

Lines changed: 35 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,14 +46,37 @@ class Parser: public langutil::ParserBase
4646
None, ForLoopPre, ForLoopPost, ForLoopBody
4747
};
4848

49+
enum class UseSourceLocationFrom
50+
{
51+
Scanner, LocationOverride, Comments,
52+
};
53+
4954
explicit Parser(
5055
langutil::ErrorReporter& _errorReporter,
5156
Dialect const& _dialect,
5257
std::optional<langutil::SourceLocation> _locationOverride = {}
5358
):
5459
ParserBase(_errorReporter),
5560
m_dialect(_dialect),
56-
m_locationOverride(std::move(_locationOverride))
61+
m_locationOverride{_locationOverride ? *_locationOverride : langutil::SourceLocation{}},
62+
m_useSourceLocationFrom{
63+
_locationOverride ?
64+
UseSourceLocationFrom::LocationOverride :
65+
UseSourceLocationFrom::Scanner
66+
}
67+
{}
68+
69+
/// Constructs a Yul parser that is using the source locations
70+
/// from the comments (via @src).
71+
explicit Parser(
72+
langutil::ErrorReporter& _errorReporter,
73+
Dialect const& _dialect,
74+
std::map<unsigned, std::shared_ptr<langutil::CharStream>> _charStreamMap
75+
):
76+
ParserBase(_errorReporter),
77+
m_dialect(_dialect),
78+
m_charStreamMap{std::move(_charStreamMap)},
79+
m_useSourceLocationFrom{UseSourceLocationFrom::Comments}
5780
{}
5881

5982
/// Parses an inline assembly block starting with `{` and ending with `}`.
@@ -64,9 +87,15 @@ class Parser: public langutil::ParserBase
6487
protected:
6588
langutil::SourceLocation currentLocation() const override
6689
{
67-
return m_locationOverride ? *m_locationOverride : ParserBase::currentLocation();
90+
if (m_useSourceLocationFrom == UseSourceLocationFrom::Scanner)
91+
return ParserBase::currentLocation();
92+
return m_locationOverride;
6893
}
6994

95+
langutil::Token advance() override;
96+
97+
void fetchSourceLocationFromComment();
98+
7099
/// Creates an inline assembly node with the current source location.
71100
template <class T> T createWithLocation() const
72101
{
@@ -97,7 +126,10 @@ class Parser: public langutil::ParserBase
97126

98127
private:
99128
Dialect const& m_dialect;
100-
std::optional<langutil::SourceLocation> m_locationOverride;
129+
130+
std::optional<std::map<unsigned, std::shared_ptr<langutil::CharStream>>> m_charStreamMap;
131+
langutil::SourceLocation m_locationOverride;
132+
UseSourceLocationFrom m_useSourceLocationFrom = UseSourceLocationFrom::Scanner;
101133
ForLoopComponent m_currentForLoopComponent = ForLoopComponent::None;
102134
bool m_insideFunction = false;
103135
};

‎scripts/error_codes.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,9 @@ def examine_id_coverage(top_dir, source_id_to_file_names, new_ids_only=False):
191191

192192
# white list of ids which are not covered by tests
193193
white_ids = {
194+
"6367", # these three are temporarily whitelisted until both PRs are merged.
195+
"5798",
196+
"2674",
194197
"3805", # "This is a pre-release compiler version, please do not use it in production."
195198
# The warning may or may not exist in a compiler build.
196199
"4591", # "There are more than 256 warnings. Ignoring the rest."

0 commit comments

Comments
 (0)