Skip to content

Commit 28845ad

Browse files
chrisethchriseth
authored
Merge pull request #11558 from ethereum/KeyValueParser
Yul: custom source locations (`@src`)
2 parents c3fa520 + f129a34 commit 28845ad

File tree

6 files changed

+558
-20
lines changed

6 files changed

+558
-20
lines changed

‎libsolidity/interface/CompilerStack.cpp

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,6 @@
8080

8181
#include <utility>
8282
#include <map>
83-
#include <range/v3/view/concat.hpp>
8483

8584
#include <boost/algorithm/string/replace.hpp>
8685

@@ -927,6 +926,19 @@ map<string, unsigned> CompilerStack::sourceIndices() const
927926
return indices;
928927
}
929928

929+
map<unsigned, shared_ptr<CharStream>> CompilerStack::indicesToCharStreams() const
930+
{
931+
map<unsigned, shared_ptr<CharStream>> result;
932+
unsigned index = 0;
933+
for (auto const& s: m_sources)
934+
result[index++] = s.second.scanner->charStream();
935+
936+
// NB: CompilerContext::yulUtilityFileName() does not have a source,
937+
result[index++] = shared_ptr<CharStream>{};
938+
939+
return result;
940+
}
941+
930942
Json::Value const& CompilerStack::contractABI(string const& _contractName) const
931943
{
932944
if (m_stackState < AnalysisPerformed)

‎libsolidity/interface/CompilerStack.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,10 @@ class CompilerStack
239239
/// by sourceNames().
240240
std::map<std::string, unsigned> sourceIndices() const;
241241

242+
/// @returns the reverse mapping of source indices to their respective
243+
/// CharStream instances.
244+
std::map<unsigned, std::shared_ptr<langutil::CharStream>> indicesToCharStreams() const;
245+
242246
/// @returns the previously used scanner, useful for counting lines during error reporting.
243247
langutil::Scanner const& scanner(std::string const& _sourceName) const;
244248

‎libyul/AsmParser.cpp

Lines changed: 93 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,12 @@
2929
#include <libsolutil/Common.h>
3030
#include <libsolutil/Visitor.h>
3131

32+
#include <range/v3/view/subrange.hpp>
33+
3234
#include <boost/algorithm/string.hpp>
3335

3436
#include <algorithm>
37+
#include <regex>
3538

3639
using namespace std;
3740
using namespace solidity;
@@ -53,6 +56,18 @@ shared_ptr<DebugData const> updateLocationEndFrom(
5356
return make_shared<DebugData const>(updatedLocation);
5457
}
5558

59+
optional<int> toInt(string const& _value)
60+
{
61+
try
62+
{
63+
return stoi(_value);
64+
}
65+
catch (...)
66+
{
67+
return nullopt;
68+
}
69+
}
70+
5671
}
5772

5873
unique_ptr<Block> Parser::parse(std::shared_ptr<Scanner> const& _scanner, bool _reuseScanner)
@@ -65,6 +80,8 @@ unique_ptr<Block> Parser::parse(std::shared_ptr<Scanner> const& _scanner, bool _
6580
try
6681
{
6782
m_scanner = _scanner;
83+
if (m_charStreamMap)
84+
fetchSourceLocationFromComment();
6885
auto block = make_unique<Block>(parseBlock());
6986
if (!_reuseScanner)
7087
expectToken(Token::EOS);
@@ -78,14 +95,66 @@ unique_ptr<Block> Parser::parse(std::shared_ptr<Scanner> const& _scanner, bool _
7895
return nullptr;
7996
}
8097

98+
langutil::Token Parser::advance()
99+
{
100+
auto const token = ParserBase::advance();
101+
if (m_useSourceLocationFrom == UseSourceLocationFrom::Comments)
102+
fetchSourceLocationFromComment();
103+
return token;
104+
}
105+
106+
void Parser::fetchSourceLocationFromComment()
107+
{
108+
solAssert(m_charStreamMap.has_value(), "");
109+
110+
if (m_scanner->currentCommentLiteral().empty())
111+
return;
112+
113+
static regex const lineRE = std::regex(
114+
R"~~~((^|\s*)@src\s+(-1|\d+):(-1|\d+):(-1|\d+)(\s+|$))~~~",
115+
std::regex_constants::ECMAScript | std::regex_constants::optimize
116+
);
117+
118+
string const text = m_scanner->currentCommentLiteral();
119+
auto from = sregex_iterator(text.begin(), text.end(), lineRE);
120+
auto to = sregex_iterator();
121+
122+
for (auto const& matchResult: ranges::make_subrange(from, to))
123+
{
124+
solAssert(matchResult.size() == 6, "");
125+
126+
auto const sourceIndex = toInt(matchResult[2].str());
127+
auto const start = toInt(matchResult[3].str());
128+
auto const end = toInt(matchResult[4].str());
129+
130+
auto const commentLocation = m_scanner->currentCommentLocation();
131+
m_debugDataOverride = DebugData::create();
132+
if (!sourceIndex || !start || !end)
133+
m_errorReporter.syntaxError(6367_error, commentLocation, "Invalid value in source location mapping. Could not parse location specification.");
134+
else if (!((*start < 0 && *end < 0) || (*start >= 0 && *start <= *end)))
135+
m_errorReporter.syntaxError(5798_error, commentLocation, "Invalid value in source location mapping. Start offset larger than end offset.");
136+
else if (sourceIndex == -1 && (0 <= *start && *start <= *end)) // Use source index -1 to indicate original source.
137+
m_debugDataOverride = DebugData::create(SourceLocation{*start, *end, ParserBase::currentLocation().source});
138+
else if (!(sourceIndex >= 0 && m_charStreamMap->count(static_cast<unsigned>(*sourceIndex))))
139+
m_errorReporter.syntaxError(2674_error, commentLocation, "Invalid source mapping. Source index not defined via @use-src.");
140+
else
141+
{
142+
shared_ptr<CharStream> charStream = m_charStreamMap->at(static_cast<unsigned>(*sourceIndex));
143+
solAssert(charStream, "");
144+
m_debugDataOverride = DebugData::create(SourceLocation{*start, *end, charStream});
145+
}
146+
}
147+
}
148+
81149
Block Parser::parseBlock()
82150
{
83151
RecursionGuard recursionGuard(*this);
84152
Block block = createWithLocation<Block>();
85153
expectToken(Token::LBrace);
86154
while (currentToken() != Token::RBrace)
87155
block.statements.emplace_back(parseStatement());
88-
block.debugData = updateLocationEndFrom(block.debugData, currentLocation());
156+
if (m_useSourceLocationFrom == UseSourceLocationFrom::Scanner)
157+
block.debugData = updateLocationEndFrom(block.debugData, currentLocation());
89158
advance();
90159
return block;
91160
}
@@ -107,7 +176,8 @@ Statement Parser::parseStatement()
107176
advance();
108177
_if.condition = make_unique<Expression>(parseExpression());
109178
_if.body = parseBlock();
110-
_if.debugData = updateLocationEndFrom(_if.debugData, _if.body.debugData->location);
179+
if (m_useSourceLocationFrom == UseSourceLocationFrom::Scanner)
180+
_if.debugData = updateLocationEndFrom(_if.debugData, _if.body.debugData->location);
111181
return Statement{move(_if)};
112182
}
113183
case Token::Switch:
@@ -125,7 +195,8 @@ Statement Parser::parseStatement()
125195
fatalParserError(4904_error, "Case not allowed after default case.");
126196
if (_switch.cases.empty())
127197
fatalParserError(2418_error, "Switch statement without any cases.");
128-
_switch.debugData = updateLocationEndFrom(_switch.debugData, _switch.cases.back().body.debugData->location);
198+
if (m_useSourceLocationFrom == UseSourceLocationFrom::Scanner)
199+
_switch.debugData = updateLocationEndFrom(_switch.debugData, _switch.cases.back().body.debugData->location);
129200
return Statement{move(_switch)};
130201
}
131202
case Token::For:
@@ -207,7 +278,8 @@ Statement Parser::parseStatement()
207278
expectToken(Token::AssemblyAssign);
208279

209280
assignment.value = make_unique<Expression>(parseExpression());
210-
assignment.debugData = updateLocationEndFrom(assignment.debugData, locationOf(*assignment.value));
281+
if (m_useSourceLocationFrom == UseSourceLocationFrom::Scanner)
282+
assignment.debugData = updateLocationEndFrom(assignment.debugData, locationOf(*assignment.value));
211283

212284
return Statement{move(assignment)};
213285
}
@@ -237,7 +309,8 @@ Case Parser::parseCase()
237309
else
238310
yulAssert(false, "Case or default case expected.");
239311
_case.body = parseBlock();
240-
_case.debugData = updateLocationEndFrom(_case.debugData, _case.body.debugData->location);
312+
if (m_useSourceLocationFrom == UseSourceLocationFrom::Scanner)
313+
_case.debugData = updateLocationEndFrom(_case.debugData, _case.body.debugData->location);
241314
return _case;
242315
}
243316

@@ -257,7 +330,8 @@ ForLoop Parser::parseForLoop()
257330
forLoop.post = parseBlock();
258331
m_currentForLoopComponent = ForLoopComponent::ForLoopBody;
259332
forLoop.body = parseBlock();
260-
forLoop.debugData = updateLocationEndFrom(forLoop.debugData, forLoop.body.debugData->location);
333+
if (m_useSourceLocationFrom == UseSourceLocationFrom::Scanner)
334+
forLoop.debugData = updateLocationEndFrom(forLoop.debugData, forLoop.body.debugData->location);
261335

262336
m_currentForLoopComponent = outerForLoopComponent;
263337

@@ -296,7 +370,7 @@ variant<Literal, Identifier> Parser::parseLiteralOrIdentifier()
296370
{
297371
case Token::Identifier:
298372
{
299-
Identifier identifier{DebugData::create(currentLocation()), YulString{currentLiteral()}};
373+
Identifier identifier{createDebugData(), YulString{currentLiteral()}};
300374
advance();
301375
return identifier;
302376
}
@@ -327,7 +401,7 @@ variant<Literal, Identifier> Parser::parseLiteralOrIdentifier()
327401
}
328402

329403
Literal literal{
330-
DebugData::create(currentLocation()),
404+
createDebugData(),
331405
kind,
332406
YulString{currentLiteral()},
333407
kind == LiteralKind::Boolean ? m_dialect.boolType : m_dialect.defaultType
@@ -336,7 +410,8 @@ variant<Literal, Identifier> Parser::parseLiteralOrIdentifier()
336410
if (currentToken() == Token::Colon)
337411
{
338412
expectToken(Token::Colon);
339-
literal.debugData = updateLocationEndFrom(literal.debugData, currentLocation());
413+
if (m_useSourceLocationFrom == UseSourceLocationFrom::Scanner)
414+
literal.debugData = updateLocationEndFrom(literal.debugData, currentLocation());
340415
literal.type = expectAsmIdentifier();
341416
}
342417

@@ -368,9 +443,10 @@ VariableDeclaration Parser::parseVariableDeclaration()
368443
{
369444
expectToken(Token::AssemblyAssign);
370445
varDecl.value = make_unique<Expression>(parseExpression());
371-
varDecl.debugData = updateLocationEndFrom(varDecl.debugData, locationOf(*varDecl.value));
446+
if (m_useSourceLocationFrom == UseSourceLocationFrom::Scanner)
447+
varDecl.debugData = updateLocationEndFrom(varDecl.debugData, locationOf(*varDecl.value));
372448
}
373-
else
449+
else if (m_useSourceLocationFrom == UseSourceLocationFrom::Scanner)
374450
varDecl.debugData = updateLocationEndFrom(varDecl.debugData, varDecl.variables.back().debugData->location);
375451

376452
return varDecl;
@@ -417,7 +493,8 @@ FunctionDefinition Parser::parseFunctionDefinition()
417493
m_insideFunction = true;
418494
funDef.body = parseBlock();
419495
m_insideFunction = preInsideFunction;
420-
funDef.debugData = updateLocationEndFrom(funDef.debugData, funDef.body.debugData->location);
496+
if (m_useSourceLocationFrom == UseSourceLocationFrom::Scanner)
497+
funDef.debugData = updateLocationEndFrom(funDef.debugData, funDef.body.debugData->location);
421498

422499
m_currentForLoopComponent = outerForLoopComponent;
423500
return funDef;
@@ -444,7 +521,8 @@ FunctionCall Parser::parseCall(variant<Literal, Identifier>&& _initialOp)
444521
ret.arguments.emplace_back(parseExpression());
445522
}
446523
}
447-
ret.debugData = updateLocationEndFrom(ret.debugData, currentLocation());
524+
if (m_useSourceLocationFrom == UseSourceLocationFrom::Scanner)
525+
ret.debugData = updateLocationEndFrom(ret.debugData, currentLocation());
448526
expectToken(Token::RParen);
449527
return ret;
450528
}
@@ -457,7 +535,8 @@ TypedName Parser::parseTypedName()
457535
if (currentToken() == Token::Colon)
458536
{
459537
expectToken(Token::Colon);
460-
typedName.debugData = updateLocationEndFrom(typedName.debugData, currentLocation());
538+
if (m_useSourceLocationFrom == UseSourceLocationFrom::Scanner)
539+
typedName.debugData = updateLocationEndFrom(typedName.debugData, currentLocation());
461540
typedName.type = expectAsmIdentifier();
462541
}
463542
else

‎libyul/AsmParser.h

Lines changed: 55 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323

2424
#pragma once
2525

26+
#include <libyul/AST.h>
2627
#include <libyul/ASTForward.h>
2728
#include <libyul/Dialect.h>
2829

@@ -45,14 +46,39 @@ class Parser: public langutil::ParserBase
4546
None, ForLoopPre, ForLoopPost, ForLoopBody
4647
};
4748

49+
enum class UseSourceLocationFrom
50+
{
51+
Scanner, LocationOverride, Comments,
52+
};
53+
4854
explicit Parser(
4955
langutil::ErrorReporter& _errorReporter,
5056
Dialect const& _dialect,
5157
std::optional<langutil::SourceLocation> _locationOverride = {}
5258
):
5359
ParserBase(_errorReporter),
5460
m_dialect(_dialect),
55-
m_locationOverride(std::move(_locationOverride))
61+
m_locationOverride{_locationOverride ? *_locationOverride : langutil::SourceLocation{}},
62+
m_debugDataOverride{},
63+
m_useSourceLocationFrom{
64+
_locationOverride ?
65+
UseSourceLocationFrom::LocationOverride :
66+
UseSourceLocationFrom::Scanner
67+
}
68+
{}
69+
70+
/// Constructs a Yul parser that is using the source locations
71+
/// from the comments (via @src).
72+
explicit Parser(
73+
langutil::ErrorReporter& _errorReporter,
74+
Dialect const& _dialect,
75+
std::map<unsigned, std::shared_ptr<langutil::CharStream>> _charStreamMap
76+
):
77+
ParserBase(_errorReporter),
78+
m_dialect(_dialect),
79+
m_charStreamMap{std::move(_charStreamMap)},
80+
m_debugDataOverride{DebugData::create()},
81+
m_useSourceLocationFrom{UseSourceLocationFrom::Comments}
5682
{}
5783

5884
/// Parses an inline assembly block starting with `{` and ending with `}`.
@@ -63,14 +89,35 @@ class Parser: public langutil::ParserBase
6389
protected:
6490
langutil::SourceLocation currentLocation() const override
6591
{
66-
return m_locationOverride ? *m_locationOverride : ParserBase::currentLocation();
92+
if (m_useSourceLocationFrom == UseSourceLocationFrom::Scanner)
93+
return ParserBase::currentLocation();
94+
return m_locationOverride;
95+
}
96+
97+
langutil::Token advance() override;
98+
99+
void fetchSourceLocationFromComment();
100+
101+
/// Creates a DebugData object with the correct source location set.
102+
std::shared_ptr<DebugData const> createDebugData() const
103+
{
104+
switch (m_useSourceLocationFrom)
105+
{
106+
case UseSourceLocationFrom::Scanner:
107+
return DebugData::create(ParserBase::currentLocation());
108+
case UseSourceLocationFrom::LocationOverride:
109+
return DebugData::create(m_locationOverride);
110+
case UseSourceLocationFrom::Comments:
111+
return m_debugDataOverride;
112+
}
113+
solAssert(false, "");
67114
}
68115

69116
/// Creates an inline assembly node with the current source location.
70117
template <class T> T createWithLocation() const
71118
{
72119
T r;
73-
r.debugData = DebugData::create(currentLocation());
120+
r.debugData = createDebugData();
74121
return r;
75122
}
76123

@@ -96,7 +143,11 @@ class Parser: public langutil::ParserBase
96143

97144
private:
98145
Dialect const& m_dialect;
99-
std::optional<langutil::SourceLocation> m_locationOverride;
146+
147+
std::optional<std::map<unsigned, std::shared_ptr<langutil::CharStream>>> m_charStreamMap;
148+
langutil::SourceLocation m_locationOverride;
149+
std::shared_ptr<DebugData const> m_debugDataOverride;
150+
UseSourceLocationFrom m_useSourceLocationFrom = UseSourceLocationFrom::Scanner;
100151
ForLoopComponent m_currentForLoopComponent = ForLoopComponent::None;
101152
bool m_insideFunction = false;
102153
};

‎scripts/error_codes.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,9 @@ def examine_id_coverage(top_dir, source_id_to_file_names, new_ids_only=False):
191191

192192
# white list of ids which are not covered by tests
193193
white_ids = {
194+
"6367", # these three are temporarily whitelisted until both PRs are merged.
195+
"5798",
196+
"2674",
194197
"3805", # "This is a pre-release compiler version, please do not use it in production."
195198
# The warning may or may not exist in a compiler build.
196199
"4591", # "There are more than 256 warnings. Ignoring the rest."

0 commit comments

Comments
 (0)