Merge pull request #11658 from ethereum/removeScannerFromCompilerStack

Remove scanner from compiler stack

Author: chriseth

liblangutil/ParserBase.h

@@ -24,7 +24,6 @@
 #pragma once
 
 #include <liblangutil/Token.h>
-#include <liblangutil/Scanner.h>
 #include <memory>
 #include <string>
 
@@ -50,8 +49,6 @@ class ParserBase
 
 	virtual ~ParserBase() = default;
 
-	std::shared_ptr<CharStream> source() const { return m_scanner->charStream(); }
-
 protected:
 	/// Utility class that creates an error and throws an exception if the
 	/// recursion depth is too deep.

liblangutil/Scanner.cpp

@@ -135,26 +135,11 @@ class LiteralScope
 	bool m_complete;
 };
 
-void Scanner::reset(CharStream _source)
-{
-	m_source = make_shared<CharStream>(std::move(_source));
-	m_sourceName = make_shared<string>(m_source->name());
-	reset();
-}
-
-void Scanner::reset(shared_ptr<CharStream> _source)
-{
-	solAssert(_source.get() != nullptr, "You MUST provide a CharStream when resetting.");
-	m_source = std::move(_source);
-	m_sourceName = make_shared<string>(m_source->name());
-	reset();
-}
-
 void Scanner::reset()
 {
-	m_source->reset();
+	m_source.reset();
 	m_kind = ScannerKind::Solidity;
-	m_char = m_source->get();
+	m_char = m_source.get();
 	skipWhitespace();
 	next();
 	next();
@@ -163,7 +148,7 @@ void Scanner::reset()
 
 void Scanner::setPosition(size_t _offset)
 {
-	m_char = m_source->setPosition(_offset);
+	m_char = m_source.setPosition(_offset);
 	scanToken();
 	next();
 	next();
@@ -229,7 +214,7 @@ void Scanner::rescan()
 		rollbackTo = static_cast<size_t>(m_tokens[Current].location.start);
 	else
 		rollbackTo = static_cast<size_t>(m_skippedComments[Current].location.start);
-	m_char = m_source->rollback(m_source->position() - rollbackTo);
+	m_char = m_source.rollback(m_source.position() - rollbackTo);
 	next();
 	next();
 	next();
@@ -324,12 +309,12 @@ Token Scanner::skipSingleLineComment()
 {
 	// Line terminator is not part of the comment. If it is a
 	// non-ascii line terminator, it will result in a parser error.
-	size_t startPosition = m_source->position();
+	size_t startPosition = m_source.position();
 	while (!isUnicodeLinebreak())
 		if (!advance())
 			break;
 
-	ScannerError unicodeDirectionError = validateBiDiMarkup(*m_source, startPosition);
+	ScannerError unicodeDirectionError = validateBiDiMarkup(m_source, startPosition);
 	if (unicodeDirectionError != ScannerError::NoError)
 		return setError(unicodeDirectionError);
 
@@ -362,28 +347,28 @@ bool Scanner::tryScanEndOfLine()
 size_t Scanner::scanSingleLineDocComment()
 {
 	LiteralScope literal(this, LITERAL_TYPE_COMMENT);
-	size_t endPosition = m_source->position();
+	size_t endPosition = m_source.position();
 
 	skipWhitespaceExceptUnicodeLinebreak();
 
 	while (!isSourcePastEndOfInput())
 	{
-		endPosition = m_source->position();
+		endPosition = m_source.position();
 		if (tryScanEndOfLine())
 		{
 			// Check if next line is also a single-line comment.
 			// If any whitespaces were skipped, use source position before.
 			if (!skipWhitespaceExceptUnicodeLinebreak())
-				endPosition = m_source->position();
+				endPosition = m_source.position();
 
-			if (!m_source->isPastEndOfInput(3) &&
-				m_source->get(0) == '/' &&
-				m_source->get(1) == '/' &&
-				m_source->get(2) == '/')
+			if (!m_source.isPastEndOfInput(3) &&
+				m_source.get(0) == '/' &&
+				m_source.get(1) == '/' &&
+				m_source.get(2) == '/')
 			{
-				if (!m_source->isPastEndOfInput(4) && m_source->get(3) == '/')
+				if (!m_source.isPastEndOfInput(4) && m_source.get(3) == '/')
 					break; // "////" is not a documentation comment
-				m_char = m_source->advanceAndGet(3);
+				m_char = m_source.advanceAndGet(3);
 				if (atEndOfLine())
 					continue;
 				addCommentLiteralChar('\n');
@@ -404,7 +389,7 @@ size_t Scanner::scanSingleLineDocComment()
 
 Token Scanner::skipMultiLineComment()
 {
-	size_t startPosition = m_source->position();
+	size_t startPosition = m_source.position();
 	while (!isSourcePastEndOfInput())
 	{
 		char prevChar = m_char;
@@ -415,7 +400,7 @@ Token Scanner::skipMultiLineComment()
 		// multi-line comments are treated as whitespace.
 		if (prevChar == '*' && m_char == '/')
 		{
-			ScannerError unicodeDirectionError = validateBiDiMarkup(*m_source, startPosition);
+			ScannerError unicodeDirectionError = validateBiDiMarkup(m_source, startPosition);
 			if (unicodeDirectionError != ScannerError::NoError)
 				return setError(unicodeDirectionError);
 
@@ -442,32 +427,32 @@ Token Scanner::scanMultiLineDocComment()
 		if (atEndOfLine())
 		{
 			skipWhitespace();
-			if (!m_source->isPastEndOfInput(1) && m_source->get(0) == '*' && m_source->get(1) == '*')
+			if (!m_source.isPastEndOfInput(1) && m_source.get(0) == '*' && m_source.get(1) == '*')
 			{ // it is unknown if this leads to the end of the comment
 				addCommentLiteralChar('*');
 				advance();
 			}
-			else if (!m_source->isPastEndOfInput(1) && m_source->get(0) == '*' && m_source->get(1) != '/')
+			else if (!m_source.isPastEndOfInput(1) && m_source.get(0) == '*' && m_source.get(1) != '/')
 			{ // skip first '*' in subsequent lines
-				m_char = m_source->advanceAndGet(1);
+				m_char = m_source.advanceAndGet(1);
 				if (atEndOfLine()) // ignores empty lines
 					continue;
 				if (charsAdded)
 					addCommentLiteralChar('\n'); // corresponds to the end of previous line
 			}
-			else if (!m_source->isPastEndOfInput(1) && m_source->get(0) == '*' && m_source->get(1) == '/')
+			else if (!m_source.isPastEndOfInput(1) && m_source.get(0) == '*' && m_source.get(1) == '/')
 			{ // if after newline the comment ends, don't insert the newline
-				m_char = m_source->advanceAndGet(2);
+				m_char = m_source.advanceAndGet(2);
 				endFound = true;
 				break;
 			}
 			else if (charsAdded)
 				addCommentLiteralChar('\n');
 		}
 
-		if (!m_source->isPastEndOfInput(1) && m_source->get(0) == '*' && m_source->get(1) == '/')
+		if (!m_source.isPastEndOfInput(1) && m_source.get(0) == '*' && m_source.get(1) == '/')
 		{
-			m_char = m_source->advanceAndGet(2);
+			m_char = m_source.advanceAndGet(2);
 			endFound = true;
 			break;
 		}
@@ -822,11 +807,11 @@ bool Scanner::isUnicodeLinebreak()
 	if (0x0a <= m_char && m_char <= 0x0d)
 		// line feed, vertical tab, form feed, carriage return
 		return true;
-	if (!m_source->isPastEndOfInput(1) && uint8_t(m_source->get(0)) == 0xc2 && uint8_t(m_source->get(1)) == 0x85)
+	if (!m_source.isPastEndOfInput(1) && uint8_t(m_source.get(0)) == 0xc2 && uint8_t(m_source.get(1)) == 0x85)
 		// NEL - U+0085, C2 85 in utf8
 		return true;
-	if (!m_source->isPastEndOfInput(2) && uint8_t(m_source->get(0)) == 0xe2 && uint8_t(m_source->get(1)) == 0x80 && (
-		uint8_t(m_source->get(2)) == 0xa8 || uint8_t(m_source->get(2)) == 0xa9
+	if (!m_source.isPastEndOfInput(2) && uint8_t(m_source.get(0)) == 0xe2 && uint8_t(m_source.get(1)) == 0x80 && (
+		uint8_t(m_source.get(2)) == 0xa8 || uint8_t(m_source.get(2)) == 0xa9
 	))
 		// LS - U+2028, E2 80 A8  in utf8
 		// PS - U+2029, E2 80 A9  in utf8
@@ -836,7 +821,7 @@ bool Scanner::isUnicodeLinebreak()
 
 Token Scanner::scanString(bool const _isUnicode)
 {
-	size_t startPosition = m_source->position();
+	size_t startPosition = m_source.position();
 	char const quote = m_char;
 	advance();  // consume quote
 	LiteralScope literal(this, LITERAL_TYPE_STRING);
@@ -867,7 +852,7 @@ Token Scanner::scanString(bool const _isUnicode)
 
 	if (_isUnicode)
 	{
-		ScannerError unicodeDirectionError = validateBiDiMarkup(*m_source, startPosition);
+		ScannerError unicodeDirectionError = validateBiDiMarkup(m_source, startPosition);
 		if (unicodeDirectionError != ScannerError::NoError)
 			return setError(unicodeDirectionError);
 	}
@@ -921,7 +906,7 @@ void Scanner::scanDecimalDigits()
 	// May continue with decimal digit or underscore for grouping.
 	do
 		addLiteralCharAndAdvance();
-	while (!m_source->isPastEndOfInput() && (isDecimalDigit(m_char) || m_char == '_'));
+	while (!m_source.isPastEndOfInput() && (isDecimalDigit(m_char) || m_char == '_'));
 
 	// Defer further validation of underscore to SyntaxChecker.
 }
@@ -967,15 +952,15 @@ Token Scanner::scanNumber(char _charSeen)
 			scanDecimalDigits();  // optional
 			if (m_char == '.')
 			{
-				if (!m_source->isPastEndOfInput(1) && m_source->get(1) == '_')
+				if (!m_source.isPastEndOfInput(1) && m_source.get(1) == '_')
 				{
 					// Assume the input may be a floating point number with leading '_' in fraction part.
 					// Recover by consuming it all but returning `Illegal` right away.
 					addLiteralCharAndAdvance(); // '.'
 					addLiteralCharAndAdvance(); // '_'
 					scanDecimalDigits();
 				}
-				if (m_source->isPastEndOfInput() || !isDecimalDigit(m_source->get(1)))
+				if (m_source.isPastEndOfInput() || !isDecimalDigit(m_source.get(1)))
 				{
 					// A '.' has to be followed by a number.
 					literal.complete();
@@ -992,7 +977,7 @@ Token Scanner::scanNumber(char _charSeen)
 		solAssert(kind != HEX, "'e'/'E' must be scanned as part of the hex number");
 		if (kind != DECIMAL)
 			return setError(ScannerError::IllegalExponent);
-		else if (!m_source->isPastEndOfInput(1) && m_source->get(1) == '_')
+		else if (!m_source.isPastEndOfInput(1) && m_source.get(1) == '_')
 		{
 			// Recover from wrongly placed underscore as delimiter in literal with scientific
 			// notation by consuming until the end.

liblangutil/Scanner.h

@@ -100,17 +100,13 @@ class Scanner
 {
 	friend class LiteralScope;
 public:
-	explicit Scanner(std::shared_ptr<CharStream> _source) { reset(std::move(_source)); }
-	explicit Scanner(CharStream _source = CharStream()) { reset(std::move(_source)); }
-
-	std::string const& source() const noexcept { return m_source->source(); }
-
-	std::shared_ptr<CharStream> charStream() noexcept { return m_source; }
-	std::shared_ptr<CharStream const> charStream() const noexcept { return m_source; }
+	explicit Scanner(CharStream& _source):
+		m_source(_source),
+		m_sourceName{std::make_shared<std::string>(_source.name())}
+	{
+		reset();
+	}
 
-	/// Resets the scanner as if newly constructed with _source as input.
-	void reset(CharStream _source);
-	void reset(std::shared_ptr<CharStream> _source);
 	/// Resets scanner to the start of input.
 	void reset();
 
@@ -123,6 +119,8 @@ class Scanner
 		rescan();
 	}
 
+	CharStream const& charStream() const noexcept { return m_source; }
+
 	/// @returns the next token and advances input
 	Token next();
 
@@ -201,8 +199,8 @@ class Scanner
 	void addUnicodeAsUTF8(unsigned codepoint);
 	///@}
 
-	bool advance() { m_char = m_source->advanceAndGet(); return !m_source->isPastEndOfInput(); }
-	void rollback(size_t _amount) { m_char = m_source->rollback(_amount); }
+	bool advance() { m_char = m_source.advanceAndGet(); return !m_source.isPastEndOfInput(); }
+	void rollback(size_t _amount) { m_char = m_source.rollback(_amount); }
 	/// Rolls back to the start of the current token and re-runs the scanner.
 	void rescan();
 
@@ -251,15 +249,15 @@ class Scanner
 	bool isUnicodeLinebreak();
 
 	/// Return the current source position.
-	size_t sourcePos() const { return m_source->position(); }
-	bool isSourcePastEndOfInput() const { return m_source->isPastEndOfInput(); }
+	size_t sourcePos() const { return m_source.position(); }
+	bool isSourcePastEndOfInput() const { return m_source.isPastEndOfInput(); }
 
 	enum TokenIndex { Current, Next, NextNext };
 
 	TokenDesc m_skippedComments[3] = {}; // desc for the current, next and nextnext skipped comment
 	TokenDesc m_tokens[3] = {}; // desc for the current, next and nextnext token
 
-	std::shared_ptr<CharStream> m_source;
+	CharStream& m_source;
 	std::shared_ptr<std::string const> m_sourceName;
 
 	ScannerKind m_kind = ScannerKind::Solidity;

libsolidity/ast/ASTJsonImporter.cpp

@@ -958,7 +958,8 @@ Json::Value ASTJsonImporter::member(Json::Value const& _node, string const& _nam
 
 Token ASTJsonImporter::scanSingleToken(Json::Value const& _node)
 {
-	langutil::Scanner scanner{langutil::CharStream(_node.asString(), "")};
+	langutil::CharStream charStream(_node.asString(), "");
+	langutil::Scanner scanner{charStream};
 	astAssert(scanner.peekNextToken() == Token::EOS, "Token string is too long.");
 	return scanner.currentToken();
 }

libsolidity/codegen/CompilerContext.cpp

@@ -434,14 +434,14 @@ void CompilerContext::appendInlineAssembly(
 
 	ErrorList errors;
 	ErrorReporter errorReporter(errors);
-	auto scanner = make_shared<langutil::Scanner>(langutil::CharStream(_assembly, _sourceName));
+	langutil::CharStream charStream(_assembly, _sourceName);
 	yul::EVMDialect const& dialect = yul::EVMDialect::strictAssemblyForEVM(m_evmVersion);
 	optional<langutil::SourceLocation> locationOverride;
 	if (!_system)
 		locationOverride = m_asm->currentSourceLocation();
 	shared_ptr<yul::Block> parserResult =
 		yul::Parser(errorReporter, dialect, std::move(locationOverride))
-		.parse(scanner, false);
+		.parse(charStream);
 #ifdef SOL_OUTPUT_ASM
 	cout << yul::AsmPrinter(&dialect)(*parserResult) << endl;
 #endif
@@ -457,7 +457,7 @@ void CompilerContext::appendInlineAssembly(
 		for (auto const& error: errorReporter.errors())
 			// TODO if we have "locationOverride", it will be the wrong char stream,
 			// but we do not have access to the solidity scanner.
-			message += SourceReferenceFormatter::formatErrorInformation(*error, *scanner->charStream());
+			message += SourceReferenceFormatter::formatErrorInformation(*error, charStream);
 		message += "-------------------------------------------\n";
 
 		solAssert(false, message);
@@ -491,8 +491,8 @@ void CompilerContext::appendInlineAssembly(
 			solAssert(m_generatedYulUtilityCode.empty(), "");
 			m_generatedYulUtilityCode = yul::AsmPrinter(dialect)(*obj.code);
 			string code = yul::AsmPrinter{dialect}(*obj.code);
-			scanner = make_shared<langutil::Scanner>(langutil::CharStream(m_generatedYulUtilityCode, _sourceName));
-			obj.code = yul::Parser(errorReporter, dialect).parse(scanner, false);
+			langutil::CharStream charStream(m_generatedYulUtilityCode, _sourceName);
+			obj.code = yul::Parser(errorReporter, dialect).parse(charStream);
 			*obj.analysisInfo = yul::AsmAnalyzer::analyzeStrictAssertCorrect(dialect, obj);
 		}