Merge pull request #14518 from ethereum/analysis-framework-refactor

`AnalysisFramework` refactor

Author: ekpyron

test/libsolidity/AnalysisFramework.cpp

@@ -22,6 +22,7 @@
 #include <test/libsolidity/AnalysisFramework.h>
 
 #include <test/libsolidity/util/Common.h>
+#include <test/libsolidity/util/SoltestErrors.h>
 #include <test/Common.h>
 
 #include <libsolidity/interface/CompilerStack.h>
@@ -41,36 +42,84 @@ using namespace solidity::langutil;
 using namespace solidity::frontend;
 using namespace solidity::frontend::test;
 
-std::pair<SourceUnit const*, ErrorList>
-AnalysisFramework::parseAnalyseAndReturnError(
+std::pair<SourceUnit const*, ErrorList> AnalysisFramework::runAnalysisAndExpectNoParsingErrors(
 	std::string const& _source,
-	bool _reportWarnings,
-	bool _insertLicenseAndVersionPragma,
-	bool _allowMultipleErrors
+	bool _includeWarningsAndInfos,
+	bool _addPreamble,
+	bool _allowMultiple
 )
 {
-	compiler().reset();
-	compiler().setSources({{"", _insertLicenseAndVersionPragma ? withPreamble(_source) : _source}});
-	compiler().setEVMVersion(solidity::test::CommonOptions::get().evmVersion());
-	if (!compiler().parse())
-	{
-		BOOST_FAIL("Parsing contract failed in analysis test suite:" + formatErrors(compiler().errors()));
-	}
+	runFramework(_addPreamble ? withPreamble(_source) : _source, PipelineStage::Analysis);
 
-	compiler().analyze();
+	if (!stageSuccessful(PipelineStage::Parsing))
+		BOOST_FAIL("Parsing contract failed in analysis test suite:" + formatErrors(m_compiler->errors()));
 
-	ErrorList errors = filteredErrors(_reportWarnings);
-	if (errors.size() > 1 && !_allowMultipleErrors)
+	ErrorList errors = filteredErrors(_includeWarningsAndInfos);
+	if (errors.size() > 1 && !_allowMultiple)
 		BOOST_FAIL("Multiple errors found: " + formatErrors(errors));
 
 	return make_pair(&compiler().ast(""), std::move(errors));
 }
 
+bool AnalysisFramework::runFramework(StringMap _sources, PipelineStage _targetStage)
+{
+	resetFramework();
+	m_targetStage = _targetStage;
+	soltestAssert(m_compiler);
+
+	m_compiler->setSources(std::move(_sources));
+	setupCompiler(*m_compiler);
+	executeCompilationPipeline();
+	return pipelineSuccessful();
+}
+
+void AnalysisFramework::resetFramework()
+{
+	compiler().reset();
+	m_targetStage = PipelineStage::Compilation;
+}
+
 std::unique_ptr<CompilerStack> AnalysisFramework::createStack() const
 {
 	return std::make_unique<CompilerStack>();
 }
 
+void AnalysisFramework::setupCompiler(CompilerStack& _compiler)
+{
+	// These are just defaults based on the (global) CLI options.
+	// Technically, every TestCase should override these with values passed to it in TestCase::Config.
+	// In practice TestCase::Config always matches global config so most test cases don't care.
+	_compiler.setEVMVersion(solidity::test::CommonOptions::get().evmVersion());
+	_compiler.setEOFVersion(solidity::test::CommonOptions::get().eofVersion());
+	_compiler.setOptimiserSettings(solidity::test::CommonOptions::get().optimize);
+}
+
+void AnalysisFramework::executeCompilationPipeline()
+{
+	soltestAssert(m_compiler);
+
+	// If you add a new stage, remember to handle it below.
+	soltestAssert(
+		m_targetStage == PipelineStage::Parsing ||
+		m_targetStage == PipelineStage::Analysis ||
+		m_targetStage == PipelineStage::Compilation
+	);
+
+	bool parsingSuccessful = m_compiler->parse();
+	soltestAssert(parsingSuccessful || !filteredErrors(false /* _includeWarningsAndInfos */).empty());
+	if (!parsingSuccessful || stageSuccessful(m_targetStage))
+		return;
+
+	bool analysisSuccessful = m_compiler->analyze();
+	soltestAssert(analysisSuccessful || !filteredErrors(false /* _includeWarningsAndInfos */).empty());
+	if (!analysisSuccessful || stageSuccessful(m_targetStage))
+		return;
+
+	bool compilationSuccessful = m_compiler->compile();
+	soltestAssert(compilationSuccessful || !filteredErrors(false /* _includeWarningsAndInfos */).empty());
+	soltestAssert(stageSuccessful(m_targetStage) == compilationSuccessful);
+}
+
 ErrorList AnalysisFramework::filterErrors(ErrorList const& _errorList, bool _includeWarningsAndInfos) const
 {
 	ErrorList errors;
@@ -113,28 +162,26 @@ ErrorList AnalysisFramework::filterErrors(ErrorList const& _errorList, bool _inc
 	return errors;
 }
 
-SourceUnit const* AnalysisFramework::parseAndAnalyse(std::string const& _source)
+bool AnalysisFramework::stageSuccessful(PipelineStage _stage) const
 {
-	auto sourceAndError = parseAnalyseAndReturnError(_source);
-	BOOST_REQUIRE(!!sourceAndError.first);
-	std::string message;
-	if (!sourceAndError.second.empty())
-		message = "Unexpected error: " + formatErrors(compiler().errors());
-	BOOST_REQUIRE_MESSAGE(sourceAndError.second.empty(), message);
-	return sourceAndError.first;
-}
-
-bool AnalysisFramework::success(std::string const& _source)
-{
-	return parseAnalyseAndReturnError(_source).second.empty();
+	switch (_stage) {
+		case PipelineStage::Parsing: return compiler().state() >= CompilerStack::Parsed;
+		case PipelineStage::Analysis: return compiler().state() >= CompilerStack::AnalysisSuccessful;
+		case PipelineStage::Compilation: return compiler().state() >= CompilerStack::CompilationSuccessful;
+	}
+	unreachable();
 }
 
-ErrorList AnalysisFramework::expectError(std::string const& _source, bool _warning, bool _allowMultiple)
+ErrorList AnalysisFramework::runAnalysisAndExpectError(
+	std::string const& _source,
+	bool _includeWarningsAndInfos,
+	bool _allowMultiple
+)
 {
-	auto sourceAndErrors = parseAnalyseAndReturnError(_source, _warning, true, _allowMultiple);
-	BOOST_REQUIRE(!sourceAndErrors.second.empty());
-	BOOST_REQUIRE_MESSAGE(!!sourceAndErrors.first, "Expected error, but no error happened.");
-	return sourceAndErrors.second;
+	auto [ast, errors] = runAnalysisAndExpectNoParsingErrors(_source, _includeWarningsAndInfos, true, _allowMultiple);
+	BOOST_REQUIRE(!errors.empty());
+	BOOST_REQUIRE_MESSAGE(ast, "Expected error, but no error happened.");
+	return errors;
 }
 
 std::string AnalysisFramework::formatErrors(

test/libsolidity/AnalysisFramework.h

@@ -39,22 +39,66 @@ using FunctionTypePointer = FunctionType const*;
 namespace solidity::frontend::test
 {
 
+enum class PipelineStage {
+	Parsing,
+	Analysis,
+	Compilation,
+};
+
 class AnalysisFramework
 {
 
 protected:
-	virtual std::pair<SourceUnit const*, langutil::ErrorList>
-	parseAnalyseAndReturnError(
+	virtual ~AnalysisFramework() = default;
+
+	/// Runs analysis via runFramework() and returns either an AST or a filtered list of errors.
+	/// Uses Boost test macros to fail if errors do not occur specifically at the analysis stage.
+	///
+	/// @deprecated This is a legacy helper. Use runFramework() directly in new tests.
+	///
+	/// @param _includeWarningsAndInfos Do not remove warning and info messages from the error list.
+	/// @param _addPreamble Apply withPreamble() to @p _source.
+	/// @param _allowMultiple When false, use Boost test macros to fail when there's more
+	///        than one item on the error list.
+	std::pair<SourceUnit const*, langutil::ErrorList> runAnalysisAndExpectNoParsingErrors(
 		std::string const& _source,
-		bool _reportWarnings = false,
-		bool _insertLicenseAndVersionPragma = true,
-		bool _allowMultipleErrors = false
+		bool _includeWarningsAndInfos = false,
+		bool _addPreamble = true,
+		bool _allowMultiple = false
 	);
-	virtual ~AnalysisFramework() = default;
 
-	SourceUnit const* parseAndAnalyse(std::string const& _source);
-	bool success(std::string const& _source);
-	langutil::ErrorList expectError(std::string const& _source, bool _warning = false, bool _allowMultiple = false);
+	/// Runs analysis via runAnalysisAndExpectNoParsingErrors() and returns the list of errors.
+	/// Uses Boost test macros to fail if there are no errors.
+	///
+	/// @deprecated This is a legacy helper. Use runFramework() directly in new tests.
+	///
+	/// @param _includeWarningsAndInfos Do not remove warning and info messages from the error list.
+	/// @param _allowMultiple When false, use Boost test macros to fail when there's more
+	///        than one item on the error list.
+	langutil::ErrorList runAnalysisAndExpectError(
+		std::string const& _source,
+		bool _includeWarningsAndInfos = false,
+		bool _allowMultiple = false
+	);
+
+public:
+	/// Runs the full compiler pipeline on specified sources. This is the main function of the
+	/// framework. Resets the stack, configures it and runs either until the first failed stage or
+	/// until the @p _targetStage is reached.
+	/// Afterwards the caller can inspect the stack via @p compiler(). The framework provides a few
+	/// convenience helpers to check the state and error list, in general the caller can freely
+	/// access the stack, including generating outputs if the compilation succeeded.
+	bool runFramework(StringMap _sources, PipelineStage _targetStage = PipelineStage::Compilation);
+	bool runFramework(std::string _source, PipelineStage _targetStage = PipelineStage::Compilation)
+	{
+		return runFramework({{"", std::move(_source)}}, _targetStage);
+	}
+
+	void resetFramework();
+
+	PipelineStage targetStage() const { return m_targetStage; }
+	bool pipelineSuccessful() const { return stageSuccessful(m_targetStage); }
+	bool stageSuccessful(PipelineStage _stage) const;
 
 	std::string formatErrors(
 		langutil::ErrorList const& _errors,
@@ -80,9 +124,6 @@ class AnalysisFramework
 		return filterErrors(compiler().errors(), _includeWarningsAndInfos);
 	}
 
-	std::vector<std::string> m_warningsToFilter = {"This is a pre-release compiler version"};
-	std::vector<std::string> m_messagesToCut = {"Source file requires different compiler version (current compiler is"};
-
 	/// @returns reference to lazy-instantiated CompilerStack.
 	solidity::frontend::CompilerStack& compiler()
 	{
@@ -99,28 +140,41 @@ class AnalysisFramework
 		return *m_compiler;
 	}
 
+protected:
 	/// Creates a new instance of @p CompilerStack. Override if your test case needs to pass in
 	/// custom constructor arguments.
 	virtual std::unique_ptr<CompilerStack> createStack() const;
 
+	/// Configures @p CompilerStack. The default implementation sets basic parameters based on
+	/// CLI options. Override if your test case needs extra configuration.
+	virtual void setupCompiler(CompilerStack& _compiler);
+
+	/// Executes the requested pipeline stages until @p m_targetStage is reached.
+	/// Stops at the first failed stage.
+	void executeCompilationPipeline();
+
+	std::vector<std::string> m_warningsToFilter = {"This is a pre-release compiler version"};
+	std::vector<std::string> m_messagesToCut = {"Source file requires different compiler version (current compiler is"};
+
 private:
 	mutable std::unique_ptr<solidity::frontend::CompilerStack> m_compiler;
+	PipelineStage m_targetStage = PipelineStage::Compilation;
 };
 
 // Asserts that the compilation down to typechecking
 // emits multiple errors of different types and messages, provided in the second argument.
 #define CHECK_ALLOW_MULTI(text, expectations) \
 do \
 { \
-	ErrorList errors = expectError((text), true, true); \
+	ErrorList errors = runAnalysisAndExpectError((text), true, true); \
 	auto message = searchErrors(errors, (expectations)); \
 	BOOST_CHECK_MESSAGE(message.empty(), message); \
 } while(0)
 
-#define CHECK_ERROR_OR_WARNING(text, typ, substrings, warning, allowMulti) \
+#define CHECK_ERROR_OR_WARNING(text, typ, substrings, includeWarningsAndInfos, allowMulti) \
 do \
 { \
-	ErrorList errors = expectError((text), (warning), (allowMulti)); \
+	ErrorList errors = runAnalysisAndExpectError((text), (includeWarningsAndInfos), (allowMulti)); \
 	std::vector<std::pair<Error::Type, std::string>> expectations; \
 	for (auto const& str: substrings) \
 		expectations.emplace_back((Error::Type::typ), str); \
@@ -154,16 +208,19 @@ CHECK_ERROR_OR_WARNING(text, Warning, std::vector<std::string>{(substring)}, tru
 CHECK_ERROR_OR_WARNING(text, Warning, substrings, true, true)
 
 // [checkSuccess(text)] asserts that the compilation down to typechecking succeeds.
-#define CHECK_SUCCESS(text) do { BOOST_CHECK(success((text))); } while(0)
+#define CHECK_SUCCESS(text) do { \
+	auto [ast, errors] = runAnalysisAndExpectNoParsingErrors((text)); \
+	BOOST_CHECK(errors.empty()); \
+} while(0)
 
 #define CHECK_SUCCESS_NO_WARNINGS(text) \
 do \
 { \
-	auto sourceAndError = parseAnalyseAndReturnError((text), true); \
+	auto [ast, errors] = runAnalysisAndExpectNoParsingErrors((text), true); \
 	std::string message; \
-	if (!sourceAndError.second.empty()) \
-		message = formatErrors(compiler().errors());\
-	BOOST_CHECK_MESSAGE(sourceAndError.second.empty(), message); \
+	if (!errors.empty()) \
+		message = formatErrors(errors);\
+	BOOST_CHECK_MESSAGE(errors.empty(), message); \
 } \
 while(0)
 

test/libsolidity/GasTest.cpp

@@ -97,24 +97,31 @@ void GasTest::printUpdatedExpectations(std::ostream& _stream, std::string const&
 	}
 }
 
-TestCase::TestResult GasTest::run(std::ostream& _stream, std::string const& _linePrefix, bool _formatted)
+void GasTest::setupCompiler(CompilerStack& _compiler)
 {
-	compiler().reset();
+	AnalysisFramework::setupCompiler(_compiler);
+
 	// Prerelease CBOR metadata varies in size due to changing version numbers and build dates.
 	// This leads to volatile creation cost estimates. Therefore we force the compiler to
 	// release mode for testing gas estimates.
-	compiler().setMetadataFormat(CompilerStack::MetadataFormat::NoMetadata);
+	_compiler.setMetadataFormat(CompilerStack::MetadataFormat::NoMetadata);
 	OptimiserSettings settings = m_optimise ? OptimiserSettings::standard() : OptimiserSettings::minimal();
 	if (m_optimiseYul)
 	{
 		settings.runYulOptimiser = m_optimise;
 		settings.optimizeStackAllocation = m_optimise;
 	}
 	settings.expectedExecutionsPerDeployment = m_optimiseRuns;
-	compiler().setOptimiserSettings(settings);
-	compiler().setSources({{"", withPreamble(m_source)}});
+	_compiler.setOptimiserSettings(settings);
+
+	// Intentionally ignoring EVM version specified on the command line.
+	// Gas expectations are only valid for the default version.
+	_compiler.setEVMVersion(EVMVersion{});
+}
 
-	if (!compiler().parseAndAnalyze() || !compiler().compile())
+TestCase::TestResult GasTest::run(std::ostream& _stream, std::string const& _linePrefix, bool _formatted)
+{
+	if (!runFramework(withPreamble(m_source), PipelineStage::Compilation))
 	{
 		_stream << formatErrors(filteredErrors(), _formatted);
 		return TestResult::FatalError;

test/libsolidity/GasTest.h

@@ -44,6 +44,9 @@ class GasTest: AnalysisFramework, public TestCase
 	void printSource(std::ostream &_stream, std::string const &_linePrefix = "", bool _formatted = false) const override;
 	void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override;
 
+protected:
+	void setupCompiler(CompilerStack& _compiler) override;
+
 private:
 	void parseExpectations(std::istream& _stream);
 

test/libsolidity/MemoryGuardTest.cpp

@@ -36,13 +36,17 @@ using namespace solidity::frontend;
 using namespace solidity::frontend::test;
 using namespace yul;
 
+void MemoryGuardTest::setupCompiler(CompilerStack& _compiler)
+{
+	AnalysisFramework::setupCompiler(_compiler);
+
+	_compiler.setViaIR(true);
+	_compiler.setOptimiserSettings(OptimiserSettings::none());
+}
+
 TestCase::TestResult MemoryGuardTest::run(std::ostream& _stream, std::string const& _linePrefix, bool _formatted)
 {
-	compiler().reset();
-	compiler().setSources(StringMap{{"", m_source}});
-	compiler().setViaIR(true);
-	compiler().setOptimiserSettings(OptimiserSettings::none());
-	if (!compiler().compile())
+	if (!runFramework(m_source, PipelineStage::Compilation))
 	{
 		_stream << formatErrors(filteredErrors(), _formatted);
 		return TestResult::FatalError;

test/libsolidity/MemoryGuardTest.h

@@ -48,6 +48,9 @@ class MemoryGuardTest: public AnalysisFramework, public TestCase
 	}
 
 	TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
+
+protected:
+	void setupCompiler(CompilerStack& _compiler) override;
 };
 
 }