diff --git a/libevmasm/Assembly.cpp b/libevmasm/Assembly.cpp index 8e06da8ae98d..852f0e4465e3 100644 --- a/libevmasm/Assembly.cpp +++ b/libevmasm/Assembly.cpp @@ -34,7 +34,7 @@ #include #include -#include +#include #include #include @@ -74,6 +74,123 @@ unsigned Assembly::codeSize(unsigned subTagSize) const } } +void Assembly::addAssemblyItemsFromJSON(Json::Value const& _code) +{ + solAssert(m_items.empty(), ""); + solAssert(_code.isArray(), ""); + for (auto const& jsonItem: _code) + m_items.emplace_back(createAssemblyItemFromJSON(jsonItem)); + + for (auto current = m_items.begin(); current != m_items.end(); ++current) + { + // During the assembly json export a `JUMPDEST` is always generated after a `tag`. + // So we just ignore exactly these `JUMPDEST`'s. + auto const next = std::next(current); + if ( + next != m_items.end() && + current->type() == AssemblyItemType::Tag && + next->type() == AssemblyItemType::Operation && + next->instruction() == Instruction::JUMPDEST + ) + m_items.erase(next); + } +} + +AssemblyItem Assembly::createAssemblyItemFromJSON(Json::Value const& _json) +{ + solAssert(ofType(_json, "name")); + solAssert(ofType(_json, "begin")); + solAssert(ofType(_json, "end")); + solAssert(ofType(_json, "source")); + solAssert(ofTypeIfExists(_json, "value")); + solAssert(ofTypeIfExists(_json, "modifierDepth")); + solAssert(ofTypeIfExists(_json, "jumpType")); + + std::string name = getOrDefault(_json, "name", ""); + solAssert(!name.empty()); + + SourceLocation location; + location.start = get(_json, "begin"); + location.end = get(_json, "end"); + int srcIndex = get(_json, "source"); + size_t modifierDepth = static_cast(getOrDefault(_json, "modifierDepth", 0)); + std::string value = getOrDefault(_json, "value", ""); + std::string jumpType = getOrDefault(_json, "jumpType", ""); + + + auto updateUsedTags = [&](u256 const& data) { + m_usedTags = max(m_usedTags, static_cast(data) + 1); + return data; + }; + + auto immutableHash = [&](string const& _immutableName) -> h256 { + h256 hash(util::keccak256(_immutableName)); + m_immutables[hash] = _immutableName; + return hash; + }; + + auto libraryHash = [&](string const& _libraryName) -> h256 { + h256 hash(util::keccak256(_libraryName)); + m_libraries[hash] = _libraryName; + return hash; + }; + + if (srcIndex > -1 && srcIndex < static_cast(sources().size())) + location.sourceName = sources()[static_cast(srcIndex)]; + + AssemblyItem result(0); + + if (c_instructions.count(name)) + { + AssemblyItem item{c_instructions.at(name), location}; + if (!jumpType.empty()) + item.setJumpType(jumpType); + result = item; + } + else + { + if (name == "PUSH") + { + AssemblyItem item{AssemblyItemType::Push, u256("0x" + value)}; + if (!jumpType.empty()) + item.setJumpType(jumpType); + result = item; + } + else if (name == "PUSH [ErrorTag]") + result = {AssemblyItemType::PushTag, 0}; + else if (name == "PUSH [tag]") + result = {AssemblyItemType::PushTag, updateUsedTags(u256(value))}; + else if (name == "PUSH [$]") + result = {AssemblyItemType::PushSub, u256("0x" + value)}; + else if (name == "PUSH #[$]") + result = {AssemblyItemType::PushSubSize, u256("0x" + value)}; + else if (name == "PUSHSIZE") + result = {AssemblyItemType::PushProgramSize, 0}; + else if (name == "PUSHLIB") + result = {AssemblyItemType::PushLibraryAddress, libraryHash(value)}; + else if (name == "PUSHDEPLOYADDRESS") + result = {AssemblyItemType::PushDeployTimeAddress, 0}; + else if (name == "PUSHIMMUTABLE") + result = {AssemblyItemType::PushImmutable, immutableHash(value)}; + else if (name == "ASSIGNIMMUTABLE") + result = {AssemblyItemType::AssignImmutable, immutableHash(value)}; + else if (name == "tag") + result = {AssemblyItemType::Tag, updateUsedTags(u256(value))}; + else if (name == "PUSH data") + result = {AssemblyItemType::PushData, u256("0x" + value)}; + else if (name == "VERBATIM") + { + AssemblyItem item(fromHex(value), 0, 0); + result = item; + } + else + assertThrow(false, InvalidOpcode, ""); + } + result.setLocation(location); + result.m_modifierDepth = modifierDepth; + return result; +} + namespace { @@ -298,6 +415,43 @@ Json::Value Assembly::assemblyJSON(map const& _sourceIndices, return root; } +std::shared_ptr Assembly::loadFromAssemblyJSON(Json::Value const& _json, std::vector const& _sourceList /* = {} */, bool _isCreation /* = true */) +{ + if (!_json[".code"].isArray()) + return {}; + + std::shared_ptr result = std::make_shared(_isCreation, ""); + vector sourceList; + if (_sourceList.empty()) + { + if (_json.isMember("sourceList")) + for (auto const& it: _json["sourceList"]) + sourceList.emplace_back(it.asString()); + } + else + sourceList = _sourceList; + result->setSources(sourceList); + result->addAssemblyItemsFromJSON(_json[".code"]); + if (_json[".auxdata"].isString()) + result->m_auxiliaryData = fromHex(_json[".auxdata"].asString()); + Json::Value const& data = _json[".data"]; + for (Json::ValueConstIterator itr = data.begin(); itr != data.end(); itr++) + { + solAssert(itr.key().isString(), ""); + std::string key = itr.key().asString(); + Json::Value const& code = data[key]; + if (code.isString()) + result->m_data[h256(fromHex(key))] = fromHex(code.asString()); + else + { + std::shared_ptr subassembly(Assembly::loadFromAssemblyJSON(code, sourceList, /* isCreation = */ false)); + assertThrow(subassembly, AssemblyException, ""); + result->m_subs.emplace_back(std::make_shared(*subassembly)); + } + } + return result; +} + AssemblyItem Assembly::namedTag(string const& _name, size_t _params, size_t _returns, optional _sourceID) { assertThrow(!_name.empty(), AssemblyException, "Empty named tag."); diff --git a/libevmasm/Assembly.h b/libevmasm/Assembly.h index 592119f17fe1..c8c1763cdb64 100644 --- a/libevmasm/Assembly.h +++ b/libevmasm/Assembly.h @@ -152,6 +152,12 @@ class Assembly bool _includeSourceList = true ) const; + /// Loads the JSON representation of assembly. + /// @param _json JSON object containing assembly + /// @param _loadSources true, if source list should be included, false otherwise. + /// @returns true on success, false otherwise + static std::shared_ptr loadFromAssemblyJSON(Json::Value const& _json, std::vector const& _sourceList = {}, bool _isCreation = true); + /// Mark this assembly as invalid. Calling ``assemble`` on it will throw. void markAsInvalid() { m_invalid = true; } @@ -160,6 +166,22 @@ class Assembly bool isCreation() const { return m_creation; } + /// Set the source name list. + void setSources(std::vector> _sources) + { + m_sources = std::move(_sources); + } + + /// Set the source name list from simple vector. + void setSources(std::vector const& _sources) + { + for (auto const& item: _sources) + m_sources.emplace_back(std::make_shared(item)); + } + + /// @returns List of source names. + std::vector> sources() const& { return m_sources; } + protected: /// Does the same operations as @a optimise, but should only be applied to a sub and /// returns the replaced tags. Also takes an argument containing the tags of this assembly @@ -168,6 +190,14 @@ class Assembly unsigned codeSize(unsigned subTagSize) const; + /// Add all assembly items from given JSON array. + void addAssemblyItemsFromJSON(Json::Value const& _code); + + /// Creates an AssemblyItem from a given JSON representation. + /// @param _json JSON representation of an assembly item + /// @returns AssemblyItem of _json argument. + AssemblyItem createAssemblyItemFromJSON(Json::Value const& _json); + private: bool m_invalid = false; @@ -214,6 +244,7 @@ class Assembly std::string m_name; langutil::SourceLocation m_currentSourceLocation; + std::vector> m_sources; public: size_t m_currentModifierDepth = 0; diff --git a/libevmasm/AssemblyItem.cpp b/libevmasm/AssemblyItem.cpp index a36e0ddb1dc9..c4bb9d80d89a 100644 --- a/libevmasm/AssemblyItem.cpp +++ b/libevmasm/AssemblyItem.cpp @@ -243,6 +243,18 @@ string AssemblyItem::getJumpTypeAsString() const } } +void AssemblyItem::setJumpType(std::string const& _jumpType) +{ + if (_jumpType == "[in]") + m_jumpType = JumpType::IntoFunction; + else if (_jumpType == "[out]") + m_jumpType = JumpType::OutOfFunction; + else if (_jumpType.empty()) + m_jumpType = JumpType::Ordinary; + else + assertThrow(false, AssemblyException, "Invalid jump type."); +} + string AssemblyItem::toAssemblyText(Assembly const& _assembly) const { string text; diff --git a/libevmasm/AssemblyItem.h b/libevmasm/AssemblyItem.h index 4aef82d8b25d..6675379432f8 100644 --- a/libevmasm/AssemblyItem.h +++ b/libevmasm/AssemblyItem.h @@ -173,6 +173,7 @@ class AssemblyItem langutil::SourceLocation const& location() const { return m_location; } void setJumpType(JumpType _jumpType) { m_jumpType = _jumpType; } + void setJumpType(std::string const& _jumpType); JumpType getJumpType() const { return m_jumpType; } std::string getJumpTypeAsString() const; diff --git a/libsolidity/interface/CompilerStack.cpp b/libsolidity/interface/CompilerStack.cpp index 1096501303e1..7539436a3e22 100644 --- a/libsolidity/interface/CompilerStack.cpp +++ b/libsolidity/interface/CompilerStack.cpp @@ -242,6 +242,7 @@ void CompilerStack::setLibraries(std::map const& _libra void CompilerStack::setOptimiserSettings(bool _optimize, size_t _runs) { OptimiserSettings settings = _optimize ? OptimiserSettings::standard() : OptimiserSettings::minimal(); + settings.enabled = _optimize; settings.expectedExecutionsPerDeployment = _runs; setOptimiserSettings(std::move(settings)); } @@ -410,11 +411,34 @@ void CompilerStack::importASTs(map const& _sources) m_sources[path] = std::move(source); } m_stackState = ParsedAndImported; - m_importedSources = true; + m_importedSourceType = ImportedSourceType::SolidityAST; storeContractDefinitions(); } +void CompilerStack::importEvmAssemblyJson(std::map const& _sources) +{ + solAssert(_sources.size() == 1, ""); + solAssert(m_sourceJsons.empty(), ""); + solAssert(m_sourceOrder.empty(), ""); + if (m_stackState != Empty) + solThrow(CompilerError, "Must call importEvmAssemblyJson only before the SourcesSet state."); + + m_sourceJsons = _sources; + Json::Value jsonValue = _sources.begin()->second; + if (jsonValue.isMember("sourceList")) + for (auto const& item: jsonValue["sourceList"]) + { + Source source; + source.charStream = std::make_shared(item.asString(), ""); + m_sources.emplace(std::make_pair(item.asString(), source)); + m_sourceOrder.push_back(&m_sources[item.asString()]); + } + m_sourceJsons[_sources.begin()->first] = std::move(jsonValue); + m_importedSourceType = ImportedSourceType::EvmAssemblyJson; + m_stackState = SourcesSet; +} + bool CompilerStack::analyze() { if (m_stackState != ParsedAndImported || m_stackState >= AnalysisPerformed) @@ -604,6 +628,9 @@ bool CompilerStack::parseAndAnalyze(State _stopAfter) { m_stopAfter = _stopAfter; + if (m_importedSourceType == ImportedSourceType::EvmAssemblyJson) + return true; + bool success = parse(); if (m_stackState >= m_stopAfter) return success; @@ -653,55 +680,83 @@ bool CompilerStack::compile(State _stopAfter) // Only compile contracts individually which have been requested. map> otherCompilers; - for (Source const* source: m_sourceOrder) - for (ASTPointer const& node: source->ast->nodes()) - if (auto contract = dynamic_cast(node.get())) - if (isRequestedContract(*contract)) - { - try + if (m_importedSourceType == ImportedSourceType::EvmAssemblyJson) + { + solAssert(m_sourceJsons.size() == 1, ""); + + string const evmSourceName = m_sourceJsons.begin()->first; + Json::Value const evmJson = m_sourceJsons.begin()->second; + + // todo: remove code duplication. + evmasm::Assembly::OptimiserSettings optimiserSettings{false, false, false, false, false, false, m_evmVersion, 0}; + optimiserSettings.runInliner = m_optimiserSettings.runInliner; + optimiserSettings.runJumpdestRemover = m_optimiserSettings.runJumpdestRemover; + optimiserSettings.runPeephole = m_optimiserSettings.runPeephole; + optimiserSettings.runDeduplicate = m_optimiserSettings.runDeduplicate; + optimiserSettings.runCSE = m_optimiserSettings.runCSE; + optimiserSettings.runConstantOptimiser = m_optimiserSettings.runConstantOptimiser; + optimiserSettings.expectedExecutionsPerDeployment = m_optimiserSettings.expectedExecutionsPerDeployment; + optimiserSettings.evmVersion = m_evmVersion; + + m_contracts[evmSourceName].evmAssembly = evmasm::Assembly::loadFromAssemblyJSON(m_sourceJsons[evmSourceName]); + if (m_optimiserSettings.enabled) + m_contracts[evmSourceName].evmAssembly->optimise(optimiserSettings); + m_contracts[evmSourceName].object = m_contracts[evmSourceName].evmAssembly->assemble(); + + m_contracts[evmSourceName].evmRuntimeAssembly = std::make_shared(m_contracts[evmSourceName].evmAssembly->sub(0)); + solAssert(m_contracts[evmSourceName].evmRuntimeAssembly->isCreation() == false, ""); + if (m_optimiserSettings.enabled) + m_contracts[evmSourceName].evmRuntimeAssembly->optimise(optimiserSettings); + m_contracts[evmSourceName].runtimeObject = m_contracts[evmSourceName].evmRuntimeAssembly->assemble(); + } + else + { + for (Source const* source: m_sourceOrder) + for (ASTPointer const& node: source->ast->nodes()) + if (auto contract = dynamic_cast(node.get())) + if (isRequestedContract(*contract)) { - if (m_viaIR || m_generateIR || m_generateEwasm) - generateIR(*contract); - if (m_generateEvmBytecode) + try { - if (m_viaIR) - generateEVMFromIR(*contract); - else - compileContract(*contract, otherCompilers); + if (m_viaIR || m_generateIR || m_generateEwasm) + generateIR(*contract); + if (m_generateEvmBytecode) + { + if (m_viaIR) + generateEVMFromIR(*contract); + else + compileContract(*contract, otherCompilers); + } + if (m_generateEwasm) + generateEwasm(*contract); } - if (m_generateEwasm) - generateEwasm(*contract); - } - catch (Error const& _error) - { - if (_error.type() != Error::Type::CodeGenerationError) - throw; - m_errorReporter.error(_error.errorId(), _error.type(), SourceLocation(), _error.what()); - return false; - } - catch (UnimplementedFeatureError const& _unimplementedError) - { - if ( - SourceLocation const* sourceLocation = - boost::get_error_info(_unimplementedError) - ) + catch (Error const& _error) { - string const* comment = _unimplementedError.comment(); - m_errorReporter.error( - 1834_error, - Error::Type::CodeGenerationError, - *sourceLocation, - "Unimplemented feature error" + - ((comment && !comment->empty()) ? ": " + *comment : string{}) + - " in " + - _unimplementedError.lineInfo() - ); + if (_error.type() != Error::Type::CodeGenerationError) + throw; + m_errorReporter.error(_error.errorId(), _error.type(), SourceLocation(), _error.what()); return false; } - else - throw; + catch (UnimplementedFeatureError const& _unimplementedError) + { + if (SourceLocation const* sourceLocation + = boost::get_error_info(_unimplementedError)) + { + string const* comment = _unimplementedError.comment(); + m_errorReporter.error( + 1834_error, + Error::Type::CodeGenerationError, + *sourceLocation, + "Unimplemented feature error" + + ((comment && !comment->empty()) ? ": " + *comment : string{}) + " in " + + _unimplementedError.lineInfo()); + return false; + } + else + throw; + } } - } + } m_stackState = CompilationSuccessful; this->link(); return true; @@ -942,9 +997,10 @@ map CompilerStack::sourceIndices() const map indices; unsigned index = 0; for (auto const& s: m_sources) - indices[s.first] = index++; - solAssert(!indices.count(CompilerContext::yulUtilityFileName()), ""); - indices[CompilerContext::yulUtilityFileName()] = index++; + if (s.first != CompilerContext::yulUtilityFileName()) + indices[s.first] = index++; + if (indices.find(CompilerContext::yulUtilityFileName()) == indices.end()) + indices[CompilerContext::yulUtilityFileName()] = index++; return indices; } @@ -1482,7 +1538,22 @@ string CompilerStack::createMetadata(Contract const& _contract, bool _forIR) con { Json::Value meta{Json::objectValue}; meta["version"] = 1; - meta["language"] = m_importedSources ? "SolidityAST" : "Solidity"; + string sourceType; + switch (m_importedSourceType) + { + case ImportedSourceType::None: + sourceType = "Solidity"; + break; + case ImportedSourceType::SolidityAST: + sourceType = "SolidityAST"; + break; + case ImportedSourceType::EvmAssemblyJson: + sourceType = "EvmAssemblyJson"; + break; + default: + solAssert(false); + } + meta["language"] = sourceType; meta["compiler"]["version"] = VersionStringStrict; /// All the source files (including self), which should be included in the metadata. diff --git a/libsolidity/interface/CompilerStack.h b/libsolidity/interface/CompilerStack.h index e5bc9b345828..cfc47cab142d 100644 --- a/libsolidity/interface/CompilerStack.h +++ b/libsolidity/interface/CompilerStack.h @@ -117,6 +117,12 @@ class CompilerStack: public langutil::CharStreamProvider None }; + enum class ImportedSourceType { + None, + SolidityAST, + EvmAssemblyJson + }; + /// Creates a new compiler stack. /// @param _readFile callback used to read files for import statements. Must return /// and must not emit exceptions. @@ -222,6 +228,10 @@ class CompilerStack: public langutil::CharStreamProvider /// Will throw errors if the import fails void importASTs(std::map const& _sources); + /// Imports given Evm Assembly Json. Leads to the same internal state as parse(). + /// Will throw errors if the import fails + void importEvmAssemblyJson(std::map const& _sources); + /// Performs the analysis steps (imports, scopesetting, syntaxCheck, referenceResolving, /// typechecking, staticAnalysis) on previously parsed sources. /// @returns false on error. @@ -511,7 +521,7 @@ class CompilerStack: public langutil::CharStreamProvider langutil::DebugInfoSelection m_debugInfoSelection = langutil::DebugInfoSelection::Default(); bool m_parserErrorRecovery = false; State m_stackState = Empty; - bool m_importedSources = false; + ImportedSourceType m_importedSourceType = ImportedSourceType::None; /// Whether or not there has been an error during processing. /// If this is true, the stack will refuse to generate code. bool m_hasError = false; diff --git a/libsolidity/interface/OptimiserSettings.h b/libsolidity/interface/OptimiserSettings.h index bae343f2d13d..096a5660dfa6 100644 --- a/libsolidity/interface/OptimiserSettings.h +++ b/libsolidity/interface/OptimiserSettings.h @@ -155,6 +155,8 @@ struct OptimiserSettings /// This specifies an estimate on how often each opcode in this assembly will be executed, /// i.e. use a small value to optimise for size and a large value to optimise for runtime gas usage. size_t expectedExecutionsPerDeployment = 200; + /// Flag reflecting whether optimizer is enabled. + bool enabled = false; }; } diff --git a/libsolutil/JSON.h b/libsolutil/JSON.h index 43dddfd4a455..9c553a9b8671 100644 --- a/libsolutil/JSON.h +++ b/libsolutil/JSON.h @@ -67,4 +67,67 @@ std::string jsonPrint(Json::Value const& _input, JsonFormat const& _format); /// \return \c true if the document was successfully parsed, \c false if an error occurred. bool jsonParseStrict(std::string const& _input, Json::Value& _json, std::string* _errs = nullptr); +namespace detail +{ + +template +struct helper; + +#define DEFINE_HELPER(TYPE, CHECK_TYPE, CONVERT_TYPE) \ + template<> \ + struct helper \ + { \ + static bool ofType(Json::Value const& _input, std::string const& _name) \ + { \ + return _input[_name].CHECK_TYPE(); \ + } \ + static TYPE get(Json::Value const& _input, std::string const& _name) \ + { \ + return _input[_name].CONVERT_TYPE(); \ + } \ + static TYPE getOrDefault(Json::Value const& _input, std::string const& _name, TYPE _default = {}) \ + { \ + TYPE result = _default; \ + if (helper::ofType(_input, _name)) \ + result = _input[_name].CONVERT_TYPE(); \ + return result; \ + } \ + }; + +DEFINE_HELPER(float, isDouble, asFloat) +DEFINE_HELPER(double, isDouble, asDouble) +DEFINE_HELPER(std::string, isString, asString) +DEFINE_HELPER(Json::Int, isInt, asInt) +DEFINE_HELPER(Json::Int64, isInt64, asInt64) +DEFINE_HELPER(Json::UInt, isUInt, asUInt) +DEFINE_HELPER(Json::UInt64, isUInt64, asUInt64) + +} // namespace detail + +template +bool ofType(Json::Value const& _input, std::string const& _name) +{ + return detail::helper::ofType(_input, _name); } + +template +bool ofTypeIfExists(Json::Value const& _input, std::string const& _name) +{ + if (_input.isMember(_name)) + return ofType(_input, _name); + return true; +} + +template +T get(Json::Value const& _input, std::string const& _name) +{ + return detail::helper::get(_input, _name); +} + +template +T getOrDefault(Json::Value const& _input, std::string const& _name, T _default = {}) +{ + return detail::helper::getOrDefault(_input, _name, _default); +} + +} // namespace solidity::util diff --git a/scripts/ASTImportTest.sh b/scripts/ASTImportTest.sh deleted file mode 100755 index daf5a61632d7..000000000000 --- a/scripts/ASTImportTest.sh +++ /dev/null @@ -1,155 +0,0 @@ -#!/usr/bin/env bash - -set -e - -# Bash script to test the ast-import option of the compiler by -# first exporting a .sol file to JSON, then loading it into the compiler -# and exporting it again. The second JSON should be identical to the first -READLINK=readlink -if [[ "$OSTYPE" == "darwin"* ]]; then - READLINK=greadlink -fi -REPO_ROOT=$(${READLINK} -f "$(dirname "$0")"/..) -SOLIDITY_BUILD_DIR=${SOLIDITY_BUILD_DIR:-${REPO_ROOT}/build} -SOLC=${SOLIDITY_BUILD_DIR}/solc/solc -SPLITSOURCES=${REPO_ROOT}/scripts/splitSources.py - -SYNTAXTESTS_DIR="${REPO_ROOT}/test/libsolidity/syntaxTests" -ASTJSONTESTS_DIR="${REPO_ROOT}/test/libsolidity/ASTJSON" -NSOURCES="$(find "$SYNTAXTESTS_DIR" -type f | wc -l)" - -# DEV_DIR="${REPO_ROOT}/../tmp/contracts/" -# NSOURCES="$(find $DEV_DIR -type f | wc -l)" #TODO use find command - -FAILED=0 -UNCOMPILABLE=0 -TESTED=0 - -if [[ "$(find . -maxdepth 0 -type d -empty)" == "" ]]; then - echo "Test directory not empty. Skipping!" - exit 1 -fi - -# function tests whether exporting and importing again leaves the JSON ast unchanged -# Results are recorded by adding to FAILED or UNCOMPILABLE. -# Also, in case of a mismatch a diff and the respective ASTs are printed -# Expected parameters: -# $1 name of the file to be exported and imported -# $2 any files needed to do so that might be in parent directories -function testImportExportEquivalence { - local nth_input_file="$1" - IFS=" " read -r -a all_input_files <<< "$2" - - if $SOLC "$nth_input_file" "${all_input_files[@]}" > /dev/null 2>&1 - then - ! [[ -e stderr.txt ]] || { echo "stderr.txt already exists. Refusing to overwrite."; exit 1; } - - # save exported json as expected result (silently) - $SOLC --combined-json ast --pretty-json "$nth_input_file" "${all_input_files[@]}" > expected.json 2> /dev/null - # import it, and export it again as obtained result (silently) - if ! $SOLC --import-ast --combined-json ast --pretty-json expected.json > obtained.json 2> stderr.txt - then - # For investigating, use exit 1 here so the script stops at the - # first failing test - # exit 1 - FAILED=$((FAILED + 1)) - echo -e "ERROR: AST reimport failed for input file $nth_input_file" - echo - echo "Compiler stderr:" - cat ./stderr.txt - echo - echo "Compiler stdout:" - cat ./obtained.json - return 1 - fi - DIFF="$(diff expected.json obtained.json)" - if [ "$DIFF" != "" ] - then - if [ "$DIFFVIEW" == "" ] - then - echo -e "ERROR: JSONS differ for $1: \n $DIFF \n" - echo "Expected:" - cat ./expected.json - echo "Obtained:" - cat ./obtained.json - else - # Use user supplied diff view binary - $DIFFVIEW expected.json obtained.json - fi - FAILED=$((FAILED + 1)) - return 2 - fi - TESTED=$((TESTED + 1)) - rm expected.json obtained.json - rm -f stderr.txt - else - # echo "contract $solfile could not be compiled " - UNCOMPILABLE=$((UNCOMPILABLE + 1)) - fi - # return 0 -} -echo "Looking at $NSOURCES .sol files..." - -WORKINGDIR=$PWD - -# for solfile in $(find $DEV_DIR -name *.sol) -# boost_filesystem_bug specifically tests a local fix for a boost::filesystem -# bug. Since the test involves a malformed path, there is no point in running -# AST tests on it. See https://github.com/boostorg/filesystem/issues/176 -# shellcheck disable=SC2044 -for solfile in $(find "$SYNTAXTESTS_DIR" "$ASTJSONTESTS_DIR" -name "*.sol" -and -not -name "boost_filesystem_bug.sol") -do - echo -n "." - # create a temporary sub-directory - FILETMP=$(mktemp -d) - cd "$FILETMP" - - set +e - OUTPUT=$("$SPLITSOURCES" "$solfile") - SPLITSOURCES_RC=$? - set -e - if [ ${SPLITSOURCES_RC} == 0 ] - then - # echo $OUTPUT - NSOURCES=$((NSOURCES - 1)) - for i in $OUTPUT; - do - testImportExportEquivalence "$i" "$OUTPUT" - NSOURCES=$((NSOURCES + 1)) - done - elif [ ${SPLITSOURCES_RC} == 1 ] - then - testImportExportEquivalence "$solfile" - elif [ ${SPLITSOURCES_RC} == 2 ] - then - # The script will exit with return code 2, if an UnicodeDecodeError occurred. - # This is the case if e.g. some tests are using invalid utf-8 sequences. We will ignore - # these errors, but print the actual output of the script. - echo -e "\n${OUTPUT}\n" - testImportExportEquivalence "$solfile" - else - # All other return codes will be treated as critical errors. The script will exit. - echo -e "\nGot unexpected return code ${SPLITSOURCES_RC} from ${SPLITSOURCES}. Aborting." - echo -e "\n${OUTPUT}\n" - - cd "$WORKINGDIR" - # Delete temporary files - rm -rf "$FILETMP" - - exit 1 - fi - - cd "$WORKINGDIR" - # Delete temporary files - rm -rf "$FILETMP" -done - -echo "" - -if [ "$FAILED" = 0 ] -then - echo "SUCCESS: $TESTED syntaxTests passed, $FAILED failed, $UNCOMPILABLE could not be compiled ($NSOURCES sources total)." -else - echo "FAILURE: Out of $NSOURCES sources, $FAILED failed, ($UNCOMPILABLE could not be compiled)." - exit 1 -fi diff --git a/scripts/ImportExportTest.sh b/scripts/ImportExportTest.sh new file mode 100755 index 000000000000..99ff78de69a0 --- /dev/null +++ b/scripts/ImportExportTest.sh @@ -0,0 +1,257 @@ +#!/usr/bin/env bash +# ------------------------------------------------------------------------------ +# vim:ts=4:et +# This file is part of solidity. +# +# solidity is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# solidity is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with solidity. If not, see +# +# (c) solidity contributors. +# ------------------------------------------------------------------------------ + +set -euo pipefail +IMPORT_TEST_TYPE="${1}" + +# Bash script to test the import/exports. +# ast import/export tests: +# - first exporting a .sol file to JSON, then loading it into the compiler +# and exporting it again. The second JSON should be identical to the first. +# evm-assembly import/export tests: +# - + +READLINK=readlink +if [[ "$OSTYPE" == "darwin"* ]]; then + READLINK=greadlink +fi +REPO_ROOT=$(${READLINK} -f "$(dirname "$0")"/..) +SOLIDITY_BUILD_DIR=${SOLIDITY_BUILD_DIR:-${REPO_ROOT}/build} +SOLC=${SOLIDITY_BUILD_DIR}/solc/solc +SPLITSOURCES=${REPO_ROOT}/scripts/splitSources.py + +# shellcheck source=scripts/common.sh +source "${REPO_ROOT}/scripts/common.sh" + +SYNTAXTESTS_DIR="${REPO_ROOT}/test/libsolidity/syntaxTests" +SEMANTICTESTS_DIR="${REPO_ROOT}/test/libsolidity/semanticTests" +ASTJSONTESTS_DIR="${REPO_ROOT}/test/libsolidity/ASTJSON" + +FAILED=0 +UNCOMPILABLE=0 +TESTED=0 + +if [[ "$(find . -maxdepth 0 -type d -empty)" == "" ]]; then + fail "Test directory not empty. Skipping!" +fi + +function ast_import_export_equivalence +{ + local sol_file="$1" + local input_files="$2" + # save exported json as expected result (silently) + $SOLC --combined-json ast --pretty-json --json-indent 4 "${input_files}" > expected.json 2> /dev/null + # import it, and export it again as obtained result (silently) + if ! $SOLC --import-ast --combined-json ast --pretty-json --json-indent 4 expected.json > obtained.json 2> stderr.txt + then + # For investigating, use exit 1 here so the script stops at the + # first failing test + # exit 1 + FAILED=$((FAILED + 1)) + printError -e "ERROR: AST reimport failed for input file $sol_file" + printError + printError "Compiler stderr:" + cat ./stderr.txt >&2 + printError + printError "Compiler stdout:" + cat ./obtained.json >&2 + return 1 + fi + if ! diff_files expected.json obtained.json + then + FAILED=$((FAILED + 1)) + fi + TESTED=$((TESTED + 1)) + rm expected.json obtained.json + rm -f stderr.txt +} + +function json_evm_asm_import_export_equivalence +{ + local sol_file="$1" + local input_files="$2" + local outputs=( "asm" "bin" "bin-runtime" "opcodes" "srcmap" "srcmap-runtime" ) + local _TESTED=1 + if ! "${SOLC}" --combined-json "$(IFS=, ; echo "${outputs[*]}")" --pretty-json --json-indent 4 "${input_files}" > expected.json 2> expected.error + then + printError + printError "$sol_file" + cat expected.error >&2 + UNCOMPILABLE=$((UNCOMPILABLE + 1)) + return 0 + fi + + # Note that we have some test files, that only consists of free functions. + # Those files doesn't define any contracts, so the resulting json does not define any + # keys. In this case `jq` returns an error like `jq: error: null (null) has no keys` + # to not get spammed by these errors, errors are redirected to /dev/null. + for contract in $(jq '.contracts | keys | .[]' expected.json 2> /dev/null) + do + for output in "${outputs[@]}" + do + jq --raw-output ".contracts.${contract}.\"${output}\"" expected.json > "expected.${output}.json" + done + + assembly=$(cat expected.asm.json) + [[ $assembly != "" && $assembly != "null" ]] || continue + + if ! "${SOLC}" --combined-json bin,bin-runtime,opcodes,asm,srcmap,srcmap-runtime --pretty-json --json-indent 4 --import-asm-json expected.asm.json > obtained.json 2> obtained.error + then + printError + printError "$sol_file" + cat obtained.error >&2 + FAILED=$((FAILED + 1)) + return 0 + fi + + for output in "${outputs[@]}" + do + for obtained_contract in $(jq '.contracts | keys | .[]' obtained.json 2> /dev/null) + do + jq --raw-output ".contracts.${obtained_contract}.\"${output}\"" obtained.json > "obtained.${output}.json" + if ! diff_files "expected.${output}.json" "obtained.${output}.json" + then + _TESTED= + FAILED=$((FAILED + 1)) + return 0 + fi + done + done + + # direct export via --asm-json, if imported with --import-asm-json. + if ! "${SOLC}" --asm-json --import-asm-json expected.asm.json --pretty-json --json-indent 4 | tail -n+4 > obtained_direct_import_export.json 2> obtained_direct_import_export.error + then + printError + printError "$sol_file" + cat obtained_direct_import_export.error >&2 + FAILED=$((FAILED + 1)) + return 0 + fi + + # reformat jsons using jq. + jq . expected.asm.json > expected.asm.json.pretty + jq . obtained_direct_import_export.json > obtained_direct_import_export.json.pretty + if ! diff_files expected.asm.json.pretty obtained_direct_import_export.json.pretty + then + _TESTED= + FAILED=$((FAILED + 1)) + return 0 + fi + done + + if [[ $_TESTED == "" ]] + then + TESTED=$((TESTED + 1)) + fi +} + +# function tests whether exporting and importing again is equivalent. +# Results are recorded by adding to FAILED or UNCOMPILABLE. +# Also, in case of a mismatch a diff is printed +# Expected parameters: +# $1 name of the file to be exported and imported +# $2 any files needed to do so that might be in parent directories +function testImportExportEquivalence { + local sol_file="$1" + local input_files="$2" + if "$SOLC" --bin "${input_files}" > /dev/null 2>&1 + then + ! [[ -e stderr.txt ]] || fail "stderr.txt already exists. Refusing to overwrite." + case "$IMPORT_TEST_TYPE" in + ast) ast_import_export_equivalence "${sol_file}" "${input_files}" ;; + evm-assembly) json_evm_asm_import_export_equivalence "${sol_file}" "${input_files}" ;; + *) fail "Unknown import test type. Aborting." ;; + esac + else + UNCOMPILABLE=$((UNCOMPILABLE + 1)) + fi +} + +WORKINGDIR=$PWD + +command_available "${SOLC}" --version +command_available jq --version + +case "$IMPORT_TEST_TYPE" in + ast) TEST_DIRS=("${SYNTAXTESTS_DIR}" "${ASTJSONTESTS_DIR}") ;; + evm-assembly) TEST_DIRS=("${SYNTAXTESTS_DIR}" "${SEMANTICTESTS_DIR}") ;; + *) fail "Unknown import test type. Aborting. Please specify ${0} [ast|evm-assembly]." ;; +esac + +# boost_filesystem_bug specifically tests a local fix for a boost::filesystem +# bug. Since the test involves a malformed path, there is no point in running +# tests on it. See https://github.com/boostorg/filesystem/issues/176 +IMPORT_TEST_FILES=$(find "${TEST_DIRS[@]}" -name "*.sol" -and -not -name "boost_filesystem_bug.sol") + +NSOURCES="$(echo "$IMPORT_TEST_FILES" | wc -l)" +echo "Looking at $NSOURCES .sol files..." + +for solfile in ${IMPORT_TEST_FILES} +do + echo -n "." + # create a temporary sub-directory + FILETMP=$(mktemp -d) + cd "$FILETMP" + + set +e + OUTPUT=$("$SPLITSOURCES" "$solfile") + SPLITSOURCES_RC=$? + set -e + if [[ ${SPLITSOURCES_RC} == 0 ]] + then + IFS=' ' read -ra OUTPUT_ARRAY <<< "${OUTPUT}" + NSOURCES=$((NSOURCES - 1 + ${#OUTPUT_ARRAY[@]})) + testImportExportEquivalence "$solfile" "${OUTPUT_ARRAY[*]}" + elif [ ${SPLITSOURCES_RC} == 1 ] + then + testImportExportEquivalence "$solfile" "$solfile" + elif [ ${SPLITSOURCES_RC} == 2 ] + then + # The script will exit with return code 2, if an UnicodeDecodeError occurred. + # This is the case if e.g. some tests are using invalid utf-8 sequences. We will ignore + # these errors, but print the actual output of the script. + printError "\n\n${OUTPUT}\n\n" + testImportExportEquivalence "$solfile" "$solfile" + else + # All other return codes will be treated as critical errors. The script will exit. + printError "\n\nGot unexpected return code ${SPLITSOURCES_RC} from ${SPLITSOURCES}. Aborting." + printError "\n\n${OUTPUT}\n\n" + + cd "$WORKINGDIR" + # Delete temporary files + rm -rf "$FILETMP" + + exit 1 + fi + + cd "$WORKINGDIR" + # Delete temporary files + rm -rf "$FILETMP" +done + +echo + +if (( FAILED == 0 )) +then + echo "SUCCESS: $TESTED tests passed, $FAILED failed, $UNCOMPILABLE could not be compiled ($NSOURCES sources total)." +else + fail "FAILURE: Out of $NSOURCES sources, $FAILED failed, ($UNCOMPILABLE could not be compiled)." +fi diff --git a/scripts/common.sh b/scripts/common.sh index 98f63ecb80c5..a79525fe4d98 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -26,18 +26,18 @@ set -e # changes directory. The paths returned by `caller` are relative to it. _initial_work_dir=$(pwd) -if [ "$CIRCLECI" ] +if [[ ${CIRCLECI:-} != "" ]] then export TERM="${TERM:-xterm}" - function printTask { echo "$(tput bold)$(tput setaf 2)$1$(tput setaf 7)"; } - function printError { >&2 echo "$(tput setaf 1)$1$(tput setaf 7)"; } - function printWarning { >&2 echo "$(tput setaf 11)$1$(tput setaf 7)"; } - function printLog { echo "$(tput setaf 3)$1$(tput setaf 7)"; } + function printTask { echo -e "$(tput bold)$(tput setaf 2)$1$(tput setaf 7)"; } + function printError { >&2 echo -e "$(tput setaf 1)$1$(tput setaf 7)"; } + function printWarning { >&2 echo -e "$(tput setaf 11)$1$(tput setaf 7)"; } + function printLog { echo -e "$(tput setaf 3)$1$(tput setaf 7)"; } else - function printTask { echo "$(tput bold)$(tput setaf 2)$1$(tput sgr0)"; } - function printError { >&2 echo "$(tput setaf 1)$1$(tput sgr0)"; } - function printWarning { >&2 echo "$(tput setaf 11)$1$(tput sgr0)"; } - function printLog { echo "$(tput setaf 3)$1$(tput sgr0)"; } + function printTask { echo -e "$(tput bold)$(tput setaf 2)$1$(tput sgr0)"; } + function printError { >&2 echo -e "$(tput setaf 1)$1$(tput sgr0)"; } + function printWarning { >&2 echo -e "$(tput setaf 11)$1$(tput sgr0)"; } + function printLog { echo -e "$(tput setaf 3)$1$(tput sgr0)"; } fi function checkDputEntries @@ -193,6 +193,7 @@ function msg_on_error fi } + function diff_values { (( $# >= 2 )) || fail "diff_values requires at least 2 arguments." @@ -202,7 +203,49 @@ function diff_values shift shift - diff --unified=0 <(echo "$value1") <(echo "$value2") "$@" + if ! diff --unified=0 <(echo "$value1") <(echo "$value2") "$@" + then + if [ "${DIFFVIEW:-}" == "" ] + then + printError "ERROR: values differ:" + printError "Expected:" + printError "${value1}" + printError "Obtained:" + printError "${value2}" + else + # Use user supplied diff view binary + printError "ERROR: values differ." + "$DIFFVIEW" "${file1}" "${file2}" + fi + return 1 + fi +} + +function diff_files +{ + (( $# >= 2 )) || fail "diff_files requires at least 2 arguments." + + local file1="$1" + local file2="$2" + shift + shift + + if ! diff "${file1}" "${file2}" + then + if [ "${DIFFVIEW:-}" == "" ] + then + printError "ERROR: files differ: ${file1} vs. ${file2}" + printError "Expected:" + cat "${file1}" >&2 + printError "Obtained:" + cat "${file2}" >&2 + else + # Use user supplied diff view binary + "$DIFFVIEW" "${file1}" "${file2}" + fi + return 1 + fi + return 0 } function safe_kill @@ -277,3 +320,14 @@ function split_on_empty_lines_into_numbered_files awk -v RS= "{print > (\"${path_prefix}_\"NR \"${path_suffix}\")}" } + +function command_available +{ + local program="$1" + local parameters=${*:2} + if ! "${program}" "${parameters}" > /dev/null 2>&1 + then + fail "'${program}' not found or not executed successfully with parameter(s) '${parameters}'. aborting." + fi +} + diff --git a/solc/CommandLineInterface.cpp b/solc/CommandLineInterface.cpp index e215405a1271..dfbd976b38eb 100644 --- a/solc/CommandLineInterface.cpp +++ b/solc/CommandLineInterface.cpp @@ -89,6 +89,17 @@ using namespace solidity; using namespace solidity::util; using namespace solidity::langutil; +namespace +{ + +std::set ValidInputModes{ + frontend::InputMode::Compiler, + frontend::InputMode::CompilerWithASTImport, + frontend::InputMode::CompilerWithEvmAssemblyJsonImport +}; + +} // anonymous namespace + namespace solidity::frontend { @@ -161,7 +172,7 @@ static bool coloredOutput(CommandLineOptions const& _options) void CommandLineInterface::handleBinary(string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert(ValidInputModes.count(m_options.input.mode) == 1); if (m_options.compiler.outputs.binary) { @@ -187,7 +198,7 @@ void CommandLineInterface::handleBinary(string const& _contract) void CommandLineInterface::handleOpcode(string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert(ValidInputModes.count(m_options.input.mode) == 1); if (!m_options.output.dir.empty()) createFile(m_compiler->filesystemFriendlyName(_contract) + ".opcode", evmasm::disassemble(m_compiler->object(_contract).bytecode)); @@ -201,7 +212,7 @@ void CommandLineInterface::handleOpcode(string const& _contract) void CommandLineInterface::handleIR(string const& _contractName) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert(ValidInputModes.count(m_options.input.mode) == 1); if (!m_options.compiler.outputs.ir) return; @@ -217,7 +228,7 @@ void CommandLineInterface::handleIR(string const& _contractName) void CommandLineInterface::handleIROptimized(string const& _contractName) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert(ValidInputModes.count(m_options.input.mode) == 1); if (!m_options.compiler.outputs.irOptimized) return; @@ -233,7 +244,7 @@ void CommandLineInterface::handleIROptimized(string const& _contractName) void CommandLineInterface::handleEwasm(string const& _contractName) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert(ValidInputModes.count(m_options.input.mode) == 1); if (!m_options.compiler.outputs.ewasm) return; @@ -256,7 +267,7 @@ void CommandLineInterface::handleEwasm(string const& _contractName) void CommandLineInterface::handleBytecode(string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert(ValidInputModes.count(m_options.input.mode) == 1); if (m_options.compiler.outputs.opcodes) handleOpcode(_contract); @@ -266,7 +277,7 @@ void CommandLineInterface::handleBytecode(string const& _contract) void CommandLineInterface::handleSignatureHashes(string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert(ValidInputModes.count(m_options.input.mode) == 1); if (!m_options.compiler.outputs.signatureHashes) return; @@ -298,7 +309,7 @@ void CommandLineInterface::handleSignatureHashes(string const& _contract) void CommandLineInterface::handleMetadata(string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert(ValidInputModes.count(m_options.input.mode) == 1); if (!m_options.compiler.outputs.metadata) return; @@ -312,7 +323,7 @@ void CommandLineInterface::handleMetadata(string const& _contract) void CommandLineInterface::handleABI(string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert(ValidInputModes.count(m_options.input.mode) == 1); if (!m_options.compiler.outputs.abi) return; @@ -326,7 +337,7 @@ void CommandLineInterface::handleABI(string const& _contract) void CommandLineInterface::handleStorageLayout(string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert(ValidInputModes.count(m_options.input.mode) == 1); if (!m_options.compiler.outputs.storageLayout) return; @@ -340,7 +351,7 @@ void CommandLineInterface::handleStorageLayout(string const& _contract) void CommandLineInterface::handleNatspec(bool _natspecDev, string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert(ValidInputModes.count(m_options.input.mode) == 1); bool enabled = false; std::string suffix; @@ -383,7 +394,7 @@ void CommandLineInterface::handleNatspec(bool _natspecDev, string const& _contra void CommandLineInterface::handleGasEstimation(string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert(ValidInputModes.count(m_options.input.mode) == 1); Json::Value estimates = m_compiler->gasEstimates(_contract); sout() << "Gas estimation:" << endl; @@ -425,13 +436,9 @@ void CommandLineInterface::handleGasEstimation(string const& _contract) void CommandLineInterface::readInputFiles() { - solAssert(!m_standardJsonInput.has_value(), ""); + solAssert(!m_standardJsonInput.has_value()); - if ( - m_options.input.mode == InputMode::Help || - m_options.input.mode == InputMode::License || - m_options.input.mode == InputMode::Version - ) + if (std::set{InputMode::Help, InputMode::License, InputMode::Version}.count(m_options.input.mode) == 1) return; m_fileReader.setBasePath(m_options.input.basePath); @@ -497,7 +504,7 @@ void CommandLineInterface::readInputFiles() string fileContent = readFileAsString(infile); if (m_options.input.mode == InputMode::StandardJson) { - solAssert(!m_standardJsonInput.has_value(), ""); + solAssert(!m_standardJsonInput.has_value()); m_standardJsonInput = std::move(fileContent); } else @@ -511,7 +518,7 @@ void CommandLineInterface::readInputFiles() { if (m_options.input.mode == InputMode::StandardJson) { - solAssert(!m_standardJsonInput.has_value(), ""); + solAssert(!m_standardJsonInput.has_value()); m_standardJsonInput = readUntilEnd(m_sin); } else @@ -528,7 +535,7 @@ void CommandLineInterface::readInputFiles() map CommandLineInterface::parseAstFromInput() { - solAssert(m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert(m_options.input.mode == InputMode::CompilerWithASTImport); map sourceJsons; map tmpSources; @@ -556,11 +563,30 @@ map CommandLineInterface::parseAstFromInput() return sourceJsons; } +map CommandLineInterface::parseEvmAssemblyJsonFromInput() +{ + solAssert(m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport); + solAssert(m_fileReader.sourceUnits().size() == 1); + + map sourceJsons; + + for (auto const& iter: m_fileReader.sourceUnits()) + { + Json::Value evmAsmJson; + astAssert(jsonParseStrict(iter.second, evmAsmJson), "Input file could not be parsed to JSON"); + astAssert(evmAsmJson.isMember(".code"), "Invalid Format for assembly-JSON: Must have '.code'-object"); + astAssert(evmAsmJson.isMember(".data"), "Invalid Format for assembly-JSON: Must have '.data'-object"); + sourceJsons[iter.first] = evmAsmJson; + } + + return sourceJsons; +} + void CommandLineInterface::createFile(string const& _fileName, string const& _data) { namespace fs = boost::filesystem; - solAssert(!m_options.output.dir.empty(), ""); + solAssert(!m_options.output.dir.empty()); // NOTE: create_directories() raises an exception if the path consists solely of '.' or '..' // (or equivalent such as './././.'). Paths like 'a/b/.' and 'a/b/..' are fine though. @@ -640,7 +666,7 @@ void CommandLineInterface::processInput() break; case InputMode::StandardJson: { - solAssert(m_standardJsonInput.has_value(), ""); + solAssert(m_standardJsonInput.has_value()); StandardCompiler compiler(m_fileReader.reader(), m_options.formatting.json); sout() << compiler.compile(std::move(m_standardJsonInput.value())) << endl; @@ -659,6 +685,7 @@ void CommandLineInterface::processInput() break; case InputMode::Compiler: case InputMode::CompilerWithASTImport: + case InputMode::CompilerWithEvmAssemblyJsonImport: compile(); outputCompilationResults(); } @@ -679,7 +706,7 @@ void CommandLineInterface::printLicense() void CommandLineInterface::compile() { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert(ValidInputModes.count(m_options.input.mode) == 1); m_compiler = make_unique(m_fileReader.reader()); @@ -726,7 +753,18 @@ void CommandLineInterface::compile() m_compiler->setOptimiserSettings(m_options.optimiserSettings()); - if (m_options.input.mode == InputMode::CompilerWithASTImport) + if (m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport) + { + try + { + m_compiler->importEvmAssemblyJson(parseEvmAssemblyJsonFromInput()); + } + catch (Exception const& _exc) + { + solThrow(CommandLineExecutionError, "Failed to import Evm Assembly JSON: "s + _exc.what()); + } + } + else if (m_options.input.mode == InputMode::CompilerWithASTImport) { try { @@ -786,7 +824,7 @@ void CommandLineInterface::compile() void CommandLineInterface::handleCombinedJSON() { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert(ValidInputModes.count(m_options.input.mode) == 1); if (!m_options.compiler.combinedJsonRequests.has_value()) return; @@ -878,7 +916,7 @@ void CommandLineInterface::handleCombinedJSON() void CommandLineInterface::handleAst() { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert(ValidInputModes.count(m_options.input.mode) == 1); if (!m_options.compiler.outputs.astCompactJson) return; @@ -919,7 +957,7 @@ void CommandLineInterface::serveLSP() void CommandLineInterface::link() { - solAssert(m_options.input.mode == InputMode::Linker, ""); + solAssert(m_options.input.mode == InputMode::Linker); // Map from how the libraries will be named inside the bytecode to their addresses. map librariesReplacements; @@ -982,7 +1020,7 @@ void CommandLineInterface::link() void CommandLineInterface::writeLinkedFiles() { - solAssert(m_options.input.mode == InputMode::Linker, ""); + solAssert(m_options.input.mode == InputMode::Linker); for (auto const& src: m_fileReader.sourceUnits()) if (src.first == g_stdinFileName) @@ -1016,14 +1054,14 @@ string CommandLineInterface::objectWithLinkRefsHex(evmasm::LinkerObject const& _ void CommandLineInterface::assemble(yul::YulStack::Language _language, yul::YulStack::Machine _targetMachine) { - solAssert(m_options.input.mode == InputMode::Assembler, ""); + solAssert(m_options.input.mode == InputMode::Assembler); bool successful = true; map yulStacks; for (auto const& src: m_fileReader.sourceUnits()) { // --no-optimize-yul option is not accepted in assembly mode. - solAssert(!m_options.optimizer.noOptimizeYul, ""); + solAssert(!m_options.optimizer.noOptimizeYul); auto& stack = yulStacks[src.first] = yul::YulStack( m_options.output.evmVersion, @@ -1120,7 +1158,7 @@ void CommandLineInterface::assemble(yul::YulStack::Language _language, yul::YulS void CommandLineInterface::outputCompilationResults() { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert(ValidInputModes.count(m_options.input.mode) == 1); handleCombinedJSON(); diff --git a/solc/CommandLineInterface.h b/solc/CommandLineInterface.h index b7ab158f9ed4..7c951495e2ea 100644 --- a/solc/CommandLineInterface.h +++ b/solc/CommandLineInterface.h @@ -115,6 +115,8 @@ class CommandLineInterface /// or standard-json output std::map parseAstFromInput(); + std::map parseEvmAssemblyJsonFromInput(); + /// Create a file in the given directory /// @arg _fileName the name of the file /// @arg _data to be written diff --git a/solc/CommandLineParser.cpp b/solc/CommandLineParser.cpp index c6ddfec67f20..db3be10e7194 100644 --- a/solc/CommandLineParser.cpp +++ b/solc/CommandLineParser.cpp @@ -52,6 +52,7 @@ static string const g_strExperimentalViaIR = "experimental-via-ir"; static string const g_strGas = "gas"; static string const g_strHelp = "help"; static string const g_strImportAst = "import-ast"; +static string const g_strImportEvmAssemblerJson = "import-asm-json"; static string const g_strInputFile = "input-file"; static string const g_strYul = "yul"; static string const g_strYulDialect = "yul-dialect"; @@ -138,6 +139,7 @@ static map const g_inputModeName = { {InputMode::StandardJson, "standard JSON"}, {InputMode::Linker, "linker"}, {InputMode::LanguageServer, "language server (LSP)"}, + {InputMode::CompilerWithEvmAssemblyJsonImport, "assembler (EVM ASM JSON import)"}, }; void CommandLineParser::checkMutuallyExclusive(vector const& _optionNames) @@ -461,6 +463,13 @@ void CommandLineParser::parseOutputSelection() CompilerOutputs::componentName(&CompilerOutputs::ewasm), CompilerOutputs::componentName(&CompilerOutputs::ewasmIR), }; + static set const evmAssemblyJsonImportModeOutputs = { + CompilerOutputs::componentName(&CompilerOutputs::asm_), + CompilerOutputs::componentName(&CompilerOutputs::binary), + CompilerOutputs::componentName(&CompilerOutputs::binaryRuntime), + CompilerOutputs::componentName(&CompilerOutputs::opcodes), + CompilerOutputs::componentName(&CompilerOutputs::asmJson), + }; switch (_mode) { @@ -472,6 +481,8 @@ void CommandLineParser::parseOutputSelection() case InputMode::Compiler: case InputMode::CompilerWithASTImport: return util::contains(compilerModeOutputs, _outputName); + case InputMode::CompilerWithEvmAssemblyJsonImport: + return util::contains(evmAssemblyJsonImportModeOutputs, _outputName); case InputMode::Assembler: return util::contains(assemblerModeOutputs, _outputName); case InputMode::StandardJson: @@ -650,6 +661,10 @@ General Information)").c_str(), "Supported Inputs is the output of the --" + g_strStandardJSON + " or the one produced by " "--" + g_strCombinedJson + " " + CombinedJsonRequests::componentName(&CombinedJsonRequests::ast)).c_str() ) + ( + g_strImportEvmAssemblerJson.c_str(), + "Import EVM Assembly JSON, assumes input holds the EVM Assembly in JSON format." + ) ( g_strLSP.c_str(), "Switch to language server mode (\"LSP\"). Allows the compiler to be used as an analysis backend " @@ -906,6 +921,8 @@ void CommandLineParser::processArgs() m_options.input.mode = InputMode::Linker; else if (m_args.count(g_strImportAst) > 0) m_options.input.mode = InputMode::CompilerWithASTImport; + else if (m_args.count(g_strImportEvmAssemblerJson) > 0) + m_options.input.mode = InputMode::CompilerWithEvmAssemblyJsonImport; else m_options.input.mode = InputMode::Compiler; @@ -970,9 +987,27 @@ void CommandLineParser::processArgs() for (auto& option: conflictingWithStopAfter) checkMutuallyExclusive({g_strStopAfter, option}); + array const conflictingWithAsmJsonImport{ + CompilerOutputs::componentName(&CompilerOutputs::ir), + CompilerOutputs::componentName(&CompilerOutputs::irOptimized), + CompilerOutputs::componentName(&CompilerOutputs::ewasm), + CompilerOutputs::componentName(&CompilerOutputs::ewasmIR), + g_strGas, + CompilerOutputs::componentName(&CompilerOutputs::metadata), + CompilerOutputs::componentName(&CompilerOutputs::natspecDev), + CompilerOutputs::componentName(&CompilerOutputs::natspecUser), + CompilerOutputs::componentName(&CompilerOutputs::signatureHashes), + CompilerOutputs::componentName(&CompilerOutputs::storageLayout), + CompilerOutputs::componentName(&CompilerOutputs::astCompactJson), + }; + + for (auto& option: conflictingWithAsmJsonImport) + checkMutuallyExclusive({g_strImportEvmAssemblerJson, option}); + if ( m_options.input.mode != InputMode::Compiler && m_options.input.mode != InputMode::CompilerWithASTImport && + m_options.input.mode != InputMode::CompilerWithEvmAssemblyJsonImport && m_options.input.mode != InputMode::Assembler ) { @@ -1289,7 +1324,11 @@ void CommandLineParser::processArgs() if (m_options.input.mode == InputMode::Compiler) m_options.input.errorRecovery = (m_args.count(g_strErrorRecovery) > 0); - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport + ); } void CommandLineParser::parseCombinedJsonOption() diff --git a/solc/CommandLineParser.h b/solc/CommandLineParser.h index 108a16cd23a6..12fec17b19c8 100644 --- a/solc/CommandLineParser.h +++ b/solc/CommandLineParser.h @@ -56,7 +56,8 @@ enum class InputMode StandardJson, Linker, Assembler, - LanguageServer + LanguageServer, + CompilerWithEvmAssemblyJsonImport }; struct CompilerOutputs @@ -293,4 +294,4 @@ class CommandLineParser boost::program_options::variables_map m_args; }; -} +} // namespace solidity::frontend diff --git a/test/cmdlineTests.sh b/test/cmdlineTests.sh index 7728f1f6deb5..c768b41dd179 100755 --- a/test/cmdlineTests.sh +++ b/test/cmdlineTests.sh @@ -662,11 +662,22 @@ SOLTMPDIR=$(mktemp -d) ) rm -r "$SOLTMPDIR" -printTask "Testing AST import..." +printTask "Testing AST import/export..." SOLTMPDIR=$(mktemp -d) ( cd "$SOLTMPDIR" - if ! "$REPO_ROOT/scripts/ASTImportTest.sh" + if ! "$REPO_ROOT/scripts/ImportExportTest.sh" ast + then + rm -r "$SOLTMPDIR" + fail + fi +) +rm -r "$SOLTMPDIR" +printTask "Testing EVM Assembly JSON import/export..." +SOLTMPDIR=$(mktemp -d) +( + cd "$SOLTMPDIR" + if ! "$REPO_ROOT/scripts/ImportExportTest.sh" evm-assembly then rm -r "$SOLTMPDIR" fail diff --git a/test/libsolutil/JSON.cpp b/test/libsolutil/JSON.cpp index 1b0ed938894b..3217541f745d 100644 --- a/test/libsolutil/JSON.cpp +++ b/test/libsolutil/JSON.cpp @@ -184,6 +184,131 @@ BOOST_AUTO_TEST_CASE(parse_json_strict) BOOST_CHECK(json[0] == "😊"); } +BOOST_AUTO_TEST_CASE(json_ofType) +{ + Json::Value json; + + json["float"] = 3.1f; + json["double"] = 3.1; + json["int"] = 2; + json["int64"] = Json::Int64{0x4000000000000000}; + json["string"] = "Hello World!"; + + BOOST_CHECK(ofType(json, "float")); + BOOST_CHECK(ofType(json, "double")); + BOOST_CHECK(ofType(json, "int")); + BOOST_CHECK(ofType(json, "int")); + BOOST_CHECK(ofType(json, "int")); + BOOST_CHECK(ofType(json, "int")); + BOOST_CHECK(ofType(json, "int64")); + BOOST_CHECK(ofType(json, "int64")); + BOOST_CHECK(ofType(json, "string")); + BOOST_CHECK(!ofType(json, "int64")); + BOOST_CHECK(!ofType(json, "double")); + BOOST_CHECK(!ofType(json, "string")); + BOOST_CHECK(!ofType(json, "string")); + BOOST_CHECK(!ofType(json, "string")); + BOOST_CHECK(!ofType(json, "string")); + BOOST_CHECK(!ofType(json, "string")); + BOOST_CHECK(!ofType(json, "string")); +} + +BOOST_AUTO_TEST_CASE(json_ofTypeIfExists) +{ + Json::Value json; + + json["float"] = 3.1f; + json["double"] = 3.1; + json["int"] = 2; + json["int64"] = Json::Int64{0x4000000000000000}; + json["string"] = "Hello World!"; + + BOOST_CHECK(ofTypeIfExists(json, "float")); + BOOST_CHECK(ofTypeIfExists(json, "double")); + BOOST_CHECK(ofTypeIfExists(json, "int")); + BOOST_CHECK(ofTypeIfExists(json, "int")); + BOOST_CHECK(ofTypeIfExists(json, "int")); + BOOST_CHECK(ofTypeIfExists(json, "int")); + BOOST_CHECK(ofTypeIfExists(json, "int64")); + BOOST_CHECK(ofTypeIfExists(json, "int64")); + BOOST_CHECK(ofTypeIfExists(json, "string")); + BOOST_CHECK(!ofTypeIfExists(json, "int64")); + BOOST_CHECK(!ofTypeIfExists(json, "double")); + BOOST_CHECK(!ofTypeIfExists(json, "string")); + BOOST_CHECK(!ofTypeIfExists(json, "string")); + BOOST_CHECK(!ofTypeIfExists(json, "string")); + BOOST_CHECK(!ofTypeIfExists(json, "string")); + BOOST_CHECK(!ofTypeIfExists(json, "string")); + BOOST_CHECK(!ofTypeIfExists(json, "string")); + BOOST_CHECK(ofTypeIfExists(json, "NOT_EXISTING")); +} + +BOOST_AUTO_TEST_CASE(json_getOrDefault) +{ + Json::Value json; + + json["float"] = 3.1f; + json["double"] = 3.1; + json["int"] = 2; + json["int64"] = Json::Int64{0x4000000000000000}; + json["uint64"] = Json::UInt64{0x5000000000000000}; + json["string"] = "Hello World!"; + + BOOST_CHECK(getOrDefault(json, "float") == 3.1f); + BOOST_CHECK(getOrDefault(json, "float", -1.1f) == 3.1f); + BOOST_CHECK(getOrDefault(json, "no_float", -1.1f) == -1.1f); + BOOST_CHECK(getOrDefault(json, "double") == 3.1); + BOOST_CHECK(getOrDefault(json, "double", -1) == 3.1); + BOOST_CHECK(getOrDefault(json, "no_double", -1.1) == -1.1); + BOOST_CHECK(getOrDefault(json, "int") == 2); + BOOST_CHECK(getOrDefault(json, "int", -1) == 2); + BOOST_CHECK(getOrDefault(json, "no_int", -1) == -1); + BOOST_CHECK(getOrDefault(json, "int") == 2); + BOOST_CHECK(getOrDefault(json, "int", -1) == 2); + BOOST_CHECK(getOrDefault(json, "no_int", -1) == -1); + BOOST_CHECK(getOrDefault(json, "int") == 2); + BOOST_CHECK(getOrDefault(json, "int", 1) == 2); + BOOST_CHECK(getOrDefault(json, "no_int", 1) == 1); + BOOST_CHECK(getOrDefault(json, "int") == 2); + BOOST_CHECK(getOrDefault(json, "int", -1) == 2); + BOOST_CHECK(getOrDefault(json, "no_int", -1) == -1); + BOOST_CHECK(getOrDefault(json, "int64") == 0x4000000000000000); + BOOST_CHECK(getOrDefault(json, "int64", -1) == 0x4000000000000000); + BOOST_CHECK(getOrDefault(json, "no_int64", -1) == -1); + BOOST_CHECK(getOrDefault(json, "int64") == 0x4000000000000000); + BOOST_CHECK(getOrDefault(json, "int64", 1) == 0x4000000000000000); + BOOST_CHECK(getOrDefault(json, "no_int64", 1) == 1); + BOOST_CHECK(getOrDefault(json, "uint64") == 0x5000000000000000); + BOOST_CHECK(getOrDefault(json, "uint64", 1) == 0x5000000000000000); + BOOST_CHECK(getOrDefault(json, "no_uint64", 1) == 1); + BOOST_CHECK(getOrDefault(json, "string", "ERROR") == "Hello World!"); + BOOST_CHECK(getOrDefault(json, "no_string").empty()); + BOOST_CHECK(getOrDefault(json, "no_string", "ERROR") == "ERROR"); +} + +BOOST_AUTO_TEST_CASE(json_get) +{ + Json::Value json; + + json["float"] = 3.1f; + json["double"] = 3.1; + json["int"] = 2; + json["int64"] = Json::Int64{0x4000000000000000}; + json["uint64"] = Json::UInt64{0x5000000000000000}; + json["string"] = "Hello World!"; + + BOOST_CHECK(get(json, "float") == 3.1f); + BOOST_CHECK(get(json, "double") == 3.1); + BOOST_CHECK(get(json, "int") == 2); + BOOST_CHECK(get(json, "int") == 2); + BOOST_CHECK(get(json, "int") == 2); + BOOST_CHECK(get(json, "int") == 2); + BOOST_CHECK(get(json, "int64") == 0x4000000000000000); + BOOST_CHECK(get(json, "int64") == 0x4000000000000000); + BOOST_CHECK(get(json, "uint64") == 0x5000000000000000); + BOOST_CHECK(get(json, "string") == "Hello World!"); +} + BOOST_AUTO_TEST_SUITE_END() }