diff --git a/libevmasm/Assembly.cpp b/libevmasm/Assembly.cpp index 12c7be23038d..3dc19dab67c7 100644 --- a/libevmasm/Assembly.cpp +++ b/libevmasm/Assembly.cpp @@ -34,7 +34,7 @@ #include #include -#include +#include #include #include @@ -222,13 +222,16 @@ string Assembly::assemblyString( return tmp.str(); } -Json::Value Assembly::createJsonValue(string _name, int _source, int _begin, int _end, string _value, string _jumpType) +Json::Value Assembly::createJsonValue( + string _name, int _sourceIndex, size_t _modifierDepth, int _begin, int _end, string _value, string _jumpType) { Json::Value value{Json::objectValue}; value["name"] = _name; - value["source"] = _source; + value["source"] = _sourceIndex; value["begin"] = _begin; value["end"] = _end; + if (_modifierDepth != 0) + value["modifierDepth"] = static_cast(_modifierDepth); if (!_value.empty()) value["value"] = _value; if (!_jumpType.empty()) @@ -243,12 +246,275 @@ string Assembly::toStringInHex(u256 _value) return hexStr.str(); } -Json::Value Assembly::assemblyJSON(map const& _sourceIndices) const +AssemblyItem Assembly::loadItemFromJSON(Json::Value const& _json) +{ + std::string name = _json["name"].isString() ? _json["name"].asString() : ""; + int begin = _json["begin"].isInt() ? _json["begin"].asInt() : -1; + int end = _json["end"].isInt() ? _json["end"].asInt() : -1; + int srcIndex = _json["source"].isInt() ? _json["source"].asInt() : -1; + size_t modifierDepth = _json["modifierDepth"].isInt() ? static_cast(_json["modifierDepth"].asInt()) : 0; + std::string value = _json["value"].isString() ? _json["value"].asString() : ""; + std::string jumpType = _json["jumpType"].isString() ? _json["jumpType"].asString() : ""; + solAssert(!name.empty(), ""); + + auto updateUsedTags = [&](u256 const& data) { + auto tag = static_cast(data); + if (this->m_usedTags <= tag) + this->m_usedTags = tag + 1; + }; + + auto updateImmutables = [&](string const& _immutableName) -> h256 { + h256 hash(util::keccak256(value)); + this->m_immutables[hash] = _immutableName; + return hash; + }; + + auto updateLibraries = [&](string const& _libraryName) -> h256 { + h256 hash(util::keccak256(_libraryName)); + this->m_libraries[hash] = _libraryName; + return hash; + }; + + SourceLocation location; + location.start = begin; + location.end = end; + if (srcIndex > -1 && srcIndex < (int) sources().size()) + location.sourceName = sources()[static_cast(srcIndex)]; + + AssemblyItem result(0); + + if (c_instructions.find(name) != c_instructions.end()) + { + AssemblyItem item{c_instructions.at(name), location}; + item.m_modifierDepth = modifierDepth; + if (!value.empty()) + item.setJumpType(value); + result = item; + } + else + { + u256 data; + if (name == "PUSH") + { + if (!value.empty()) + data = u256("0x" + value); + AssemblyItem item{AssemblyItemType::Push, data, location}; + if (!jumpType.empty()) + item.setJumpType(jumpType); + result = item; + } + else if (name == "PUSH [ErrorTag]") + result = {AssemblyItemType::PushTag, data, location}; + else if (name == "PUSH [tag]") + { + if (!value.empty()) + data = u256(value); + updateUsedTags(data); + result = {AssemblyItemType::PushTag, data, location}; + } + else if (name == "PUSH [$]") + { + if (!value.empty()) + data = u256("0x" + value); + result = {AssemblyItemType::PushSub, data, location}; + } + else if (name == "PUSH #[$]") + { + if (!value.empty()) + data = u256("0x" + value); + result = {AssemblyItemType::PushSubSize, data, location}; + } + else if (name == "PUSHSIZE") + result = {AssemblyItemType::PushProgramSize, data, location}; + else if (name == "PUSHLIB") + { + h256 hash = updateLibraries(value); + result = {AssemblyItemType::PushLibraryAddress, hash, location}; + } + else if (name == "PUSHDEPLOYADDRESS") + result = {AssemblyItemType::PushDeployTimeAddress, data, location}; + else if (name == "PUSHIMMUTABLE") + { + h256 hash = updateImmutables(value); + result = {AssemblyItemType::PushImmutable, hash, location}; + } + else if (name == "ASSIGNIMMUTABLE") + { + h256 hash = updateImmutables(value); + result = {AssemblyItemType::AssignImmutable, hash, location}; + } + else if (name == "tag") + { + if (!value.empty()) + data = u256(value); + result = {AssemblyItemType::Tag, data, location}; + } + else if (name == "PUSH data") + { + if (!value.empty()) + data = u256("0x" + value); + result = {AssemblyItemType::PushData, data, location}; + } + else if (name == "VERBATIM") + { + AssemblyItem item(fromHex(value), 0, 0); + item.setLocation(location); + result = item; + } + else + assertThrow(false, InvalidOpcode, ""); + } + result.m_modifierDepth = modifierDepth; + return result; +} + +vector Assembly::assemblyItemAsJSON(AssemblyItem const& _item, int _sourceIndex) const +{ + vector result; + + switch (_item.type()) + { + case Operation: + result.emplace_back(createJsonValue( + instructionInfo(_item.instruction()).name, + _sourceIndex, + _item.m_modifierDepth, + _item.location().start, + _item.location().end, + _item.getJumpTypeAsString())); + break; + case Push: + result.emplace_back(createJsonValue( + "PUSH", + _sourceIndex, + _item.m_modifierDepth, + _item.location().start, + _item.location().end, + toStringInHex(_item.data()), + _item.getJumpTypeAsString())); + break; + case PushTag: + if (_item.data() == 0) + result.emplace_back(createJsonValue( + "PUSH [ErrorTag]", + _sourceIndex, + _item.m_modifierDepth, + _item.location().start, + _item.location().end, + "")); + else + result.emplace_back(createJsonValue( + "PUSH [tag]", + _sourceIndex, + _item.m_modifierDepth, + _item.location().start, + _item.location().end, + toString(_item.data()))); + break; + case PushSub: + result.emplace_back(createJsonValue( + "PUSH [$]", + _sourceIndex, + _item.m_modifierDepth, + _item.location().start, + _item.location().end, + toString(h256(_item.data())))); + break; + case PushSubSize: + result.emplace_back(createJsonValue( + "PUSH #[$]", + _sourceIndex, + _item.m_modifierDepth, + _item.location().start, + _item.location().end, + toString(h256(_item.data())))); + break; + case PushProgramSize: + result.emplace_back(createJsonValue( + "PUSHSIZE", _sourceIndex, _item.m_modifierDepth, _item.location().start, _item.location().end)); + break; + case PushLibraryAddress: + result.emplace_back(createJsonValue( + "PUSHLIB", + _sourceIndex, + _item.m_modifierDepth, + _item.location().start, + _item.location().end, + m_libraries.at(h256(_item.data())))); + break; + case PushDeployTimeAddress: + result.emplace_back(createJsonValue( + "PUSHDEPLOYADDRESS", _sourceIndex, _item.m_modifierDepth, _item.location().start, _item.location().end)); + break; + case PushImmutable: + result.emplace_back(createJsonValue( + "PUSHIMMUTABLE", + _sourceIndex, + _item.m_modifierDepth, + _item.location().start, + _item.location().end, + m_immutables.at(h256(_item.data())))); + break; + case AssignImmutable: + result.emplace_back(createJsonValue( + "ASSIGNIMMUTABLE", + _sourceIndex, + _item.m_modifierDepth, + _item.location().start, + _item.location().end, + m_immutables.at(h256(_item.data())))); + break; + case Tag: + result.emplace_back(createJsonValue( + "tag", + _sourceIndex, + _item.m_modifierDepth, + _item.location().start, + _item.location().end, + toString(_item.data()))); + result.emplace_back(createJsonValue( + "JUMPDEST", _sourceIndex, _item.m_modifierDepth, _item.location().start, _item.location().end)); + break; + case PushData: + result.emplace_back(createJsonValue( + "PUSH data", + _sourceIndex, + _item.m_modifierDepth, + _item.location().start, + _item.location().end, + toStringInHex(_item.data()))); + break; + case VerbatimBytecode: + result.emplace_back(createJsonValue( + "VERBATIM", + _sourceIndex, + _item.m_modifierDepth, + _item.location().start, + _item.location().end, + util::toHex(_item.verbatimData()))); + break; + default: + assertThrow(false, InvalidOpcode, ""); + } + return result; +} + +Json::Value Assembly::assemblyJSON(map const& _sourceIndices, bool _includeSourceList) const { Json::Value root; - root[".code"] = Json::arrayValue; + if (_includeSourceList) + { + root["sourceList"] = Json::arrayValue; + Json::Value& sourceList = root["sourceList"]; + std::vector sources(_sourceIndices.size()); + for (auto const& item: _sourceIndices) + sources[item.second] = item.first; + for (auto const& item: sources) + sourceList.append(item); + } - Json::Value& collection = root[".code"]; + root[".code"] = Json::arrayValue; + Json::Value& code = root[".code"]; for (AssemblyItem const& i: m_items) { int sourceIndex = -1; @@ -259,85 +525,8 @@ Json::Value Assembly::assemblyJSON(map const& _sourceIndices) sourceIndex = static_cast(iter->second); } - switch (i.type()) - { - case Operation: - collection.append( - createJsonValue( - instructionInfo(i.instruction()).name, - sourceIndex, - i.location().start, - i.location().end, - i.getJumpTypeAsString()) - ); - break; - case Push: - collection.append( - createJsonValue("PUSH", sourceIndex, i.location().start, i.location().end, toStringInHex(i.data()), i.getJumpTypeAsString())); - break; - case PushTag: - if (i.data() == 0) - collection.append( - createJsonValue("PUSH [ErrorTag]", sourceIndex, i.location().start, i.location().end, "")); - else - collection.append( - createJsonValue("PUSH [tag]", sourceIndex, i.location().start, i.location().end, toString(i.data()))); - break; - case PushSub: - collection.append( - createJsonValue("PUSH [$]", sourceIndex, i.location().start, i.location().end, toString(h256(i.data())))); - break; - case PushSubSize: - collection.append( - createJsonValue("PUSH #[$]", sourceIndex, i.location().start, i.location().end, toString(h256(i.data())))); - break; - case PushProgramSize: - collection.append( - createJsonValue("PUSHSIZE", sourceIndex, i.location().start, i.location().end)); - break; - case PushLibraryAddress: - collection.append( - createJsonValue("PUSHLIB", sourceIndex, i.location().start, i.location().end, m_libraries.at(h256(i.data()))) - ); - break; - case PushDeployTimeAddress: - collection.append( - createJsonValue("PUSHDEPLOYADDRESS", sourceIndex, i.location().start, i.location().end) - ); - break; - case PushImmutable: - collection.append(createJsonValue( - "PUSHIMMUTABLE", - sourceIndex, - i.location().start, - i.location().end, - m_immutables.at(h256(i.data())) - )); - break; - case AssignImmutable: - collection.append(createJsonValue( - "ASSIGNIMMUTABLE", - sourceIndex, - i.location().start, - i.location().end, - m_immutables.at(h256(i.data())) - )); - break; - case Tag: - collection.append( - createJsonValue("tag", sourceIndex, i.location().start, i.location().end, toString(i.data()))); - collection.append( - createJsonValue("JUMPDEST", sourceIndex, i.location().start, i.location().end)); - break; - case PushData: - collection.append(createJsonValue("PUSH data", sourceIndex, i.location().start, i.location().end, toStringInHex(i.data()))); - break; - case VerbatimBytecode: - collection.append(createJsonValue("VERBATIM", sourceIndex, i.location().start, i.location().end, util::toHex(i.verbatimData()))); - break; - default: - assertThrow(false, InvalidOpcode, ""); - } + for (Json::Value const& item: assemblyItemAsJSON(i, sourceIndex)) + code.append(item); } if (!m_data.empty() || !m_subs.empty()) @@ -352,16 +541,75 @@ Json::Value Assembly::assemblyJSON(map const& _sourceIndices) { std::stringstream hexStr; hexStr << hex << i; - data[hexStr.str()] = m_subs[i]->assemblyJSON(_sourceIndices); + data[hexStr.str()] = m_subs[i]->assemblyJSON(_sourceIndices, false); } } - if (m_auxiliaryData.size() > 0) + if (!m_auxiliaryData.empty()) root[".auxdata"] = util::toHex(m_auxiliaryData); return root; } +bool Assembly::addAssemblyItemsFromJSON(Json::Value const& _code) +{ + solAssert(_code.isArray(), ""); + for (auto const& it: _code) + this->m_items.emplace_back(loadItemFromJSON(it)); + + for (auto current = this->m_items.begin(); current != this->m_items.end(); ++current) + { + // During the assembly json export a `JUMPDEST` is always generated after a `tag`. + // So we just ignore exactly these `JUMPDEST`'s. + auto const next = std::next(current); + if ( + current->type() == AssemblyItemType::Tag && + next->type() == AssemblyItemType::Operation && + next->instruction() == Instruction::JUMPDEST + ) + this->m_items.erase(next); + } + + return true; +} + +bool Assembly::loadFromAssemblyJSON(Json::Value const& _json, bool _loadSources /* = true */) +{ + if (!_json[".code"].isArray()) + return false; + bool result{true}; + + if (_loadSources) + { + vector sourceList; + if (_json.isMember("sourceList")) + for (auto const& it: _json["sourceList"]) + sourceList.emplace_back(it.asString()); + setSources(sourceList); + } + + addAssemblyItemsFromJSON(_json[".code"]); + if (_json[".auxdata"].isString()) + this->m_auxiliaryData = fromHex(_json[".auxdata"].asString()); + Json::Value const& data = _json[".data"]; + for (Json::ValueConstIterator itr = data.begin(); itr != data.end(); itr++) + { + solAssert(itr.key().isString(), ""); + std::string key = itr.key().asString(); + Json::Value const& code = data[key]; + if (code.isString()) + this->m_data[h256(fromHex(key))] = fromHex(code.asString()); + else + { + shared_ptr subassembly = make_shared(); + subassembly->setSources(this->sources()); + result &= subassembly->loadFromAssemblyJSON(code, false); + this->m_subs.emplace_back(subassembly); + } + } + return result; +} + AssemblyItem Assembly::namedTag(string const& _name, size_t _params, size_t _returns, optional _sourceID) { assertThrow(!_name.empty(), AssemblyException, "Empty named tag."); @@ -416,7 +664,6 @@ Assembly& Assembly::optimise(bool _enable, EVMVersion _evmVersion, bool _isCreat return *this; } - Assembly& Assembly::optimise(OptimiserSettings const& _settings) { optimiseInternal(_settings, {}); diff --git a/libevmasm/Assembly.h b/libevmasm/Assembly.h index f768ffe31414..f1d3edcccecd 100644 --- a/libevmasm/Assembly.h +++ b/libevmasm/Assembly.h @@ -39,6 +39,7 @@ #include #include #include +#include namespace solidity::evmasm { @@ -155,15 +156,28 @@ class Assembly /// Create a JSON representation of the assembly. Json::Value assemblyJSON( - std::map const& _sourceIndices = std::map() + std::map const& _sourceIndices = std::map(), + bool _includeSourceList = true ) const; + bool loadFromAssemblyJSON(Json::Value const& _json, bool _loadSources = true); + /// Mark this assembly as invalid. Calling ``assemble`` on it will throw. void markAsInvalid() { m_invalid = true; } std::vector decodeSubPath(size_t _subObjectId) const; size_t encodeSubPath(std::vector const& _subPath); + void setSources(std::vector> _sources) { + m_sources = std::move(_sources); + } + + void setSources(std::vector const& _sources) { + for (auto const& item: _sources) + m_sources.emplace_back(std::make_shared(item)); + } + std::vector> sources() const& { return m_sources; } + protected: /// Does the same operations as @a optimise, but should only be applied to a sub and /// returns the replaced tags. Also takes an argument containing the tags of this assembly @@ -172,10 +186,15 @@ class Assembly unsigned codeSize(unsigned subTagSize) const; + AssemblyItem loadItemFromJSON(Json::Value const& _json); + std::vector assemblyItemAsJSON(AssemblyItem const& _item, int _sourceIndex) const; + private: + bool addAssemblyItemsFromJSON(Json::Value const& _code); static Json::Value createJsonValue( std::string _name, - int _source, + int _sourceIndex, + size_t _modifierDepth, int _begin, int _end, std::string _value = std::string(), @@ -226,6 +245,8 @@ class Assembly std::string m_name; langutil::SourceLocation m_currentSourceLocation; + std::vector> m_sources; + public: size_t m_currentModifierDepth = 0; }; diff --git a/libevmasm/AssemblyItem.cpp b/libevmasm/AssemblyItem.cpp index eeeadb3ef3ea..2cb4eaa1eaf5 100644 --- a/libevmasm/AssemblyItem.cpp +++ b/libevmasm/AssemblyItem.cpp @@ -192,6 +192,18 @@ string AssemblyItem::getJumpTypeAsString() const } } +void AssemblyItem::setJumpType(std::string const& _jumpType) +{ + if (_jumpType == "[in]") + m_jumpType = JumpType::IntoFunction; + else if (_jumpType == "[out]") + m_jumpType = JumpType::OutOfFunction; + else if (_jumpType.empty()) + m_jumpType = JumpType::Ordinary; + else + assertThrow(false, AssemblyException, "Invalid jump type."); +} + string AssemblyItem::toAssemblyText(Assembly const& _assembly) const { string text; diff --git a/libevmasm/AssemblyItem.h b/libevmasm/AssemblyItem.h index 1cbfe768d6ed..652a83208501 100644 --- a/libevmasm/AssemblyItem.h +++ b/libevmasm/AssemblyItem.h @@ -165,6 +165,7 @@ class AssemblyItem langutil::SourceLocation const& location() const { return m_location; } void setJumpType(JumpType _jumpType) { m_jumpType = _jumpType; } + void setJumpType(std::string const& _jumpType); JumpType getJumpType() const { return m_jumpType; } std::string getJumpTypeAsString() const; diff --git a/libsolidity/interface/CompilerStack.cpp b/libsolidity/interface/CompilerStack.cpp index 4b7faf4966e2..c1adf5e78c9f 100644 --- a/libsolidity/interface/CompilerStack.cpp +++ b/libsolidity/interface/CompilerStack.cpp @@ -388,6 +388,28 @@ bool CompilerStack::parse() return !m_hasError; } +void CompilerStack::importEvmAssemblyJson(map const& _sources) +{ + solAssert(_sources.size() == 1, ""); + solAssert(m_sources.empty(), ""); + solAssert(m_sourceOrder.empty(), ""); + if (m_stackState != Empty) + solThrow(CompilerError, "Must call importEvmAssemblyJson only before the SourcesSet state."); + + Json::Value jsonValue = _sources.begin()->second; + if (jsonValue.isMember("sourceList")) + for (auto const& item: jsonValue["sourceList"]) + { + Source source; + source.charStream = std::make_shared(item.asString(), ""); + m_sources.emplace(std::make_pair(item.asString(), source)); + m_sourceOrder.push_back(&m_sources[item.asString()]); + } + m_evmAssemblyJson[_sources.begin()->first] = jsonValue; + m_importedSources = true; + m_stackState = SourcesSet; +} + void CompilerStack::importASTs(map const& _sources) { if (m_stackState != Empty) @@ -597,6 +619,9 @@ bool CompilerStack::parseAndAnalyze(State _stopAfter) { m_stopAfter = _stopAfter; + if (!m_evmAssemblyJson.empty()) + return true; + bool success = parse(); if (m_stackState >= m_stopAfter) return success; @@ -643,58 +668,88 @@ bool CompilerStack::compile(State _stopAfter) if (m_hasError) solThrow(CompilerError, "Called compile with errors."); - // Only compile contracts individually which have been requested. - map> otherCompilers; + if (!m_evmAssemblyJson.empty()) + { + solAssert(m_importedSources, ""); + solAssert(m_evmAssemblyJson.size() == 1, ""); + + string const evmAssemblyJsonSource = m_evmAssemblyJson.begin()->first; + + evmasm::Assembly::OptimiserSettings optimiserSettings; + optimiserSettings.evmVersion = m_evmVersion; + optimiserSettings.expectedExecutionsPerDeployment = m_optimiserSettings.expectedExecutionsPerDeployment; + optimiserSettings.runCSE = m_optimiserSettings.runCSE; + optimiserSettings.runConstantOptimiser = m_optimiserSettings.runConstantOptimiser; + optimiserSettings.runDeduplicate = m_optimiserSettings.runDeduplicate; + optimiserSettings.runInliner = m_optimiserSettings.runInliner; + optimiserSettings.runJumpdestRemover = m_optimiserSettings.runJumpdestRemover; + optimiserSettings.runPeephole = m_optimiserSettings.runPeephole; + + m_contracts[evmAssemblyJsonSource].evmAssembly = make_shared(evmAssemblyJsonSource); + m_contracts[evmAssemblyJsonSource].evmAssembly->loadFromAssemblyJSON(m_evmAssemblyJson[evmAssemblyJsonSource]); + if (m_optimiserSettings.enabled) + m_contracts[evmAssemblyJsonSource].evmAssembly->optimise(optimiserSettings); + m_contracts[evmAssemblyJsonSource].object = m_contracts[evmAssemblyJsonSource].evmAssembly->assemble(); + + m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly = make_shared(evmAssemblyJsonSource); + m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly->setSources(m_contracts[evmAssemblyJsonSource].evmAssembly->sources()); + m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly->loadFromAssemblyJSON(m_evmAssemblyJson[evmAssemblyJsonSource][".data"]["0"], false); + if (m_optimiserSettings.enabled) + m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly->optimise(optimiserSettings); + m_contracts[evmAssemblyJsonSource].runtimeObject = m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly->assemble(); + } + else + { + // Only compile contracts individually which have been requested. + map> otherCompilers; - for (Source const* source: m_sourceOrder) - for (ASTPointer const& node: source->ast->nodes()) - if (auto contract = dynamic_cast(node.get())) - if (isRequestedContract(*contract)) - { - try + for (Source const* source: m_sourceOrder) + for (ASTPointer const& node: source->ast->nodes()) + if (auto contract = dynamic_cast(node.get())) + if (isRequestedContract(*contract)) { - if (m_viaIR || m_generateIR || m_generateEwasm) - generateIR(*contract); - if (m_generateEvmBytecode) + try { - if (m_viaIR) - generateEVMFromIR(*contract); - else - compileContract(*contract, otherCompilers); + if (m_viaIR || m_generateIR || m_generateEwasm) + generateIR(*contract); + if (m_generateEvmBytecode) + { + if (m_viaIR) + generateEVMFromIR(*contract); + else + compileContract(*contract, otherCompilers); + } + if (m_generateEwasm) + generateEwasm(*contract); } - if (m_generateEwasm) - generateEwasm(*contract); - } - catch (Error const& _error) - { - if (_error.type() != Error::Type::CodeGenerationError) - throw; - m_errorReporter.error(_error.errorId(), _error.type(), SourceLocation(), _error.what()); - return false; - } - catch (UnimplementedFeatureError const& _unimplementedError) - { - if ( - SourceLocation const* sourceLocation = - boost::get_error_info(_unimplementedError) - ) + catch (Error const& _error) { - string const* comment = _unimplementedError.comment(); - m_errorReporter.error( - 1834_error, - Error::Type::CodeGenerationError, - *sourceLocation, - "Unimplemented feature error" + - ((comment && !comment->empty()) ? ": " + *comment : string{}) + - " in " + - _unimplementedError.lineInfo() - ); + if (_error.type() != Error::Type::CodeGenerationError) + throw; + m_errorReporter.error(_error.errorId(), _error.type(), SourceLocation(), _error.what()); return false; } - else - throw; + catch (UnimplementedFeatureError const& _unimplementedError) + { + if (SourceLocation const* sourceLocation + = boost::get_error_info(_unimplementedError)) + { + string const* comment = _unimplementedError.comment(); + m_errorReporter.error( + 1834_error, + Error::Type::CodeGenerationError, + *sourceLocation, + "Unimplemented feature error" + + ((comment && !comment->empty()) ? ": " + *comment : string{}) + " in " + + _unimplementedError.lineInfo()); + return false; + } + else + throw; + } } - } + } + m_stackState = CompilationSuccessful; this->link(); return true; @@ -924,20 +979,28 @@ Json::Value CompilerStack::assemblyJSON(string const& _contractName) const vector CompilerStack::sourceNames() const { - vector names; - for (auto const& s: m_sources) - names.push_back(s.first); + map indices = sourceIndices(false); + vector names(indices.size()); + for (auto const& s: indices) + names[s.second] = s.first; return names; } -map CompilerStack::sourceIndices() const +map CompilerStack::sourceIndices(bool _includeInternalSources /* = true */) const { map indices; unsigned index = 0; - for (auto const& s: m_sources) - indices[s.first] = index++; - solAssert(!indices.count(CompilerContext::yulUtilityFileName()), ""); - indices[CompilerContext::yulUtilityFileName()] = index++; + if (m_evmAssemblyJson.empty()) + { + for (auto const& s: m_sources) + indices[s.first] = index++; + solAssert(!indices.count(CompilerContext::yulUtilityFileName()), ""); + if (_includeInternalSources) + indices[CompilerContext::yulUtilityFileName()] = index++; + } + else + for (auto const& s: m_sourceOrder) + indices[s->charStream->source()] = index++; return indices; } diff --git a/libsolidity/interface/CompilerStack.h b/libsolidity/interface/CompilerStack.h index 3609662e02de..26be9da83aaa 100644 --- a/libsolidity/interface/CompilerStack.h +++ b/libsolidity/interface/CompilerStack.h @@ -222,6 +222,8 @@ class CompilerStack: public langutil::CharStreamProvider /// Will throw errors if the import fails void importASTs(std::map const& _sources); + void importEvmAssemblyJson(std::map const& _sources); + /// Performs the analysis steps (imports, scopesetting, syntaxCheck, referenceResolving, /// typechecking, staticAnalysis) on previously parsed sources. /// @returns false on error. @@ -240,7 +242,7 @@ class CompilerStack: public langutil::CharStreamProvider /// @returns a mapping assigning each source name its index inside the vector returned /// by sourceNames(). - std::map sourceIndices() const; + std::map sourceIndices(bool _includeInternalSources = true) const; /// @returns the previously used character stream, useful for counting lines during error reporting. langutil::CharStream const& charStream(std::string const& _sourceName) const override; @@ -499,6 +501,7 @@ class CompilerStack: public langutil::CharStreamProvider std::map m_sources; // if imported, store AST-JSONS for each filename std::map m_sourceJsons; + std::map m_evmAssemblyJson; std::vector m_unhandledSMTLib2Queries; std::map m_smtlib2Responses; std::shared_ptr m_globalContext; diff --git a/libsolidity/interface/OptimiserSettings.h b/libsolidity/interface/OptimiserSettings.h index 5317acee4bd4..528730eca74b 100644 --- a/libsolidity/interface/OptimiserSettings.h +++ b/libsolidity/interface/OptimiserSettings.h @@ -121,6 +121,8 @@ struct OptimiserSettings expectedExecutionsPerDeployment == _other.expectedExecutionsPerDeployment; } + /// Optimizer enabled. + bool enabled = false; /// Move literals to the right of commutative binary operators during code generation. /// This helps exploiting associativity. bool runOrderLiterals = false; diff --git a/scripts/AsmJsonImportTest.sh b/scripts/AsmJsonImportTest.sh new file mode 100755 index 000000000000..fe4b345d656a --- /dev/null +++ b/scripts/AsmJsonImportTest.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash + +# Bash script to test the asm-json-import input mode of the compiler by +# first exporting a .sol file to JSON that containing assembly json, deploy & runtime bytecode, opcodes and source mappings, +# then the compiler is invoked in assembly json import mode `--import-asm-json` and uses the previously +# generated assembly. This output will be stored. +# Finally, the originally generated outputs (bin, bin-runtime, opcodes, asm, srcmap and srcmap-runtime) +# will be compared with the outputs that where generated by using the assembly json file as input. + +set -eo pipefail +READLINK=readlink +if [[ "$OSTYPE" == "darwin"* ]]; then + READLINK=greadlink +fi +REPO_ROOT=$(${READLINK} -f "$(dirname "$0")"/..) +# shellcheck source=scripts/common_import.sh +source "${REPO_ROOT}/scripts/common_import.sh" + +SEMANTICTESTS_DIR="${REPO_ROOT}/test/libsolidity/semanticTests" +NSOURCES="$(find "$SEMANTICTESTS_DIR" -type f | wc -l)" + +init_import_tests + +# function tests whether importing an assembly json file creates identical bytecode. +# Results are recorded by adding to FAILED or UNCOMPILABLE. +# Also, in case of a mismatch a diff and the respective ASTs are printed +# Expected parameters: +# $1 name of the file to be exported and imported +# $2 any files needed to do so that might be in parent directories +function testImportExportEquivalence { + local nth_input_file="$1" + IFS=" " read -r -a all_input_files <<< "$2" + + if $SOLC "$nth_input_file" "${all_input_files[@]}" --combined-json asm,bin > /dev/null 2>&1 + then + local types=( "bin" "bin-runtime" "opcodes" "asm" "srcmap" "srcmap-runtime" ) + + # save exported json as expected result (silently) + $SOLC --combined-json asm,opcodes,bin,srcmap,srcmap-runtime,bin-runtime --pretty-json "$nth_input_file" "${all_input_files[@]}" > expected.json 2> /dev/null + for contract in $(jq '.contracts | keys | .[]' expected.json 2> /dev/null) + do + for type in "${types[@]}" + do + jq --raw-output ".contracts.${contract}.\"${type}\"" expected.json > "expected.${type}" + done + expected_bin=$(cat expected.bin) + if [[ $expected_bin == "" ]] + then + continue + fi + + if ! "$SOLC" --import-asm-json expected.asm --combined-json asm,opcodes,bin,srcmap,srcmap-runtime,bin-runtime > imported.json 2> /dev/null + then + # For investigating, use exit 1 here so the script stops at the + # first failing test + # exit 1 + echo "" + echo "Failed with contract ${contract}!?" + echo "" + FAILED=$((FAILED + 1)) + return 1 + fi + + for type in "${types[@]}" + do + jq --raw-output ".contracts.\"expected.asm\".\"${type}\"" imported.json > "imported.${type}" + if ! diff "expected.${type}" "imported.${type}" + then + echo "" + echo "Failed with contract ${contract} (${type})." + echo "" + if [ "$DIFFVIEW" == "" ] + then + echo "Expected:" + cat "./expected.${type}" + echo "Obtained:" + cat "./imported.${type}" + else + # Use user supplied diff view mismatched output + $DIFFVIEW "expected.${type}" "imported.${type}" + fi + FAILED=$((FAILED + 1)) + return 2 + fi + done + done + TESTED=$((TESTED + 1)) + rm -f expected.json + rm -f imported.json + for type in "${types[@]}" + do + rm -f "expected.${type}" + rm -f "imported.${type}" + done + else + # echo "contract $solfile could not be compiled " + UNCOMPILABLE=$((UNCOMPILABLE + 1)) + fi +} +echo "Looking at $NSOURCES .sol files..." + +TEST_FILES=$(find "$SEMANTICTESTS_DIR" -name "*.sol") +run_import_tests "$TEST_FILES" "$SPLITSOURCES" "$NSOURCES" "$PWD" diff --git a/scripts/common_import.sh b/scripts/common_import.sh new file mode 100644 index 000000000000..37cbbe30703d --- /dev/null +++ b/scripts/common_import.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash + +set -eo pipefail + +function init_import_tests() +{ + export SOLIDITY_BUILD_DIR=${SOLIDITY_BUILD_DIR:-${REPO_ROOT}/build} + export SOLC=${SOLIDITY_BUILD_DIR}/solc/solc + export SPLITSOURCES=${REPO_ROOT}/scripts/splitSources.py + export FAILED=0 + export UNCOMPILABLE=0 + export TESTED=0 + + if [[ "$(find . -maxdepth 0 -type d -empty)" == "" ]]; then + echo "Test directory not empty. Skipping!" + exit 1 + fi +} + +function run_import_tests() +{ + local TEST_FILES=$1 + local SPLITSOURCES=$2 + local NSOURCES=$3 + local WORKINGDIR=$4 + + for solfile in $TEST_FILES; do + echo -n "." + # create a temporary sub-directory + local FILETMP + FILETMP=$(mktemp -d) + cd "$FILETMP" + + set +e + local OUTPUT + OUTPUT=$("$SPLITSOURCES" "$solfile") + local SPLITSOURCES_RC=$? + set -e + if [ ${SPLITSOURCES_RC} == 0 ]; then + # echo $OUTPUT + NSOURCES=$((NSOURCES - 1)) + for i in $OUTPUT; do + testImportExportEquivalence "$i" "$OUTPUT" + NSOURCES=$((NSOURCES + 1)) + done + elif [ ${SPLITSOURCES_RC} == 1 ]; then + testImportExportEquivalence "$solfile" + elif [ ${SPLITSOURCES_RC} == 2 ]; then + # The script will exit with return code 2, if an UnicodeDecodeError occurred. + # This is the case if e.g. some tests are using invalid utf-8 sequences. We will ignore + # these errors, but print the actual output of the script. + echo -e "\n${OUTPUT}\n" + testImportExportEquivalence "$solfile" + else + # All other return codes will be treated as critical errors. The script will exit. + echo -e "\nGot unexpected return code ${SPLITSOURCES_RC} from ${SPLITSOURCES}. Aborting." + echo -e "\n${OUTPUT}\n" + + cd "$WORKINGDIR" + # Delete temporary files + rm -rf "$FILETMP" + + exit 1 + fi + + cd "$WORKINGDIR" + # Delete temporary files + rm -rf "$FILETMP" + done + + echo "" + + if [ "$FAILED" = 0 ]; then + echo "SUCCESS: $TESTED tests passed, $FAILED failed, $UNCOMPILABLE could not be compiled ($NSOURCES sources total)." + else + echo "FAILURE: Out of $NSOURCES sources, $FAILED failed, ($UNCOMPILABLE could not be compiled)." + exit 1 + fi +} diff --git a/solc/CommandLineInterface.cpp b/solc/CommandLineInterface.cpp index 1d069f925152..490a0996c287 100644 --- a/solc/CommandLineInterface.cpp +++ b/solc/CommandLineInterface.cpp @@ -155,7 +155,11 @@ static bool coloredOutput(CommandLineOptions const& _options) void CommandLineInterface::handleBinary(string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport , "" + ); if (m_options.compiler.outputs.binary) { @@ -181,7 +185,11 @@ void CommandLineInterface::handleBinary(string const& _contract) void CommandLineInterface::handleOpcode(string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport , "" + ); if (!m_options.output.dir.empty()) createFile(m_compiler->filesystemFriendlyName(_contract) + ".opcode", evmasm::disassemble(m_compiler->object(_contract).bytecode)); @@ -195,7 +203,10 @@ void CommandLineInterface::handleOpcode(string const& _contract) void CommandLineInterface::handleIR(string const& _contractName) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport + ); if (!m_options.compiler.outputs.ir) return; @@ -211,7 +222,10 @@ void CommandLineInterface::handleIR(string const& _contractName) void CommandLineInterface::handleIROptimized(string const& _contractName) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport + ); if (!m_options.compiler.outputs.irOptimized) return; @@ -227,7 +241,10 @@ void CommandLineInterface::handleIROptimized(string const& _contractName) void CommandLineInterface::handleEwasm(string const& _contractName) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport + ); if (!m_options.compiler.outputs.ewasm) return; @@ -250,7 +267,11 @@ void CommandLineInterface::handleEwasm(string const& _contractName) void CommandLineInterface::handleBytecode(string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport , "" + ); if (m_options.compiler.outputs.opcodes) handleOpcode(_contract); @@ -260,7 +281,10 @@ void CommandLineInterface::handleBytecode(string const& _contract) void CommandLineInterface::handleSignatureHashes(string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport + ); if (!m_options.compiler.outputs.signatureHashes) return; @@ -278,7 +302,11 @@ void CommandLineInterface::handleSignatureHashes(string const& _contract) void CommandLineInterface::handleMetadata(string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport , "" + ); if (!m_options.compiler.outputs.metadata) return; @@ -292,7 +320,10 @@ void CommandLineInterface::handleMetadata(string const& _contract) void CommandLineInterface::handleABI(string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport + ); if (!m_options.compiler.outputs.abi) return; @@ -306,7 +337,10 @@ void CommandLineInterface::handleABI(string const& _contract) void CommandLineInterface::handleStorageLayout(string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport + ); if (!m_options.compiler.outputs.storageLayout) return; @@ -320,7 +354,10 @@ void CommandLineInterface::handleStorageLayout(string const& _contract) void CommandLineInterface::handleNatspec(bool _natspecDev, string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport + ); bool enabled = false; std::string suffix; @@ -546,6 +583,25 @@ map CommandLineInterface::parseAstFromInput() return sourceJsons; } +map CommandLineInterface::parseEvmAssemblyJsonFromInput() +{ + solAssert(m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport, ""); + solAssert(m_fileReader.sourceCodes().size() == 1, ""); + + map sourceJsons; + + for (auto const& iter: m_fileReader.sourceCodes()) + { + Json::Value evmAsmJson; + astAssert(jsonParseStrict(iter.second, evmAsmJson), "Input file could not be parsed to JSON"); + astAssert(evmAsmJson.isMember(".code"), "Invalid Format for assembly-JSON: Must have '.code'-object"); + astAssert(evmAsmJson.isMember(".data"), "Invalid Format for assembly-JSON: Must have '.data'-object"); + sourceJsons[iter.first] = evmAsmJson; + } + + return sourceJsons; +} + void CommandLineInterface::createFile(string const& _fileName, string const& _data) { namespace fs = boost::filesystem; @@ -634,6 +690,7 @@ bool CommandLineInterface::processInput() break; case InputMode::Compiler: case InputMode::CompilerWithASTImport: + case InputMode::CompilerWithEvmAssemblyJsonImport: if (!compile()) return false; outputCompilationResults(); @@ -657,7 +714,11 @@ void CommandLineInterface::printLicense() bool CommandLineInterface::compile() { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport, "" + ); m_compiler = make_unique(m_fileReader.reader()); @@ -722,6 +783,19 @@ bool CommandLineInterface::compile() return false; } } + else if (m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport) + { + solAssert(m_fileReader.sourceCodes().size() == 1, ""); + try + { + m_compiler->importEvmAssemblyJson(parseEvmAssemblyJsonFromInput()); + } + catch (Exception const& _exc) + { + serr() << string("Failed to import Assembly JSON: ") << _exc.what() << endl; + return false; + } + } else { m_compiler->setSources(m_fileReader.sourceCodes()); @@ -763,7 +837,11 @@ bool CommandLineInterface::compile() void CommandLineInterface::handleCombinedJSON() { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport, "" + ); if (!m_options.compiler.combinedJsonRequests.has_value()) return; @@ -855,7 +933,10 @@ void CommandLineInterface::handleCombinedJSON() void CommandLineInterface::handleAst() { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport + ); if (!m_options.compiler.outputs.astCompactJson) return; @@ -1095,12 +1176,17 @@ bool CommandLineInterface::assemble(yul::AssemblyStack::Language _language, yul: void CommandLineInterface::outputCompilationResults() { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport, "" + ); handleCombinedJSON(); // do we need AST output? - handleAst(); + if (m_options.input.mode != InputMode::CompilerWithEvmAssemblyJsonImport) + handleAst(); if ( !m_compiler->compilationSuccessful() && @@ -1140,15 +1226,19 @@ void CommandLineInterface::outputCompilationResults() handleGasEstimation(contract); handleBytecode(contract); - handleIR(contract); - handleIROptimized(contract); - handleEwasm(contract); - handleSignatureHashes(contract); - handleMetadata(contract); - handleABI(contract); - handleStorageLayout(contract); - handleNatspec(true, contract); - handleNatspec(false, contract); + + if (m_options.input.mode != InputMode::CompilerWithEvmAssemblyJsonImport) + { + handleIR(contract); + handleIROptimized(contract); + handleEwasm(contract); + handleSignatureHashes(contract); + handleMetadata(contract); + handleABI(contract); + handleStorageLayout(contract); + handleNatspec(true, contract); + handleNatspec(false, contract); + } } // end of contracts iteration if (!m_hasOutput) diff --git a/solc/CommandLineInterface.h b/solc/CommandLineInterface.h index ee5057468273..250d30de902e 100644 --- a/solc/CommandLineInterface.h +++ b/solc/CommandLineInterface.h @@ -98,6 +98,8 @@ class CommandLineInterface /// or standard-json output std::map parseAstFromInput(); + std::map parseEvmAssemblyJsonFromInput(); + /// Create a file in the given directory /// @arg _fileName the name of the file /// @arg _data to be written diff --git a/solc/CommandLineParser.cpp b/solc/CommandLineParser.cpp index 9356c96625d0..11e86f2e0146 100644 --- a/solc/CommandLineParser.cpp +++ b/solc/CommandLineParser.cpp @@ -46,6 +46,7 @@ static string const g_strAllowPaths = "allow-paths"; static string const g_strBasePath = "base-path"; static string const g_strIncludePath = "include-path"; static string const g_strAssemble = "assemble"; +static string const g_strImportEvmAssemblerJson = "import-asm-json"; static string const g_strCombinedJson = "combined-json"; static string const g_strErrorRecovery = "error-recovery"; static string const g_strEVM = "evm"; @@ -134,6 +135,7 @@ static map const g_inputModeName = { {InputMode::Compiler, "compiler"}, {InputMode::CompilerWithASTImport, "compiler (AST import)"}, {InputMode::Assembler, "assembler"}, + {InputMode::CompilerWithEvmAssemblyJsonImport, "assembler (EVM ASM JSON import)"}, {InputMode::StandardJson, "standard JSON"}, {InputMode::Linker, "linker"}, }; @@ -265,6 +267,8 @@ OptimiserSettings CommandLineOptions::optimiserSettings() const if (optimizer.yulSteps.has_value()) settings.yulOptimiserSteps = optimizer.yulSteps.value(); + settings.enabled = optimizer.enabled; + return settings; } @@ -321,7 +325,20 @@ bool CommandLineParser::parseInputPathsAndRemappings() m_options.input.paths.insert(positionalArg); } - if (m_options.input.mode == InputMode::StandardJson) + if (m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport) + { + if (m_options.input.paths.size() > 1 || (m_options.input.paths.size() == 1 && m_options.input.addStdin)) + { + serr() << "Too many input files for --" << g_strImportEvmAssemblerJson << "." << endl; + serr() << "Please either specify a single file name or provide its content on standard input." << endl; + return false; + } + else if (m_options.input.paths.size() == 0) + // Standard JSON mode input used to be handled separately and zero files meant "read from stdin". + // Keep it working that way for backwards-compatibility. + m_options.input.addStdin = true; + } + else if (m_options.input.mode == InputMode::StandardJson) { if (m_options.input.paths.size() > 1 || (m_options.input.paths.size() == 1 && m_options.input.addStdin)) { @@ -458,6 +475,12 @@ bool CommandLineParser::parseOutputSelection() CompilerOutputs::componentName(&CompilerOutputs::ewasm), CompilerOutputs::componentName(&CompilerOutputs::ewasmIR), }; + static set const assemblyJsonImportModeOutputs = { + CompilerOutputs::componentName(&CompilerOutputs::asm_), + CompilerOutputs::componentName(&CompilerOutputs::binary), + CompilerOutputs::componentName(&CompilerOutputs::binaryRuntime), + CompilerOutputs::componentName(&CompilerOutputs::opcodes), + }; switch (_mode) { @@ -468,6 +491,8 @@ bool CommandLineParser::parseOutputSelection() case InputMode::Compiler: case InputMode::CompilerWithASTImport: return contains(compilerModeOutputs, _outputName); + case InputMode::CompilerWithEvmAssemblyJsonImport: + return contains(assemblyJsonImportModeOutputs, _outputName); case InputMode::Assembler: return contains(assemblerModeOutputs, _outputName); case InputMode::StandardJson: @@ -644,6 +669,10 @@ General Information)").c_str(), "Supported Inputs is the output of the --" + g_strStandardJSON + " or the one produced by " "--" + g_strCombinedJson + " " + CombinedJsonRequests::componentName(&CombinedJsonRequests::ast)).c_str() ) + ( + g_strImportEvmAssemblerJson.c_str(), + "Import evm assembler json to be compiled, assumes input holds the evm assembly in JSON format." + ) ; desc.add(alternativeInputModes); @@ -879,6 +908,7 @@ bool CommandLineParser::processArgs() g_strStrictAssembly, g_strYul, g_strImportAst, + g_strImportEvmAssemblerJson, })) return false; @@ -892,6 +922,8 @@ bool CommandLineParser::processArgs() m_options.input.mode = InputMode::StandardJson; else if (m_args.count(g_strAssemble) > 0 || m_args.count(g_strStrictAssembly) > 0 || m_args.count(g_strYul) > 0) m_options.input.mode = InputMode::Assembler; + else if (m_args.count(g_strImportEvmAssemblerJson) > 0) + m_options.input.mode = InputMode::CompilerWithEvmAssemblyJsonImport; else if (m_args.count(g_strLink) > 0) m_options.input.mode = InputMode::Linker; else if (m_args.count(g_strImportAst) > 0) @@ -946,8 +978,8 @@ bool CommandLineParser::processArgs() if ( m_options.input.mode != InputMode::Compiler && m_options.input.mode != InputMode::CompilerWithASTImport && - m_options.input.mode != InputMode::Assembler - ) + m_options.input.mode != InputMode::Assembler && + m_options.input.mode != InputMode::CompilerWithEvmAssemblyJsonImport) { if (!m_args[g_strOptimizeRuns].defaulted()) { @@ -1127,16 +1159,18 @@ bool CommandLineParser::processArgs() m_options.optimizer.yulSteps = m_args[g_strYulOptimizations].as(); } - if (m_options.input.mode == InputMode::Assembler) + if (m_options.input.mode == InputMode::Assembler || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport) { - vector const nonAssemblyModeOptions = { + vector nonAssemblyModeOptions = { // TODO: The list is not complete. Add more. g_strOutputDir, g_strGas, - g_strCombinedJson, g_strOptimizeYul, g_strNoOptimizeYul, }; + if (m_options.input.mode == InputMode::Assembler) + nonAssemblyModeOptions.emplace_back(g_strCombinedJson); if (countEnabledOptions(nonAssemblyModeOptions) >= 1) { auto optionEnabled = [&](string const& name){ return m_args.count(name) > 0; }; @@ -1193,12 +1227,15 @@ bool CommandLineParser::processArgs() } if (m_options.optimizer.enabled && (m_options.assembly.inputLanguage != Input::StrictAssembly && m_options.assembly.inputLanguage != Input::Ewasm)) { - serr() << - "Optimizer can only be used for strict assembly. Use --" << - g_strStrictAssembly << - "." << - endl; - return false; + if (m_options.input.mode != InputMode::CompilerWithEvmAssemblyJsonImport) + { + serr() << + "Optimizer can only be used for strict assembly or with assembly import. Use --" << + g_strStrictAssembly << + " or --" << g_strImportEvmAssemblerJson << "." << + endl; + return false; + } } if (m_options.assembly.targetMachine == Machine::Ewasm && m_options.assembly.inputLanguage != Input::StrictAssembly && m_options.assembly.inputLanguage != Input::Ewasm) { @@ -1206,9 +1243,10 @@ bool CommandLineParser::processArgs() serr() << "and automatic translation is not available." << endl; return false; } - serr() << - "Warning: Yul is still experimental. Please use the output with care." << - endl; + if (m_options.input.mode == InputMode::Assembler) + serr() << + "Warning: Yul is still experimental. Please use the output with care." << + endl; return true; } @@ -1329,7 +1367,13 @@ bool CommandLineParser::parseCombinedJsonOption() set requests; for (string const& item: boost::split(requests, m_args[g_strCombinedJson].as(), boost::is_any_of(","))) - if (CombinedJsonRequests::componentMap().count(item) == 0) + if (m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport && + CombinedJsonRequests::componentMapAssemblyJsonImport().count(item) == 0) + { + serr() << "Invalid option to --" << g_strCombinedJson << ": " << item << ", for --" << g_strImportEvmAssemblerJson << endl; + return false; + } + else if (CombinedJsonRequests::componentMap().count(item) == 0) { serr() << "Invalid option to --" << g_strCombinedJson << ": " << item << endl; return false; diff --git a/solc/CommandLineParser.h b/solc/CommandLineParser.h index a1d13c690756..0545023e1983 100644 --- a/solc/CommandLineParser.h +++ b/solc/CommandLineParser.h @@ -56,6 +56,7 @@ enum class InputMode StandardJson, Linker, Assembler, + CompilerWithEvmAssemblyJsonImport, }; struct CompilerOutputs @@ -113,6 +114,18 @@ struct CombinedJsonRequests friend std::ostream& operator<<(std::ostream& _out, CombinedJsonRequests const& _requests); static std::string const& componentName(bool CombinedJsonRequests::* _component); + static auto const& componentMapAssemblyJsonImport() + { + static std::map const components = { + {"bin", &CombinedJsonRequests::binary}, + {"bin-runtime", &CombinedJsonRequests::binaryRuntime}, + {"opcodes", &CombinedJsonRequests::opcodes}, + {"asm", &CombinedJsonRequests::asm_}, + {"srcmap", &CombinedJsonRequests::srcMap}, + {"srcmap-runtime", &CombinedJsonRequests::srcMapRuntime}, + }; + return components; + } static auto const& componentMap() { static std::map const components = { diff --git a/test/cmdlineTests.sh b/test/cmdlineTests.sh index 2076eb993e5a..87820bf64e6b 100755 --- a/test/cmdlineTests.sh +++ b/test/cmdlineTests.sh @@ -154,7 +154,7 @@ function ask_expectation_update # General helper function for testing SOLC behaviour, based on file name, compile opts, exit code, stdout and stderr. # An failure is expected. -function test_solc_behaviour +function test_solc_behaviour() { local filename="${1}" local solc_args @@ -288,7 +288,7 @@ EOF } -function test_solc_assembly_output +function test_solc_assembly_output() { local input="${1}" local expected="${2}" @@ -572,6 +572,18 @@ SOLTMPDIR=$(mktemp -d) ) rm -r "$SOLTMPDIR" +printTask "Testing ASM-JSON import..." +SOLTMPDIR=$(mktemp -d) +( + cd "$SOLTMPDIR" + if ! "$REPO_ROOT/scripts/AsmJsonImportTest.sh" + then + rm -rf "$SOLTMPDIR" + exit 1 + fi +) +rm -rf "$SOLTMPDIR" + printTask "Testing AST export with stop-after=parsing..." "$REPO_ROOT/test/stopAfterParseTests.sh" diff --git a/test/cmdlineTests/asm_json/output b/test/cmdlineTests/asm_json/output index 26ab87147a30..0128c541c474 100644 --- a/test/cmdlineTests/asm_json/output +++ b/test/cmdlineTests/asm_json/output @@ -1582,5 +1582,10 @@ EVM assembly: } ] } - } + }, + "sourceList": + [ + "asm_json/input.sol", + "#utility.yul" + ] } diff --git a/test/libevmasm/Assembler.cpp b/test/libevmasm/Assembler.cpp index c7f28f654cd9..4a974ac27b54 100644 --- a/test/libevmasm/Assembler.cpp +++ b/test/libevmasm/Assembler.cpp @@ -55,7 +55,8 @@ BOOST_AUTO_TEST_CASE(all_assembly_items) { map indices = { { "root.asm", 0 }, - { "sub.asm", 1 } + { "sub.asm", 1 }, + { "verbatim.asm", 2 } }; Assembly _assembly; auto root_asm = make_shared("root.asm"); @@ -64,11 +65,22 @@ BOOST_AUTO_TEST_CASE(all_assembly_items) Assembly _subAsm; auto sub_asm = make_shared("sub.asm"); _subAsm.setSourceLocation({6, 8, sub_asm}); + + Assembly _verbatimAsm; + auto verbatim_asm = make_shared("verbatim.asm"); + _verbatimAsm.setSourceLocation({8, 18, verbatim_asm}); + // PushImmutable _subAsm.appendImmutable("someImmutable"); + _subAsm.append(AssemblyItem(PushTag, 0)); _subAsm.append(Instruction::INVALID); shared_ptr _subAsmPtr = make_shared(_subAsm); + _verbatimAsm.appendVerbatim({0xff,0xff}, 0, 0); + _verbatimAsm.appendVerbatim({0x74, 0x65, 0x73, 0x74}, 0, 1); + _verbatimAsm.append(Instruction::MSTORE); + shared_ptr _verbatimAsmPtr = make_shared(_verbatimAsm); + // Tag auto tag = _assembly.newTag(); _assembly.append(tag); @@ -89,6 +101,10 @@ BOOST_AUTO_TEST_CASE(all_assembly_items) auto sub = _assembly.appendSubroutine(_subAsmPtr); // PushSub _assembly.pushSubroutineOffset(static_cast(sub.data())); + // PushSubSize + auto verbatim_sub = _assembly.appendSubroutine(_verbatimAsmPtr); + // PushSub + _assembly.pushSubroutineOffset(static_cast(verbatim_sub.data())); // PushDeployTimeAddress _assembly.append(PushDeployTimeAddress); // AssignImmutable. @@ -105,12 +121,12 @@ BOOST_AUTO_TEST_CASE(all_assembly_items) BOOST_CHECK_EQUAL( _assembly.assemble().toHex(), - "5b6001600220606f73__$bf005014d9d0f534b8fcb268bd84c491a2$__" - "6000566067602260457300000000000000000000000000000000000000005050" + "5b6001600220607c73__$bf005014d9d0f534b8fcb268bd84c491a2$__" + "6000566074602460496007606d7300000000000000000000000000000000000000005050" "600260010152" "00fe" "7f0000000000000000000000000000000000000000000000000000000000000000" - "fe010203044266eeaa" + "6000feffff7465737452010203044266eeaa" ); BOOST_CHECK_EQUAL( _assembly.assemblyString(), @@ -123,6 +139,8 @@ BOOST_AUTO_TEST_CASE(all_assembly_items) " data_a6885b3731702da62e8e4a8f584ac46a7f6822f4e2ba50fba902f67b1588d23b\n" " dataSize(sub_0)\n" " dataOffset(sub_0)\n" + " dataSize(sub_1)\n" + " dataOffset(sub_1)\n" " deployTimeAddress()\n" " assignImmutable(\"0xc3978657661c4d8e32e3d5f42597c009f0d3859e9f9d0d94325268f9799e2bfb\")\n" " 0x02\n" @@ -134,13 +152,20 @@ BOOST_AUTO_TEST_CASE(all_assembly_items) "sub_0: assembly {\n" " /* \"sub.asm\":6:8 */\n" " immutable(\"0x26f2c0195e9d408feff3abd77d83f2971f3c9a18d1e8a9437c7835ae4211fc9f\")\n" + " tag_0\n" " invalid\n" "}\n" "\n" + "sub_1: assembly {\n" + " /* \"verbatim.asm\":8:18 */\n" + " verbatimbytecode_ffff\n" + " verbatimbytecode_74657374\n" + " mstore\n" + "}\n" + "\n" "auxdata: 0x4266eeaa\n" ); - BOOST_CHECK_EQUAL( - util::jsonCompactPrint(_assembly.assemblyJSON(indices)), + string json{ "{\".auxdata\":\"4266eeaa\",\".code\":[" "{\"begin\":1,\"end\":3,\"name\":\"tag\",\"source\":0,\"value\":\"1\"}," "{\"begin\":1,\"end\":3,\"name\":\"JUMPDEST\",\"source\":0}," @@ -154,6 +179,8 @@ BOOST_AUTO_TEST_CASE(all_assembly_items) "{\"begin\":1,\"end\":3,\"name\":\"PUSH data\",\"source\":0,\"value\":\"A6885B3731702DA62E8E4A8F584AC46A7F6822F4E2BA50FBA902F67B1588D23B\"}," "{\"begin\":1,\"end\":3,\"name\":\"PUSH #[$]\",\"source\":0,\"value\":\"0000000000000000000000000000000000000000000000000000000000000000\"}," "{\"begin\":1,\"end\":3,\"name\":\"PUSH [$]\",\"source\":0,\"value\":\"0000000000000000000000000000000000000000000000000000000000000000\"}," + "{\"begin\":1,\"end\":3,\"name\":\"PUSH #[$]\",\"source\":0,\"value\":\"0000000000000000000000000000000000000000000000000000000000000001\"}," + "{\"begin\":1,\"end\":3,\"name\":\"PUSH [$]\",\"source\":0,\"value\":\"0000000000000000000000000000000000000000000000000000000000000001\"}," "{\"begin\":1,\"end\":3,\"name\":\"PUSHDEPLOYADDRESS\",\"source\":0}," "{\"begin\":1,\"end\":3,\"name\":\"ASSIGNIMMUTABLE\",\"source\":0,\"value\":\"someOtherImmutable\"}," "{\"begin\":1,\"end\":3,\"name\":\"PUSH\",\"source\":0,\"value\":\"2\"}," @@ -161,9 +188,23 @@ BOOST_AUTO_TEST_CASE(all_assembly_items) "{\"begin\":1,\"end\":3,\"name\":\"STOP\",\"source\":0}" "],\".data\":{\"0\":{\".code\":[" "{\"begin\":6,\"end\":8,\"name\":\"PUSHIMMUTABLE\",\"source\":1,\"value\":\"someImmutable\"}," + "{\"begin\":6,\"end\":8,\"name\":\"PUSH [ErrorTag]\",\"source\":1}," "{\"begin\":6,\"end\":8,\"name\":\"INVALID\",\"source\":1}" - "]},\"A6885B3731702DA62E8E4A8F584AC46A7F6822F4E2BA50FBA902F67B1588D23B\":\"01020304\"}}" - ); + "]}," + "\"1\":{\".code\":[" + "{\"begin\":8,\"end\":18,\"name\":\"VERBATIM\",\"source\":2,\"value\":\"ffff\"}," + "{\"begin\":8,\"end\":18,\"name\":\"VERBATIM\",\"source\":2,\"value\":\"74657374\"}," + "{\"begin\":8,\"end\":18,\"name\":\"MSTORE\",\"source\":2}" + "]},\"A6885B3731702DA62E8E4A8F584AC46A7F6822F4E2BA50FBA902F67B1588D23B\":\"01020304\"},\"sourceList\":[\"root.asm\",\"sub.asm\",\"verbatim.asm\"]}" + }; + Json::Value jsonValue; + BOOST_CHECK(util::jsonParseStrict(json, jsonValue)); + BOOST_CHECK_EQUAL(util::jsonCompactPrint(_assembly.assemblyJSON(indices)), util::jsonCompactPrint(jsonValue)); + + Assembly _assemblyFromJson; + _assemblyFromJson.loadFromAssemblyJSON(_assembly.assemblyJSON(indices)); + BOOST_CHECK_EQUAL(util::jsonCompactPrint(_assemblyFromJson.assemblyJSON(indices)), util::jsonCompactPrint(jsonValue)); + BOOST_CHECK_EQUAL(_assembly.assemble().toHex(), _assemblyFromJson.assemble().toHex()); } BOOST_AUTO_TEST_CASE(immutables_and_its_source_maps) @@ -342,7 +383,7 @@ BOOST_AUTO_TEST_CASE(immutable) "{\"begin\":6,\"end\":8,\"name\":\"PUSHIMMUTABLE\",\"source\":1,\"value\":\"someImmutable\"}," "{\"begin\":6,\"end\":8,\"name\":\"PUSHIMMUTABLE\",\"source\":1,\"value\":\"someOtherImmutable\"}," "{\"begin\":6,\"end\":8,\"name\":\"PUSHIMMUTABLE\",\"source\":1,\"value\":\"someImmutable\"}" - "]}}}" + "]}},\"sourceList\":[\"root.asm\",\"sub.asm\"]}" ); } diff --git a/test/solc/CommandLineInterface.cpp b/test/solc/CommandLineInterface.cpp index 0269ec462fc6..fcb3d6d67c48 100644 --- a/test/solc/CommandLineInterface.cpp +++ b/test/solc/CommandLineInterface.cpp @@ -157,7 +157,7 @@ BOOST_AUTO_TEST_CASE(multiple_input_modes) }; string expectedMessage = "The following options are mutually exclusive: " - "--help, --license, --version, --standard-json, --link, --assemble, --strict-assembly, --yul, --import-ast. " + "--help, --license, --version, --standard-json, --link, --assemble, --strict-assembly, --yul, --import-ast, --import-asm-json. " "Select at most one.\n"; for (string const& mode1: inputModeOptions)