Merge pull request #210 from ethereum/develop

version 0.3.1 Develop

Author: winsvega

CMakeLists.txt

@@ -4,6 +4,9 @@ set(CMAKE_TOOLCHAIN_FILE ${CMAKE_SOURCE_DIR}/toolchain.cmake)
 
 include("./cmake/Hunter/HunterGate.cmake")
 
+if ("${CMAKE_BUILD_TYPE}" STREQUAL "")
+    set(CMAKE_BUILD_TYPE "Release")
+endif()
 if (NOT "${CMAKE_BUILD_TYPE}" STREQUAL "Release")
     add_compile_definitions("__DEBUG__")
 endif()
@@ -23,7 +26,7 @@ HunterGate(
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 
 project(retesteth VERSION 0.3.1)
-set(VERSION_SUFFIX "shanghai")
+set(VERSION_SUFFIX "cancun")
 
 set(Boost_USE_STATIC_LIBS ON)
 set(Boost_USE_MULTITHREADED ON)

Dockerfile

@@ -7,6 +7,7 @@ ARG RETESTETH_SRC="https://github.com/ethereum/retesteth.git"
 ARG GETH_SRC="https://github.com/ethereum/go-ethereum.git"
 ARG NIMBUS_SRC="https://github.com/status-im/nimbus-eth1.git"
 ARG EVMONE_SRC="https://github.com/ethereum/evmone.git"
+ARG PYT8N_SRC="https://github.com/ethereum/execution-specs.git"
 
 # Leave empty to disable the build, can point to commit hash as well
 ARG BESU="main"
@@ -16,6 +17,7 @@ ARG ETHEREUMJS="master"
 ARG RETESTETH="develop"
 ARG PYSPECS="main"
 ARG EVMONE="master"
+ARG PYT8N="master"
 
 SHELL ["/bin/bash", "-c"]
 ENV TZ=Etc/UTC
@@ -27,10 +29,11 @@ RUN apt-get update \
     && add-apt-repository -y ppa:ubuntu-toolchain-r/test \
     && add-apt-repository -y ppa:deadsnakes/ppa  \
     && add-apt-repository ppa:linuxuprising/java \
-    && apt-get install --yes git cmake make perl psmisc curl wget gcc-11 g++-11 python3.10 python3.10-venv python3-pip python3-dev \
+    && apt-get install --yes jq lsof git cmake make perl psmisc curl wget gcc-11 g++-11 python3.10 python3.10-venv python3-pip python3-dev \
     && apt-get install --yes libboost-filesystem-dev libboost-system-dev libboost-program-options-dev libboost-test-dev \
     && echo oracle-java17-installer shared/accepted-oracle-license-v1-3 select true | /usr/bin/debconf-set-selections  \
     && apt-get install --yes oracle-java17-installer oracle-java17-set-default \
+    && apt-get install --yes uuid-runtime \
     && rm -rf /var/lib/apt/lists/*
 RUN rm /usr/bin/python3 && ln -s /usr/bin/python3.10 /usr/bin/python3 \
     && rm /usr/bin/gcc && rm /usr/bin/g++ \
@@ -50,7 +53,7 @@ RUN test -n "$RETESTETH" \
     || echo "Retesteth is empty, skip LLLC"
 
 # Solidity solc
-RUN wget https://github.com/ethereum/solidity/releases/download/v0.8.17/solc-static-linux \
+RUN wget https://github.com/ethereum/solidity/releases/download/v0.8.21/solc-static-linux \
    && cp solc-static-linux /bin/solc \
    && chmod +x /bin/solc
 
@@ -64,16 +67,23 @@ RUN cd /execution-spec-tests && git fetch && git checkout $PYSPECS \
     && cp tfinit.sh /usr/bin/tfinit.sh \
     && chmod +x /usr/bin/tfinit.sh
 
+# PYT8N
+RUN test -n "$PYT8N" \
+     && git clone $PYT8N_SRC /pyt8n \
+     && cd /pyt8n && git fetch && git checkout $PYT8N \
+     && python3 -m venv ./venv/ \
+     && source ./venv/bin/activate \
+     && pip install -e . \
+    || echo "Pyt8n is empty"
 
 # Geth
 RUN test -n "$GETH" \
      && git clone $GETH_SRC /geth \
      && cd /geth && git fetch && git checkout $GETH \
-     && wget https://dl.google.com/go/go1.19.linux-amd64.tar.gz \
-     && tar -xvf go1.19.linux-amd64.tar.gz \
+     && wget https://dl.google.com/go/go1.20.linux-amd64.tar.gz \
+     && tar -xvf go1.20.linux-amd64.tar.gz \
      && mv go /usr/local && ln -s /usr/local/go/bin/go /bin/go \
-     && make all && cp /geth/build/bin/evm /bin/evm \
-     && cp /geth/build/bin/geth /bin/geth \
+     && go build ./cmd/evm  && cp evm /bin/evm \
      && rm -rf /geth && rm -rf /usr/local/go \
     || echo "Geth is empty"
 

circle.yml

@@ -44,8 +44,8 @@ defaults:
       name: "LinuxInstallGO and Tests"
       working_directory: ~/project
       command: |
-          curl -L --output go1.19.linux-amd64.tar.gz "https://dl.google.com/go/go1.19.linux-amd64.tar.gz"
-          tar -xf go1.19.linux-amd64.tar.gz
+          curl -L --output go1.20.linux-amd64.tar.gz "https://dl.google.com/go/go1.20.linux-amd64.tar.gz"
+          tar -xf go1.20.linux-amd64.tar.gz
           sudo mv go /usr/local
           sudo ln -s /usr/local/go/bin/go /usr/local/bin/go
           git clone --depth 1 https://github.com/ethereum/go-ethereum.git
@@ -165,9 +165,9 @@ defaults:
             git clone --depth 1 https://github.com/ethereum/go-ethereum.git
             git clone --depth 1 https://github.com/ethereum/tests.git
 
-            curl -L https://go.dev/dl/go1.19.1.darwin-amd64.pkg -o go1.19.pkg
-            chmod +x go1.19.pkg
-            sudo installer -pkg go1.19.pkg -target /
+            curl -L https://go.dev/dl/go1.20.1.darwin-amd64.pkg -o go1.20.pkg
+            chmod +x go1.20.pkg
+            sudo installer -pkg go1.20.pkg -target /
             sudo ln -s /usr/local/go/bin/go /usr/local/bin
 
             cd go-ethereum
@@ -238,7 +238,7 @@ jobs:
       - BUILD_PARALLEL_JOBS: 4
       - TEST_PARALLEL_JOBS: 4
     macos:
-      xcode: "14.0.1"
+      xcode: "14.2.0"
     steps: *mac-steps
 
 # TODO: Run GCC6 build only in develop branch.

dretesteth.sh

@@ -45,6 +45,31 @@ cleanDocker () {
     exit 0
 }
 
+lllc () {
+    testpath=""
+    testpaths=0
+    argstring=""
+    for var in "$@"
+    do
+        if [ "$var" = "--testpath" ]; then
+            testpaths=1
+            continue
+        fi
+        if [ "$testpaths" -eq "1" ]; then
+            testpaths=0
+            testpath=$var
+            continue
+        fi
+        argstring=$argstring" "$var
+    done
+    if [ -z $testpath ]; then
+        echo "Provide --testpath var'to mount into docker '/test'"
+        exit 1
+    fi
+    docker run --entrypoint /bin/lllc -it -v $testpath:/tests -w /tests retesteth $argstring
+    exit 0
+}
+
 
 case $1 in
     "build")
@@ -59,6 +84,9 @@ case $1 in
     "clean")
         cleanDocker
         ;;
+    "lllc")
+        lllc $2 $3 $4 $5 $6 $7 $8
+        ;;
 esac
 
 
@@ -124,7 +152,7 @@ if [ "$SCRIPT_NAME" = "dtf.sh" ]; then
       dockerwdir="/"
     fi
 
-    docker run --entrypoint /usr/bin/tfinit.sh -it -w $dockerwdir -e ETHEREUMJS_PATH=/ethereumjs -e PYSPECS_PATH=/execution-spec-tests -v $testpath:/tests retesteth $argstring
+    docker run --entrypoint /usr/bin/tfinit.sh -it -w $dockerwdir -e BESU_PATH=/besu -e ETHEREUMJS_PATH=/ethereumjs -e PYSPECS_PATH=/execution-spec-tests -e PYT8N_PATH=/pyt8n -v $testpath:/tests retesteth $argstring
     exit 0
 fi
 
@@ -182,5 +210,5 @@ fi
 if [ "$helpversion" -eq 1 ]; then
     docker run retesteth $argstring
 else
-    docker run -w $dockerwdir -e ETHEREUMJS_PATH=/ethereumjs -e PYSPECS_PATH=/execution-spec-tests -v $testpath:/tests retesteth $argstring --testpath /tests $defaultclient
+    docker run -w $dockerwdir -e BESU_PATH=/besu -e ETHEREUMJS_PATH=/ethereumjs -e PYSPECS_PATH=/execution-spec-tests -e PYT8N_PATH=/pyt8n -v $testpath:/tests retesteth $argstring --testpath /tests $defaultclient
 fi

libdataobj/DataObject.cpp

@@ -523,14 +523,11 @@ std::string DataObject::asJson(int level, bool pretty, bool nokey) const
         break;
     case DataType::String:
         printLevel();
-        if (pretty)
+        if (!m_strKey.empty() && !nokey)
         {
-            if (!m_strKey.empty() && !nokey)
+            if(pretty)
                 out << "\"" << m_strKey << "\" : ";
-        }
-        else
-        {
-            if (!m_strKey.empty() && !nokey)
+            else
                 out << "\"" << m_strKey << "\":";
         }
 
@@ -541,6 +538,11 @@ std::string DataObject::asJson(int level, bool pretty, bool nokey) const
                 buffer += "\\n";
             else if (ch == 9)
                 buffer += "\\t";
+            else if (ch == '"')
+            {
+                buffer += "\\";
+                buffer += "\"";
+            }
             else
                 buffer += ch;
         }

libdataobj/JsonParser.cpp

@@ -13,11 +13,12 @@ JsonParser::JsonParser(std::string const& _input, CJOptions const& _opt)
     if (_input.size() < 2 || _input.find("{") == string::npos || _input.rfind("}") == string::npos)
         throw DataObjectException() << "ConvertJsoncppStringToData can't read json structure in file: `" + _input.substr(0, 50);
 
+    m_applyDepth.clear();
     m_root.getContent().setAutosort(_opt.autosort);
     m_actualRoot = &m_root.getContent();
 }
 
-string JsonParser::printDebug(size_t const& _i)
+string JsonParser::printDebug(size_t const& _i) const
 {
     static const short c_debugSize = 120;
     string debug;
@@ -55,13 +56,24 @@ void JsonParser::parse()
     }
 }
 
+void JsonParser::checkJsonCommaEnding(size_t& _i) const
+{
+    if (m_input.at(_i) == '}' || m_input.at(_i) == ']')
+        _i--;  // because cycle iteration we need to process ending clouse
+    else if (m_input.at(_i) != ',')
+        throw DataObjectException() << errorPrefix
+            + "Dataobject array/object expected ',' when listing elements, but got `" + m_input.at(_i) + "`"
+            + "around: " + printDebug(_i);
+}
+
 JsonParser::RET JsonParser::tryParseKeyValue(size_t& _i)
 {
     const bool escapeChar = (_i > 0 && m_input.at(_i - 1) == '\\');
     if (m_input.at(_i) == '"' && !escapeChar)
     {
         spDataObject obj;
         string key = parseKeyValue(_i);
+
         _i = skipSpaces(_i);
         if (m_input.at(_i) == ':')
         {
@@ -104,14 +116,12 @@ JsonParser::RET JsonParser::tryParseKeyValue(size_t& _i)
             if (m_actualRoot->type() == DataType::Array)
             {
                 m_actualRoot->addArrayObject(sDataObject(std::move(key)));
-                if (m_input.at(_i) != ',')
-                    _i--;  // because cycle iteration we need to process ending clouse
+                checkJsonCommaEnding(_i);
                 return RET::CONTINUE;
             }
             else
                 m_actualRoot->setString(std::move(key));
-            if (m_input.at(_i) != ',')
-                _i--;  // because cycle iteration we need to process ending clouse
+            checkJsonCommaEnding(_i);
             m_actualRoot = m_applyDepth.at(m_applyDepth.size() - 1);
             m_applyDepth.pop_back();
             return RET::CONTINUE;
@@ -263,25 +273,25 @@ size_t JsonParser::skipSpaces(size_t const& _i) const
 string JsonParser::parseKeyValue(size_t& _i) const
 {
     if (_i + 1 > m_input.size())
-        throw DataObjectException() << errorPrefix + "reached EOF before reading char: `\"`";
+        throw DataObjectException() << errorPrefix + "reached EOF before reading char: `\"` around: " + printDebug(_i);
 
     bool escapeChar = true;
     size_t endPos = m_input.find('"', _i + 1);
-    while (escapeChar)
-    {
-        escapeChar = (m_input[endPos - 1] == '\\');
-        if (escapeChar)
-            endPos = m_input.find('"', endPos + 1);
-    }
-
     if (endPos != string::npos)
     {
+        while (escapeChar)
+        {
+            escapeChar = (m_input[endPos - 1] == '\\');
+            if (escapeChar)
+                endPos = m_input.find('"', endPos + 1);
+        }
+
         const string key = m_input.substr(_i + 1, endPos - _i - 1);
         _i = endPos + 1;
         return key;
     }
     else
-        throw DataObjectException() << errorPrefix + "not found key ending char: `\"`";
+        throw DataObjectException() << errorPrefix + "not found key ending char: `\"` around: " + printDebug(_i);
     return string();
 }
 

libdataobj/JsonParser.h

@@ -20,7 +20,7 @@ class JsonParser
     };
 private:
     void keyEncountered() { m_keyEncountered = true; }
-    std::string printDebug(size_t const& _i);
+    std::string printDebug(size_t const& _i) const;
     bool isEmptyChar(char const& _char) const;
     RET tryParseKeyValue(size_t& _i);
     RET tryParseArrayBegin(size_t const& _i);
@@ -34,13 +34,14 @@ class JsonParser
     std::string parseKeyValue(size_t& _i) const;
     bool readBoolOrNull(size_t& _i, bool& _result, bool& _readNull) const;
     bool readDigit(size_t& _i, int& _result) const;
+    void checkJsonCommaEnding(size_t& _i) const;
 private:
     std::string const& m_input;
     CJOptions const m_opt;
 
     std::vector<DataObject*> m_applyDepth;  // indexes at root array of objects that we are reading into
     spDataObject m_root;
-    DataObject* m_actualRoot;
+    DataObject* m_actualRoot = nullptr;
     bool m_keyEncountered = false;
     std::string const errorPrefix = "Error parsing json: ";
 };

retesteth/Constants.cpp

@@ -47,7 +47,14 @@ const string c_transactionsRoot = "transactionsRoot";
 const string c_transactionsTrie = "transactionsTrie";
 const string c_baseFeePerGas = "baseFeePerGas";
 const string c_withdrawalsRoot = "withdrawalsRoot";
-const string c_excessDataGas = "excessDataGas";
+const string c_parentBeaconBlockRoot = "parentBeaconBlockRoot";
+const string c_excessBlobGas = "excessBlobGas";
+const string c_blobGasUsed = "blobGasUsed";
+const string c_currentExcessBlobGas = "currentExcessBlobGas";
+const string c_parentExcessBlobGas = "parentExcessBlobGas";
+const string c_currentBlobGasUsed = "currentBlobGasUsed";
+const string c_parentBlobGasUsed = "parentBlobGasUsed";
+const string c_currentBeaconRoot = "currentBeaconRoot";
 
 // Transactions
 const string c_data = "data";
@@ -61,7 +68,7 @@ const string c_r = "r";
 const string c_s = "s";
 const string c_maxFeePerGas = "maxFeePerGas";
 const string c_maxPriorityFeePerGas = "maxPriorityFeePerGas";
-const string c_maxFeePerDataGas = "maxFeePerDataGas";
+const string c_maxFeePerBlobGas = "maxFeePerBlobGas";
 const string c_blobVersionedHashes = "blobVersionedHashes";
 const string c_accessLists = "accessLists";
 const string c_accessList = "accessList";
@@ -70,7 +77,7 @@ const string c_code = "code";
 const string c_balance = "balance";
 const string c_storage = "storage";
 const string c_transactionHash = "transactionHash";
-const string c_blockHash = "transactionHash";
+const string c_blockHash = "blockHash";
 
 }}}
 

retesteth/Constants.h

@@ -42,7 +42,14 @@ extern const std::string c_transactionsRoot;
 extern const std::string c_transactionsTrie;
 extern const std::string c_baseFeePerGas;
 extern const std::string c_withdrawalsRoot;
-extern const std::string c_excessDataGas;
+extern const std::string c_parentBeaconBlockRoot;
+extern const std::string c_excessBlobGas;
+extern const std::string c_blobGasUsed;
+extern const std::string c_currentExcessBlobGas;
+extern const std::string c_currentBeaconRoot;
+extern const std::string c_parentExcessBlobGas;
+extern const std::string c_currentBlobGasUsed;
+extern const std::string c_parentBlobGasUsed;
 
 // Transactions
 extern const std::string c_data;
@@ -56,7 +63,7 @@ extern const std::string c_r;
 extern const std::string c_s;
 extern const std::string c_maxFeePerGas;
 extern const std::string c_maxPriorityFeePerGas;
-extern const std::string c_maxFeePerDataGas;
+extern const std::string c_maxFeePerBlobGas;
 extern const std::string c_blobVersionedHashes;
 extern const std::string c_accessLists;
 extern const std::string c_accessList;