Merge branch 'fix-vm-memory-leak' into eth-wire-protocol-p62

Author: Jan Xie

.release_notify.sh

@@ -1,3 +1,6 @@
 #!/usr/bin/env sh
 PAYLOAD="{\"text\": \"<https://pypi.python.org/pypi/ethereum|ethereum $TRAVIS_TAG> was released on pypi!\"}"
 curl -s -X POST --data-urlencode "payload=$PAYLOAD" $SLACK_WEBHOOK_URL
+
+PAYLOAD="{\"attachments\":[{\"text\":\"[ethereum $TRAVIS_TAG](https://pypi.org/project/ethereum) was released on PyPI!\",\"color\":\"good\"}]}"
+curl -s -X POST --data-urlencode "payload=$PAYLOAD" $ROCKETCHAT_URL

.travis.yml

@@ -14,6 +14,7 @@ env:
   global:
     - COVERAGE_APPEND="--append"
     - secure: cKbIgpTJ1yjKLBxpCEiT6IH7NShDWZUE+BvnrAfc+ujCsR6LyLJcKxFQmKnWryJCqg7fp82Ep2bF2oDKzanAROar2xDY1SFGbai42seYMaFCw53YPGJ6u3VNCcfT0rN9BWgE7el/m4fjcD6CRsZYKArNNJbMX8csRt3uXXCFLso=
+    - secure: "QyFPrxQHd2LlNQ6zNeTFYhgPmZaOHoWuywcgn8qaSOh6PklyFxHbexkwg0bl23JvtgNEZ1mCD8j0x1/ydSdtxgCFwK9SEL0h7aYuAq+OAIa/G18OPeTJMf7ASsb2QZdfkt9reFpUnjbadzHkuv+rqqb4bFnTJBKwB2LWzHPLhWg="
 install:
   - travis_retry pip install setuptools --upgrade
   - travis_retry pip install tox
@@ -27,11 +28,12 @@ script:
 after_success:
   - travis_retry pip install coveralls
   - cd .tox/$TOX_ENV && coveralls
-after_script:
-  - cat .tox/$TOX_ENV/log/*.log; true
 notifications:
   slack:
     secure: W/UAhQ/GgYwMWrl3aiVAVOWr4WGdWrxUOX/rTB3ZgwDwGqDYLzQO5UqbsQlo1JXPZ6JOWfIPMURhHu7DSfue9dBW6xQ+NL+bFHe9lSXG4nqFK3IjezYyTBzNRJRDbGUvSSqgj6D5cwhJ8BjfUIRPbJz3CxL64KmsNXezEaMY60w=
+  webhooks:
+    - secure: "KBw4iPJVsw3qv0qbvkZ2tjams/280aEovJ88ylR9ClI2krtxJFoVlAB0YdkztpKxcAxbIZg8nd4S3XpeOUsxfoZq5mZ2wHNU7G8LjqjwZ7eOhj76K+NAnEhRQT2evqZkWqP0h5fxuXQaT4KjwxGkxGNDEMJUVH5RO2cnFUQO98s="
+
 before_deploy:
 - cd $TRAVIS_BUILD_DIR
 deploy:

ethereum/_solidity.py

@@ -3,6 +3,7 @@
 import re
 import subprocess
 import warnings
+import shlex
 
 import yaml
 
@@ -22,6 +23,11 @@ def get_compiler_path():
     This funtion will search for the solc binary in the $PATH and return the
     path of the first executable occurence.
     """
+    # If the user provides a specific solc binary let's use that
+    given_binary = os.environ.get('SOLC_BINARY')
+    if given_binary:
+        return given_binary
+
     for path in os.getenv('PATH', '').split(os.pathsep):
         path = path.strip('"')
         executable_path = os.path.join(path, BINARY)
@@ -40,16 +46,22 @@ def get_solidity():
     return solc_wrapper
 
 
-def solc_arguments(libraries=None, combined='bin,abi', optimize=True):
+def solc_arguments(libraries=None, combined='bin,abi', optimize=True, extra_args=None):
     """ Build the arguments to call the solc binary. """
     args = [
         '--combined-json', combined,
-        '--add-std',
+        '--add-std'
     ]
 
     if optimize:
         args.append('--optimize')
 
+    if extra_args:
+        try:
+            args.extend(shlex.split(extra_args))
+        except:  # if not a parseable string then treat it as a list
+            args.extend(extra_args)
+
     if libraries is not None and len(libraries):
         addresses = [
             '{name}:{address}'.format(name=name, address=address.decode('utf8'))
@@ -65,6 +77,10 @@ def solc_arguments(libraries=None, combined='bin,abi', optimize=True):
 
 def solc_parse_output(compiler_output):
     """ Parses the compiler output. """
+    # At the moment some solc output like --hashes or -- gas will not output
+    # json at all so if used with those arguments the logic here will break.
+    # Perhaps solidity will slowly switch to a json only output and this comment
+    # can eventually go away and we will not need to add more logic here at all.
     result = yaml.safe_load(compiler_output)['contracts']
 
     if 'bin' in tuple(result.values())[0]:
@@ -93,7 +109,7 @@ def solc_parse_output(compiler_output):
 def compiler_version():
     """ Return the version of the installed solc. """
     version_info = subprocess.check_output(['solc', '--version'])
-    match = re.search('^Version: ([0-9a-z.-]+)/', version_info, re.MULTILINE)
+    match = re.search(b'^Version: ([0-9a-z.-]+)/', version_info, re.MULTILINE)
 
     if match:
         return match.group(1)
@@ -185,7 +201,7 @@ def solidity_resolve_address(hex_code, library_symbol, library_address):
         raise ValueError('Address should not contain the 0x prefix')
 
     try:
-        _ = decode_hex(library_address)
+        decode_hex(library_address)
     except TypeError:
         raise ValueError('library_address contains invalid characters, it must be hex encoded.')
 
@@ -221,7 +237,7 @@ def solidity_unresolved_symbols(hex_code):
     return set(re.findall(r"_.{39}", hex_code))
 
 
-def compile_file(filepath, libraries=None, combined='bin,abi', optimize=True):
+def compile_file(filepath, libraries=None, combined='bin,abi', optimize=True, extra_args=None):
     """ Return the compile contract code.
 
     Args:
@@ -236,7 +252,7 @@ def compile_file(filepath, libraries=None, combined='bin,abi', optimize=True):
 
     workdir, filename = os.path.split(filepath)
 
-    args = solc_arguments(libraries=libraries, combined=combined, optimize=optimize)
+    args = solc_arguments(libraries=libraries, combined=combined, optimize=optimize, extra_args=extra_args)
     args.insert(0, get_compiler_path())
     args.append(filename)
 
@@ -245,18 +261,19 @@ def compile_file(filepath, libraries=None, combined='bin,abi', optimize=True):
     return solc_parse_output(output)
 
 
-def compile_contract(filepath, contract_name, libraries=None, combined='bin,abi', optimize=True):
+def compile_contract(filepath, contract_name, libraries=None, combined='bin,abi', optimize=True, extra_args=None):
     all_contracts = compile_file(
         filepath,
         libraries=libraries,
         combined=combined,
         optimize=optimize,
+        extra_args=extra_args
     )
 
     return all_contracts[contract_name]
 
 
-def compile_last_contract(filepath, libraries=None, combined='bin,abi', optimize=True):
+def compile_last_contract(filepath, libraries=None, combined='bin,abi', optimize=True, extra_args=None):
     with open(filepath) as handler:
         all_names = solidity_names(handler.read())
 
@@ -273,11 +290,12 @@ def compile_last_contract(filepath, libraries=None, combined='bin,abi', optimize
         libraries=libraries,
         combined=combined,
         optimize=optimize,
+        extra_args=extra_args
     )
 
 
-def compile_code(sourcecode, libraries=None, combined='bin,abi', optimize=True):
-    args = solc_arguments(libraries=libraries, combined=combined, optimize=optimize)
+def compile_code(sourcecode, libraries=None, combined='bin,abi', optimize=True, extra_args=None):
+    args = solc_arguments(libraries=libraries, combined=combined, optimize=optimize, extra_args=extra_args)
     args.insert(0, get_compiler_path())
 
     process = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -297,17 +315,17 @@ class Solc(object):
     compiler_version = staticmethod(compiler_version)
 
     @staticmethod
-    def _code_or_path(sourcecode, path, contract_name, libraries, combined):
+    def _code_or_path(sourcecode, path, contract_name, libraries, combined, extra_args):
         warnings.warn('solc_wrapper is deprecated, please use the functions compile_file or compile_code')
 
         if sourcecode and path:
             raise ValueError('sourcecode and path are mutually exclusive.')
 
         if path and contract_name:
-            return compile_contract(path, contract_name, libraries=libraries, combined=combined)
+            return compile_contract(path, contract_name, libraries=libraries, combined=combined, extra_args=extra_args)
 
         if path:
-            return compile_last_contract(path, libraries=libraries, combined=combined)
+            return compile_last_contract(path, libraries=libraries, combined=combined, extra_args=extra_args)
 
         all_names = solidity_names(sourcecode)
         all_contract_names = [
@@ -316,41 +334,44 @@ def _code_or_path(sourcecode, path, contract_name, libraries, combined):
         ]
         last_contract = all_contract_names[-1]
 
-        result = compile_code(sourcecode, libraries=libraries, combined=combined)
+        result = compile_code(sourcecode, libraries=libraries, combined=combined, extra_args=extra_args)
         return result[last_contract]
 
     @classmethod
-    def compile(cls, code, path=None, libraries=None, contract_name=''):
+    def compile(cls, code, path=None, libraries=None, contract_name='', extra_args=None):
         """ Return the binary of last contract in code. """
-        result = cls._code_or_path(code, path, contract_name, libraries, 'bin')
+        result = cls._code_or_path(code, path, contract_name, libraries, 'bin', extra_args)
         return result['bin']
 
     @classmethod
-    def mk_full_signature(cls, code, path=None, libraries=None, contract_name=''):
+    def mk_full_signature(cls, code, path=None, libraries=None, contract_name='', extra_args=None):
         "returns signature of last contract in code"
 
-        result = cls._code_or_path(code, path, contract_name, libraries, 'abi')
+        result = cls._code_or_path(code, path, contract_name, libraries, 'abi', extra_args)
         return result['abi']
 
     @classmethod
-    def combined(cls, code, path=None):
+    def combined(cls, code, path=None, extra_args=None):
         """ Compile combined-json with abi,bin,devdoc,userdoc.
 
         @param code: literal solidity code as a string.
-        @param path: absolute path to solidity-file. Note: code & path are exclusive!
+        @param path: absolute path to solidity-file. Note: code & path are
+                     mutually exclusive!
+        @param extra_args: Either a space separated string or a list of extra
+                           arguments to be passed to the solidity compiler.
         """
 
         if code and path:
             raise ValueError('sourcecode and path are mutually exclusive.')
 
         if path:
-            contracts = compile_file(path)
+            contracts = compile_file(path, extra_args=extra_args)
 
             with open(path) as handler:
                 code = handler.read()
 
         elif code:
-            contracts = compile_code(code)
+            contracts = compile_code(code, extra_args=extra_args)
 
         else:
             raise ValueError('either code or path needs to be supplied.')
@@ -361,7 +382,7 @@ def combined(cls, code, path=None):
         return sorted_contracts
 
     @classmethod
-    def compile_rich(cls, code, path=None):
+    def compile_rich(cls, code, path=None, extra_args=None):
         """full format as returned by jsonrpc"""
 
         return {
@@ -378,7 +399,7 @@ def compile_rich(cls, code, path=None):
                 },
             }
             for contract_name, contract
-            in cls.combined(code, path=path)
+            in cls.combined(code, path=path, extra_args=extra_args)
         }
 
 

ethereum/config.py

@@ -64,6 +64,9 @@
     DAO_FORK_BLKEXTRA=decode_hex('64616f2d686172642d666f726b'),
     CHILD_DAO_LIST=map(utils.normalize_address, child_dao_list),
     DAO_WITHDRAWER=utils.normalize_address('0xbf4ed7b27f1d666546e30d74d50d173d20bca754'),
+    # Anti-DoS fork
+    ANTI_DOS_FORK_BLKNUM=2457000,
+    CLEARING_FORK_BLKNUM=2 ** 98,
 )
 assert default_config['NEPHEW_REWARD'] == \
     default_config['BLOCK_REWARD'] // 32

ethereum/fastvm.py

@@ -8,14 +8,17 @@
 import sys
 
 from ethereum import utils
-from ethereum.abi import is_numeric
 import copy
 from ethereum import opcodes
 import time
 from ethereum.slogging import get_logger
 from rlp.utils import encode_hex, ascii_chr
 from ethereum.utils import to_string
-import numpy
+
+if sys.version_info.major == 2:
+    from repoze.lru import lru_cache
+else:
+    from functools import lru_cache
 
 log_log = get_logger('eth.vm.log')
 log_vm_exit = get_logger('eth.vm.exit')
@@ -99,6 +102,7 @@ def __init__(self, **kwargs):
 
 # Preprocesses code, and determines which locations are in the middle
 # of pushdata and thus invalid
+@lru_cache(128)
 def preprocess_code(code):
     assert isinstance(code, bytes)
     code = memoryview(code).tolist()
@@ -185,8 +189,6 @@ def peaceful_exit(cause, gas, data, **kargs):
     log_vm_exit.trace('EXIT', cause=cause, **kargs)
     return 1, gas, data
 
-code_cache = {}
-
 
 def vm_execute(ext, msg, code):
     # precompute trace flag
@@ -197,11 +199,7 @@ def vm_execute(ext, msg, code):
     stk = compustate.stack
     mem = compustate.memory
 
-    if code in code_cache:
-        processed_code = code_cache[code]
-    else:
-        processed_code = preprocess_code(code)
-        code_cache[code] = processed_code
+    processed_code = preprocess_code(code)
 
     s = time.time()
     op = None

ethereum/opcodes.py

@@ -111,3 +111,12 @@
 
 GCALLNEWACCOUNT = 25000
 GSUICIDEREFUND = 24000
+
+# Anti-DoS HF changes
+SLOAD_SUPPLEMENTAL_GAS = 150
+CALL_SUPPLEMENTAL_GAS = 660
+EXTCODELOAD_SUPPLEMENTAL_GAS = 680
+BALANCE_SUPPLEMENTAL_GAS = 380
+CALL_CHILD_LIMIT_NUM = 63
+CALL_CHILD_LIMIT_DENOM = 64
+SUICIDE_SUPPLEMENTAL_GAS = 5000

ethereum/processblock.py

@@ -248,7 +248,8 @@ def __init__(self, block, tx):
         self.create = lambda msg: create_contract(self, msg)
         self.msg = lambda msg: _apply_msg(self, msg, self.get_code(msg.code_address))
         self.account_exists = block.account_exists
-        self.post_homestead_hardfork = lambda: block.number >= block.config['HOMESTEAD_FORK_BLKNUM']
+        self.post_homestead_hardfork = block.number >= block.config['HOMESTEAD_FORK_BLKNUM']
+        self.post_anti_dos_hardfork = block.number >= block.config['ANTI_DOS_FORK_BLKNUM']
 
 
 def apply_msg(ext, msg):