Merge pull request #751 from gsalgado/develop

Revert deletion of pow.ethash and pow.ethash_utils

Author: Jan Xie

Makefile

@@ -40,7 +40,7 @@ lint:
 	flake8 ethereum tests --ignore=E501
 
 lint-minimal:
-	python -m flake8 --ignore=F401,F841,F811 --select=F --exclude=todo,experimental ethereum
+	python -m flake8 --ignore=F401,F841,F811 --select=F --exclude=todo,experimental,ethash.py,ethash_utils.py ethereum
 
 test:
 	py.test --tb=no ethereum/tests/

ethereum/pow/__init__.py

@@ -1 +1 @@
-from ethereum.pow import chain, consensus, ethpow
+from ethereum.pow import chain, consensus, ethash, ethash_utils, ethpow

ethereum/pow/consensus.py

@@ -1,4 +1,4 @@
-from ethereum.pow import ethpow
+from ethereum.pow import ethash, ethash_utils, ethpow
 from ethereum import utils
 from ethereum.common import update_block_env_variables, calc_difficulty
 from ethereum.exceptions import VerificationFailed

ethereum/pow/ethash.py

@@ -0,0 +1,100 @@
+import copy
+import sys
+
+
+if sys.version_info.major == 2:
+    from repoze.lru import lru_cache
+else:
+    from functools import lru_cache
+
+
+cache_seeds = [b'\x00' * 32]
+
+
+def mkcache(block_number):
+    while len(cache_seeds) <= block_number // EPOCH_LENGTH:
+        cache_seeds.append(sha3.sha3_256(cache_seeds[-1]).digest())
+
+    seed = cache_seeds[block_number // EPOCH_LENGTH]
+
+    n = get_cache_size(block_number) // HASH_BYTES
+    return _get_cache(seed, n)
+
+
+@lru_cache(5)
+def _get_cache(seed, n):
+    # Sequentially produce the initial dataset
+    o = [sha3_512(seed)]
+    for i in range(1, n):
+        o.append(sha3_512(o[-1]))
+
+    for _ in range(CACHE_ROUNDS):
+        for i in range(n):
+            v = o[i][0] % n
+            o[i] = sha3_512(list(map(xor, o[(i - 1 + n) % n], o[v])))
+
+    return o
+
+
+def calc_dataset_item(cache, i):
+    n = len(cache)
+    r = HASH_BYTES // WORD_BYTES
+    mix = copy.copy(cache[i % n])
+    mix[0] ^= i
+    mix = sha3_512(mix)
+    for j in range(DATASET_PARENTS):
+        cache_index = fnv(i ^ j, mix[j % r])
+        mix = list(map(fnv, mix, cache[cache_index % n]))
+    return sha3_512(mix)
+
+
+def calc_dataset(full_size, cache):
+    o = []
+    percent = (full_size // HASH_BYTES) // 100
+    for i in range(full_size // HASH_BYTES):
+        if i % percent == 0:
+            sys.stderr.write("Completed %d items, %d percent\n" % (i, i // percent))
+        o.append(calc_dataset_item(cache, i))
+    return o
+
+
+def hashimoto(header, nonce, full_size, dataset_lookup):
+    n = full_size // HASH_BYTES
+    w = MIX_BYTES // WORD_BYTES
+    mixhashes = MIX_BYTES // HASH_BYTES
+    s = sha3_512(header + nonce[::-1])
+    mix = []
+    for _ in range(MIX_BYTES // HASH_BYTES):
+        mix.extend(s)
+    for i in range(ACCESSES):
+        p = fnv(i ^ s[0], mix[i % w]) % (n // mixhashes) * mixhashes
+        newdata = []
+        for j in range(mixhashes):
+            newdata.extend(dataset_lookup(p + j))
+        mix = list(map(fnv, mix, newdata))
+    cmix = []
+    for i in range(0, len(mix), 4):
+        cmix.append(fnv(fnv(fnv(mix[i], mix[i + 1]), mix[i + 2]), mix[i + 3]))
+    return {
+        "mix digest": serialize_hash(cmix),
+        "result": serialize_hash(sha3_256(s + cmix))
+    }
+
+
+def hashimoto_light(block_number, cache, header, nonce):
+    return hashimoto(header, nonce, get_full_size(block_number),
+                     lambda x: calc_dataset_item(cache, x))
+
+
+def hashimoto_full(dataset, header, nonce):
+    return hashimoto(header, nonce, len(dataset) * HASH_BYTES,
+                     lambda x: dataset[x])
+
+
+def mine(full_size, dataset, header, difficulty):
+    from random import randint
+    nonce = randint(0, 2**64)
+    while decode_int(hashimoto_full(full_size, dataset, header, nonce)) < difficulty:
+        nonce += 1
+        nonce %= 2**64
+    return nonce

ethereum/pow/ethash_utils.py

@@ -0,0 +1,136 @@
+try:
+    from Crypto.Hash import keccak
+    sha3_256 = lambda x: keccak.new(digest_bits=256, data=x).digest()
+    sha3_512 = lambda x: keccak.new(digest_bits=512, data=x)
+except:
+    import sha3 as _sha3
+    sha3_256 = lambda x: _sha3.sha3_256(x).digest()
+    sha3_512 = lambda x: _sha3.sha3_512(x).digest()
+from rlp.utils import decode_hex
+from ethereum.utils import encode_hex
+import sys
+
+WORD_BYTES = 4                    # bytes in word
+DATASET_BYTES_INIT = 2**30        # bytes in dataset at genesis
+DATASET_BYTES_GROWTH = 2**23      # growth per epoch (~7 GB per year)
+CACHE_BYTES_INIT = 2**24          # Size of the dataset relative to the cache
+CACHE_BYTES_GROWTH = 2**17        # Size of the dataset relative to the cache
+EPOCH_LENGTH = 30000              # blocks per epoch
+MIX_BYTES = 128                   # width of mix
+HASH_BYTES = 64                   # hash length in bytes
+DATASET_PARENTS = 256             # number of parents of each dataset element
+CACHE_ROUNDS = 3                  # number of rounds in cache production
+ACCESSES = 64                     # number of accesses in hashimoto loop
+
+
+FNV_PRIME = 0x01000193
+
+
+def fnv(v1, v2):
+    return (v1 * FNV_PRIME ^ v2) % 2**32
+
+
+# Assumes little endian bit ordering (same as Intel architectures)
+def decode_int(s):
+    return int(encode_hex(s[::-1]), 16) if s else 0
+
+
+def encode_int(s):
+    a = "%x" % s
+    return b'' if s == 0 else decode_hex('0' * (len(a) % 2) + a)[::-1]
+
+
+def zpad(s, length):
+    return s + b'\x00' * max(0, length - len(s))
+
+
+def serialize_hash(h):
+    return b''.join([zpad(encode_int(x), 4) for x in h])
+
+
+def deserialize_hash(h):
+    return [decode_int(h[i:i+WORD_BYTES]) for i in range(0, len(h), WORD_BYTES)]
+
+
+def hash_words(h, sz, x):
+    if isinstance(x, list):
+        x = serialize_hash(x)
+    y = h(x)
+    return deserialize_hash(y)
+
+
+def to_bytes(x):
+    if sys.version_info.major > 2 and isinstance(x, str):
+        x = bytes(x, 'utf-8')
+    return x
+
+
+# sha3 hash function, outputs 64 bytes
+def sha3_512(x):
+    return hash_words(sha3_512(to_bytes(v)).digest(), 64, x)
+
+
+def sha3_256(x):
+    return hash_words(sha3_256(to_bytes(v)).digest(), 32, x)
+
+
+def xor(a, b):
+    return a ^ b
+
+
+# Works for dataset and cache
+def serialize_cache(ds):
+    return b''.join([serialize_hash(h) for h in ds])
+
+serialize_dataset = serialize_cache
+
+
+def deserialize_cache(ds):
+    return [deserialize_hash(ds[i:i+HASH_BYTES])
+            for i in range(0, len(ds), HASH_BYTES)]
+
+deserialize_dataset = deserialize_cache
+
+
+class ListWrapper(list):
+    def __init__(self, data):
+        self.data = data
+        self.len = len(data) // HASH_BYTES
+
+    def __len__(self):
+        return self.len
+
+    def __getitem__(self, i):
+        if i >= self.len:
+            raise Exception("listwrap access out of range")
+        return deserialize_hash(self.data[i*HASH_BYTES:(i+1)*HASH_BYTES])
+
+    def __iter__(self):
+        for i in range(self.len):
+            yield self[i]
+
+    def __repr__(self):
+        return repr([x for x in self])
+
+
+def isprime(x):
+    for i in range(2, int(x**0.5)):
+        if not x % i:
+            return False
+    return True
+
+
+def get_cache_size(block_number):
+    sz = CACHE_BYTES_INIT + CACHE_BYTES_GROWTH * (block_number // EPOCH_LENGTH)
+    sz -= HASH_BYTES
+    while not isprime(sz // HASH_BYTES):
+        sz -= 2 * HASH_BYTES
+    return sz
+
+
+def get_full_size(block_number):
+    sz = DATASET_BYTES_INIT + DATASET_BYTES_GROWTH * (block_number // EPOCH_LENGTH)
+    sz -= MIX_BYTES
+    while not isprime(sz // MIX_BYTES):
+        sz -= 2 * MIX_BYTES
+    return sz

ethereum/pow/ethpow.py

@@ -1,8 +1,12 @@
+from ethereum.pow import ethash
 from ethereum import utils
+import time
 import sys
+import warnings
 from collections import OrderedDict
+from ethereum import utils
 from ethereum.slogging import get_logger
-import pyethash
+import rlp
 
 log = get_logger('eth.pow')
 
@@ -11,10 +15,24 @@
 else:
     from functools import lru_cache
 
-mkcache = pyethash.mkcache_bytes
-EPOCH_LENGTH = 30000
-def hashimoto_light(s, c, h, n):
-    return pyethash.hashimoto_light(s, c, h, utils.big_endian_to_int(n))
+try:
+    import pyethash
+    ETHASH_LIB = 'pyethash'  # the C++ based implementation
+except ImportError:
+    ETHASH_LIB = 'ethash'
+    warnings.warn('using pure python implementation', ImportWarning)
+
+if ETHASH_LIB == 'ethash':
+    mkcache = ethash.mkcache
+    EPOCH_LENGTH = 30000
+    hashimoto_light = ethash.hashimoto_light
+elif ETHASH_LIB == 'pyethash':
+    mkcache = pyethash.mkcache_bytes
+    EPOCH_LENGTH = 30000
+    hashimoto_light = lambda s, c, h, n: \
+        pyethash.hashimoto_light(s, c, h, utils.big_endian_to_int(n))
+else:
+    raise Exception("invalid ethash library set")
 
 TT64M1 = 2**64 - 1
 cache_seeds = ['\x00' * 32]
@@ -23,6 +41,7 @@ def hashimoto_light(s, c, h, n):
 
 
 def get_cache(block_number):
+    import sha3
     while len(cache_seeds) <= block_number // EPOCH_LENGTH:
         cache_seeds.append(utils.sha3(cache_seeds[-1]))
     seed = cache_seeds[block_number // EPOCH_LENGTH]
@@ -105,3 +124,5 @@ def mine(block_number, difficulty, mining_hash, start_nonce=0, rounds=1000):
             assert len(o[b"mix digest"]) == 32
             return bin_nonce, o[b"mix digest"]
     return None, None
+
+