diff --git a/configure.ac b/configure.ac index 35bb0c0231525..5780c5cd163ef 100644 --- a/configure.ac +++ b/configure.ac @@ -662,6 +662,9 @@ if test x$use_lcov_branch != xno; then AC_SUBST(LCOV_OPTS, "$LCOV_OPTS --rc lcov_branch_coverage=1") fi +dnl Check for __int128 +AC_CHECK_TYPES([__int128]) + dnl Check for endianness AC_C_BIGENDIAN diff --git a/src/Makefile.am b/src/Makefile.am index 477b1300bc651..fad4cd0a87857 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -135,6 +135,7 @@ BITCOIN_CORE_H = \ index/base.h \ index/blockfilterindex.h \ index/txindex.h \ + index/utxosethash.h \ indirectmap.h \ init.h \ interfaces/chain.h \ @@ -270,6 +271,7 @@ libbitcoin_server_a_SOURCES = \ index/base.cpp \ index/blockfilterindex.cpp \ index/txindex.cpp \ + index/utxosethash.cpp \ interfaces/chain.cpp \ interfaces/node.cpp \ init.cpp \ @@ -369,6 +371,8 @@ crypto_libbitcoin_crypto_base_a_SOURCES = \ crypto/hmac_sha512.h \ crypto/poly1305.h \ crypto/poly1305.cpp \ + crypto/muhash.cpp \ + crypto/muhash.h \ crypto/ripemd160.cpp \ crypto/ripemd160.h \ crypto/sha1.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index d3fe138133e7b..847d6c296ff3f 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -148,6 +148,7 @@ BITCOIN_TESTS =\ test/txvalidationcache_tests.cpp \ test/uint256_tests.cpp \ test/util_tests.cpp \ + test/utxosethash_tests.cpp \ test/validation_block_tests.cpp \ test/versionbits_tests.cpp diff --git a/src/bench/crypto_hash.cpp b/src/bench/crypto_hash.cpp index fb2bab9dee7a1..9eee06fe21ed9 100644 --- a/src/bench/crypto_hash.cpp +++ b/src/bench/crypto_hash.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -92,6 +93,18 @@ static void FastRandom_1bit(benchmark::State& state) } } +static void MuHash(benchmark::State& state) +{ + FastRandomContext rng(true); + MuHash3072 acc; + unsigned char key[32] = {0}; + int i = 0; + while (state.KeepRunning()) { + key[0] = ++i; + acc *= MuHash3072(key); + } +} + BENCHMARK(RIPEMD160, 440); BENCHMARK(SHA1, 570); BENCHMARK(SHA256, 340); @@ -102,3 +115,5 @@ BENCHMARK(SipHash_32b, 40 * 1000 * 1000); BENCHMARK(SHA256D64_1024, 7400); BENCHMARK(FastRandom_32bit, 110 * 1000 * 1000); BENCHMARK(FastRandom_1bit, 440 * 1000 * 1000); + +BENCHMARK(MuHash, 5000); diff --git a/src/crypto/muhash.cpp b/src/crypto/muhash.cpp new file mode 100644 index 0000000000000..cd0b1febf28a3 --- /dev/null +++ b/src/crypto/muhash.cpp @@ -0,0 +1,323 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "muhash.h" + +#include +#include "common.h" +#include "chacha20.h" + +#include +#include + +namespace { + +/** Extract the lowest limb of [c0,c1,c2] into n, and left shift the number by 1 limb. */ +#define extract3(c0,c1,c2,n) { \ + (n) = c0; \ + c0 = c1; \ + c1 = c2; \ + c2 = 0; \ +} + +/** Extract the lowest limb of [c0,c1] into n, and left shift the number by 1 limb. */ +#define extract2(c0,c1,n) { \ + (n) = c0; \ + c0 = c1; \ + c1 = 0; \ +} + +#if defined(__amd64__) || defined(__x86_64__) + +/** [c0,c1] = a * b */ +#define mul(c0,c1,a,b) { \ + __asm__ ("mulq %3" : "=d"(c1), "=a"(c0) : "a"(a), "g"(b) : "cc"); \ +} + +/** [c0,c1,c2] += a * b */ +#define muladd3(c0,c1,c2,a,b) { \ + uint64_t tl, th; \ + __asm__ ("mulq %3" : "=a"(tl), "=d"(th) : "a"(a), "g"(b) : "cc"); \ + __asm__ ("addq %3,%0; adcq %4,%1; adcq $0,%2" : "+r"(c0), "+r"(c1), "+r"(c2) : "a"(tl), "d"(th) : "cc"); \ +} + +/** [c0,c1,c2] += 2 * a * b */ +#define muldbladd3(c0,c1,c2,a,b) { \ + uint64_t tl, th; \ + __asm__ ("mulq %3" : "=a"(tl), "=d"(th) : "a"(a), "g"(b) : "cc"); \ + __asm__ ("addq %3,%0; adcq %4,%1; adcq $0,%2" : "+r"(c0), "+r"(c1), "+r"(c2) : "a"(tl), "d"(th) : "cc"); \ + __asm__ ("addq %3,%0; adcq %4,%1; adcq $0,%2" : "+r"(c0), "+r"(c1), "+r"(c2) : "a"(tl), "d"(th) : "cc"); \ +} + +/* [c0,c1,c2] += n * [d0,d1,d2]. c0 is initially 0 */ +#define mulnadd3(c0,c1,c2,d0,d1,d2,n) { \ + uint64_t tl1, th1, tl2, th2, tl3; \ + __asm__ ("mulq %3" : "=a"(tl1), "=d"(th1) : "a"(d0), "r"((Num3072::limb_type)n) : "cc"); \ + __asm__ ("addq %3,%0; adcq %4,%1; adcq $0,%2" : "+r"(c0), "+r"(c1), "+r"(c2) : "g"(tl1), "g"(th1) : "cc"); \ + __asm__ ("mulq %3" : "=a"(tl2), "=d"(th2) : "a"(d1), "r"((Num3072::limb_type)n) : "cc"); \ + __asm__ ("addq %2,%0; adcq %3,%1" : "+r"(c1), "+r"(c2) : "g"(tl2), "g"(th2) : "cc"); \ + __asm__ ("imulq %2,%1,%0" : "=r"(tl3) : "g"(d2), "i"(n) : "cc"); \ + __asm__ ("addq %1,%0" : "+r"(c2) : "g"(tl3) : "cc"); \ +} + +/* [c0,c1] *= n */ +#define muln2(c0,c1,n) { \ + uint64_t th; \ + __asm__ ("mulq %2" : "+a"(c0), "=d"(th) : "r"((Num3072::limb_type)n) : "cc"); \ + __asm__ ("imul %1,%0,%0" : "+r"(c1) : "i"(n) : "cc"); \ + __asm__ ("addq %1,%0" : "+r"(c1) : "g"(th) : "cc"); \ +} + +/** [c0,c1] += a */ +#define add2(c0,c1,a) { \ + __asm__ ("add %2,%0; adc $0,%1" : "+r"(c0), "+r"(c1) : "r"(a) : "cc"); \ +} + +#else + +/** [c0,c1] = a * b */ +#define mul(c0,c1,a,b) { \ + Num3072::double_limb_type t = (Num3072::double_limb_type)a * b; \ + c2 = 0; \ + c1 = t >> Num3072::LIMB_SIZE; \ + c0 = t; \ +} + +/* [c0,c1,c2] += n * [d0,d1,d2]. c2 is 0 initially */ +#define mulnadd3(c0,c1,c2,d0,d1,d2,n) { \ + Num3072::double_limb_type t = (Num3072::double_limb_type)d0 * n + c0; \ + c0 = t; \ + t >>= Num3072::LIMB_SIZE; \ + t += (Num3072::double_limb_type)d1 * n + c1; \ + c1 = t; \ + t >>= Num3072::LIMB_SIZE; \ + c2 = t + d2 * n; \ +} + +/* [c0,c1] *= n */ +#define muln2(c0,c1,n) { \ + Num3072::double_limb_type t = (Num3072::double_limb_type)c0 * n; \ + c0 = t; \ + t >>= Num3072::LIMB_SIZE; \ + t += (Num3072::double_limb_type)c1 * n; \ + c1 = t; \ + t >>= Num3072::LIMB_SIZE; \ +} + +/** [c0,c1,c2] += a * b */ +#define muladd3(c0,c1,c2,a,b) { \ + Num3072::limb_type tl, th; \ + { \ + Num3072::double_limb_type t = (Num3072::double_limb_type)a * b; \ + th = t >> Num3072::LIMB_SIZE; \ + tl = t; \ + } \ + c0 += tl; \ + th += (c0 < tl) ? 1 : 0; \ + c1 += th; \ + c2 += (c1 < th) ? 1 : 0; \ +} + +/** [c0,c1,c2] += 2 * a * b */ +#define muldbladd3(c0,c1,c2,a,b) { \ + Num3072::limb_type tl, th; \ + { \ + Num3072::double_limb_type t = (Num3072::double_limb_type)a * b; \ + th = t >> Num3072::LIMB_SIZE; \ + tl = t; \ + } \ + c0 += tl; \ + Num3072::limb_type tt = th + ((c0 < tl) ? 1 : 0); \ + c1 += tt; \ + c2 += (c1 < tt) ? 1 : 0; \ + c0 += tl; \ + th += (c0 < tl) ? 1 : 0; \ + c1 += th; \ + c2 += (c1 < th) ? 1 : 0; \ +} + +/** [c0,c1] += a */ +#define add2(c0,c1,a) { \ + c0 += (a); \ + c1 += (c0 < (a)) ? 1 : 0; \ +} + +#endif + +bool IsOverflow(const Num3072* d) +{ + for (int i = 1; i < Num3072::LIMBS - 1; ++i) { + if (d->limbs[i] != std::numeric_limits::max()) return false; + } + if (d->limbs[0] <= std::numeric_limits::max() - 1103717) return false; + return true; +} + +void FullReduce(Num3072* d) +{ + Num3072::limb_type c0 = 1103717; + for (int i = 0; i < Num3072::LIMBS; ++i) { + Num3072::limb_type c1 = 0; + add2(c0, c1, d->limbs[i]); + extract2(c0, c1, d->limbs[i]); + } +} + +void Multiply(Num3072* out, const Num3072* a, const Num3072* b) +{ + Num3072::limb_type c0 = 0, c1 = 0; + Num3072 tmp; + + /* Compute limbs 0..N-2 of a*b into tmp, including one reduction. */ + for (int j = 0; j < Num3072::LIMBS - 1; ++j) { + Num3072::limb_type d0 = 0, d1 = 0, d2 = 0, c2 = 0; + mul(d0, d1, a->limbs[1 + j], b->limbs[Num3072::LIMBS + j - (1 + j)]); + for (int i = 2 + j; i < Num3072::LIMBS; ++i) muladd3(d0, d1, d2, a->limbs[i], b->limbs[Num3072::LIMBS + j - i]); + mulnadd3(c0, c1, c2, d0, d1, d2, 1103717); + for (int i = 0; i < j + 1; ++i) muladd3(c0, c1, c2, a->limbs[i], b->limbs[j - i]); + extract3(c0, c1, c2, tmp.limbs[j]); + } + /* Compute limb N-1 of a*b into tmp. */ + { + Num3072::limb_type c2 = 0; + for (int i = 0; i < Num3072::LIMBS; ++i) muladd3(c0, c1, c2, a->limbs[i], b->limbs[Num3072::LIMBS - 1 - i]); + extract3(c0, c1, c2, tmp.limbs[Num3072::LIMBS - 1]); + } + /* Perform a second reduction. */ + muln2(c0, c1, 1103717); + for (int j = 0; j < Num3072::LIMBS; ++j) { + add2(c0, c1, tmp.limbs[j]); + extract2(c0, c1, out->limbs[j]); + } + assert(c1 == 0); + assert(c0 == 0 || c0 == 1); + /* Perform a potential third reduction. */ + if (c0) FullReduce(out); +} + +void Square(Num3072* out, const Num3072* a) +{ + Num3072::limb_type c0 = 0, c1 = 0; + Num3072 tmp; + + /* Compute limbs 0..N-2 of a*a into tmp, including one reduction. */ + for (int j = 0; j < Num3072::LIMBS - 1; ++j) { + Num3072::limb_type d0 = 0, d1 = 0, d2 = 0, c2 = 0; + for (int i = 0; i < (Num3072::LIMBS - 1 - j) / 2; ++i) muldbladd3(d0, d1, d2, a->limbs[i + j + 1], a->limbs[Num3072::LIMBS - 1 - i]); + if ((j + 1) & 1) muladd3(d0, d1, d2, a->limbs[(Num3072::LIMBS - 1 - j) / 2 + j + 1], a->limbs[Num3072::LIMBS - 1 - (Num3072::LIMBS - 1 - j) / 2]); + mulnadd3(c0, c1, c2, d0, d1, d2, 1103717); + for (int i = 0; i < (j + 1) / 2; ++i) muldbladd3(c0, c1, c2, a->limbs[i], a->limbs[j - i]); + if ((j + 1) & 1) muladd3(c0, c1, c2, a->limbs[(j + 1) / 2], a->limbs[j - (j + 1) / 2]); + extract3(c0, c1, c2, tmp.limbs[j]); + } + { + Num3072::limb_type c2 = 0; + for (int i = 0; i < Num3072::LIMBS / 2; ++i) muldbladd3(c0, c1, c2, a->limbs[i], a->limbs[Num3072::LIMBS - 1 - i]); + extract3(c0, c1, c2, tmp.limbs[Num3072::LIMBS - 1]); + } + /* Perform a second reduction. */ + muln2(c0, c1, 1103717); + for (int j = 0; j < Num3072::LIMBS; ++j) { + add2(c0, c1, tmp.limbs[j]); + extract2(c0, c1, out->limbs[j]); + } + assert(c1 == 0); + assert(c0 == 0 || c0 == 1); + /* Perform a potential third reduction. */ + if (c0) FullReduce(out); +} + +void Inverse(Num3072* out, const Num3072* a) +{ + Num3072 p[12]; // p[i] = a^(2^(2^i)-1) + Num3072 x; + + p[0] = *a; + + for (int i = 0; i < 11; ++i) { + p[i + 1] = p[i]; + for (int j = 0; j < (1 << i); ++j) Square(&p[i + 1], &p[i + 1]); + Multiply(&p[i + 1], &p[i + 1], &p[i]); + } + + x = p[11]; + + for (int j = 0; j < 512; ++j) Square(&x, &x); + Multiply(&x, &x, &p[9]); + for (int j = 0; j < 256; ++j) Square(&x, &x); + Multiply(&x, &x, &p[8]); + for (int j = 0; j < 128; ++j) Square(&x, &x); + Multiply(&x, &x, &p[7]); + for (int j = 0; j < 64; ++j) Square(&x, &x); + Multiply(&x, &x, &p[6]); + for (int j = 0; j < 32; ++j) Square(&x, &x); + Multiply(&x, &x, &p[5]); + for (int j = 0; j < 8; ++j) Square(&x, &x); + Multiply(&x, &x, &p[3]); + for (int j = 0; j < 2; ++j) Square(&x, &x); + Multiply(&x, &x, &p[1]); + for (int j = 0; j < 1; ++j) Square(&x, &x); + Multiply(&x, &x, &p[0]); + for (int j = 0; j < 5; ++j) Square(&x, &x); + Multiply(&x, &x, &p[2]); + for (int j = 0; j < 3; ++j) Square(&x, &x); + Multiply(&x, &x, &p[0]); + for (int j = 0; j < 2; ++j) Square(&x, &x); + Multiply(&x, &x, &p[0]); + for (int j = 0; j < 4; ++j) Square(&x, &x); + Multiply(&x, &x, &p[0]); + for (int j = 0; j < 4; ++j) Square(&x, &x); + Multiply(&x, &x, &p[1]); + for (int j = 0; j < 3; ++j) Square(&x, &x); + Multiply(&x, &x, &p[0]); + + *out = x; +} + +} + +MuHash3072::MuHash3072() noexcept +{ + data.limbs[0] = 1; + for (int i = 1; i < Num3072::LIMBS; ++i) data.limbs[i] = 0; +} + +MuHash3072::MuHash3072(const unsigned char* key32) noexcept +{ + unsigned char tmp[384]; + ChaCha20(key32, 32).Keystream(tmp, 384); + for (int i = 0; i < Num3072::LIMBS; ++i) { + if (sizeof(Num3072::limb_type) == 4) { + data.limbs[i] = ReadLE32(tmp + 4 * i); + } else if (sizeof(Num3072::limb_type) == 8) { + data.limbs[i] = ReadLE64(tmp + 8 * i); + } + } +} + +void MuHash3072::Finalize(unsigned char* hash384) noexcept +{ + if (IsOverflow(&data)) FullReduce(&data); + for (int i = 0; i < Num3072::LIMBS; ++i) { + if (sizeof(Num3072::limb_type) == 4) { + WriteLE32(hash384 + i * 4, data.limbs[i]); + } else if (sizeof(Num3072::limb_type) == 8) { + WriteLE64(hash384 + i * 8, data.limbs[i]); + } + } +} + +MuHash3072& MuHash3072::operator*=(const MuHash3072& x) noexcept +{ + Multiply(&this->data, &this->data, &x.data); + return *this; +} + +MuHash3072& MuHash3072::operator/=(const MuHash3072& x) noexcept +{ + Num3072 tmp; + Inverse(&tmp, &x.data); + Multiply(&this->data, &this->data, &tmp); + return *this; +} diff --git a/src/crypto/muhash.h b/src/crypto/muhash.h new file mode 100644 index 0000000000000..c9df5cd959cf0 --- /dev/null +++ b/src/crypto/muhash.h @@ -0,0 +1,62 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CRYPTO_MUHASH_H +#define BITCOIN_CRYPTO_MUHASH_H + +#if defined(HAVE_CONFIG_H) +#include "bitcoin-config.h" +#endif + +#include +#include + +struct Num3072 { +#ifdef HAVE___INT128 + typedef unsigned __int128 double_limb_type; + typedef uint64_t limb_type; + static constexpr int LIMBS = 48; + static constexpr int LIMB_SIZE = 64; +#else + typedef uint64_t double_limb_type; + typedef uint32_t limb_type; + static constexpr int LIMBS = 96; + static constexpr int LIMB_SIZE = 32; +#endif + limb_type limbs[LIMBS]; +}; + +/** A class representing MuHash sets */ +class MuHash3072 +{ +private: + Num3072 data; + +public: + /* The empty set. */ + MuHash3072() noexcept; + + /* A singleton with a single 32-byte key in it. */ + explicit MuHash3072(const unsigned char* key32) noexcept; + + /* Multiply (resulting in a hash for the union of the sets) */ + MuHash3072& operator*=(const MuHash3072& add) noexcept; + + /* Divide (resulting in a hash for the difference of the sets) */ + MuHash3072& operator/=(const MuHash3072& sub) noexcept; + + /* Finalize into a 384-byte hash. Does not change this object's value. */ + void Finalize(unsigned char* hash384) noexcept; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) { + for(int i = 0; i #include #include +#include #include #include #include @@ -191,6 +192,41 @@ class CHashVerifier : public CHashWriter } }; +/** A writer stream that computes a 256-bit truncated SHA512. */ +class TruncatedSHA512Writer +{ +private: + CSHA512 ctx; + + const int nType; + const int nVersion; +public: + + TruncatedSHA512Writer(int nTypeIn, int nVersionIn) : nType(nTypeIn), nVersion(nVersionIn) {} + + int GetType() const { return nType; } + int GetVersion() const { return nVersion; } + + void write(const char *pch, size_t size) { + ctx.Write((const unsigned char*)pch, size); + } + + uint256 GetHash() { + unsigned char out[64]; + ctx.Finalize(out); + uint256 result; + memcpy((unsigned char*)&result, out, 32); + return result; + } + + template + TruncatedSHA512Writer& operator<<(const T& obj) { + // Serialize to this stream + ::Serialize(*this, obj); + return (*this); + } +}; + /** Compute the 256-bit hash of an object's serialization. */ template uint256 SerializeHash(const T& obj, int nType=SER_GETHASH, int nVersion=PROTOCOL_VERSION) diff --git a/src/index/utxosethash.cpp b/src/index/utxosethash.cpp new file mode 100644 index 0000000000000..b33d47b1d4197 --- /dev/null +++ b/src/index/utxosethash.cpp @@ -0,0 +1,342 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +constexpr char DB_BLOCK_HASH = 's'; +constexpr char DB_BLOCK_HEIGHT = 't'; +constexpr char DB_MUHASH = 'M'; + +namespace { + +struct DBVal { + uint256 muhash; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITE(muhash); + } +}; + +struct DBHeightKey { + int height; + + DBHeightKey() : height(0) {} + explicit DBHeightKey(int height_in) : height(height_in) {} + + template + void Serialize(Stream& s) const + { + ser_writedata8(s, DB_BLOCK_HEIGHT); + ser_writedata32be(s, height); + } + + template + void Unserialize(Stream& s) + { + char prefix = ser_readdata8(s); + if (prefix != DB_BLOCK_HEIGHT) { + throw std::ios_base::failure("Invalid format for block filter index DB height key"); + } + height = ser_readdata32be(s); + } +}; + +struct DBHashKey { + uint256 block_hash; + + explicit DBHashKey(const uint256& hash_in) : block_hash(hash_in) {} + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) { + char prefix = DB_BLOCK_HASH; + READWRITE(prefix); + if (prefix != DB_BLOCK_HASH) { + throw std::ios_base::failure("Invalid format for block filter index DB hash key"); + } + + READWRITE(block_hash); + } +}; + +struct DBMuhash { + MuHash3072 hash; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITE(hash); + } +}; + +}; // namespace + +std::unique_ptr g_utxo_set_hash; + +UtxoSetHash::UtxoSetHash(size_t n_cache_size, bool f_memory, bool f_wipe) +{ + fs::path path = GetDataDir() / "indexes" / "utxo_set_hash"; + fs::create_directories(path); + + m_db = MakeUnique(path / "db", n_cache_size, f_memory, f_wipe); +} + +bool UtxoSetHash::Init() +{ + if (!m_db->Read(DB_MUHASH, m_muhash)) { + // Check that the cause of the read failure is that the key does not exist. Any other errors + // indicate database corruption or a disk failure, and starting the index would cause + // further corruption. + if (m_db->Exists(DB_MUHASH)) { + return error("%s: Cannot read current %s state; index may be corrupted", + __func__, GetName()); + } + + // If the DB_MUHASH is not set, initialize empty muhash + m_muhash = MuHash3072(); + } + + return BaseIndex::Init(); +} + +bool UtxoSetHash::WriteBlock(const CBlock& block, const CBlockIndex* pindex) +{ + CBlockUndo block_undo; + + if (pindex->nHeight > 0) { + if (!UndoReadFromDisk(block_undo, pindex)) { + return false; + } + + std::pair read_out; + if (!m_db->Read(DBHeightKey(pindex->nHeight - 1), read_out)) { + return false; + } + + uint256 expected_block_hash = pindex->pprev->GetBlockHash(); + if (read_out.first != expected_block_hash) { + return error("%s: previous block header belongs to unexpected block %s; expected %s", + __func__, read_out.first.ToString(), expected_block_hash.ToString()); + } + } + + // Add the new utxos created from the block + for (size_t i = 0; i < block.vtx.size(); ++i) { + const auto& tx = block.vtx.at(i); + + for (size_t j = 0; j < tx->vout.size(); ++j) { + const CTxOut& out = tx->vout[j]; + COutPoint outpoint = COutPoint(tx->GetHash(), j); + Coin coin = Coin(out, pindex->nHeight, tx->IsCoinBase()); + + TruncatedSHA512Writer ss(SER_DISK, 0); + ss << outpoint; + ss << (uint32_t)(coin.nHeight * 2 + coin.fCoinBase); + ss << coin.out; + m_muhash *= MuHash3072(ss.GetHash().begin()); + } + + // The coinbase tx has no undo data since no former output is spent + if (i > 0) { + const auto& tx_undo = block_undo.vtxundo.at(i-1); + + for (size_t j = 0; j < tx_undo.vprevout.size(); ++j) { + Coin coin = tx_undo.vprevout[j]; + COutPoint outpoint = COutPoint(tx->vin[j].prevout.hash, tx->vin[j].prevout.n); + + TruncatedSHA512Writer ss(SER_DISK, 0); + ss << outpoint; + ss << (uint32_t)(coin.nHeight * 2 + coin.fCoinBase); + ss << coin.out; + m_muhash /= MuHash3072(ss.GetHash().begin()); + } + } + } + + std::pair value; + value.first = pindex->GetBlockHash(); + value.second.muhash = currentHashInternal(); + + if (!m_db->Write(DBHeightKey(pindex->nHeight), value)) { + return false; + } + + if (!m_db->Write(DB_MUHASH, m_muhash)) { + return false; + } + + return true; +} + +static bool CopyHeightIndexToHashIndex(CDBIterator& db_it, CDBBatch& batch, + const std::string& index_name, + int start_height, int stop_height) +{ + DBHeightKey key(start_height); + db_it.Seek(key); + + for (int height = start_height; height <= stop_height; ++height) { + if (!db_it.GetKey(key) || key.height != height) { + return error("%s: unexpected key in %s: expected (%c, %d)", + __func__, index_name, DB_BLOCK_HEIGHT, height); + } + + std::pair value; + if (!db_it.GetValue(value)) { + return error("%s: unable to read value in %s at key (%c, %d)", + __func__, index_name, DB_BLOCK_HEIGHT, height); + } + + batch.Write(DBHashKey(value.first), std::move(value.second)); + + db_it.Next(); + } + return true; +} + +bool UtxoSetHash::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) +{ + assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip); + + CDBBatch batch(*m_db); + std::unique_ptr db_it(m_db->NewIterator()); + + CBlockIndex* iter_tip = LookupBlockIndex(current_tip->GetBlockHash()); + auto& consensus_params = Params().GetConsensus(); + + do { + CBlock block; + + if (!ReadBlockFromDisk(block, iter_tip, consensus_params)) { + return error("%s: Failed to read block %s from disk", + __func__, iter_tip->GetBlockHash().ToString()); + } + + ReverseBlock(block, iter_tip); + + iter_tip = iter_tip->GetAncestor(iter_tip->nHeight - 1); + } while (new_tip != iter_tip); + + // During a reorg, we need to copy all hash digests for blocks that are getting disconnected from the + // height index to the hash index so we can still find them when the height index entries are + // overwritten. + if (!CopyHeightIndexToHashIndex(*db_it, batch, m_name, new_tip->nHeight, current_tip->nHeight)) { + return false; + } + + if (!m_db->WriteBatch(batch)) return false; + + return BaseIndex::Rewind(current_tip, new_tip); +} + +static bool LookupOne(const CDBWrapper& db, const CBlockIndex* block_index, DBVal& result) +{ + // First check if the result is stored under the height index and the value there matches the + // block hash. This should be the case if the block is on the active chain. + std::pair read_out; + if (!db.Read(DBHeightKey(block_index->nHeight), read_out)) { + return false; + } + if (read_out.first == block_index->GetBlockHash()) { + result = std::move(read_out.second); + return true; + } + + // If value at the height index corresponds to an different block, the result will be stored in + // the hash index. + return db.Read(DBHashKey(block_index->GetBlockHash()), result); +} + +bool UtxoSetHash::LookupHash(const CBlockIndex* block_index, uint256& utxo_set_hash) const +{ + DBVal entry; + if (!LookupOne(*m_db, block_index, entry)) { + return false; + } + + utxo_set_hash = entry.muhash; + return true; +} + +uint256 UtxoSetHash::currentHashInternal() +{ + unsigned char out[384]; + m_muhash.Finalize(out); + return (TruncatedSHA512Writer(SER_DISK, 0) << out).GetHash(); +} + +// Reverse Block in case of reorg +bool UtxoSetHash::ReverseBlock(const CBlock& block, const CBlockIndex* pindex) +{ + CBlockUndo block_undo; + + if (pindex->nHeight > 0) { + if (!UndoReadFromDisk(block_undo, pindex)) { + return false; + } + + std::pair read_out; + if (!m_db->Read(DBHeightKey(pindex->nHeight - 1), read_out)) { + return false; + } + + uint256 expected_block_hash = pindex->pprev->GetBlockHash(); + if (read_out.first != expected_block_hash) { + return error("%s: previous block header belongs to unexpected block %s; expected %s", + __func__, read_out.first.ToString(), expected_block_hash.ToString()); + } + } + + // Add the new utxos created from the block + for (size_t i = 0; i < block.vtx.size(); ++i) { + const auto& tx = block.vtx.at(i); + + for (size_t j = 0; j < tx->vout.size(); ++j) { + const CTxOut& out = tx->vout[j]; + COutPoint outpoint = COutPoint(tx->GetHash(), j); + Coin coin = Coin(out, pindex->nHeight, tx->IsCoinBase()); + + TruncatedSHA512Writer ss(SER_DISK, 0); + ss << outpoint; + ss << (uint32_t)(coin.nHeight * 2 + coin.fCoinBase); + ss << coin.out; + m_muhash /= MuHash3072(ss.GetHash().begin()); + } + + // The coinbase tx has no undo data since no former output is spent + if (i > 0) { + const auto& tx_undo = block_undo.vtxundo.at(i-1); + + for (size_t j = 0; j < tx_undo.vprevout.size(); ++j) { + Coin coin = tx_undo.vprevout[j]; + COutPoint outpoint = COutPoint(tx->vin[j].prevout.hash, tx->vin[j].prevout.n); + + TruncatedSHA512Writer ss(SER_DISK, 0); + ss << outpoint; + ss << (uint32_t)(coin.nHeight * 2 + coin.fCoinBase); + ss << coin.out; + m_muhash *= MuHash3072(ss.GetHash().begin()); + } + } + } + + if (!m_db->Write(DB_MUHASH, m_muhash)) { + return false; + } + + return true; +} diff --git a/src/index/utxosethash.h b/src/index/utxosethash.h new file mode 100644 index 0000000000000..57af68c6ce394 --- /dev/null +++ b/src/index/utxosethash.h @@ -0,0 +1,51 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_INDEX_UTXOSETHASH_H +#define BITCOIN_INDEX_UTXOSETHASH_H + +#include +#include +#include +#include + +/** + * UtxoHashSet maintains a rolling hash of the utxo set and + * caches a hash digest for every block. + */ +class UtxoSetHash final : public BaseIndex +{ +private: + std::string m_name; + std::unique_ptr m_db; + MuHash3072 m_muhash; + + // Digest of the current Muhash object + uint256 currentHashInternal(); + + // Roll back the Muhash of a particular block + bool ReverseBlock(const CBlock& block, const CBlockIndex* pindex); +protected: + bool Init() override; + + bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override; + + bool Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) override; + + BaseIndex::DB& GetDB() const override { return *m_db; } + + const char* GetName() const override { return "utxosethash"; } + +public: + // Constructs the index, which becomes available to be queried. + explicit UtxoSetHash(size_t n_cache_size, bool f_memory = false, bool f_wipe = false); + + // Look up hash digest for a specific block using CBlockIndex + bool LookupHash(const CBlockIndex* block_index, uint256& utxo_set_hash) const; +}; + +/// The global UTXO set hash object. +extern std::unique_ptr g_utxo_set_hash; + +#endif // BITCOIN_INDEX_UTXOSETHASH_H diff --git a/src/init.cpp b/src/init.cpp index 042335b8e74c8..34300e428e901 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -1443,6 +1444,7 @@ bool AppInitMain(InitInterfaces& interfaces) filter_index_cache = max_cache / n_indexes; nTotalCache -= filter_index_cache * n_indexes; } + int64_t utsh_cache = 0; int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache nCoinDBCache = std::min(nCoinDBCache, nMaxCoinsDBCache << 20); // cap total coins db cache nTotalCache -= nCoinDBCache; @@ -1661,6 +1663,9 @@ bool AppInitMain(InitInterfaces& interfaces) GetBlockFilterIndex(filter_type)->Start(); } + g_utxo_set_hash = MakeUnique(utsh_cache, false, fReindex); + g_utxo_set_hash->Start(); + // ********************************************************* Step 9: load wallet for (const auto& client : interfaces.chain_clients) { if (!client->load()) { diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index a74003149d374..397570141f657 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -35,6 +36,7 @@ #include #include + #include #include @@ -916,29 +918,20 @@ struct CCoinsStats uint64_t nTransactions; uint64_t nTransactionOutputs; uint64_t nBogoSize; - uint256 hashSerialized; + uint256 muhash; uint64_t nDiskSize; CAmount nTotalAmount; CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nBogoSize(0), nDiskSize(0), nTotalAmount(0) {} }; -static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, const std::map& outputs) +static void ApplyStats(CCoinsStats &stats, const COutPoint outpoint, const Coin& coin) { - assert(!outputs.empty()); - ss << hash; - ss << VARINT(outputs.begin()->second.nHeight * 2 + outputs.begin()->second.fCoinBase ? 1u : 0u); stats.nTransactions++; - for (const auto& output : outputs) { - ss << VARINT(output.first + 1); - ss << output.second.out.scriptPubKey; - ss << VARINT(output.second.out.nValue, VarIntMode::NONNEGATIVE_SIGNED); - stats.nTransactionOutputs++; - stats.nTotalAmount += output.second.out.nValue; - stats.nBogoSize += 32 /* txid */ + 4 /* vout index */ + 4 /* height + coinbase */ + 8 /* amount */ + - 2 /* scriptPubKey len */ + output.second.out.scriptPubKey.size() /* scriptPubKey */; - } - ss << VARINT(0u); + stats.nTransactionOutputs++; + stats.nTotalAmount += coin.out.nValue; + stats.nBogoSize += 32 /* txid */ + 4 /* vout index */ + 4 /* height + coinbase */ + 8 /* amount */ + + 2 /* scriptPubKey len */ + coin.out.scriptPubKey.size() /* scriptPubKey */; } //! Calculate statistics about the unspent transaction output set @@ -947,35 +940,42 @@ static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) std::unique_ptr pcursor(view->Cursor()); assert(pcursor); - CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); + uint256 muhash_buf; + const CBlockIndex* block_index; stats.hashBlock = pcursor->GetBestBlock(); { LOCK(cs_main); - stats.nHeight = LookupBlockIndex(stats.hashBlock)->nHeight; + block_index = LookupBlockIndex(stats.hashBlock); + } + + stats.nHeight = block_index->nHeight; + + bool index_ready = g_utxo_set_hash->BlockUntilSyncedToCurrentChain(); + + if (!g_utxo_set_hash->LookupHash(block_index, muhash_buf)) { + int err_code = RPC_MISC_ERROR; + + if (!index_ready) { + throw JSONRPCError(err_code, "UTXO set hash is still in the process of being indexed."); + } else { + throw JSONRPCError(err_code, "Unable to get UTXO set hash"); + } } - ss << stats.hashBlock; - uint256 prevkey; - std::map outputs; + + stats.muhash = muhash_buf; + while (pcursor->Valid()) { boost::this_thread::interruption_point(); COutPoint key; Coin coin; if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { - if (!outputs.empty() && key.hash != prevkey) { - ApplyStats(stats, ss, prevkey, outputs); - outputs.clear(); - } - prevkey = key.hash; - outputs[key.n] = std::move(coin); + ApplyStats(stats, key, coin); } else { return error("%s: unable to read value", __func__); } pcursor->Next(); } - if (!outputs.empty()) { - ApplyStats(stats, ss, prevkey, outputs); - } - stats.hashSerialized = ss.GetHash(); + stats.nDiskSize = view->EstimateSize(); return true; } @@ -1049,7 +1049,7 @@ static UniValue gettxoutsetinfo(const JSONRPCRequest& request) " \"transactions\": n, (numeric) The number of transactions with unspent outputs\n" " \"txouts\": n, (numeric) The number of unspent transaction outputs\n" " \"bogosize\": n, (numeric) A meaningless metric for UTXO set size\n" - " \"hash_serialized_2\": \"hash\", (string) The serialized hash\n" + " \"muhash\": \"hash\", (string) Rolling UTXO set hash\n" " \"disk_size\": n, (numeric) The estimated size of the chainstate on disk\n" " \"total_amount\": x.xxx (numeric) The total amount\n" "}\n" @@ -1072,7 +1072,7 @@ static UniValue gettxoutsetinfo(const JSONRPCRequest& request) ret.pushKV("transactions", (int64_t)stats.nTransactions); ret.pushKV("txouts", (int64_t)stats.nTransactionOutputs); ret.pushKV("bogosize", (int64_t)stats.nBogoSize); - ret.pushKV("hash_serialized_2", stats.hashSerialized.GetHex()); + ret.pushKV("muhash", stats.muhash.GetHex()); ret.pushKV("disk_size", stats.nDiskSize); ret.pushKV("total_amount", ValueFromAmount(stats.nTotalAmount)); } else { diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index 4ac12bf9698c9..a01ed23fafca7 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -749,4 +750,58 @@ BOOST_AUTO_TEST_CASE(sha256d64) } } +static MuHash3072 FromInt(unsigned char i) { + unsigned char tmp[32] = {i, 0}; + return MuHash3072(tmp); +} + +BOOST_AUTO_TEST_CASE(muhash_tests) +{ + unsigned char out[384]; + + for (int iter = 0; iter < 10; ++iter) { + unsigned char res[384]; + int table[4]; + for (int i = 0; i < 4; ++i) { + table[i] = g_insecure_rand_ctx.randbits(3); + } + for (int order = 0; order < 4; ++order) { + MuHash3072 acc; + for (int i = 0; i < 4; ++i) { + int t = table[i ^ order]; + if (t & 4) { + acc /= FromInt(t & 3); + } else { + acc *= FromInt(t & 3); + } + } + acc.Finalize(out); + if (order == 0) { + memcpy(res, out, 384); + } else { + BOOST_CHECK(memcmp(res, out, 384) == 0); + } + } + + MuHash3072 x = FromInt(g_insecure_rand_ctx.randbits(4)); // x=X + MuHash3072 y = FromInt(g_insecure_rand_ctx.randbits(4)); // x=X, y=Y + MuHash3072 z; // x=X, y=Y, z=1 + z *= x; // x=X, y=Y, z=X + z /= y; // x=X, y=Y, z=X/Y + y /= x; // x=X, y=Y/X, z=X/Y + z *= y; // x=X, y=Y/X, z=1 + z.Finalize(out); + for (int i = 0; i < 384; ++i) { + BOOST_CHECK_EQUAL(out[i], i == 0); + } + } + + MuHash3072 acc = FromInt(0); + acc *= FromInt(1); + acc /= FromInt(2); + acc.Finalize(out); + uint256 x = (TruncatedSHA512Writer(SER_DISK, 0) << out).GetHash(); + BOOST_CHECK(x == uint256S("0e94c56c180f27fd6b182f091c5b007e2d6eba5ae28daa5aa92d2af8c26ea9a6")); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/utxosethash_tests.cpp b/src/test/utxosethash_tests.cpp new file mode 100644 index 0000000000000..66d977028cfde --- /dev/null +++ b/src/test/utxosethash_tests.cpp @@ -0,0 +1,70 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +// +#include + +BOOST_AUTO_TEST_SUITE(utxosethash_tests) + +BOOST_FIXTURE_TEST_CASE(utxosethash_initial_sync, TestChain100Setup) +{ + UtxoSetHash utxo_set_hash(0, false); + + uint256 hash_digest; + CBlockIndex* block_index = ::ChainActive().Tip(); + + // UTXO set hash should not be found before it is started. + BOOST_CHECK(!utxo_set_hash.LookupHash(block_index, hash_digest)); + + // BlockUntilSyncedToCurrentChain should return false before utxo_set_hash is started. + BOOST_CHECK(!utxo_set_hash.BlockUntilSyncedToCurrentChain()); + + utxo_set_hash.Start(); + + // Allow the UTXO set hash to catch up with the block index. + constexpr int64_t timeout_ms = 10 * 1000; + int64_t time_start = GetTimeMillis(); + while (!utxo_set_hash.BlockUntilSyncedToCurrentChain()) { + BOOST_REQUIRE(time_start + timeout_ms > GetTimeMillis()); + MilliSleep(100); + } + + // Check that UTXO set hash works for genesis block. + CBlockIndex* genesis_block_index = ::ChainActive().Genesis(); + BOOST_CHECK(utxo_set_hash.LookupHash(genesis_block_index, hash_digest)); + + // Check that UTXO set hash updates with new blocks. + block_index = ::ChainActive().Tip(); + utxo_set_hash.LookupHash(block_index, hash_digest); + + CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG; + std::vector noTxns; + CreateAndProcessBlock(noTxns, scriptPubKey); + + time_start = GetTimeMillis(); + while (!utxo_set_hash.BlockUntilSyncedToCurrentChain()) { + BOOST_REQUIRE(time_start + timeout_ms > GetTimeMillis()); + MilliSleep(100); + } + + uint256 new_hash_digest; + CBlockIndex* new_block_index = ::ChainActive().Tip(); + utxo_set_hash.LookupHash(new_block_index, new_hash_digest); + + BOOST_CHECK(block_index != new_block_index); + BOOST_CHECK(hash_digest != new_hash_digest); + + // shutdown sequence (c.f. Shutdown() in init.cpp) + utxo_set_hash.Stop(); + + threadGroup.interrupt_all(); + threadGroup.join_all(); + + // Rest of shutdown sequence and destructors happen in ~TestingSetup() +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index 6c30e050847e1..ce39fe6371d82 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -31,6 +31,7 @@ assert_raises_rpc_error, assert_is_hex_string, assert_is_hash_string, + wait_until, ) from test_framework.blocktools import ( create_block, @@ -62,6 +63,7 @@ def run_test(self): self._test_getnetworkhashps() self._test_stopatheight() self._test_waitforblockheight() + self._test_muhash() assert self.nodes[0].verifychain(4, 0) def mine_chain(self): @@ -73,6 +75,10 @@ def mine_chain(self): self.nodes[0].generatetoaddress(1, address) assert_equal(self.nodes[0].getblockchaininfo()['blocks'], 200) + def mine_block(self): + address = self.nodes[0].get_deterministic_priv_key().address + self.nodes[0].generatetoaddress(1, address) + def _test_getblockchaininfo(self): self.log.info("Test getblockchaininfo") @@ -202,6 +208,8 @@ def _test_getchaintxstats(self): def _test_gettxoutsetinfo(self): node = self.nodes[0] + + wait_until(lambda: self.nodes[0].getblockcount() == 200) res = node.gettxoutsetinfo() assert_equal(res['total_amount'], Decimal('8725.00000000')) @@ -214,7 +222,7 @@ def _test_gettxoutsetinfo(self): assert size > 6400 assert size < 64000 assert_equal(len(res['bestblock']), 64) - assert_equal(len(res['hash_serialized_2']), 64) + assert_equal(len(res['muhash']), 64) self.log.info("Test that gettxoutsetinfo() works for blockchain with just the genesis block") b1hash = node.getblockhash(1) @@ -227,11 +235,10 @@ def _test_gettxoutsetinfo(self): assert_equal(res2['txouts'], 0) assert_equal(res2['bogosize'], 0), assert_equal(res2['bestblock'], node.getblockhash(0)) - assert_equal(len(res2['hash_serialized_2']), 64) + assert_equal(len(res2['muhash']), 64) self.log.info("Test that gettxoutsetinfo() returns the same result after invalidate/reconsider block") node.reconsiderblock(b1hash) - res3 = node.gettxoutsetinfo() # The field 'disk_size' is non-deterministic and can thus not be # compared between res and res3. Everything else should be the same. @@ -331,6 +338,31 @@ def assert_waitforheight(height, timeout=2): assert_waitforheight(current_height) assert_waitforheight(current_height + 1) + def _test_muhash(self): + self.restart_node(0) + node = self.nodes[0] + + self.log.info("Test that gettxoutsetinfo() muhash is unchanged when rolling back a new block") + + # Test consistency of hashing + res = node.gettxoutsetinfo() + muhash_at_207 = res['muhash'] + assert(node.gettxoutsetinfo()['muhash'] == muhash_at_207) + + # Hash is updated with new block + self.mine_block() + assert(node.gettxoutsetinfo()['muhash'] != muhash_at_207) + + # Hash is rolled back to previous block if invalidated + b208hash = node.getblockhash(208) + node.invalidateblock(b208hash) + assert(node.gettxoutsetinfo()['muhash'] == muhash_at_207) + + # Hash persists restart + self.stop_node(0) + self.start_node(0) + assert(node.gettxoutsetinfo()['muhash'] == muhash_at_207) + if __name__ == '__main__': BlockchainTest().main()