From d55a51d49a3fbcef2211b9c9f3acaa2514f6c887 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Thu, 18 May 2017 15:50:31 -0700 Subject: [PATCH 01/10] MuHash3072 implementation --- configure.ac | 3 + src/Makefile.am | 2 + src/bench/crypto_hash.cpp | 16 +++ src/crypto/muhash.cpp | 273 ++++++++++++++++++++++++++++++++++++++ src/crypto/muhash.h | 52 ++++++++ src/test/crypto_tests.cpp | 55 ++++++++ 6 files changed, 401 insertions(+) create mode 100644 src/crypto/muhash.cpp create mode 100644 src/crypto/muhash.h diff --git a/configure.ac b/configure.ac index 35bb0c0231525..5780c5cd163ef 100644 --- a/configure.ac +++ b/configure.ac @@ -662,6 +662,9 @@ if test x$use_lcov_branch != xno; then AC_SUBST(LCOV_OPTS, "$LCOV_OPTS --rc lcov_branch_coverage=1") fi +dnl Check for __int128 +AC_CHECK_TYPES([__int128]) + dnl Check for endianness AC_C_BIGENDIAN diff --git a/src/Makefile.am b/src/Makefile.am index 477b1300bc651..d2ecb4ead2683 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -369,6 +369,8 @@ crypto_libbitcoin_crypto_base_a_SOURCES = \ crypto/hmac_sha512.h \ crypto/poly1305.h \ crypto/poly1305.cpp \ + crypto/muhash.cpp \ + crypto/muhash.h \ crypto/ripemd160.cpp \ crypto/ripemd160.h \ crypto/sha1.cpp \ diff --git a/src/bench/crypto_hash.cpp b/src/bench/crypto_hash.cpp index fb2bab9dee7a1..dffc3d73de7b4 100644 --- a/src/bench/crypto_hash.cpp +++ b/src/bench/crypto_hash.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -92,6 +93,19 @@ static void FastRandom_1bit(benchmark::State& state) } } +static void MuHash(benchmark::State& state) +{ + FastRandomContext rng(true); + MuHash3072 acc; + unsigned char key[32] = {0}; + while (state.KeepRunning()) { + for (int i = 0; i < 1000; i++) { + key[0] = i; + acc *= MuHash3072(key); + } + } +} + BENCHMARK(RIPEMD160, 440); BENCHMARK(SHA1, 570); BENCHMARK(SHA256, 340); @@ -102,3 +116,5 @@ BENCHMARK(SipHash_32b, 40 * 1000 * 1000); BENCHMARK(SHA256D64_1024, 7400); BENCHMARK(FastRandom_32bit, 110 * 1000 * 1000); BENCHMARK(FastRandom_1bit, 440 * 1000 * 1000); + +BENCHMARK(MuHash); diff --git a/src/crypto/muhash.cpp b/src/crypto/muhash.cpp new file mode 100644 index 0000000000000..91491c3d006bc --- /dev/null +++ b/src/crypto/muhash.cpp @@ -0,0 +1,273 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "muhash.h" + +#include +#include "common.h" +#include "chacha20.h" + +#include +#include + +namespace { + +/** Extract the lowest limb of [c0,c1,c2] into n, and left shift the number by 1 limb. */ +#define extract3(c0,c1,c2,n) { \ + (n) = c0; \ + c0 = c1; \ + c1 = c2; \ + c2 = 0; \ +} + +/** Extract the lowest limb of [c0,c1] into n, and left shift the number by 1 limb. */ +#define extract2(c0,c1,n) { \ + (n) = c0; \ + c0 = c1; \ + c1 = 0; \ +} + +/** [c0,c1] = a * b */ +#define mul(c0,c1,a,b) { \ + Num3072::double_limb_type t = (Num3072::double_limb_type)a * b; \ + c2 = 0; \ + c1 = t >> Num3072::LIMB_SIZE; \ + c0 = t; \ +} + +/* [c0,c1,c2] += n * [d0,d1,d2]. c2 is 0 initially */ +#define mulnadd3(c0,c1,c2,d0,d1,d2,n) { \ + Num3072::double_limb_type t = (Num3072::double_limb_type)d0 * n + c0; \ + c0 = t; \ + t >>= Num3072::LIMB_SIZE; \ + t += (Num3072::double_limb_type)d1 * n + c1; \ + c1 = t; \ + t >>= Num3072::LIMB_SIZE; \ + c2 = t + d2 * n; \ +} + +/* [c0,c1] *= n */ +#define muln2(c0,c1,n) { \ + Num3072::double_limb_type t = (Num3072::double_limb_type)c0 * n; \ + c0 = t; \ + t >>= Num3072::LIMB_SIZE; \ + t += (Num3072::double_limb_type)c1 * n; \ + c1 = t; \ + t >>= Num3072::LIMB_SIZE; \ +} + +/** [c0,c1,c2] += a * b */ +#define muladd3(c0,c1,c2,a,b) { \ + Num3072::limb_type tl, th; \ + { \ + Num3072::double_limb_type t = (Num3072::double_limb_type)a * b; \ + th = t >> Num3072::LIMB_SIZE; \ + tl = t; \ + } \ + c0 += tl; \ + th += (c0 < tl) ? 1 : 0; \ + c1 += th; \ + c2 += (c1 < th) ? 1 : 0; \ +} + +/** [c0,c1,c2] += 2 * a * b */ +#define muldbladd3(c0,c1,c2,a,b) { \ + Num3072::limb_type tl, th; \ + { \ + Num3072::double_limb_type t = (Num3072::double_limb_type)a * b; \ + th = t >> Num3072::LIMB_SIZE; \ + tl = t; \ + } \ + c0 += tl; \ + Num3072::limb_type tt = th + ((c0 < tl) ? 1 : 0); \ + c1 += tt; \ + c2 += (c1 < tt) ? 1 : 0; \ + c0 += tl; \ + th += (c0 < tl) ? 1 : 0; \ + c1 += th; \ + c2 += (c1 < th) ? 1 : 0; \ +} + +/** [c0,c1] += a */ +#define add2(c0,c1,a) { \ + c0 += (a); \ + c1 += (c0 < (a)) ? 1 : 0; \ +} + +bool IsOverflow(const Num3072* d) +{ + for (int i = 1; i < Num3072::LIMBS - 1; ++i) { + if (d->limbs[i] != std::numeric_limits::max()) return false; + } + if (d->limbs[0] <= std::numeric_limits::max() - 1103717) return false; + return true; +} + +void FullReduce(Num3072* d) +{ + Num3072::limb_type c0 = 1103717; + for (int i = 0; i < Num3072::LIMBS; ++i) { + Num3072::limb_type c1 = 0; + add2(c0, c1, d->limbs[i]); + extract2(c0, c1, d->limbs[i]); + } +} + +void Multiply(Num3072* out, const Num3072* a, const Num3072* b) +{ + Num3072::limb_type c0 = 0, c1 = 0; + Num3072 tmp; + + /* Compute limbs 0..N-2 of a*b into tmp, including one reduction. */ + for (int j = 0; j < Num3072::LIMBS - 1; ++j) { + Num3072::limb_type d0 = 0, d1 = 0, d2 = 0, c2 = 0; + mul(d0, d1, a->limbs[1 + j], b->limbs[Num3072::LIMBS + j - (1 + j)]); + for (int i = 2 + j; i < Num3072::LIMBS; ++i) muladd3(d0, d1, d2, a->limbs[i], b->limbs[Num3072::LIMBS + j - i]); + mulnadd3(c0, c1, c2, d0, d1, d2, 1103717); + for (int i = 0; i < j + 1; ++i) muladd3(c0, c1, c2, a->limbs[i], b->limbs[j - i]); + extract3(c0, c1, c2, tmp.limbs[j]); + } + /* Compute limb N-1 of a*b into tmp. */ + { + Num3072::limb_type c2 = 0; + for (int i = 0; i < Num3072::LIMBS; ++i) muladd3(c0, c1, c2, a->limbs[i], b->limbs[Num3072::LIMBS - 1 - i]); + extract3(c0, c1, c2, tmp.limbs[Num3072::LIMBS - 1]); + } + /* Perform a second reduction. */ + muln2(c0, c1, 1103717); + for (int j = 0; j < Num3072::LIMBS; ++j) { + add2(c0, c1, tmp.limbs[j]); + extract2(c0, c1, out->limbs[j]); + } + assert(c1 == 0); + assert(c0 == 0 || c0 == 1); + /* Perform a potential third reduction. */ + if (c0) FullReduce(out); +} + +void Square(Num3072* out, const Num3072* a) +{ + Num3072::limb_type c0 = 0, c1 = 0; + Num3072 tmp; + + /* Compute limbs 0..N-2 of a*a into tmp, including one reduction. */ + for (int j = 0; j < Num3072::LIMBS - 1; ++j) { + Num3072::limb_type d0 = 0, d1 = 0, d2 = 0, c2 = 0; + for (int i = 0; i < (Num3072::LIMBS - 1 - j) / 2; ++i) muldbladd3(d0, d1, d2, a->limbs[i + j + 1], a->limbs[Num3072::LIMBS - 1 - i]); + if ((j + 1) & 1) muladd3(d0, d1, d2, a->limbs[(Num3072::LIMBS - 1 - j) / 2 + j + 1], a->limbs[Num3072::LIMBS - 1 - (Num3072::LIMBS - 1 - j) / 2]); + mulnadd3(c0, c1, c2, d0, d1, d2, 1103717); + for (int i = 0; i < (j + 1) / 2; ++i) muldbladd3(c0, c1, c2, a->limbs[i], a->limbs[j - i]); + if ((j + 1) & 1) muladd3(c0, c1, c2, a->limbs[(j + 1) / 2], a->limbs[j - (j + 1) / 2]); + extract3(c0, c1, c2, tmp.limbs[j]); + } + { + Num3072::limb_type c2 = 0; + for (int i = 0; i < Num3072::LIMBS / 2; ++i) muldbladd3(c0, c1, c2, a->limbs[i], a->limbs[Num3072::LIMBS - 1 - i]); + extract3(c0, c1, c2, tmp.limbs[Num3072::LIMBS - 1]); + } + /* Perform a second reduction. */ + muln2(c0, c1, 1103717); + for (int j = 0; j < Num3072::LIMBS; ++j) { + add2(c0, c1, tmp.limbs[j]); + extract2(c0, c1, out->limbs[j]); + } + assert(c1 == 0); + assert(c0 == 0 || c0 == 1); + /* Perform a potential third reduction. */ + if (c0) FullReduce(out); +} + +void Inverse(Num3072* out, const Num3072* a) +{ + Num3072 p[12]; // p[i] = a^(2^(2^i)-1) + Num3072 x; + + p[0] = *a; + + for (int i = 0; i < 11; ++i) { + p[i + 1] = p[i]; + for (int j = 0; j < (1 << i); ++j) Square(&p[i + 1], &p[i + 1]); + Multiply(&p[i + 1], &p[i + 1], &p[i]); + } + + x = p[11]; + + for (int j = 0; j < 512; ++j) Square(&x, &x); + Multiply(&x, &x, &p[9]); + for (int j = 0; j < 256; ++j) Square(&x, &x); + Multiply(&x, &x, &p[8]); + for (int j = 0; j < 128; ++j) Square(&x, &x); + Multiply(&x, &x, &p[7]); + for (int j = 0; j < 64; ++j) Square(&x, &x); + Multiply(&x, &x, &p[6]); + for (int j = 0; j < 32; ++j) Square(&x, &x); + Multiply(&x, &x, &p[5]); + for (int j = 0; j < 8; ++j) Square(&x, &x); + Multiply(&x, &x, &p[3]); + for (int j = 0; j < 2; ++j) Square(&x, &x); + Multiply(&x, &x, &p[1]); + for (int j = 0; j < 1; ++j) Square(&x, &x); + Multiply(&x, &x, &p[0]); + for (int j = 0; j < 5; ++j) Square(&x, &x); + Multiply(&x, &x, &p[2]); + for (int j = 0; j < 3; ++j) Square(&x, &x); + Multiply(&x, &x, &p[0]); + for (int j = 0; j < 2; ++j) Square(&x, &x); + Multiply(&x, &x, &p[0]); + for (int j = 0; j < 4; ++j) Square(&x, &x); + Multiply(&x, &x, &p[0]); + for (int j = 0; j < 4; ++j) Square(&x, &x); + Multiply(&x, &x, &p[1]); + for (int j = 0; j < 3; ++j) Square(&x, &x); + Multiply(&x, &x, &p[0]); + + *out = x; +} + +} + +MuHash3072::MuHash3072() noexcept +{ + data.limbs[0] = 1; + for (int i = 1; i < Num3072::LIMBS; ++i) data.limbs[i] = 0; +} + +MuHash3072::MuHash3072(const unsigned char* key32) noexcept +{ + unsigned char tmp[384]; + ChaCha20(key32, 32).Output(tmp, 384); + for (int i = 0; i < Num3072::LIMBS; ++i) { + if (sizeof(Num3072::limb_type) == 4) { + data.limbs[i] = ReadLE32(tmp + 4 * i); + } else if (sizeof(Num3072::limb_type) == 8) { + data.limbs[i] = ReadLE64(tmp + 8 * i); + } + } +} + +void MuHash3072::Finalize(unsigned char* hash384) noexcept +{ + if (IsOverflow(&data)) FullReduce(&data); + for (int i = 0; i < Num3072::LIMBS; ++i) { + if (sizeof(Num3072::limb_type) == 4) { + WriteLE32(hash384 + i * 4, data.limbs[i]); + } else if (sizeof(Num3072::limb_type) == 8) { + WriteLE64(hash384 + i * 8, data.limbs[i]); + } + } +} + +MuHash3072& MuHash3072::operator*=(const MuHash3072& x) noexcept +{ + Multiply(&this->data, &this->data, &x.data); + return *this; +} + +MuHash3072& MuHash3072::operator/=(const MuHash3072& x) noexcept +{ + Num3072 tmp; + Inverse(&tmp, &x.data); + Multiply(&this->data, &this->data, &tmp); + return *this; +} diff --git a/src/crypto/muhash.h b/src/crypto/muhash.h new file mode 100644 index 0000000000000..f29d3f9e1b355 --- /dev/null +++ b/src/crypto/muhash.h @@ -0,0 +1,52 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CRYPTO_MUHASH_H +#define BITCOIN_CRYPTO_MUHASH_H + +#if defined(HAVE_CONFIG_H) +#include "bitcoin-config.h" +#endif + +#include + +struct Num3072 { +#ifdef HAVE___INT128 + typedef unsigned __int128 double_limb_type; + typedef uint64_t limb_type; + static constexpr int LIMBS = 48; + static constexpr int LIMB_SIZE = 64; +#else + typedef uint64_t double_limb_type; + typedef uint32_t limb_type; + static constexpr int LIMBS = 96; + static constexpr int LIMB_SIZE = 32; +#endif + limb_type limbs[LIMBS]; +}; + +/** A class representing MuHash sets */ +class MuHash3072 +{ +private: + Num3072 data; + +public: + /* The empty set. */ + MuHash3072() noexcept; + + /* A singleton with a single 32-byte key in it. */ + explicit MuHash3072(const unsigned char* key32) noexcept; + + /* Multiply (resulting in a hash for the union of the sets) */ + MuHash3072& operator*=(const MuHash3072& add) noexcept; + + /* Divide (resulting in a hash for the difference of the sets) */ + MuHash3072& operator/=(const MuHash3072& sub) noexcept; + + /* Finalize into a 384-byte hash. Does not change this object's value. */ + void Finalize(unsigned char* hash384) noexcept; +}; + +#endif // BITCOIN_CRYPTO_MUHASH_H diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index 4ac12bf9698c9..355ba8106b494 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -749,4 +750,58 @@ BOOST_AUTO_TEST_CASE(sha256d64) } } +static MuHash3072 FromInt(unsigned char i) { + unsigned char tmp[32] = {i, 0}; + return MuHash3072(tmp); +} + +BOOST_AUTO_TEST_CASE(muhash_tests) +{ + unsigned char out[384]; + + for (int iter = 0; iter < 10; ++iter) { + unsigned char res[384]; + int table[4]; + for (int i = 0; i < 4; ++i) { + table[i] = insecure_rand_ctx.randbits(3); + } + for (int order = 0; order < 4; ++order) { + MuHash3072 acc; + for (int i = 0; i < 4; ++i) { + int t = table[i ^ order]; + if (t & 4) { + acc /= FromInt(t & 3); + } else { + acc *= FromInt(t & 3); + } + } + acc.Finalize(out); + if (order == 0) { + memcpy(res, out, 384); + } else { + BOOST_CHECK(memcmp(res, out, 384) == 0); + } + } + + MuHash3072 x = FromInt(insecure_rand_ctx.randbits(4)); // x=X + MuHash3072 y = FromInt(insecure_rand_ctx.randbits(4)); // x=X, y=Y + MuHash3072 z; // x=X, y=Y, z=1 + z *= x; // x=X, y=Y, z=X + z /= y; // x=X, y=Y, z=X/Y + y /= x; // x=X, y=Y/X, z=X/Y + z *= y; // x=X, y=Y/X, z=1 + z.Finalize(out); + for (int i = 0; i < 384; ++i) { + BOOST_CHECK_EQUAL(out[i], i == 0); + } + } + + MuHash3072 acc = FromInt(0); + acc *= FromInt(1); + acc /= FromInt(2); + acc.Finalize(out); + uint256 x = (TruncatedSHA512Writer(SER_DISK, 0) << FLATDATA(out)).GetHash(); + BOOST_CHECK(x == uint256S("0e94c56c180f27fd6b182f091c5b007e2d6eba5ae28daa5aa92d2af8c26ea9a6")); +} + BOOST_AUTO_TEST_SUITE_END() From 569cff505a0c6738ee52c273c398ce3b24cf6f44 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Fri, 19 May 2017 18:20:44 -0700 Subject: [PATCH 02/10] muhash in gettxoutsetinfo --- src/hash.h | 36 ++++++++++++++++++++++++++++++++++++ src/rpc/blockchain.cpp | 38 +++++++++++++++++++++----------------- 2 files changed, 57 insertions(+), 17 deletions(-) diff --git a/src/hash.h b/src/hash.h index c295568a3e733..4ff5d4446cb70 100644 --- a/src/hash.h +++ b/src/hash.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -191,6 +192,41 @@ class CHashVerifier : public CHashWriter } }; +/** A writer stream that computes a 256-bit truncated SHA512. */ +class TruncatedSHA512Writer +{ +private: + CSHA512 ctx; + + const int nType; + const int nVersion; +public: + + TruncatedSHA512Writer(int nTypeIn, int nVersionIn) : nType(nTypeIn), nVersion(nVersionIn) {} + + int GetType() const { return nType; } + int GetVersion() const { return nVersion; } + + void write(const char *pch, size_t size) { + ctx.Write((const unsigned char*)pch, size); + } + + uint256 GetHash() { + unsigned char out[64]; + ctx.Finalize(out); + uint256 result; + memcpy((unsigned char*)&result, out, 32); + return result; + } + + template + TruncatedSHA512Writer& operator<<(const T& obj) { + // Serialize to this stream + ::Serialize(*this, obj); + return (*this); + } +}; + /** Compute the 256-bit hash of an object's serialization. */ template uint256 SerializeHash(const T& obj, int nType=SER_GETHASH, int nVersion=PROTOCOL_VERSION) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index a74003149d374..2ef18641d55c1 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -35,6 +36,7 @@ #include #include + #include #include @@ -916,29 +918,28 @@ struct CCoinsStats uint64_t nTransactions; uint64_t nTransactionOutputs; uint64_t nBogoSize; - uint256 hashSerialized; + uint256 muhash; uint64_t nDiskSize; CAmount nTotalAmount; CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nBogoSize(0), nDiskSize(0), nTotalAmount(0) {} }; -static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, const std::map& outputs) +static void ApplyStats(CCoinsStats &stats, MuHash3072& acc, const uint256& hash, const std::map& outputs) { assert(!outputs.empty()); - ss << hash; - ss << VARINT(outputs.begin()->second.nHeight * 2 + outputs.begin()->second.fCoinBase ? 1u : 0u); stats.nTransactions++; - for (const auto& output : outputs) { - ss << VARINT(output.first + 1); - ss << output.second.out.scriptPubKey; - ss << VARINT(output.second.out.nValue, VarIntMode::NONNEGATIVE_SIGNED); + for (const auto output : outputs) { + TruncatedSHA512Writer ss(SER_DISK, 0); + ss << COutPoint(hash, output.first); + ss << (uint32_t)(output.second.nHeight * 2 + output.second.fCoinBase); + ss << output.second.out; + acc *= MuHash3072(ss.GetHash().begin()); stats.nTransactionOutputs++; stats.nTotalAmount += output.second.out.nValue; stats.nBogoSize += 32 /* txid */ + 4 /* vout index */ + 4 /* height + coinbase */ + 8 /* amount */ + 2 /* scriptPubKey len */ + output.second.out.scriptPubKey.size() /* scriptPubKey */; } - ss << VARINT(0u); } //! Calculate statistics about the unspent transaction output set @@ -947,22 +948,21 @@ static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) std::unique_ptr pcursor(view->Cursor()); assert(pcursor); - CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); stats.hashBlock = pcursor->GetBestBlock(); { LOCK(cs_main); stats.nHeight = LookupBlockIndex(stats.hashBlock)->nHeight; } - ss << stats.hashBlock; - uint256 prevkey; + MuHash3072 acc; std::map outputs; + uint256 prevkey; while (pcursor->Valid()) { boost::this_thread::interruption_point(); COutPoint key; Coin coin; if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { if (!outputs.empty() && key.hash != prevkey) { - ApplyStats(stats, ss, prevkey, outputs); + ApplyStats(stats, acc, prevkey, outputs); outputs.clear(); } prevkey = key.hash; @@ -973,9 +973,13 @@ static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) pcursor->Next(); } if (!outputs.empty()) { - ApplyStats(stats, ss, prevkey, outputs); + ApplyStats(stats, acc, prevkey, outputs); } - stats.hashSerialized = ss.GetHash(); + unsigned char data[384]; + acc.Finalize(data); + TruncatedSHA512Writer ss(SER_DISK, 0); + ss << FLATDATA(data); + stats.muhash = ss.GetHash(); stats.nDiskSize = view->EstimateSize(); return true; } @@ -1049,7 +1053,7 @@ static UniValue gettxoutsetinfo(const JSONRPCRequest& request) " \"transactions\": n, (numeric) The number of transactions with unspent outputs\n" " \"txouts\": n, (numeric) The number of unspent transaction outputs\n" " \"bogosize\": n, (numeric) A meaningless metric for UTXO set size\n" - " \"hash_serialized_2\": \"hash\", (string) The serialized hash\n" + " \"muhash\": \"hash\", (string) Rolling UTXO set hash\n" " \"disk_size\": n, (numeric) The estimated size of the chainstate on disk\n" " \"total_amount\": x.xxx (numeric) The total amount\n" "}\n" @@ -1072,7 +1076,7 @@ static UniValue gettxoutsetinfo(const JSONRPCRequest& request) ret.pushKV("transactions", (int64_t)stats.nTransactions); ret.pushKV("txouts", (int64_t)stats.nTransactionOutputs); ret.pushKV("bogosize", (int64_t)stats.nBogoSize); - ret.pushKV("hash_serialized_2", stats.hashSerialized.GetHex()); + ret.pushKV("muhash", stats.muhash.GetHex()); ret.pushKV("disk_size", stats.nDiskSize); ret.pushKV("total_amount", ValueFromAmount(stats.nTotalAmount)); } else { From f8bcc45c8a278f17b76d2f2d439b703371a47774 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Fri, 30 Jun 2017 11:11:20 -0700 Subject: [PATCH 03/10] Simplification --- src/rpc/blockchain.cpp | 35 +++++++++++------------------------ 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 2ef18641d55c1..defe0de63284e 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -925,21 +925,18 @@ struct CCoinsStats CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nBogoSize(0), nDiskSize(0), nTotalAmount(0) {} }; -static void ApplyStats(CCoinsStats &stats, MuHash3072& acc, const uint256& hash, const std::map& outputs) +static void ApplyStats(CCoinsStats &stats, MuHash3072& acc, const COutPoint outpoint, const Coin& coin) { - assert(!outputs.empty()); stats.nTransactions++; - for (const auto output : outputs) { - TruncatedSHA512Writer ss(SER_DISK, 0); - ss << COutPoint(hash, output.first); - ss << (uint32_t)(output.second.nHeight * 2 + output.second.fCoinBase); - ss << output.second.out; - acc *= MuHash3072(ss.GetHash().begin()); - stats.nTransactionOutputs++; - stats.nTotalAmount += output.second.out.nValue; - stats.nBogoSize += 32 /* txid */ + 4 /* vout index */ + 4 /* height + coinbase */ + 8 /* amount */ + - 2 /* scriptPubKey len */ + output.second.out.scriptPubKey.size() /* scriptPubKey */; - } + TruncatedSHA512Writer ss(SER_DISK, 0); + ss << outpoint; + ss << (uint32_t)(coin.nHeight * 2 + coin.fCoinBase); + ss << coin.out; + acc *= MuHash3072(ss.GetHash().begin()); + stats.nTransactionOutputs++; + stats.nTotalAmount += coin.out.nValue; + stats.nBogoSize += 32 /* txid */ + 4 /* vout index */ + 4 /* height + coinbase */ + 8 /* amount */ + + 2 /* scriptPubKey len */ + coin.out.scriptPubKey.size() /* scriptPubKey */; } //! Calculate statistics about the unspent transaction output set @@ -954,27 +951,17 @@ static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) stats.nHeight = LookupBlockIndex(stats.hashBlock)->nHeight; } MuHash3072 acc; - std::map outputs; - uint256 prevkey; while (pcursor->Valid()) { boost::this_thread::interruption_point(); COutPoint key; Coin coin; if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { - if (!outputs.empty() && key.hash != prevkey) { - ApplyStats(stats, acc, prevkey, outputs); - outputs.clear(); - } - prevkey = key.hash; - outputs[key.n] = std::move(coin); + ApplyStats(stats, acc, key, coin); } else { return error("%s: unable to read value", __func__); } pcursor->Next(); } - if (!outputs.empty()) { - ApplyStats(stats, acc, prevkey, outputs); - } unsigned char data[384]; acc.Finalize(data); TruncatedSHA512Writer ss(SER_DISK, 0); From ab9fa6b3b22b594326913730551bf19ba840c498 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Fri, 30 Jun 2017 11:15:17 -0700 Subject: [PATCH 04/10] Add x86_64 assembly optimization for MuHash --- src/crypto/muhash.cpp | 50 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/src/crypto/muhash.cpp b/src/crypto/muhash.cpp index 91491c3d006bc..4ac1d88619419 100644 --- a/src/crypto/muhash.cpp +++ b/src/crypto/muhash.cpp @@ -28,6 +28,54 @@ namespace { c1 = 0; \ } +#if defined(__amd64__) || defined(__x86_64__) + +/** [c0,c1] = a * b */ +#define mul(c0,c1,a,b) { \ + __asm__ ("mulq %3" : "=d"(c1), "=a"(c0) : "a"(a), "g"(b) : "cc"); \ +} + +/** [c0,c1,c2] += a * b */ +#define muladd3(c0,c1,c2,a,b) { \ + uint64_t tl, th; \ + __asm__ ("mulq %3" : "=a"(tl), "=d"(th) : "a"(a), "g"(b) : "cc"); \ + __asm__ ("addq %3,%0; adcq %4,%1; adcq $0,%2" : "+r"(c0), "+r"(c1), "+r"(c2) : "a"(tl), "d"(th) : "cc"); \ +} + +/** [c0,c1,c2] += 2 * a * b */ +#define muldbladd3(c0,c1,c2,a,b) { \ + uint64_t tl, th; \ + __asm__ ("mulq %3" : "=a"(tl), "=d"(th) : "a"(a), "g"(b) : "cc"); \ + __asm__ ("addq %3,%0; adcq %4,%1; adcq $0,%2" : "+r"(c0), "+r"(c1), "+r"(c2) : "a"(tl), "d"(th) : "cc"); \ + __asm__ ("addq %3,%0; adcq %4,%1; adcq $0,%2" : "+r"(c0), "+r"(c1), "+r"(c2) : "a"(tl), "d"(th) : "cc"); \ +} + +/* [c0,c1,c2] += n * [d0,d1,d2]. c0 is initially 0 */ +#define mulnadd3(c0,c1,c2,d0,d1,d2,n) { \ + uint64_t tl1, th1, tl2, th2, tl3; \ + __asm__ ("mulq %3" : "=a"(tl1), "=d"(th1) : "a"(d0), "r"((Num3072::limb_type)n) : "cc"); \ + __asm__ ("addq %3,%0; adcq %4,%1; adcq $0,%2" : "+r"(c0), "+r"(c1), "+r"(c2) : "g"(tl1), "g"(th1) : "cc"); \ + __asm__ ("mulq %3" : "=a"(tl2), "=d"(th2) : "a"(d1), "r"((Num3072::limb_type)n) : "cc"); \ + __asm__ ("addq %2,%0; adcq %3,%1" : "+r"(c1), "+r"(c2) : "g"(tl2), "g"(th2) : "cc"); \ + __asm__ ("imulq %2,%1,%0" : "=r"(tl3) : "g"(d2), "i"(n) : "cc"); \ + __asm__ ("addq %1,%0" : "+r"(c2) : "g"(tl3) : "cc"); \ +} + +/* [c0,c1] *= n */ +#define muln2(c0,c1,n) { \ + uint64_t th; \ + __asm__ ("mulq %2" : "+a"(c0), "=d"(th) : "r"((Num3072::limb_type)n) : "cc"); \ + __asm__ ("imul %1,%0,%0" : "+r"(c1) : "i"(n) : "cc"); \ + __asm__ ("addq %1,%0" : "+r"(c1) : "g"(th) : "cc"); \ +} + +/** [c0,c1] += a */ +#define add2(c0,c1,a) { \ + __asm__ ("add %2,%0; adc $0,%1" : "+r"(c0), "+r"(c1) : "r"(a) : "cc"); \ +} + +#else + /** [c0,c1] = a * b */ #define mul(c0,c1,a,b) { \ Num3072::double_limb_type t = (Num3072::double_limb_type)a * b; \ @@ -95,6 +143,8 @@ namespace { c1 += (c0 < (a)) ? 1 : 0; \ } +#endif + bool IsOverflow(const Num3072* d) { for (int i = 1; i < Num3072::LIMBS - 1; ++i) { From 00a8423099c73bf95b24f305dc1f150909c94fc1 Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Mon, 26 Aug 2019 09:59:48 -0400 Subject: [PATCH 05/10] Update original Muhash PR and fix functional test and benchmarks --- src/bench/crypto_hash.cpp | 9 ++++----- src/crypto/muhash.cpp | 2 +- src/test/crypto_tests.cpp | 8 ++++---- test/functional/rpc_blockchain.py | 4 ++-- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/bench/crypto_hash.cpp b/src/bench/crypto_hash.cpp index dffc3d73de7b4..9eee06fe21ed9 100644 --- a/src/bench/crypto_hash.cpp +++ b/src/bench/crypto_hash.cpp @@ -98,11 +98,10 @@ static void MuHash(benchmark::State& state) FastRandomContext rng(true); MuHash3072 acc; unsigned char key[32] = {0}; + int i = 0; while (state.KeepRunning()) { - for (int i = 0; i < 1000; i++) { - key[0] = i; - acc *= MuHash3072(key); - } + key[0] = ++i; + acc *= MuHash3072(key); } } @@ -117,4 +116,4 @@ BENCHMARK(SHA256D64_1024, 7400); BENCHMARK(FastRandom_32bit, 110 * 1000 * 1000); BENCHMARK(FastRandom_1bit, 440 * 1000 * 1000); -BENCHMARK(MuHash); +BENCHMARK(MuHash, 5000); diff --git a/src/crypto/muhash.cpp b/src/crypto/muhash.cpp index 4ac1d88619419..cd0b1febf28a3 100644 --- a/src/crypto/muhash.cpp +++ b/src/crypto/muhash.cpp @@ -286,7 +286,7 @@ MuHash3072::MuHash3072() noexcept MuHash3072::MuHash3072(const unsigned char* key32) noexcept { unsigned char tmp[384]; - ChaCha20(key32, 32).Output(tmp, 384); + ChaCha20(key32, 32).Keystream(tmp, 384); for (int i = 0; i < Num3072::LIMBS; ++i) { if (sizeof(Num3072::limb_type) == 4) { data.limbs[i] = ReadLE32(tmp + 4 * i); diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index 355ba8106b494..a01ed23fafca7 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -763,7 +763,7 @@ BOOST_AUTO_TEST_CASE(muhash_tests) unsigned char res[384]; int table[4]; for (int i = 0; i < 4; ++i) { - table[i] = insecure_rand_ctx.randbits(3); + table[i] = g_insecure_rand_ctx.randbits(3); } for (int order = 0; order < 4; ++order) { MuHash3072 acc; @@ -783,8 +783,8 @@ BOOST_AUTO_TEST_CASE(muhash_tests) } } - MuHash3072 x = FromInt(insecure_rand_ctx.randbits(4)); // x=X - MuHash3072 y = FromInt(insecure_rand_ctx.randbits(4)); // x=X, y=Y + MuHash3072 x = FromInt(g_insecure_rand_ctx.randbits(4)); // x=X + MuHash3072 y = FromInt(g_insecure_rand_ctx.randbits(4)); // x=X, y=Y MuHash3072 z; // x=X, y=Y, z=1 z *= x; // x=X, y=Y, z=X z /= y; // x=X, y=Y, z=X/Y @@ -800,7 +800,7 @@ BOOST_AUTO_TEST_CASE(muhash_tests) acc *= FromInt(1); acc /= FromInt(2); acc.Finalize(out); - uint256 x = (TruncatedSHA512Writer(SER_DISK, 0) << FLATDATA(out)).GetHash(); + uint256 x = (TruncatedSHA512Writer(SER_DISK, 0) << out).GetHash(); BOOST_CHECK(x == uint256S("0e94c56c180f27fd6b182f091c5b007e2d6eba5ae28daa5aa92d2af8c26ea9a6")); } diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index 6c30e050847e1..cf1c6d841f37b 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -214,7 +214,7 @@ def _test_gettxoutsetinfo(self): assert size > 6400 assert size < 64000 assert_equal(len(res['bestblock']), 64) - assert_equal(len(res['hash_serialized_2']), 64) + assert_equal(len(res['muhash']), 64) self.log.info("Test that gettxoutsetinfo() works for blockchain with just the genesis block") b1hash = node.getblockhash(1) @@ -227,7 +227,7 @@ def _test_gettxoutsetinfo(self): assert_equal(res2['txouts'], 0) assert_equal(res2['bogosize'], 0), assert_equal(res2['bestblock'], node.getblockhash(0)) - assert_equal(len(res2['hash_serialized_2']), 64) + assert_equal(len(res2['muhash']), 64) self.log.info("Test that gettxoutsetinfo() returns the same result after invalidate/reconsider block") node.reconsiderblock(b1hash) From 08b0a8a5f6369879a5f3408df1e44892ca367010 Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Mon, 26 Aug 2019 10:34:03 -0400 Subject: [PATCH 06/10] Add UtxoSetHash file to manage UTXO set hashes The UtxoSetHash is implemented as an index (subclass of BaseIndex) and maintains data in LevelDB. It keeps the muhash for each block in the database and also allows to search them by block hash or height. It also maintains the global Muhash object which is being modified as new blocks get added to the chain or as reorgs happen. This commit also adds a serialization method to the Muhash object to be able to persist it between restarts of the node. --- src/Makefile.am | 2 + src/crypto/muhash.h | 10 ++ src/index/utxosethash.cpp | 342 ++++++++++++++++++++++++++++++++++++++ src/index/utxosethash.h | 51 ++++++ 4 files changed, 405 insertions(+) create mode 100644 src/index/utxosethash.cpp create mode 100644 src/index/utxosethash.h diff --git a/src/Makefile.am b/src/Makefile.am index d2ecb4ead2683..fad4cd0a87857 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -135,6 +135,7 @@ BITCOIN_CORE_H = \ index/base.h \ index/blockfilterindex.h \ index/txindex.h \ + index/utxosethash.h \ indirectmap.h \ init.h \ interfaces/chain.h \ @@ -270,6 +271,7 @@ libbitcoin_server_a_SOURCES = \ index/base.cpp \ index/blockfilterindex.cpp \ index/txindex.cpp \ + index/utxosethash.cpp \ interfaces/chain.cpp \ interfaces/node.cpp \ init.cpp \ diff --git a/src/crypto/muhash.h b/src/crypto/muhash.h index f29d3f9e1b355..c9df5cd959cf0 100644 --- a/src/crypto/muhash.h +++ b/src/crypto/muhash.h @@ -10,6 +10,7 @@ #endif #include +#include struct Num3072 { #ifdef HAVE___INT128 @@ -47,6 +48,15 @@ class MuHash3072 /* Finalize into a 384-byte hash. Does not change this object's value. */ void Finalize(unsigned char* hash384) noexcept; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) { + for(int i = 0; i +#include +#include +#include +#include +#include +#include +#include +#include + +constexpr char DB_BLOCK_HASH = 's'; +constexpr char DB_BLOCK_HEIGHT = 't'; +constexpr char DB_MUHASH = 'M'; + +namespace { + +struct DBVal { + uint256 muhash; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITE(muhash); + } +}; + +struct DBHeightKey { + int height; + + DBHeightKey() : height(0) {} + explicit DBHeightKey(int height_in) : height(height_in) {} + + template + void Serialize(Stream& s) const + { + ser_writedata8(s, DB_BLOCK_HEIGHT); + ser_writedata32be(s, height); + } + + template + void Unserialize(Stream& s) + { + char prefix = ser_readdata8(s); + if (prefix != DB_BLOCK_HEIGHT) { + throw std::ios_base::failure("Invalid format for block filter index DB height key"); + } + height = ser_readdata32be(s); + } +}; + +struct DBHashKey { + uint256 block_hash; + + explicit DBHashKey(const uint256& hash_in) : block_hash(hash_in) {} + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) { + char prefix = DB_BLOCK_HASH; + READWRITE(prefix); + if (prefix != DB_BLOCK_HASH) { + throw std::ios_base::failure("Invalid format for block filter index DB hash key"); + } + + READWRITE(block_hash); + } +}; + +struct DBMuhash { + MuHash3072 hash; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITE(hash); + } +}; + +}; // namespace + +std::unique_ptr g_utxo_set_hash; + +UtxoSetHash::UtxoSetHash(size_t n_cache_size, bool f_memory, bool f_wipe) +{ + fs::path path = GetDataDir() / "indexes" / "utxo_set_hash"; + fs::create_directories(path); + + m_db = MakeUnique(path / "db", n_cache_size, f_memory, f_wipe); +} + +bool UtxoSetHash::Init() +{ + if (!m_db->Read(DB_MUHASH, m_muhash)) { + // Check that the cause of the read failure is that the key does not exist. Any other errors + // indicate database corruption or a disk failure, and starting the index would cause + // further corruption. + if (m_db->Exists(DB_MUHASH)) { + return error("%s: Cannot read current %s state; index may be corrupted", + __func__, GetName()); + } + + // If the DB_MUHASH is not set, initialize empty muhash + m_muhash = MuHash3072(); + } + + return BaseIndex::Init(); +} + +bool UtxoSetHash::WriteBlock(const CBlock& block, const CBlockIndex* pindex) +{ + CBlockUndo block_undo; + + if (pindex->nHeight > 0) { + if (!UndoReadFromDisk(block_undo, pindex)) { + return false; + } + + std::pair read_out; + if (!m_db->Read(DBHeightKey(pindex->nHeight - 1), read_out)) { + return false; + } + + uint256 expected_block_hash = pindex->pprev->GetBlockHash(); + if (read_out.first != expected_block_hash) { + return error("%s: previous block header belongs to unexpected block %s; expected %s", + __func__, read_out.first.ToString(), expected_block_hash.ToString()); + } + } + + // Add the new utxos created from the block + for (size_t i = 0; i < block.vtx.size(); ++i) { + const auto& tx = block.vtx.at(i); + + for (size_t j = 0; j < tx->vout.size(); ++j) { + const CTxOut& out = tx->vout[j]; + COutPoint outpoint = COutPoint(tx->GetHash(), j); + Coin coin = Coin(out, pindex->nHeight, tx->IsCoinBase()); + + TruncatedSHA512Writer ss(SER_DISK, 0); + ss << outpoint; + ss << (uint32_t)(coin.nHeight * 2 + coin.fCoinBase); + ss << coin.out; + m_muhash *= MuHash3072(ss.GetHash().begin()); + } + + // The coinbase tx has no undo data since no former output is spent + if (i > 0) { + const auto& tx_undo = block_undo.vtxundo.at(i-1); + + for (size_t j = 0; j < tx_undo.vprevout.size(); ++j) { + Coin coin = tx_undo.vprevout[j]; + COutPoint outpoint = COutPoint(tx->vin[j].prevout.hash, tx->vin[j].prevout.n); + + TruncatedSHA512Writer ss(SER_DISK, 0); + ss << outpoint; + ss << (uint32_t)(coin.nHeight * 2 + coin.fCoinBase); + ss << coin.out; + m_muhash /= MuHash3072(ss.GetHash().begin()); + } + } + } + + std::pair value; + value.first = pindex->GetBlockHash(); + value.second.muhash = currentHashInternal(); + + if (!m_db->Write(DBHeightKey(pindex->nHeight), value)) { + return false; + } + + if (!m_db->Write(DB_MUHASH, m_muhash)) { + return false; + } + + return true; +} + +static bool CopyHeightIndexToHashIndex(CDBIterator& db_it, CDBBatch& batch, + const std::string& index_name, + int start_height, int stop_height) +{ + DBHeightKey key(start_height); + db_it.Seek(key); + + for (int height = start_height; height <= stop_height; ++height) { + if (!db_it.GetKey(key) || key.height != height) { + return error("%s: unexpected key in %s: expected (%c, %d)", + __func__, index_name, DB_BLOCK_HEIGHT, height); + } + + std::pair value; + if (!db_it.GetValue(value)) { + return error("%s: unable to read value in %s at key (%c, %d)", + __func__, index_name, DB_BLOCK_HEIGHT, height); + } + + batch.Write(DBHashKey(value.first), std::move(value.second)); + + db_it.Next(); + } + return true; +} + +bool UtxoSetHash::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) +{ + assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip); + + CDBBatch batch(*m_db); + std::unique_ptr db_it(m_db->NewIterator()); + + CBlockIndex* iter_tip = LookupBlockIndex(current_tip->GetBlockHash()); + auto& consensus_params = Params().GetConsensus(); + + do { + CBlock block; + + if (!ReadBlockFromDisk(block, iter_tip, consensus_params)) { + return error("%s: Failed to read block %s from disk", + __func__, iter_tip->GetBlockHash().ToString()); + } + + ReverseBlock(block, iter_tip); + + iter_tip = iter_tip->GetAncestor(iter_tip->nHeight - 1); + } while (new_tip != iter_tip); + + // During a reorg, we need to copy all hash digests for blocks that are getting disconnected from the + // height index to the hash index so we can still find them when the height index entries are + // overwritten. + if (!CopyHeightIndexToHashIndex(*db_it, batch, m_name, new_tip->nHeight, current_tip->nHeight)) { + return false; + } + + if (!m_db->WriteBatch(batch)) return false; + + return BaseIndex::Rewind(current_tip, new_tip); +} + +static bool LookupOne(const CDBWrapper& db, const CBlockIndex* block_index, DBVal& result) +{ + // First check if the result is stored under the height index and the value there matches the + // block hash. This should be the case if the block is on the active chain. + std::pair read_out; + if (!db.Read(DBHeightKey(block_index->nHeight), read_out)) { + return false; + } + if (read_out.first == block_index->GetBlockHash()) { + result = std::move(read_out.second); + return true; + } + + // If value at the height index corresponds to an different block, the result will be stored in + // the hash index. + return db.Read(DBHashKey(block_index->GetBlockHash()), result); +} + +bool UtxoSetHash::LookupHash(const CBlockIndex* block_index, uint256& utxo_set_hash) const +{ + DBVal entry; + if (!LookupOne(*m_db, block_index, entry)) { + return false; + } + + utxo_set_hash = entry.muhash; + return true; +} + +uint256 UtxoSetHash::currentHashInternal() +{ + unsigned char out[384]; + m_muhash.Finalize(out); + return (TruncatedSHA512Writer(SER_DISK, 0) << out).GetHash(); +} + +// Reverse Block in case of reorg +bool UtxoSetHash::ReverseBlock(const CBlock& block, const CBlockIndex* pindex) +{ + CBlockUndo block_undo; + + if (pindex->nHeight > 0) { + if (!UndoReadFromDisk(block_undo, pindex)) { + return false; + } + + std::pair read_out; + if (!m_db->Read(DBHeightKey(pindex->nHeight - 1), read_out)) { + return false; + } + + uint256 expected_block_hash = pindex->pprev->GetBlockHash(); + if (read_out.first != expected_block_hash) { + return error("%s: previous block header belongs to unexpected block %s; expected %s", + __func__, read_out.first.ToString(), expected_block_hash.ToString()); + } + } + + // Add the new utxos created from the block + for (size_t i = 0; i < block.vtx.size(); ++i) { + const auto& tx = block.vtx.at(i); + + for (size_t j = 0; j < tx->vout.size(); ++j) { + const CTxOut& out = tx->vout[j]; + COutPoint outpoint = COutPoint(tx->GetHash(), j); + Coin coin = Coin(out, pindex->nHeight, tx->IsCoinBase()); + + TruncatedSHA512Writer ss(SER_DISK, 0); + ss << outpoint; + ss << (uint32_t)(coin.nHeight * 2 + coin.fCoinBase); + ss << coin.out; + m_muhash /= MuHash3072(ss.GetHash().begin()); + } + + // The coinbase tx has no undo data since no former output is spent + if (i > 0) { + const auto& tx_undo = block_undo.vtxundo.at(i-1); + + for (size_t j = 0; j < tx_undo.vprevout.size(); ++j) { + Coin coin = tx_undo.vprevout[j]; + COutPoint outpoint = COutPoint(tx->vin[j].prevout.hash, tx->vin[j].prevout.n); + + TruncatedSHA512Writer ss(SER_DISK, 0); + ss << outpoint; + ss << (uint32_t)(coin.nHeight * 2 + coin.fCoinBase); + ss << coin.out; + m_muhash *= MuHash3072(ss.GetHash().begin()); + } + } + } + + if (!m_db->Write(DB_MUHASH, m_muhash)) { + return false; + } + + return true; +} diff --git a/src/index/utxosethash.h b/src/index/utxosethash.h new file mode 100644 index 0000000000000..57af68c6ce394 --- /dev/null +++ b/src/index/utxosethash.h @@ -0,0 +1,51 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_INDEX_UTXOSETHASH_H +#define BITCOIN_INDEX_UTXOSETHASH_H + +#include +#include +#include +#include + +/** + * UtxoHashSet maintains a rolling hash of the utxo set and + * caches a hash digest for every block. + */ +class UtxoSetHash final : public BaseIndex +{ +private: + std::string m_name; + std::unique_ptr m_db; + MuHash3072 m_muhash; + + // Digest of the current Muhash object + uint256 currentHashInternal(); + + // Roll back the Muhash of a particular block + bool ReverseBlock(const CBlock& block, const CBlockIndex* pindex); +protected: + bool Init() override; + + bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override; + + bool Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) override; + + BaseIndex::DB& GetDB() const override { return *m_db; } + + const char* GetName() const override { return "utxosethash"; } + +public: + // Constructs the index, which becomes available to be queried. + explicit UtxoSetHash(size_t n_cache_size, bool f_memory = false, bool f_wipe = false); + + // Look up hash digest for a specific block using CBlockIndex + bool LookupHash(const CBlockIndex* block_index, uint256& utxo_set_hash) const; +}; + +/// The global UTXO set hash object. +extern std::unique_ptr g_utxo_set_hash; + +#endif // BITCOIN_INDEX_UTXOSETHASH_H From 704ce2e701fa1b4a35d636e4750759bc67fb8a0e Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Mon, 26 Aug 2019 10:35:17 -0400 Subject: [PATCH 07/10] Maintain UTXO Set hash index using Muhash --- src/init.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/init.cpp b/src/init.cpp index 042335b8e74c8..34300e428e901 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -1443,6 +1444,7 @@ bool AppInitMain(InitInterfaces& interfaces) filter_index_cache = max_cache / n_indexes; nTotalCache -= filter_index_cache * n_indexes; } + int64_t utsh_cache = 0; int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache nCoinDBCache = std::min(nCoinDBCache, nMaxCoinsDBCache << 20); // cap total coins db cache nTotalCache -= nCoinDBCache; @@ -1661,6 +1663,9 @@ bool AppInitMain(InitInterfaces& interfaces) GetBlockFilterIndex(filter_type)->Start(); } + g_utxo_set_hash = MakeUnique(utsh_cache, false, fReindex); + g_utxo_set_hash->Start(); + // ********************************************************* Step 9: load wallet for (const auto& client : interfaces.chain_clients) { if (!client->load()) { From d9e18096daf5f05fa1ded14e1491fa9428776d79 Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Mon, 26 Aug 2019 15:46:55 -0400 Subject: [PATCH 08/10] rpc: Use rolling Muhash in gettxoutsetinfo --- src/rpc/blockchain.cpp | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index defe0de63284e..397570141f657 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -11,10 +11,10 @@ #include #include #include -#include #include #include #include +#include #include #include #include @@ -925,14 +925,9 @@ struct CCoinsStats CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nBogoSize(0), nDiskSize(0), nTotalAmount(0) {} }; -static void ApplyStats(CCoinsStats &stats, MuHash3072& acc, const COutPoint outpoint, const Coin& coin) +static void ApplyStats(CCoinsStats &stats, const COutPoint outpoint, const Coin& coin) { stats.nTransactions++; - TruncatedSHA512Writer ss(SER_DISK, 0); - ss << outpoint; - ss << (uint32_t)(coin.nHeight * 2 + coin.fCoinBase); - ss << coin.out; - acc *= MuHash3072(ss.GetHash().begin()); stats.nTransactionOutputs++; stats.nTotalAmount += coin.out.nValue; stats.nBogoSize += 32 /* txid */ + 4 /* vout index */ + 4 /* height + coinbase */ + 8 /* amount */ + @@ -945,28 +940,42 @@ static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) std::unique_ptr pcursor(view->Cursor()); assert(pcursor); + uint256 muhash_buf; + const CBlockIndex* block_index; stats.hashBlock = pcursor->GetBestBlock(); { LOCK(cs_main); - stats.nHeight = LookupBlockIndex(stats.hashBlock)->nHeight; + block_index = LookupBlockIndex(stats.hashBlock); } - MuHash3072 acc; + + stats.nHeight = block_index->nHeight; + + bool index_ready = g_utxo_set_hash->BlockUntilSyncedToCurrentChain(); + + if (!g_utxo_set_hash->LookupHash(block_index, muhash_buf)) { + int err_code = RPC_MISC_ERROR; + + if (!index_ready) { + throw JSONRPCError(err_code, "UTXO set hash is still in the process of being indexed."); + } else { + throw JSONRPCError(err_code, "Unable to get UTXO set hash"); + } + } + + stats.muhash = muhash_buf; + while (pcursor->Valid()) { boost::this_thread::interruption_point(); COutPoint key; Coin coin; if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { - ApplyStats(stats, acc, key, coin); + ApplyStats(stats, key, coin); } else { return error("%s: unable to read value", __func__); } pcursor->Next(); } - unsigned char data[384]; - acc.Finalize(data); - TruncatedSHA512Writer ss(SER_DISK, 0); - ss << FLATDATA(data); - stats.muhash = ss.GetHash(); + stats.nDiskSize = view->EstimateSize(); return true; } From ebb4432141d749812a4dfbadd88df566ff975854 Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Mon, 26 Aug 2019 15:49:57 -0400 Subject: [PATCH 09/10] test: Add functional test for rolling utxo set hash --- test/functional/rpc_blockchain.py | 34 ++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index cf1c6d841f37b..ce39fe6371d82 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -31,6 +31,7 @@ assert_raises_rpc_error, assert_is_hex_string, assert_is_hash_string, + wait_until, ) from test_framework.blocktools import ( create_block, @@ -62,6 +63,7 @@ def run_test(self): self._test_getnetworkhashps() self._test_stopatheight() self._test_waitforblockheight() + self._test_muhash() assert self.nodes[0].verifychain(4, 0) def mine_chain(self): @@ -73,6 +75,10 @@ def mine_chain(self): self.nodes[0].generatetoaddress(1, address) assert_equal(self.nodes[0].getblockchaininfo()['blocks'], 200) + def mine_block(self): + address = self.nodes[0].get_deterministic_priv_key().address + self.nodes[0].generatetoaddress(1, address) + def _test_getblockchaininfo(self): self.log.info("Test getblockchaininfo") @@ -202,6 +208,8 @@ def _test_getchaintxstats(self): def _test_gettxoutsetinfo(self): node = self.nodes[0] + + wait_until(lambda: self.nodes[0].getblockcount() == 200) res = node.gettxoutsetinfo() assert_equal(res['total_amount'], Decimal('8725.00000000')) @@ -231,7 +239,6 @@ def _test_gettxoutsetinfo(self): self.log.info("Test that gettxoutsetinfo() returns the same result after invalidate/reconsider block") node.reconsiderblock(b1hash) - res3 = node.gettxoutsetinfo() # The field 'disk_size' is non-deterministic and can thus not be # compared between res and res3. Everything else should be the same. @@ -331,6 +338,31 @@ def assert_waitforheight(height, timeout=2): assert_waitforheight(current_height) assert_waitforheight(current_height + 1) + def _test_muhash(self): + self.restart_node(0) + node = self.nodes[0] + + self.log.info("Test that gettxoutsetinfo() muhash is unchanged when rolling back a new block") + + # Test consistency of hashing + res = node.gettxoutsetinfo() + muhash_at_207 = res['muhash'] + assert(node.gettxoutsetinfo()['muhash'] == muhash_at_207) + + # Hash is updated with new block + self.mine_block() + assert(node.gettxoutsetinfo()['muhash'] != muhash_at_207) + + # Hash is rolled back to previous block if invalidated + b208hash = node.getblockhash(208) + node.invalidateblock(b208hash) + assert(node.gettxoutsetinfo()['muhash'] == muhash_at_207) + + # Hash persists restart + self.stop_node(0) + self.start_node(0) + assert(node.gettxoutsetinfo()['muhash'] == muhash_at_207) + if __name__ == '__main__': BlockchainTest().main() From e278ecef10c36fed36be1d2f21a70c2f7fd67f91 Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Wed, 28 Aug 2019 15:58:53 -0400 Subject: [PATCH 10/10] test: Add unit test for UtxoSetHash --- src/Makefile.test.include | 1 + src/test/utxosethash_tests.cpp | 70 ++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 src/test/utxosethash_tests.cpp diff --git a/src/Makefile.test.include b/src/Makefile.test.include index d3fe138133e7b..847d6c296ff3f 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -148,6 +148,7 @@ BITCOIN_TESTS =\ test/txvalidationcache_tests.cpp \ test/uint256_tests.cpp \ test/util_tests.cpp \ + test/utxosethash_tests.cpp \ test/validation_block_tests.cpp \ test/versionbits_tests.cpp diff --git a/src/test/utxosethash_tests.cpp b/src/test/utxosethash_tests.cpp new file mode 100644 index 0000000000000..66d977028cfde --- /dev/null +++ b/src/test/utxosethash_tests.cpp @@ -0,0 +1,70 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +// +#include + +BOOST_AUTO_TEST_SUITE(utxosethash_tests) + +BOOST_FIXTURE_TEST_CASE(utxosethash_initial_sync, TestChain100Setup) +{ + UtxoSetHash utxo_set_hash(0, false); + + uint256 hash_digest; + CBlockIndex* block_index = ::ChainActive().Tip(); + + // UTXO set hash should not be found before it is started. + BOOST_CHECK(!utxo_set_hash.LookupHash(block_index, hash_digest)); + + // BlockUntilSyncedToCurrentChain should return false before utxo_set_hash is started. + BOOST_CHECK(!utxo_set_hash.BlockUntilSyncedToCurrentChain()); + + utxo_set_hash.Start(); + + // Allow the UTXO set hash to catch up with the block index. + constexpr int64_t timeout_ms = 10 * 1000; + int64_t time_start = GetTimeMillis(); + while (!utxo_set_hash.BlockUntilSyncedToCurrentChain()) { + BOOST_REQUIRE(time_start + timeout_ms > GetTimeMillis()); + MilliSleep(100); + } + + // Check that UTXO set hash works for genesis block. + CBlockIndex* genesis_block_index = ::ChainActive().Genesis(); + BOOST_CHECK(utxo_set_hash.LookupHash(genesis_block_index, hash_digest)); + + // Check that UTXO set hash updates with new blocks. + block_index = ::ChainActive().Tip(); + utxo_set_hash.LookupHash(block_index, hash_digest); + + CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG; + std::vector noTxns; + CreateAndProcessBlock(noTxns, scriptPubKey); + + time_start = GetTimeMillis(); + while (!utxo_set_hash.BlockUntilSyncedToCurrentChain()) { + BOOST_REQUIRE(time_start + timeout_ms > GetTimeMillis()); + MilliSleep(100); + } + + uint256 new_hash_digest; + CBlockIndex* new_block_index = ::ChainActive().Tip(); + utxo_set_hash.LookupHash(new_block_index, new_hash_digest); + + BOOST_CHECK(block_index != new_block_index); + BOOST_CHECK(hash_digest != new_hash_digest); + + // shutdown sequence (c.f. Shutdown() in init.cpp) + utxo_set_hash.Stop(); + + threadGroup.interrupt_all(); + threadGroup.join_all(); + + // Rest of shutdown sequence and destructors happen in ~TestingSetup() +} + +BOOST_AUTO_TEST_SUITE_END()