diff --git a/include/bitcoin/database/impl/primitives/arraymap.ipp b/include/bitcoin/database/impl/primitives/arraymap.ipp index a2181598b..f698cde3d 100644 --- a/include/bitcoin/database/impl/primitives/arraymap.ipp +++ b/include/bitcoin/database/impl/primitives/arraymap.ipp @@ -202,7 +202,7 @@ bool CLASS::read(const memory_ptr& ptr, const Link& link, iostream stream{ offset, size - position }; reader source{ stream }; - if constexpr (!is_slab) { BC_DEBUG_ONLY(source.set_limit(Size);) } + if constexpr (!is_slab) { BC_DEBUG_ONLY(source.set_limit(Size * element.count());) } return element.from_data(source); } diff --git a/include/bitcoin/database/impl/query/confirm.ipp b/include/bitcoin/database/impl/query/confirm.ipp index bb9345a0b..fee07e7ea 100644 --- a/include/bitcoin/database/impl/query/confirm.ipp +++ b/include/bitcoin/database/impl/query/confirm.ipp @@ -277,7 +277,10 @@ error::error_t CLASS::spent_prevout(const point_link& link, index index, // But if we hold the spend.pk/prevout.tx we can just read the // spend.hash/index, so we don't need to store the index, and we need to // read the spend.hash anyway, so index is free (no paging). So that's now - // just spend[4] + tx[4], back to 8 bytes (19GB). + // just spend[4] + tx[4], back to 8 bytes (19GB). But getting spends is + // relatively cheap, just search txs[hash] and navigate puts. The downside + // is the extra search and puts is > 2x prevouts and can't be pruned. + // The upside is half the prevout size (read/write/page) and store increase. // Iterate points by point hash (of output tx) because may be conflicts. auto point = store_.point.it(get_point_key(link)); @@ -705,13 +708,14 @@ bool CLASS::set_unstrong(const header_link& link) NOEXCEPT } TEMPLATE -bool CLASS::set_prevouts(const header_link&, const block&) NOEXCEPT +bool CLASS::set_prevouts(size_t height, const block& block) NOEXCEPT { // ======================================================================== const auto scope = store_.get_transactor(); - // TODO: implement. - return {}; + // Clean single allocation failure (e.g. disk full). + const table::prevout::record_put_ref prevouts{ {}, block }; + return store_.prevout.put(height, prevouts); // ======================================================================== } diff --git a/include/bitcoin/database/query.hpp b/include/bitcoin/database/query.hpp index b83dddf41..c2db01aa5 100644 --- a/include/bitcoin/database/query.hpp +++ b/include/bitcoin/database/query.hpp @@ -510,7 +510,7 @@ class query /// Block association relies on strong (confirmed or pending). bool set_strong(const header_link& link) NOEXCEPT; bool set_unstrong(const header_link& link) NOEXCEPT; - bool set_prevouts(const header_link& link, const block& block) NOEXCEPT; + bool set_prevouts(size_t height, const block& block) NOEXCEPT; code block_confirmable(const header_link& link) const NOEXCEPT; ////code tx_confirmable(const tx_link& link, const context& ctx) const NOEXCEPT; code unspent_duplicates(const header_link& coinbase, diff --git a/include/bitcoin/database/tables/caches/prevout.hpp b/include/bitcoin/database/tables/caches/prevout.hpp index 1414760cb..bf3da50af 100644 --- a/include/bitcoin/database/tables/caches/prevout.hpp +++ b/include/bitcoin/database/tables/caches/prevout.hpp @@ -29,44 +29,161 @@ namespace database { namespace table { /// prevout is an array map index of previous outputs by block. +/// The coinbase flag is merged into the tx field, reducing it's domain. +/// Masking is from the right in order to accomodate non-integral domain. struct prevout : public array_map { using tx = linkage; - using spend = linkage; using array_map::arraymap; + static constexpr size_t offset = sub1(to_bits(tx::size)); struct record : public schema::prevout { + inline bool coinbase() const NOEXCEPT + { + return system::get_right(value, offset); + } + + inline tx::integer output_tx_fk() const NOEXCEPT + { + return system::set_right(value, offset, false); + } + + inline void set(bool coinbase, tx::integer output_tx_fk) NOEXCEPT + { + using namespace system; + BC_ASSERT_MSG(!get_right(output_tx_fk, offset), "overflow"); + value = set_right(output_tx_fk, offset, coinbase); + } + inline bool from_data(reader& source) NOEXCEPT { - coinbase = source.read_byte(); - spend_fk = source.read_little_endian(); - output_tx_fk = source.read_little_endian(); + value = source.read_little_endian(); BC_ASSERT(!source || source.get_read_position() == minrow); return source; } inline bool to_data(finalizer& sink) const NOEXCEPT { - sink.write_byte(coinbase); - sink.write_little_endian(spend_fk); - sink.write_little_endian(output_tx_fk); + sink.write_little_endian(value); BC_ASSERT(!sink || sink.get_write_position() == minrow); return sink; } inline bool operator==(const record& other) const NOEXCEPT { - return coinbase == other.coinbase - && spend_fk == other.spend_fk - && output_tx_fk == other.output_tx_fk; + return coinbase() == other.coinbase() + && output_tx_fk() == other.output_tx_fk(); + } + + tx::integer value{}; + }; + + struct record_put_ref + : public schema::prevout + { + static constexpr tx::integer merge(bool coinbase, + tx::integer output_tx_fk) NOEXCEPT + { + using namespace system; + BC_ASSERT_MSG(!get_right(output_tx_fk, offset), "overflow"); + return system::set_right(output_tx_fk, offset, coinbase); + } + + // This is called once by put(), and hides base count(). + inline link count() const NOEXCEPT + { + const auto spends = block.spends(); + BC_ASSERT(spends < link::terminal); + return system::possible_narrow_cast(spends); + } + + inline bool to_data(finalizer& sink) const NOEXCEPT + { + const auto txs = *block.transactions_ptr(); + if (txs.size() <= one) + { + // Empty or coinbase only implies no spends. + sink.invalidate(); + } + else + { + const auto write_spend = [&](const auto& in) NOEXCEPT + { + // Sets terminal sentinel for block-internal spends. + const auto value = in->metadata.inside ? tx::terminal : + merge(in->metadata.coinbase, in->metadata.parent); + + sink.write_little_endian(value); + }; + + const auto write_tx = [&](const auto& tx) NOEXCEPT + { + const auto& ins = tx->inputs_ptr(); + return std::for_each(ins->begin(), ins->end(), write_spend); + }; + + std::for_each(std::next(txs.begin()), txs.end(), write_tx); + } + + BC_ASSERT(!sink || (sink.get_write_position() == count() * minrow)); + return sink; + } + + const system::chain::block& block{}; + }; + + struct record_get + : public schema::prevout + { + // This is called once by assert, and hides base class count(). + inline link count() const NOEXCEPT + { + BC_ASSERT(values.size() < link::terminal); + return system::possible_narrow_cast(values.size()); + } + + inline bool from_data(reader& source) NOEXCEPT + { + // Values must be set to read size (i.e. using knowledge of spends). + std::for_each(values.begin(), values.end(), [&](auto& value) NOEXCEPT + { + value = source.read_little_endian(); + }); + + BC_ASSERT(!source || source.get_read_position() == count() * minrow); + return source; + } + + inline bool inside(size_t index) const NOEXCEPT + { + BC_ASSERT(index < count()); + + // Identifies terminal sentinel as block-internal spend. + return values.at(index) == tx::terminal; + } + + inline bool coinbase(size_t index) const NOEXCEPT + { + BC_ASSERT(index < count()); + + // Inside are always reflected as coinbase. + return system::get_right(values.at(index), offset); + } + + inline tx::integer output_tx_fk(size_t index) const NOEXCEPT + { + BC_ASSERT(index < count()); + + // Inside are always mapped to terminal. + return inside(index) ? tx::terminal : + system::set_right(values.at(index), offset, false); } - bool coinbase{}; - spend::integer spend_fk{}; - tx::integer output_tx_fk{}; + // Spend count is derived in confirmation by summing block.txs.puts. + std::vector values{}; }; }; diff --git a/include/bitcoin/database/tables/schema.hpp b/include/bitcoin/database/tables/schema.hpp index 5327f3bb9..08be20358 100644 --- a/include/bitcoin/database/tables/schema.hpp +++ b/include/bitcoin/database/tables/schema.hpp @@ -319,6 +319,22 @@ namespace schema /// Cache tables. /// ----------------------------------------------------------------------- + // record arraymap + struct prevout + { + static constexpr size_t pk = schema::spend_; + static constexpr size_t minsize = + ////schema::bit + // merged bit into tx. + schema::tx; + static constexpr size_t minrow = minsize; + static constexpr size_t size = minsize; + + // This is hidden by derivatives, to avoid virtual methods. + inline linkage count() const NOEXCEPT { return one; } + static_assert(minsize == 4u); + static_assert(minrow == 4u); + }; + // slab hashmap struct validated_bk { @@ -355,22 +371,6 @@ namespace schema static_assert(minrow == 23u); }; - // record arraymap - struct prevout - { - static constexpr size_t pk = schema::spend_; - ////static constexpr size_t sk = zero; - static constexpr size_t minsize = - schema::bit + // TODO: merge bit. - schema::spend_ + - schema::tx; - static constexpr size_t minrow = minsize; - static constexpr size_t size = minsize; - static constexpr linkage count() NOEXCEPT { return 1; } - static_assert(minsize == 9u); - static_assert(minrow == 9u); - }; - // slab hashmap struct neutrino { diff --git a/test/tables/caches/prevout.cpp b/test/tables/caches/prevout.cpp index 99c70580c..e80582c7c 100644 --- a/test/tables/caches/prevout.cpp +++ b/test/tables/caches/prevout.cpp @@ -21,13 +21,124 @@ BOOST_AUTO_TEST_SUITE(prevout_tests) +// Setting block metadata on a shared instance creates test side effects. +// Chain objects such as blocks cannot be copied for side-effect-free metadata tests, since +// block copy takes shared pointer references. So create new test blocks for each metadata test. +#define DECLARE_BOGUS_BLOCK \ + const block bogus_block \ + { \ + header \ + { \ + 0x31323334, \ + system::null_hash, \ + system::one_hash, \ + 0x41424344, \ + 0x51525354, \ + 0x61626364 \ + }, \ + transactions \ + { \ + transaction \ + { \ + 0x01, \ + inputs \ + { \ + input \ + { \ + point{}, \ + script{}, \ + witness{}, \ + 0x02 \ + }, \ + input \ + { \ + point{}, \ + script{}, \ + witness{}, \ + 0x03 \ + } \ + }, \ + outputs \ + { \ + output \ + { \ + 0x04, \ + script{} \ + } \ + }, \ + 0x05 \ + }, \ + transaction \ + { \ + 0x06, \ + inputs \ + { \ + input \ + { \ + point{}, \ + script{}, \ + witness{}, \ + 0x07 \ + }, \ + input \ + { \ + point{}, \ + script{}, \ + witness{}, \ + 0x08 \ + } \ + }, \ + outputs \ + { \ + output \ + { \ + 0x09, \ + script{} \ + } \ + }, \ + 0x0a \ + }, \ + transaction \ + { \ + 0x0b, \ + inputs \ + { \ + input \ + { \ + point{}, \ + script{}, \ + witness{}, \ + 0x0c \ + }, \ + input \ + { \ + point{}, \ + script{}, \ + witness{}, \ + 0x0d \ + } \ + }, \ + outputs \ + { \ + output \ + { \ + 0x0e, \ + script{} \ + } \ + }, \ + 0x0f \ + } \ + } \ + } + using namespace system; -constexpr table::prevout::record record1{ {}, true, 0x01020304, 0xbaadf00d }; -constexpr table::prevout::record record2{ {}, false, 0xbaadf00d, 0x01020304 }; +using namespace system::chain; +constexpr auto terminal = linkage::terminal; +constexpr table::prevout::record record1{ {}, 0x01020304_u32 }; +constexpr table::prevout::record record2{ {}, 0xbaadf00d_u32 }; -BOOST_AUTO_TEST_CASE(header__put__at1__expected) +BOOST_AUTO_TEST_CASE(prevout__put__at1__expected) { - table::prevout::record element{}; test::chunk_storage head_store{}; test::chunk_storage body_store{}; table::prevout instance{ head_store, body_store, 5 }; @@ -43,7 +154,7 @@ BOOST_AUTO_TEST_CASE(header__put__at1__expected) BOOST_REQUIRE_EQUAL(instance.at(42), 1u); } -BOOST_AUTO_TEST_CASE(header__put__at2__expected) +BOOST_AUTO_TEST_CASE(prevout__put__at2__expected) { table::prevout::record element{}; test::chunk_storage head_store{}; @@ -63,9 +174,8 @@ BOOST_AUTO_TEST_CASE(header__put__at2__expected) BOOST_REQUIRE(element == record2); } -BOOST_AUTO_TEST_CASE(header__put__exists__expected) +BOOST_AUTO_TEST_CASE(prevout__put__exists__expected) { - table::prevout::record element{}; test::chunk_storage head_store{}; test::chunk_storage body_store{}; table::prevout instance{ head_store, body_store, 5 }; @@ -81,7 +191,7 @@ BOOST_AUTO_TEST_CASE(header__put__exists__expected) BOOST_REQUIRE(instance.exists(42)); } -BOOST_AUTO_TEST_CASE(header__put__get__expected) +BOOST_AUTO_TEST_CASE(prevout__put__get__expected) { table::prevout::record element{}; test::chunk_storage head_store{}; @@ -101,4 +211,196 @@ BOOST_AUTO_TEST_CASE(header__put__get__expected) BOOST_REQUIRE(element == record2); } +// values + +BOOST_AUTO_TEST_CASE(prevout__put__isolated_values__expected) +{ + constexpr auto bits = sub1(to_bits(linkage::size)); + + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + table::prevout instance{ head_store, body_store, 5 }; + BOOST_REQUIRE(instance.create()); + + constexpr auto cb_only = table::prevout::record{ {}, 0b10000000'00000000'00000000'00000000_u32 }; + BOOST_REQUIRE(instance.put(3, cb_only)); + + constexpr auto tx_only = table::prevout::record{ {}, 0b01010101'01010101'01010101'01010101_u32 }; + BOOST_REQUIRE(instance.put(42, tx_only)); + + table::prevout::record element1{}; + BOOST_REQUIRE(instance.at(3, element1)); + BOOST_REQUIRE(element1.coinbase()); + BOOST_REQUIRE_EQUAL(element1.coinbase(), cb_only.coinbase()); + BOOST_REQUIRE_EQUAL(element1.output_tx_fk(), cb_only.output_tx_fk()); + BOOST_REQUIRE_EQUAL(element1.output_tx_fk(), set_right(cb_only.value, bits, false)); + + table::prevout::record element2{}; + BOOST_REQUIRE(instance.at(42, element2)); + BOOST_REQUIRE(!element2.coinbase()); + BOOST_REQUIRE_EQUAL(element2.coinbase(), tx_only.coinbase()); + BOOST_REQUIRE_EQUAL(element2.output_tx_fk(), tx_only.output_tx_fk()); + BOOST_REQUIRE_EQUAL(element2.output_tx_fk(), set_right(tx_only.value, bits, false)); +} + +BOOST_AUTO_TEST_CASE(prevout__put__merged_values__expected) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + table::prevout instance{ head_store, body_store, 5 }; + BOOST_REQUIRE(instance.create()); + + constexpr auto expected_cb = true; + constexpr auto expected_tx = 0b01010101'01010101'01010101'01010101_u32; + auto record = table::prevout::record{ {}, 0_u32 }; + record.set(expected_cb, expected_tx); + BOOST_REQUIRE(instance.put(3, record)); + + table::prevout::record element{}; + BOOST_REQUIRE(instance.at(3, element)); + BOOST_REQUIRE_EQUAL(element.coinbase(), expected_cb); + BOOST_REQUIRE_EQUAL(element.output_tx_fk(), expected_tx); +} + +// record_put_ref + +BOOST_AUTO_TEST_CASE(prevout__record_put_ref__empty_block__false) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + table::prevout instance{ head_store, body_store, 5 }; + BOOST_REQUIRE(instance.create()); + + const auto genesis = system::settings(selection::mainnet).genesis_block; + const auto record = table::prevout::record_put_ref{ {}, genesis }; + BOOST_REQUIRE(!instance.put(4, record)); +} + +BOOST_AUTO_TEST_CASE(prevout__put_ref__get_non_empty_block_with_default_metadata__inside_spend_terminals) +{ + DECLARE_BOGUS_BLOCK; + + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + table::prevout instance{ head_store, body_store, 5 }; + BOOST_REQUIRE(instance.create()); + + const auto record = table::prevout::record_put_ref{ {}, bogus_block }; + BOOST_REQUIRE(instance.put(2, record)); + + table::prevout::record_get element{}; + const auto spends = bogus_block.spends(); + BOOST_REQUIRE_EQUAL(spends, 4u); + + element.values.resize(spends); + BOOST_REQUIRE(instance.at(2, element)); + BOOST_REQUIRE_EQUAL(element.count(), 4u); + + // First block.tx is coinbase, no spends, so only 4, all terminal (defaults). + BOOST_REQUIRE_EQUAL(element.values.at(0), terminal); + BOOST_REQUIRE_EQUAL(element.values.at(1), terminal); + BOOST_REQUIRE_EQUAL(element.values.at(2), terminal); + BOOST_REQUIRE_EQUAL(element.values.at(3), terminal); + + // Block-internal spend. + // Positionally identifies spends not requiring a double spend check. + // Blocks are guarded agianst internal double spends, and tx previously + // confirmed would imply a double spend of it or one of its ancestors. + BOOST_REQUIRE(element.inside(0)); + BOOST_REQUIRE(element.inside(1)); + BOOST_REQUIRE(element.inside(2)); + BOOST_REQUIRE(element.inside(3)); + + // Inside are always reflected as coinbase. + BOOST_REQUIRE(element.coinbase(0)); + BOOST_REQUIRE(element.coinbase(1)); + BOOST_REQUIRE(element.coinbase(2)); + BOOST_REQUIRE(element.coinbase(3)); + + // Inside are always mapped to terminal. + BOOST_REQUIRE_EQUAL(element.output_tx_fk(0), terminal); + BOOST_REQUIRE_EQUAL(element.output_tx_fk(1), terminal); + BOOST_REQUIRE_EQUAL(element.output_tx_fk(2), terminal); + BOOST_REQUIRE_EQUAL(element.output_tx_fk(3), terminal); +} + +BOOST_AUTO_TEST_CASE(prevout__put_ref__get_non_empty_block_with_metadata__expected) +{ + DECLARE_BOGUS_BLOCK; + + const auto tx0 = bogus_block.transactions_ptr()->at(0); + const auto tx1 = bogus_block.transactions_ptr()->at(1); + const auto tx2 = bogus_block.transactions_ptr()->at(2); + + const auto& in0_0 = *tx0->inputs_ptr()->at(0); + const auto& in0_1 = *tx0->inputs_ptr()->at(1); + const auto& in1_0 = *tx1->inputs_ptr()->at(0); + const auto& in1_1 = *tx1->inputs_ptr()->at(1); + const auto& in2_0 = *tx2->inputs_ptr()->at(0); + const auto& in2_1 = *tx2->inputs_ptr()->at(1); + + // Coinbase identifies parent of prevout (spent), not input (spender). + in0_0.metadata.parent = 0x01234560_u32; + in0_1.metadata.parent = 0x01234561_u32; + in1_0.metadata.parent = 0x01234562_u32; + in1_1.metadata.parent = 0x01234563_u32; + in2_0.metadata.parent = 0x01234564_u32; + in2_1.metadata.parent = 0x01234565_u32; + + // Coinbase identifies prevout (spent), not input (spender). + in0_0.metadata.coinbase = false; + in0_1.metadata.coinbase = true; + in1_0.metadata.coinbase = false; + in1_1.metadata.coinbase = false; + in2_0.metadata.coinbase = false; + in2_1.metadata.coinbase = true; + + // Inside implies prevout spent in block (terminal). + in0_0.metadata.inside = false; + in0_1.metadata.inside = true; + in1_0.metadata.inside = false; + in1_1.metadata.inside = false; + in2_0.metadata.inside = true; + in2_1.metadata.inside = false; + + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + table::prevout instance{ head_store, body_store, 5 }; + BOOST_REQUIRE(instance.create()); + + const auto record = table::prevout::record_put_ref{ {}, bogus_block }; + BOOST_REQUIRE(instance.put(2, record)); + + table::prevout::record_get element{}; + const auto spends = bogus_block.spends(); + BOOST_REQUIRE_EQUAL(spends, 4u); + + element.values.resize(spends); + BOOST_REQUIRE(instance.at(2, element)); + BOOST_REQUIRE_EQUAL(element.count(), spends); + + // First block.tx is coinbase, no spends, so only 4, none terminal. + BOOST_REQUIRE_NE(element.values.at(0), terminal); + BOOST_REQUIRE_NE(element.values.at(1), terminal); + BOOST_REQUIRE_EQUAL(element.values.at(2), terminal); + BOOST_REQUIRE_NE(element.values.at(3), terminal); + + // Inside spends are used as positional placeholders for double spend check bypass. + BOOST_REQUIRE(!element.inside(0)); + BOOST_REQUIRE(!element.inside(1)); + BOOST_REQUIRE(element.inside(2)); + BOOST_REQUIRE(!element.inside(3)); + + BOOST_REQUIRE(!element.coinbase(0)); + BOOST_REQUIRE(!element.coinbase(1)); + BOOST_REQUIRE(element.coinbase(2)); + BOOST_REQUIRE(element.coinbase(3)); + + // Spend ordinal position relative to the block must be preserved (coinbase excluded). + BOOST_REQUIRE_EQUAL(element.output_tx_fk(0), in1_0.metadata.parent); + BOOST_REQUIRE_EQUAL(element.output_tx_fk(1), in1_1.metadata.parent); + BOOST_REQUIRE_EQUAL(element.output_tx_fk(2), terminal); + BOOST_REQUIRE_EQUAL(element.output_tx_fk(3), in2_1.metadata.parent); +} + BOOST_AUTO_TEST_SUITE_END()