diff --git a/Makefile.am b/Makefile.am index e055901dd..3c6f3ac6c 100644 --- a/Makefile.am +++ b/Makefile.am @@ -78,7 +78,9 @@ test_libbitcoin_database_test_SOURCES = \ test/mocks/map_store.hpp \ test/primitives/arraymap.cpp \ test/primitives/hashmap.cpp \ + test/primitives/hashmap2.cpp \ test/primitives/head.cpp \ + test/primitives/head2.cpp \ test/primitives/iterator.cpp \ test/primitives/linkage.cpp \ test/primitives/manager.cpp \ @@ -95,17 +97,17 @@ test_libbitcoin_database_test_SOURCES = \ test/tables/archives/output.cpp \ test/tables/archives/point.cpp \ test/tables/archives/puts.cpp \ + test/tables/archives/spend.cpp \ test/tables/archives/transaction.cpp \ test/tables/archives/txs.cpp \ - test/tables/caches/bootstrap.cpp \ - test/tables/caches/buffer.cpp \ - test/tables/caches/neutrino.cpp \ + test/tables/caches/prevouts.cpp \ test/tables/caches/validated_bk.cpp \ test/tables/caches/validated_tx.cpp \ - test/tables/indexes/address.cpp \ test/tables/indexes/height.cpp \ - test/tables/indexes/spend.cpp \ - test/tables/indexes/strong_tx.cpp + test/tables/indexes/strong_tx.cpp \ + test/tables/optional/address.cpp \ + test/tables/optional/bootstrap.cpp \ + test/tables/optional/neutrino.cpp endif WITH_TESTS @@ -159,7 +161,9 @@ include_bitcoin_database_impl_primitivesdir = ${includedir}/bitcoin/database/imp include_bitcoin_database_impl_primitives_HEADERS = \ include/bitcoin/database/impl/primitives/arraymap.ipp \ include/bitcoin/database/impl/primitives/hashmap.ipp \ + include/bitcoin/database/impl/primitives/hashmap2.ipp \ include/bitcoin/database/impl/primitives/head.ipp \ + include/bitcoin/database/impl/primitives/head2.ipp \ include/bitcoin/database/impl/primitives/iterator.ipp \ include/bitcoin/database/impl/primitives/linkage.ipp \ include/bitcoin/database/impl/primitives/manager.ipp @@ -204,7 +208,9 @@ include_bitcoin_database_primitivesdir = ${includedir}/bitcoin/database/primitiv include_bitcoin_database_primitives_HEADERS = \ include/bitcoin/database/primitives/arraymap.hpp \ include/bitcoin/database/primitives/hashmap.hpp \ + include/bitcoin/database/primitives/hashmap2.hpp \ include/bitcoin/database/primitives/head.hpp \ + include/bitcoin/database/primitives/head2.hpp \ include/bitcoin/database/primitives/iterator.hpp \ include/bitcoin/database/primitives/linkage.hpp \ include/bitcoin/database/primitives/manager.hpp \ @@ -231,6 +237,7 @@ include_bitcoin_database_tables_archives_HEADERS = \ include_bitcoin_database_tables_cachesdir = ${includedir}/bitcoin/database/tables/caches include_bitcoin_database_tables_caches_HEADERS = \ + include/bitcoin/database/tables/caches/prevouts.hpp \ include/bitcoin/database/tables/caches/validated_bk.hpp \ include/bitcoin/database/tables/caches/validated_tx.hpp @@ -243,7 +250,6 @@ include_bitcoin_database_tables_optionalsdir = ${includedir}/bitcoin/database/ta include_bitcoin_database_tables_optionals_HEADERS = \ include/bitcoin/database/tables/optionals/address.hpp \ include/bitcoin/database/tables/optionals/bootstrap.hpp \ - include/bitcoin/database/tables/optionals/buffer.hpp \ include/bitcoin/database/tables/optionals/neutrino.hpp diff --git a/builds/cmake/CMakeLists.txt b/builds/cmake/CMakeLists.txt index 58b947e8b..36fee6177 100644 --- a/builds/cmake/CMakeLists.txt +++ b/builds/cmake/CMakeLists.txt @@ -286,7 +286,9 @@ if (with-tests) "../../test/mocks/map_store.hpp" "../../test/primitives/arraymap.cpp" "../../test/primitives/hashmap.cpp" + "../../test/primitives/hashmap2.cpp" "../../test/primitives/head.cpp" + "../../test/primitives/head2.cpp" "../../test/primitives/iterator.cpp" "../../test/primitives/linkage.cpp" "../../test/primitives/manager.cpp" @@ -303,17 +305,17 @@ if (with-tests) "../../test/tables/archives/output.cpp" "../../test/tables/archives/point.cpp" "../../test/tables/archives/puts.cpp" + "../../test/tables/archives/spend.cpp" "../../test/tables/archives/transaction.cpp" "../../test/tables/archives/txs.cpp" - "../../test/tables/caches/bootstrap.cpp" - "../../test/tables/caches/buffer.cpp" - "../../test/tables/caches/neutrino.cpp" + "../../test/tables/caches/prevouts.cpp" "../../test/tables/caches/validated_bk.cpp" "../../test/tables/caches/validated_tx.cpp" - "../../test/tables/indexes/address.cpp" "../../test/tables/indexes/height.cpp" - "../../test/tables/indexes/spend.cpp" - "../../test/tables/indexes/strong_tx.cpp" ) + "../../test/tables/indexes/strong_tx.cpp" + "../../test/tables/optional/address.cpp" + "../../test/tables/optional/bootstrap.cpp" + "../../test/tables/optional/neutrino.cpp" ) add_test( NAME libbitcoin-database-test COMMAND libbitcoin-database-test --run_test=* diff --git a/builds/msvc/vs2022/libbitcoin-database-test/libbitcoin-database-test.vcxproj b/builds/msvc/vs2022/libbitcoin-database-test/libbitcoin-database-test.vcxproj index ebbd2ed54..ee72c818f 100644 --- a/builds/msvc/vs2022/libbitcoin-database-test/libbitcoin-database-test.vcxproj +++ b/builds/msvc/vs2022/libbitcoin-database-test/libbitcoin-database-test.vcxproj @@ -87,7 +87,9 @@ + + @@ -106,17 +108,17 @@ + - - - + - - + + + diff --git a/builds/msvc/vs2022/libbitcoin-database-test/libbitcoin-database-test.vcxproj.filters b/builds/msvc/vs2022/libbitcoin-database-test/libbitcoin-database-test.vcxproj.filters index 3fc88056b..4f7eb488a 100644 --- a/builds/msvc/vs2022/libbitcoin-database-test/libbitcoin-database-test.vcxproj.filters +++ b/builds/msvc/vs2022/libbitcoin-database-test/libbitcoin-database-test.vcxproj.filters @@ -40,6 +40,9 @@ {5C4DA53A-8C06-4DA6-0000-00000000000A} + + {5C4DA53A-8C06-4DA6-0000-00000000000B} + @@ -81,9 +84,15 @@ src\primitives + + src\primitives + src\primitives + + src\primitives + src\primitives @@ -138,19 +147,16 @@ src\tables\archives + + src\tables\archives + src\tables\archives src\tables\archives - - src\tables\caches - - - src\tables\caches - - + src\tables\caches @@ -159,18 +165,21 @@ src\tables\caches - - src\tables\indexes - src\tables\indexes - - src\tables\indexes - src\tables\indexes + + src\tables\optional + + + src\tables\optional + + + src\tables\optional + src diff --git a/builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj b/builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj index c4853d9d1..b5a8344fb 100644 --- a/builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj +++ b/builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj @@ -115,7 +115,9 @@ + + @@ -131,6 +133,7 @@ + @@ -139,7 +142,6 @@ - @@ -153,7 +155,9 @@ + + diff --git a/builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj.filters b/builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj.filters index 537a8dd86..7ab8c117c 100644 --- a/builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj.filters +++ b/builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj.filters @@ -188,9 +188,15 @@ include\bitcoin\database\primitives + + include\bitcoin\database\primitives + include\bitcoin\database\primitives + + include\bitcoin\database\primitives + include\bitcoin\database\primitives @@ -236,6 +242,9 @@ include\bitcoin\database\tables\archives + + include\bitcoin\database\tables\caches + include\bitcoin\database\tables\caches @@ -260,9 +269,6 @@ include\bitcoin\database\tables\optionals - - include\bitcoin\database\tables\optionals - include\bitcoin\database\tables\optionals @@ -298,9 +304,15 @@ include\bitcoin\database\impl\primitives + + include\bitcoin\database\impl\primitives + include\bitcoin\database\impl\primitives + + include\bitcoin\database\impl\primitives + include\bitcoin\database\impl\primitives diff --git a/include/bitcoin/database.hpp b/include/bitcoin/database.hpp index 2696fb79b..fb615bb16 100644 --- a/include/bitcoin/database.hpp +++ b/include/bitcoin/database.hpp @@ -44,7 +44,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -62,13 +64,13 @@ #include #include #include +#include #include #include #include #include #include #include -#include #include #endif diff --git a/include/bitcoin/database/impl/primitives/hashmap.ipp b/include/bitcoin/database/impl/primitives/hashmap.ipp index c68d84a0a..51b7079ad 100644 --- a/include/bitcoin/database/impl/primitives/hashmap.ipp +++ b/include/bitcoin/database/impl/primitives/hashmap.ipp @@ -155,6 +155,7 @@ typename CLASS::iterator CLASS::it(const Key& key) const NOEXCEPT { return { get_memory(), head_.top(key), key }; } + TEMPLATE Link CLASS::allocate(const Link& size) NOEXCEPT { @@ -307,7 +308,6 @@ bool CLASS::put(const Link& link, const Key& key, const Element& element) NOEXCEPT { using namespace system; - const auto count = element.count(); const auto ptr = manager_.get(link); if (!ptr) return false; @@ -318,7 +318,11 @@ bool CLASS::put(const Link& link, const Key& key, sink.skip_bytes(Link::size); sink.write_bytes(key); - if constexpr (!is_slab) { BC_DEBUG_ONLY(sink.set_limit(Size * count);) } + if constexpr (!is_slab) + { + BC_DEBUG_ONLY(sink.set_limit(Size * element.count());) + } + auto& next = unsafe_array_cast(ptr->begin()); return element.to_data(sink) && head_.push(link, next, head_.index(key)); } diff --git a/include/bitcoin/database/impl/primitives/hashmap2.ipp b/include/bitcoin/database/impl/primitives/hashmap2.ipp new file mode 100644 index 000000000..60e15de4f --- /dev/null +++ b/include/bitcoin/database/impl/primitives/hashmap2.ipp @@ -0,0 +1,226 @@ +/** + * Copyright (c) 2011-2025 libbitcoin developers (see AUTHORS) + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef LIBBITCOIN_DATABASE_PRIMITIVES_HASHMAP2_IPP +#define LIBBITCOIN_DATABASE_PRIMITIVES_HASHMAP2_IPP + +#include +#include +#include + +namespace libbitcoin { +namespace database { + +TEMPLATE +CLASS::hashmap2(storage& header, storage& body, const Link& buckets) NOEXCEPT + : head_(header, buckets), manager_(body) +{ +} + +// not thread safe +// ---------------------------------------------------------------------------- + +TEMPLATE +bool CLASS::create() NOEXCEPT +{ + Link count{}; + return head_.create() && head_.get_body_count(count) && + manager_.truncate(count); +} + +TEMPLATE +bool CLASS::close() NOEXCEPT +{ + return head_.set_body_count(manager_.count()); +} + +TEMPLATE +bool CLASS::backup() NOEXCEPT +{ + return head_.set_body_count(manager_.count()); +} + +TEMPLATE +bool CLASS::restore() NOEXCEPT +{ + Link count{}; + return head_.verify() && head_.get_body_count(count) && + manager_.truncate(count); +} + +TEMPLATE +bool CLASS::verify() const NOEXCEPT +{ + Link count{}; + return head_.verify() && head_.get_body_count(count) && + (count == manager_.count()); +} + +// sizing +// ---------------------------------------------------------------------------- + +TEMPLATE +bool CLASS::enabled() const NOEXCEPT +{ + return head_.enabled(); +} + +TEMPLATE +size_t CLASS::buckets() const NOEXCEPT +{ + return head_.buckets(); +} + +TEMPLATE +size_t CLASS::head_size() const NOEXCEPT +{ + return head_.size(); +} + +TEMPLATE +size_t CLASS::body_size() const NOEXCEPT +{ + return manager_.size(); +} + +TEMPLATE +Link CLASS::count() const NOEXCEPT +{ + return manager_.count(); +} + +// query interface +// ---------------------------------------------------------------------------- + +TEMPLATE +code CLASS::get_fault() const NOEXCEPT +{ + return manager_.get_fault(); +} + +TEMPLATE +size_t CLASS::get_space() const NOEXCEPT +{ + return manager_.get_space(); +} + +TEMPLATE +code CLASS::reload() NOEXCEPT +{ + return manager_.reload(); +} + +// query interface +// ---------------------------------------------------------------------------- + +TEMPLATE +Link CLASS::top(const Link& link) const NOEXCEPT +{ + if (link >= head_.buckets()) + return {}; + + return head_.top(link); +} + +TEMPLATE +bool CLASS::exists(const Key& key) const NOEXCEPT +{ + return !first(key).is_terminal(); +} + +TEMPLATE +Link CLASS::first(const Key& key) const NOEXCEPT +{ + return head_.top(key); +} + +TEMPLATE +template > +bool CLASS::find(const Key& key, Element& element) const NOEXCEPT +{ + // This override avoids duplicated memory_ptr construct in get(first()). + const auto ptr = manager_.get(); + return read(ptr, first(ptr, head_.top(key), key), element); +} + +TEMPLATE +template > +bool CLASS::get(const Link& link, Element& element) const NOEXCEPT +{ + // This override is the normal form. + return read(manager_.get(), link, element); +} + +TEMPLATE +template > +bool CLASS::put(const Key& key, const Element& element) NOEXCEPT +{ + using namespace system; + const auto count = element.count(); + const auto link = allocate(count); + const auto ptr = manager_.get(link); + if (!ptr) + return false; + + // iostream.flush is a nop (direct copy). + iostream stream{ *ptr }; + finalizer sink{ stream }; + sink.skip_bytes(Link::size); + sink.write_bytes(key); + + if constexpr (!is_slab) { BC_DEBUG_ONLY(sink.set_limit(Size * count);) } + return element.to_data(sink) && head_.push(link, head_.index(key)); +} + +// protected +// ---------------------------------------------------------------------------- + +TEMPLATE +template > +bool CLASS::read(const memory_ptr& ptr, const Link& link, + Element& element) NOEXCEPT +{ + if (!ptr || link.is_terminal()) + return false; + + using namespace system; + const auto start = manager::link_to_position(link); + if (is_limited(start)) + return false; + + const auto size = ptr->size(); + const auto position = possible_narrow_and_sign_cast(start); + if (position > size) + return false; + + const auto offset = ptr->offset(position); + if (is_null(offset)) + return false; + + // Stream starts at record and the index is skipped for reader convenience. + iostream stream{ offset, size - position }; + reader source{ stream }; + + if constexpr (!is_slab) { BC_DEBUG_ONLY(source.set_limit(Size);) } + return element.from_data(source); +} + +} // namespace database +} // namespace libbitcoin + +#endif diff --git a/include/bitcoin/database/impl/primitives/head.ipp b/include/bitcoin/database/impl/primitives/head.ipp index 7363c6dd3..5634768be 100644 --- a/include/bitcoin/database/impl/primitives/head.ipp +++ b/include/bitcoin/database/impl/primitives/head.ipp @@ -38,7 +38,7 @@ CLASS::head(storage& head, const Link& buckets) NOEXCEPT TEMPLATE size_t CLASS::size() const NOEXCEPT { - return offset(buckets_); + return link_to_position(buckets_); } TEMPLATE @@ -74,7 +74,7 @@ bool CLASS::create() NOEXCEPT const auto allocation = size(); const auto start = file_.allocate(allocation); - // This guards addition overflow in file_.get (start must be valid). + // Guards addition overflow in file_.get (start must be valid). if (start == storage::eof) return false; @@ -85,8 +85,8 @@ bool CLASS::create() NOEXCEPT BC_ASSERT_MSG(verify(), "unexpected body size"); // std::memset/fill_n have identical performance (on win32). - ////std::memset(ptr->begin(), system::bit_all, allocation); - std::fill_n(ptr->begin(), allocation, system::bit_all); + ////std::memset(ptr->data(), system::bit_all, allocation); + std::fill_n(ptr->data(), allocation, system::bit_all); return set_body_count(zero); } @@ -103,7 +103,7 @@ bool CLASS::get_body_count(Link& count) const NOEXCEPT if (!ptr) return false; - count = array_cast(ptr->begin()); + count = array_cast(ptr->data()); return true; } @@ -114,7 +114,7 @@ bool CLASS::set_body_count(const Link& count) NOEXCEPT if (!ptr) return false; - array_cast(ptr->begin()) = count; + array_cast(ptr->data()) = count; return true; } @@ -127,7 +127,7 @@ Link CLASS::top(const Key& key) const NOEXCEPT TEMPLATE Link CLASS::top(const Link& index) const NOEXCEPT { - const auto raw = file_.get_raw(offset(index)); + const auto raw = file_.get_raw(link_to_position(index)); if (is_null(raw)) return {}; @@ -148,7 +148,7 @@ bool CLASS::push(const bytes& current, bytes& next, const Key& key) NOEXCEPT TEMPLATE bool CLASS::push(const bytes& current, bytes& next, const Link& index) NOEXCEPT { - const auto raw = file_.get_raw(offset(index)); + const auto raw = file_.get_raw(link_to_position(index)); if (is_null(raw)) return false; diff --git a/include/bitcoin/database/impl/primitives/head2.ipp b/include/bitcoin/database/impl/primitives/head2.ipp new file mode 100644 index 000000000..2eebc233b --- /dev/null +++ b/include/bitcoin/database/impl/primitives/head2.ipp @@ -0,0 +1,162 @@ +/** + * Copyright (c) 2011-2025 libbitcoin developers (see AUTHORS) + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef LIBBITCOIN_DATABASE_PRIMITIVES_HEAD2_IPP +#define LIBBITCOIN_DATABASE_PRIMITIVES_HEAD2_IPP + +#include +#include +#include + +namespace libbitcoin { +namespace database { + +TEMPLATE +CLASS::head2(storage& head, const Link& buckets) NOEXCEPT + : file_(head), initial_buckets_(buckets) +{ +} + +TEMPLATE +size_t CLASS::size() const NOEXCEPT +{ + return file_.size(); +} + +TEMPLATE +size_t CLASS::buckets() const NOEXCEPT +{ + const auto count = position_to_link(size()).value; + BC_ASSERT(count < Link::terminal); + return system::possible_narrow_cast(count); +} + +TEMPLATE +bool CLASS::enabled() const NOEXCEPT +{ + return initial_buckets_ > one; +} + +TEMPLATE +Link CLASS::index(const Key& key) const NOEXCEPT +{ + // Key is the logical bucket index (no-hash). + if (key < buckets()) + return manager, Link::size>:: + cast_link(key); + + return {}; +} + +TEMPLATE +bool CLASS::create() NOEXCEPT +{ + if (is_nonzero(file_.size())) + return false; + + const auto allocation = link_to_position(initial_buckets_); + const auto start = file_.allocate(allocation); + + // Guards addition overflow in manager_.get (start must be valid). + if (start == storage::eof) + return false; + + const auto ptr = file_.get(start); + if (!ptr) + return false; + + BC_ASSERT_MSG(verify(), "unexpected body size"); + + // std::memset/fill_n have identical performance (on win32). + ////std::memset(ptr->data(), system::bit_all, size()); + std::fill_n(ptr->data(), size(), system::bit_all); + return set_body_count(zero); +} + +TEMPLATE +bool CLASS::verify() const NOEXCEPT +{ + return buckets() >= initial_buckets_; +} + +TEMPLATE +bool CLASS::get_body_count(Link& count) const NOEXCEPT +{ + const auto ptr = file_.get(); + if (!ptr) + return false; + + count = array_cast(ptr->data()); + return true; +} + +TEMPLATE +bool CLASS::set_body_count(const Link& count) NOEXCEPT +{ + const auto ptr = file_.get(); + if (!ptr) + return false; + + array_cast(ptr->data()) = count; + return true; +} + +TEMPLATE +Link CLASS::top(const Key& key) const NOEXCEPT +{ + return top(index(key)); +} + +TEMPLATE +Link CLASS::top(const Link& index) const NOEXCEPT +{ + const auto ptr = file_.get(link_to_position(index)); + if (is_null(ptr)) + return {}; + + const auto& head = array_cast(ptr->data()); + + mutex_.lock_shared(); + const auto top = head; + mutex_.unlock_shared(); + return top; +} + +TEMPLATE +bool CLASS::push(const bytes& current, const Link& index) NOEXCEPT +{ + constexpr auto fill = system::bit_all; + + // Allocate as necessary and fill allocations. + const auto ptr = file_.set(link_to_position(index), Link::size, fill); + + if (is_null(ptr)) + return false; + + auto& head = array_cast(ptr->data()); + + mutex_.lock(); + head = current; + mutex_.unlock(); + return true; +} + +} // namespace database +} // namespace libbitcoin + +#endif diff --git a/include/bitcoin/database/impl/primitives/iterator.ipp b/include/bitcoin/database/impl/primitives/iterator.ipp index c48467da6..a2f677c3c 100644 --- a/include/bitcoin/database/impl/primitives/iterator.ipp +++ b/include/bitcoin/database/impl/primitives/iterator.ipp @@ -19,8 +19,8 @@ #ifndef LIBBITCOIN_DATABASE_PRIMITIVES_ELEMENT_IPP #define LIBBITCOIN_DATABASE_PRIMITIVES_ELEMENT_IPP -#include -#include +#include +#include #include #include @@ -78,13 +78,13 @@ Link CLASS::to_match(Link link) const NOEXCEPT // element key matches (found) const auto key_ptr = std::next(offset, Link::size); if (is_zero(std::memcmp(key_.data(), key_ptr, array_count))) - return std::move(link); + return link; // set next element link (loop) link = system::unsafe_array_cast(offset); } - return std::move(link); + return link; } TEMPLATE @@ -100,7 +100,7 @@ Link CLASS::to_next(Link link) const NOEXCEPT // set next element link (loop) link = { system::unsafe_array_cast(offset) }; if (link.is_terminal()) - return std::move(link); + return link; // get next element offset (fault) offset = memory_->offset(manager::link_to_position(link)); @@ -110,10 +110,10 @@ Link CLASS::to_next(Link link) const NOEXCEPT // next element key matches (found) const auto key_ptr = std::next(offset, Link::size); if (is_zero(std::memcmp(key_.data(), key_ptr, array_count))) - return std::move(link); + return link; } - return std::move(link); + return link; } } // namespace database diff --git a/include/bitcoin/database/impl/primitives/manager.ipp b/include/bitcoin/database/impl/primitives/manager.ipp index dc09b188a..0986bdbbe 100644 --- a/include/bitcoin/database/impl/primitives/manager.ipp +++ b/include/bitcoin/database/impl/primitives/manager.ipp @@ -134,19 +134,6 @@ constexpr size_t CLASS::link_to_position(const Link& link) NOEXCEPT } } -// private -TEMPLATE -constexpr typename Link::integer CLASS::cast_link(size_t link) NOEXCEPT -{ - using namespace system; - using integer = typename Link::integer; - constexpr auto terminal = Link::terminal; - - // link limit is sub1(terminal), where terminal is 2^((8*Link::bytes)-1). - // It is ok for the payload to exceed link limit (link is identity only). - return link >= terminal ? terminal : possible_narrow_cast(link); -} - // private TEMPLATE constexpr Link CLASS::position_to_link(size_t position) NOEXCEPT @@ -172,6 +159,19 @@ constexpr Link CLASS::position_to_link(size_t position) NOEXCEPT } } +// private +TEMPLATE +constexpr typename Link::integer CLASS::cast_link(size_t link) NOEXCEPT +{ + using namespace system; + using integer = typename Link::integer; + constexpr auto terminal = Link::terminal; + + // link limit is sub1(terminal), where terminal is 2^((8*Link::bytes)-1). + // It is ok for the payload to exceed link limit (link is identity only). + return link >= terminal ? terminal : possible_narrow_cast(link); +} + } // namespace database } // namespace libbitcoin diff --git a/include/bitcoin/database/memory/interfaces/storage.hpp b/include/bitcoin/database/memory/interfaces/storage.hpp index c508346d2..f116ffe48 100644 --- a/include/bitcoin/database/memory/interfaces/storage.hpp +++ b/include/bitcoin/database/memory/interfaces/storage.hpp @@ -67,6 +67,10 @@ class storage /// Allocate bytes and return offset to first allocated (or eof). virtual size_t allocate(size_t chunk) NOEXCEPT = 0; + /// Get remap-protected r/w access to offset (or null) allocated to size. + virtual memory_ptr set(size_t offset, size_t size, + uint8_t backfill) NOEXCEPT = 0; + /// Get remap-protected r/w access to start/offset of memory map (or null). virtual memory_ptr get(size_t offset=zero) const NOEXCEPT = 0; diff --git a/include/bitcoin/database/memory/map.hpp b/include/bitcoin/database/memory/map.hpp index 43eb3a5f2..54a67ce0f 100644 --- a/include/bitcoin/database/memory/map.hpp +++ b/include/bitcoin/database/memory/map.hpp @@ -90,6 +90,10 @@ class BCD_API map /// Allocate bytes and return offset to first allocated (or eof). size_t allocate(size_t chunk) NOEXCEPT override; + /// Get remap-protected r/w access to offset (or null) allocated to size. + memory_ptr set(size_t offset, size_t size, + uint8_t backfill) NOEXCEPT override; + /// Get remap-protected r/w access to start/offset of memory map (or null). memory_ptr get(size_t offset=zero) const NOEXCEPT override; diff --git a/include/bitcoin/database/primitives/arraymap.hpp b/include/bitcoin/database/primitives/arraymap.hpp index 216fae9c9..03d7190bb 100644 --- a/include/bitcoin/database/primitives/arraymap.hpp +++ b/include/bitcoin/database/primitives/arraymap.hpp @@ -16,8 +16,8 @@ /// You should have received a copy of the GNU Affero General Public License /// along with this program. If not, see . */ -#ifndef LIBBITCOIN_DATABASE_PRIMITIVES_ARRAY_HPP -#define LIBBITCOIN_DATABASE_PRIMITIVES_ARRAY_HPP +#ifndef LIBBITCOIN_DATABASE_PRIMITIVES_ARRAYMAP_HPP +#define LIBBITCOIN_DATABASE_PRIMITIVES_ARRAYMAP_HPP #include #include diff --git a/include/bitcoin/database/primitives/hashmap.hpp b/include/bitcoin/database/primitives/hashmap.hpp index 1781c1076..66d78b17c 100644 --- a/include/bitcoin/database/primitives/hashmap.hpp +++ b/include/bitcoin/database/primitives/hashmap.hpp @@ -95,7 +95,7 @@ class hashmap /// True if an instance of object with key exists. bool exists(const Key& key) const NOEXCEPT; - /// Return first element link or terimnal if not found/error. + /// Return first element link or terminal if not found/error. Link first(const Key& key) const NOEXCEPT; /// Iterator holds shared lock on storage remap. diff --git a/include/bitcoin/database/primitives/hashmap2.hpp b/include/bitcoin/database/primitives/hashmap2.hpp new file mode 100644 index 000000000..b92e2531e --- /dev/null +++ b/include/bitcoin/database/primitives/hashmap2.hpp @@ -0,0 +1,149 @@ +/** +/// Copyright (c) 2011-2025 libbitcoin developers (see AUTHORS) + * +/// This file is part of libbitcoin. + * +/// This program is free software: you can redistribute it and/or modify +/// it under the terms of the GNU Affero General Public License as published by +/// the Free Software Foundation, either version 3 of the License, or +/// (at your option) any later version. + * +/// This program is distributed in the hope that it will be useful, +/// but WITHOUT ANY WARRANTY; without even the implied warranty of +/// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +/// GNU Affero General Public License for more details. + * +/// You should have received a copy of the GNU Affero General Public License +/// along with this program. If not, see . + */ +#ifndef LIBBITCOIN_DATABASE_PRIMITIVES_HASHMAP2_HPP +#define LIBBITCOIN_DATABASE_PRIMITIVES_HASHMAP2_HPP + +#include +#include +#include +#include +#include +#include +#include + +namespace libbitcoin { +namespace database { + +/// Caution: iterator/reader/finalizer hold body remap lock until disposed. +/// These handles should be used for serialization and immediately disposed. +/// Readers and writers are always prepositioned at data, and are limited to +/// the extent the record/slab size is known (limit can always be removed). +/// Streams are always initialized from first element byte up to file limit. +template +class hashmap2 +{ +public: + DEFAULT_COPY_MOVE_DESTRUCT(hashmap2); + + using key = Key; + using link = Link; + using iterator = database::iterator; + + hashmap2(storage& header, storage& body, const Link& buckets) NOEXCEPT; + + /// Setup, not thread safe. + /// ----------------------------------------------------------------------- + + bool create() NOEXCEPT; + bool close() NOEXCEPT; + bool backup() NOEXCEPT; + bool restore() NOEXCEPT; + bool verify() const NOEXCEPT; + + /// Sizing. + /// ----------------------------------------------------------------------- + + /// The instance is enabled (more than 1 bucket). + bool enabled() const NOEXCEPT; + + /// Hash table bucket count. + size_t buckets() const NOEXCEPT; + + /// Head file bytes. + size_t head_size() const NOEXCEPT; + + /// Body file bytes. + size_t body_size() const NOEXCEPT; + + /// Count of records (or body file bytes if slab). + Link count() const NOEXCEPT; + + /// Errors. + /// ----------------------------------------------------------------------- + + /// Get the fault condition. + code get_fault() const NOEXCEPT; + + /// Get the space required to clear the disk full condition. + size_t get_space() const NOEXCEPT; + + /// Resume from disk full condition. + code reload() NOEXCEPT; + + /// Query interface, iterator is not thread safe. + /// ----------------------------------------------------------------------- + + /// Return the link at the top of the conflict list (for table scanning). + Link top(const Link& list) const NOEXCEPT; + + /// True if an instance of object with key exists. + bool exists(const Key& key) const NOEXCEPT; + + /// Return first element link or terminal if not found/error. + Link first(const Key& key) const NOEXCEPT; + + /// Get first element matching the search key, false if not found/error. + template = true> + bool find(const Key& key, Element& element) const NOEXCEPT; + + /// Get element at link, false if deserialize error. + template = true> + bool get(const Link& link, Element& element) const NOEXCEPT; + + /// Allocate, set, commit element to key. + /// Expands table AND HEADER as necessary. + template = true> + bool put(const Key& key, const Element& element) NOEXCEPT; + +protected: + /// Get element at link using memory object, false if deserialize error. + template = true> + static bool read(const memory_ptr& ptr, const Link& link, + Element& element) NOEXCEPT; + +private: + static constexpr auto is_slab = (Size == max_size_t); + + using head = database::head2; + using manager = database::manager; + + // Thread safe (index/top/push). + // Not thread safe (create/open/close/backup/restore). + head head_; + + // Thread safe. + manager manager_; +}; + +template +using hash_map2 = hashmap2, system::data_array, + Element::size>; + +} // namespace database +} // namespace libbitcoin + +#define TEMPLATE template +#define CLASS hashmap2 + +#include + +#undef CLASS +#undef TEMPLATE + +#endif diff --git a/include/bitcoin/database/primitives/head.hpp b/include/bitcoin/database/primitives/head.hpp index ecfa2f57c..c540b88ff 100644 --- a/include/bitcoin/database/primitives/head.hpp +++ b/include/bitcoin/database/primitives/head.hpp @@ -19,7 +19,6 @@ #ifndef LIBBITCOIN_DATABASE_PRIMITIVES_HEAD_HPP #define LIBBITCOIN_DATABASE_PRIMITIVES_HEAD_HPP -#include #include #include #include @@ -69,14 +68,13 @@ class head return system::unsafe_array_cast(buffer); } - static constexpr size_t offset(const Link& index) NOEXCEPT + // Byte offset of bucket index within head file. + // [body_size][[bucket[0]...bucket[buckets-1]]] + static constexpr size_t link_to_position(const Link& index) NOEXCEPT { using namespace system; BC_ASSERT(!is_multiply_overflow(index, Link::size)); BC_ASSERT(!is_add_overflow(Link::size, index * Link::size)); - - // Byte offset of bucket index within head file. - // [body_size][[bucket[0]...bucket[buckets-1]]] return possible_narrow_cast(Link::size + index * Link::size); } @@ -88,7 +86,6 @@ class head } // namespace database } // namespace libbitcoin - #define TEMPLATE template #define CLASS head diff --git a/include/bitcoin/database/primitives/head2.hpp b/include/bitcoin/database/primitives/head2.hpp new file mode 100644 index 000000000..cfab95188 --- /dev/null +++ b/include/bitcoin/database/primitives/head2.hpp @@ -0,0 +1,109 @@ +/** + * Copyright (c) 2011-2025 libbitcoin developers (see AUTHORS) + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef LIBBITCOIN_DATABASE_PRIMITIVES_HEAD2_HPP +#define LIBBITCOIN_DATABASE_PRIMITIVES_HEAD2_HPP + +#include +#include +#include +#include + +namespace libbitcoin { +namespace database { + +/// Dynamically expanding array map header. +/// Less efficient than a fixed-size header. +template +class head2 +{ +public: + DEFAULT_COPY_MOVE_DESTRUCT(head2); + + using bytes = typename Link::bytes; + + /// An array head has zero buckets (and cannot call index()). + head2(storage& head, const Link& buckets) NOEXCEPT; + + /// Sizing is dynamic (thread safe). + size_t size() const NOEXCEPT; + size_t buckets() const NOEXCEPT; + + /// Configure initial buckets to zero to disable the table. + bool enabled() const NOEXCEPT; + + /// Create from empty head file (not thread safe). + bool create() NOEXCEPT; + + /// False if head file size incorrect (not thread safe). + bool verify() const NOEXCEPT; + + /// Unsafe if verify false (not thread safe). + bool get_body_count(Link& count) const NOEXCEPT; + bool set_body_count(const Link& count) NOEXCEPT; + + /// Convert natural key to head bucket index. + Link index(const Key& key) const NOEXCEPT; + + /// Unsafe if verify false. + Link top(const Key& key) const NOEXCEPT; + Link top(const Link& index) const NOEXCEPT; + bool push(const bytes& current, const Link& index) NOEXCEPT; + +private: + template + static auto& array_cast(memory::iterator buffer) NOEXCEPT + { + return system::unsafe_array_cast(buffer); + } + + static constexpr Link position_to_link(size_t position) NOEXCEPT + { + using namespace system; + static_assert(is_nonzero(Link::size)); + const auto link = floored_subtract(position / Link::size, one); + return possible_narrow_cast(link); + } + + // Byte offset of bucket index within head file. + // [body_size][[bucket[0]...bucket[buckets-1]]] + static constexpr size_t link_to_position(const Link& index) NOEXCEPT + { + using namespace system; + BC_ASSERT(!is_multiply_overflow(index, Link::size)); + BC_ASSERT(!is_add_overflow(Link::size, index * Link::size)); + return possible_narrow_cast(Link::size + index * Link::size); + } + + storage& file_; + const Link initial_buckets_; + mutable std::shared_mutex mutex_{}; +}; + +} // namespace database +} // namespace libbitcoin + +#define TEMPLATE template +#define CLASS head2 + +#include + +#undef CLASS +#undef TEMPLATE + +#endif diff --git a/include/bitcoin/database/primitives/manager.hpp b/include/bitcoin/database/primitives/manager.hpp index 85fbc69e4..aecace10b 100644 --- a/include/bitcoin/database/primitives/manager.hpp +++ b/include/bitcoin/database/primitives/manager.hpp @@ -34,7 +34,9 @@ template class manager { public: + static constexpr Link position_to_link(size_t position) NOEXCEPT; static constexpr size_t link_to_position(const Link& link) NOEXCEPT; + static constexpr typename Link::integer cast_link(size_t link) NOEXCEPT; DEFAULT_COPY_MOVE_DESTRUCT(manager); @@ -74,8 +76,6 @@ class manager private: static constexpr auto is_slab = (Size == max_size_t); - static constexpr Link position_to_link(size_t position) NOEXCEPT; - static constexpr typename Link::integer cast_link(size_t link) NOEXCEPT; // Thread and remap safe. storage& file_; diff --git a/include/bitcoin/database/primitives/primitives.hpp b/include/bitcoin/database/primitives/primitives.hpp index c0dbab4f1..f1ef80732 100644 --- a/include/bitcoin/database/primitives/primitives.hpp +++ b/include/bitcoin/database/primitives/primitives.hpp @@ -21,7 +21,9 @@ #include #include +#include #include +#include #include #include #include diff --git a/include/bitcoin/database/tables/optionals/buffer.hpp b/include/bitcoin/database/tables/caches/prevouts.hpp similarity index 100% rename from include/bitcoin/database/tables/optionals/buffer.hpp rename to include/bitcoin/database/tables/caches/prevouts.hpp diff --git a/src/memory/map.cpp b/src/memory/map.cpp index be7a6c207..179ba88e6 100644 --- a/src/memory/map.cpp +++ b/src/memory/map.cpp @@ -242,13 +242,13 @@ size_t map::allocate(size_t chunk) NOEXCEPT auto end = logical_ + chunk; if (end > capacity_) { - const auto size = to_capacity(end); + const auto new_capacity = to_capacity(end); // TODO: Could loop over a try lock here and log deadlock warning. std::unique_lock remap_lock(remap_mutex_); // Disk full condition leaves store in valid state despite eof return. - if (!remap_(size)) + if (!remap_(new_capacity)) return storage::eof; } @@ -256,6 +256,40 @@ size_t map::allocate(size_t chunk) NOEXCEPT return end; } +memory_ptr map::set(size_t offset, size_t size, uint8_t backfill) NOEXCEPT +{ + { + std::unique_lock field_lock(field_mutex_); + + if (fault_ || !loaded_ || is_add_overflow(offset, size)) + return {}; + + const auto end = std::max(logical_, offset + size); + if (end > capacity_) + { + const auto old_capacity = capacity_; + const auto new_capacity = to_capacity(end); + + // TODO: Could loop over a try lock here and log deadlock warning. + std::unique_lock remap_lock(remap_mutex_); + + // Disk full condition leaves store in valid state despite null. + if (!remap_(new_capacity)) + return {}; + + // Fill new capacity as offset may not be at end due to expansion. + BC_PUSH_WARNING(NO_POINTER_ARITHMETIC) + std::fill_n(memory_map_ + logical_, new_capacity - logical_, + backfill); + BC_POP_WARNING() + } + + logical_ = end; + } + + return get(offset); +} + memory_ptr map::get(size_t offset) const NOEXCEPT { // Obtaining logical before access prevents mutual mutex wait (deadlock). diff --git a/test/memory/map.cpp b/test/memory/map.cpp index 4c87213c7..333c72630 100644 --- a/test/memory/map.cpp +++ b/test/memory/map.cpp @@ -77,6 +77,7 @@ BOOST_AUTO_TEST_CASE(map__close__open__true) { const std::string file = TEST_PATH; BOOST_REQUIRE(test::create(file)); + map instance(file); BOOST_REQUIRE(!instance.open()); BOOST_REQUIRE(instance.is_open()); @@ -88,6 +89,7 @@ BOOST_AUTO_TEST_CASE(map__close__closed__true) { const std::string file = TEST_PATH; BOOST_REQUIRE(test::create(file)); + map instance(file); BOOST_REQUIRE(!instance.is_open()); BOOST_REQUIRE(!instance.close()); @@ -97,6 +99,7 @@ BOOST_AUTO_TEST_CASE(map__close__loaded__false) { const std::string file = TEST_PATH; BOOST_REQUIRE(test::create(file)); + map instance(file); BOOST_REQUIRE(!instance.open()); BOOST_REQUIRE(!instance.load()); @@ -169,6 +172,7 @@ BOOST_AUTO_TEST_CASE(map__load__unloaded__true) { const std::string file = TEST_PATH; BOOST_REQUIRE(test::create(file)); + map instance(file); BOOST_REQUIRE(!instance.open()); BOOST_REQUIRE(!instance.load()); @@ -181,12 +185,15 @@ BOOST_AUTO_TEST_CASE(map__load__shared__false) { const std::string file = TEST_PATH; BOOST_REQUIRE(test::create(file)); + map instance(file); BOOST_REQUIRE(!instance.open()); BOOST_REQUIRE(!instance.load()); auto memory = instance.get(instance.allocate(1)); + BOOST_REQUIRE(memory); BOOST_REQUIRE_EQUAL(instance.load(), error::load_locked); + memory.reset(); BOOST_REQUIRE(!instance.unload()); BOOST_REQUIRE(!instance.close()); @@ -197,6 +204,7 @@ BOOST_AUTO_TEST_CASE(map__load__loaded__false) { const std::string file = TEST_PATH; BOOST_REQUIRE(test::create(file)); + map instance(file); BOOST_REQUIRE(!instance.open()); BOOST_REQUIRE(!instance.load()); @@ -210,6 +218,7 @@ BOOST_AUTO_TEST_CASE(map__unload__unloaded__true) { const std::string file = TEST_PATH; BOOST_REQUIRE(test::create(file)); + map instance(file); BOOST_REQUIRE(!instance.open()); BOOST_REQUIRE(!instance.unload()); @@ -221,6 +230,7 @@ BOOST_AUTO_TEST_CASE(map__unload__loaded__true) { const std::string file = TEST_PATH; BOOST_REQUIRE(test::create(file)); + map instance(file); BOOST_REQUIRE(!instance.open()); BOOST_REQUIRE(!instance.load()); @@ -232,9 +242,10 @@ BOOST_AUTO_TEST_CASE(map__unload__loaded__true) BOOST_AUTO_TEST_CASE(map__capacity__default__expected) { - constexpr auto default_minimum_capacity = 1u; + constexpr auto default_minimum_capacity = 1_size; const std::string file = TEST_PATH; BOOST_REQUIRE(test::create(file)); + map instance(file); BOOST_REQUIRE(!instance.open()); BOOST_REQUIRE_EQUAL(instance.capacity(), zero); @@ -249,6 +260,7 @@ BOOST_AUTO_TEST_CASE(map__truncate__unloaded__false) { const std::string file = TEST_PATH; BOOST_REQUIRE(test::create(file)); + map instance(file); BOOST_REQUIRE(!instance.open()); BOOST_REQUIRE(!instance.truncate(42)); @@ -279,6 +291,7 @@ BOOST_AUTO_TEST_CASE(map__allocate__unloaded__false) { const std::string file = TEST_PATH; BOOST_REQUIRE(test::create(file)); + map instance(file); BOOST_REQUIRE(!instance.open()); BOOST_REQUIRE_EQUAL(instance.allocate(42), map::eof); @@ -288,13 +301,18 @@ BOOST_AUTO_TEST_CASE(map__allocate__unloaded__false) BOOST_AUTO_TEST_CASE(map__allocate__loaded__expected_capacity) { - constexpr auto size = 100u; + constexpr auto half_rate = 50_size; + constexpr auto minimum = 42_size; + constexpr auto size = 100_size; const std::string file = TEST_PATH; BOOST_REQUIRE(test::create(file)); - map instance(file, 1, 50); + + map instance(file, minimum, half_rate); BOOST_REQUIRE(!instance.open()); BOOST_REQUIRE(!instance.load()); + BOOST_REQUIRE_EQUAL(instance.capacity(), minimum); BOOST_REQUIRE_EQUAL(instance.allocate(size), zero); + constexpr auto capacity = size + to_half(size); BOOST_REQUIRE_EQUAL(instance.capacity(), capacity); BOOST_REQUIRE(!instance.unload()); @@ -306,6 +324,7 @@ BOOST_AUTO_TEST_CASE(map__allocate__add_overflow__eof) { const std::string file = TEST_PATH; BOOST_REQUIRE(test::create(file)); + map instance(file); BOOST_REQUIRE(!instance.open()); BOOST_REQUIRE(!instance.load()); @@ -318,14 +337,16 @@ BOOST_AUTO_TEST_CASE(map__allocate__add_overflow__eof) BOOST_AUTO_TEST_CASE(map__allocate__minimum_no_expansion__expected_capacity) { - constexpr auto size = 42u; - constexpr auto minimum = 100u; + constexpr auto size = 42_size; + constexpr auto minimum = 100_size; const std::string file = TEST_PATH; BOOST_REQUIRE(test::create(file)); + map instance(file, minimum, 0); BOOST_REQUIRE(!instance.open()); BOOST_REQUIRE(!instance.load()); BOOST_REQUIRE_EQUAL(instance.allocate(size), zero); + constexpr auto capacity = std::max(minimum, size); BOOST_REQUIRE_EQUAL(instance.capacity(), capacity); BOOST_REQUIRE(!instance.unload()); @@ -336,10 +357,9 @@ BOOST_AUTO_TEST_CASE(map__allocate__minimum_no_expansion__expected_capacity) BOOST_AUTO_TEST_CASE(map__allocate__no_minimum_expansion__expected_capacity) { // map will fail if minimum is zero. - constexpr auto minimum = 1u; - constexpr auto rate = 42u; - constexpr auto size = 100u; - + constexpr auto minimum = 1_size; + constexpr auto rate = 42_size; + constexpr auto size = 100_size; const std::string file = TEST_PATH; BOOST_REQUIRE(test::create(file)); @@ -356,10 +376,215 @@ BOOST_AUTO_TEST_CASE(map__allocate__no_minimum_expansion__expected_capacity) BOOST_REQUIRE(!instance.get_fault()); } +BOOST_AUTO_TEST_CASE(map__set__unloaded__false) +{ + const std::string file = TEST_PATH; + BOOST_REQUIRE(test::create(file)); + map instance(file); + BOOST_REQUIRE(!instance.open()); + BOOST_REQUIRE(!instance.set(42, 24, 0xff)); + BOOST_REQUIRE(!instance.close()); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(map__set__loaded__expected_capacity) +{ + constexpr auto half_rate = 50_size; + constexpr auto minimum = 42_size; + constexpr auto size = 100_size; + constexpr auto offset = 42_size; + constexpr auto fill = 0b01010101; + const std::string file = TEST_PATH; + BOOST_REQUIRE(test::create(file)); + + map instance(file, minimum, half_rate); + BOOST_REQUIRE(!instance.open()); + BOOST_REQUIRE(!instance.load()); + BOOST_REQUIRE_EQUAL(instance.capacity(), minimum); + + auto memory = instance.set(offset, size, fill); + BOOST_REQUIRE(memory); + + const auto expected = std::next(instance.get()->data(), offset); + BOOST_REQUIRE(memory->data() == expected); + + constexpr auto capacity = offset + size + to_half(offset + size); + BOOST_REQUIRE_EQUAL(instance.capacity(), capacity); + BOOST_REQUIRE_EQUAL(instance.unload(), error::unload_locked); + + memory.reset(); + BOOST_REQUIRE(!instance.unload()); + BOOST_REQUIRE(!instance.close()); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(map__set__add_overflow__eof) +{ + const std::string file = TEST_PATH; + BOOST_REQUIRE(test::create(file)); + + map instance(file); + BOOST_REQUIRE(!instance.open()); + BOOST_REQUIRE(!instance.load()); + BOOST_REQUIRE(instance.set(100, 24, 0xff)); + BOOST_REQUIRE(!instance.set(max_size_t, 24, 0xff)); + BOOST_REQUIRE(!instance.unload()); + BOOST_REQUIRE(!instance.close()); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(map__set__minimum_no_expansion__expected_capacity) +{ + constexpr auto rate = 0_size; + constexpr auto minimum = 42_size; + constexpr auto size = 100_size; + constexpr auto offset = 42_size; + constexpr auto fill = 0b01010101; + const std::string file = TEST_PATH; + BOOST_REQUIRE(test::create(file)); + + map instance(file, minimum, rate); + BOOST_REQUIRE(!instance.open()); + BOOST_REQUIRE(!instance.load()); + BOOST_REQUIRE(instance.set(offset, size, fill)); + + constexpr auto capacity = std::max(minimum, offset + size); + BOOST_REQUIRE_EQUAL(instance.capacity(), capacity); + BOOST_REQUIRE(!instance.unload()); + BOOST_REQUIRE(!instance.close()); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(map__set__no_minimum_expansion__expected_capacity) +{ + // map will fail if minimum is zero. + constexpr auto rate = 42_size; + constexpr auto minimum = 1_size; + constexpr auto size = 51_size; + constexpr auto offset = 49_size; + constexpr auto fill = 0b01010101; + const std::string file = TEST_PATH; + BOOST_REQUIRE(test::create(file)); + + map instance(file, minimum, rate); + BOOST_REQUIRE(!instance.open()); + BOOST_REQUIRE(!instance.load()); + BOOST_REQUIRE(instance.set(offset, size, fill)); + + // These add only because offset + size is 100. + constexpr auto capacity = offset + size + rate; + BOOST_REQUIRE_EQUAL(instance.capacity(), capacity); + BOOST_REQUIRE(!instance.unload()); + BOOST_REQUIRE(!instance.close()); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(map__set__loaded__expected_fill) +{ + BC_PUSH_WARNING(NO_POINTER_ARITHMETIC) + + constexpr auto half_rate = 50_size; + constexpr auto minimum = 1_size; + constexpr auto size1 = 3_size; + constexpr auto offset1 = 5_size; + constexpr auto fill1 = 0b0101'0101; + const std::string file = TEST_PATH; + BOOST_REQUIRE(test::create(file)); + + map instance(file, minimum, half_rate); + BOOST_REQUIRE(!instance.open()); + BOOST_REQUIRE(!instance.load()); + + auto memory = instance.set(offset1, size1, fill1); + BOOST_REQUIRE(memory); + + constexpr auto capacity = offset1 + size1 + to_half(offset1 + size1); + BOOST_REQUIRE_EQUAL(instance.capacity(), capacity); + BOOST_REQUIRE_EQUAL(capacity, 12u); + + auto data = instance.get()->data(); + ////BOOST_REQUIRE_EQUAL(data[ 0], 0x00_u8); // cannot assume mmap default fill + ////BOOST_REQUIRE_EQUAL(data[ 1], 0x00_u8); // cannot assume mmap default fill + ////BOOST_REQUIRE_EQUAL(data[ 2], 0x00_u8); // cannot assume mmap default fill + ////BOOST_REQUIRE_EQUAL(data[ 3], 0x00_u8); // cannot assume mmap default fill + ////BOOST_REQUIRE_EQUAL(data[ 4], 0x00_u8); // cannot assume mmap default fill + + BOOST_REQUIRE_EQUAL(data[ 5], fill1); // offset + 0 + BOOST_REQUIRE_EQUAL(data[ 6], fill1); // offset + 1 + BOOST_REQUIRE_EQUAL(data[ 7], fill1); // offset + 2 + BOOST_REQUIRE_EQUAL(data[ 8], fill1); // expansion + BOOST_REQUIRE_EQUAL(data[ 9], fill1); // expansion + BOOST_REQUIRE_EQUAL(data[10], fill1); // expansion + BOOST_REQUIRE_EQUAL(data[11], fill1); // sub1(offset1 + size1 + to_half(offset1 + size1)) + + data[5] = 'a'; + data[6] = 'b'; + data[7] = 'c'; + + constexpr auto size2 = 5_size; + constexpr auto offset2 = 15_size; + constexpr auto fill2 = 0b1111'0000; + memory.reset(); + memory = instance.set(offset2, size2, fill2); + BOOST_REQUIRE(memory); + + constexpr auto capacity2 = offset2 + size2 + to_half(offset2 + size2); + BOOST_REQUIRE_EQUAL(instance.capacity(), capacity2); + BOOST_REQUIRE_EQUAL(capacity2, 30u); + + data[15] = 'd'; + data[16] = 'e'; + data[17] = 'f'; + data[18] = 'g'; + data[19] = 'h'; + + // Get data again in case it has been remapped by set(). + data = instance.get()->data(); + ////BOOST_REQUIRE_EQUAL(data[ 0], 0x00_u8); // cannot assume mmap default fill + ////BOOST_REQUIRE_EQUAL(data[ 1], 0x00_u8); // cannot assume mmap default fill + ////BOOST_REQUIRE_EQUAL(data[ 2], 0x00_u8); // cannot assume mmap default fill + ////BOOST_REQUIRE_EQUAL(data[ 3], 0x00_u8); // cannot assume mmap default fill + ////BOOST_REQUIRE_EQUAL(data[ 4], 0x00_u8); // cannot assume mmap default fill + + BOOST_REQUIRE_EQUAL(data[ 5], 'a'); // offset + 0 + BOOST_REQUIRE_EQUAL(data[ 6], 'b'); // offset + 1 + BOOST_REQUIRE_EQUAL(data[ 7], 'c'); // offset + 2 + BOOST_REQUIRE_EQUAL(data[ 8], fill2); // expansion, resize_ (macos) trims on remap... + BOOST_REQUIRE_EQUAL(data[ 9], fill2); // expansion, so it goes to zero if not refilled. + BOOST_REQUIRE_EQUAL(data[10], fill2); // expansion + BOOST_REQUIRE_EQUAL(data[11], fill2); // sub1(offset + size + to_half(offset + size)) + BOOST_REQUIRE_EQUAL(data[12], fill2); // cannot assume mmap default fill + BOOST_REQUIRE_EQUAL(data[13], fill2); // cannot assume mmap default fill + BOOST_REQUIRE_EQUAL(data[14], fill2); // cannot assume mmap default fill + BOOST_REQUIRE_EQUAL(data[15], 'd'); // offset2 + 0 + BOOST_REQUIRE_EQUAL(data[16], 'e'); // offset2 + 1 + BOOST_REQUIRE_EQUAL(data[17], 'f'); // offset2 + 2 + BOOST_REQUIRE_EQUAL(data[18], 'g'); // offset2 + 3 + BOOST_REQUIRE_EQUAL(data[19], 'h'); // offset2 + 4 + BOOST_REQUIRE_EQUAL(data[20], fill2); // expansion + BOOST_REQUIRE_EQUAL(data[21], fill2); // expansion + BOOST_REQUIRE_EQUAL(data[22], fill2); // expansion + BOOST_REQUIRE_EQUAL(data[23], fill2); // expansion + BOOST_REQUIRE_EQUAL(data[24], fill2); // expansion + BOOST_REQUIRE_EQUAL(data[25], fill2); // expansion + BOOST_REQUIRE_EQUAL(data[26], fill2); // expansion + BOOST_REQUIRE_EQUAL(data[27], fill2); // expansion + BOOST_REQUIRE_EQUAL(data[28], fill2); // expansion + BOOST_REQUIRE_EQUAL(data[29], fill2); // expansion + + memory.reset(); + BOOST_REQUIRE(!instance.unload()); + BOOST_REQUIRE(!instance.close()); + BOOST_REQUIRE(!instance.get_fault()); + + BC_POP_WARNING() +} + BOOST_AUTO_TEST_CASE(map__get__unloaded__false) { const std::string file = TEST_PATH; BOOST_REQUIRE(test::create(file)); + map instance(file); BOOST_REQUIRE(!instance.open()); BOOST_REQUIRE(!instance.get()); @@ -371,6 +596,7 @@ BOOST_AUTO_TEST_CASE(map__get__loaded__success) { const std::string file = TEST_PATH; BOOST_REQUIRE(test::create(file)); + map instance(file); BOOST_REQUIRE(!instance.open()); BOOST_REQUIRE(!instance.load()); @@ -384,6 +610,7 @@ BOOST_AUTO_TEST_CASE(map__flush__unloaded__false) { const std::string file = TEST_PATH; BOOST_REQUIRE(test::create(file)); + map instance(file); BOOST_REQUIRE(!instance.open()); BOOST_REQUIRE_EQUAL(instance.flush(), error::flush_unloaded); @@ -395,6 +622,7 @@ BOOST_AUTO_TEST_CASE(map__flush__loaded__true) { const std::string file = TEST_PATH; BOOST_REQUIRE(test::create(file)); + map instance(file); BOOST_REQUIRE(!instance.open()); BOOST_REQUIRE(!instance.load()); @@ -409,17 +637,22 @@ BOOST_AUTO_TEST_CASE(map__write__read__expected) constexpr uint64_t expected = 0x0102030405060708_u64; const std::string file = TEST_PATH; BOOST_REQUIRE(test::create(file)); + map instance(file); BOOST_REQUIRE(!instance.open()); BOOST_REQUIRE(!instance.load()); + auto memory = instance.get(instance.allocate(sizeof(uint64_t))); BOOST_REQUIRE(memory); + system::unsafe_to_little_endian(memory->begin(), expected); memory.reset(); BOOST_REQUIRE(!instance.flush()); + memory = instance.get(); BOOST_REQUIRE(memory); BOOST_REQUIRE_EQUAL(system::unsafe_from_little_endian(memory->begin()), expected); + memory.reset(); BOOST_REQUIRE(!instance.unload()); BOOST_REQUIRE(!instance.close()); @@ -430,12 +663,15 @@ BOOST_AUTO_TEST_CASE(map__unload__shared__false) { const std::string file = TEST_PATH; BOOST_REQUIRE(test::create(file)); + map instance(file); BOOST_REQUIRE(!instance.open()); BOOST_REQUIRE(!instance.load()); + auto memory = instance.get(instance.allocate(1)); BOOST_REQUIRE(memory); BOOST_REQUIRE_EQUAL(instance.unload(), error::unload_locked); + memory.reset(); BOOST_REQUIRE(!instance.unload()); BOOST_REQUIRE(!instance.close()); diff --git a/test/mocks/chunk_storage.cpp b/test/mocks/chunk_storage.cpp index 8fb71b201..e0ccbc291 100644 --- a/test/mocks/chunk_storage.cpp +++ b/test/mocks/chunk_storage.cpp @@ -18,6 +18,7 @@ */ #include "../test.hpp" #include "chunk_storage.hpp" +#include #include #include #include @@ -120,6 +121,21 @@ size_t chunk_storage::allocate(size_t chunk) NOEXCEPT return link; } +memory_ptr chunk_storage::set(size_t offset, size_t size, + uint8_t backfill) NOEXCEPT +{ + std::unique_lock field_lock(field_mutex_); + if (system::is_add_overflow(offset, size)) + return {}; + + std::unique_lock map_lock(map_mutex_); + const auto minimum = offset + size; + if (minimum > buffer_.size()) + buffer_.resize(minimum, backfill); + + return get(offset); +} + memory_ptr chunk_storage::get(size_t offset) const NOEXCEPT { const auto ptr = std::make_shared>(map_mutex_); diff --git a/test/mocks/chunk_storage.hpp b/test/mocks/chunk_storage.hpp index 6564889d2..4f5c21667 100644 --- a/test/mocks/chunk_storage.hpp +++ b/test/mocks/chunk_storage.hpp @@ -49,6 +49,8 @@ class chunk_storage size_t size() const NOEXCEPT override; bool truncate(size_t size) NOEXCEPT override; size_t allocate(size_t chunk) NOEXCEPT override; + memory_ptr set(size_t offset, size_t size, + uint8_t backfill) NOEXCEPT override; memory_ptr get(size_t offset=zero) const NOEXCEPT override; memory::iterator get_raw(size_t offset=zero) const NOEXCEPT override; code get_fault() const NOEXCEPT override; diff --git a/test/primitives/hashmap2.cpp b/test/primitives/hashmap2.cpp new file mode 100644 index 000000000..5bccfe9c5 --- /dev/null +++ b/test/primitives/hashmap2.cpp @@ -0,0 +1,1264 @@ +/** + * Copyright (c) 2011-2025 libbitcoin developers (see AUTHORS) + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "../test.hpp" +#include "../mocks/chunk_storage.hpp" + +BOOST_AUTO_TEST_SUITE(hashmap2_tests) + +template +class hashmap_ + : public hashmap2 +{ +public: + using base = hashmap2; + using hashmap2::hashmap2; +}; + +using namespace system; +using link5 = linkage<5>; +using key1 = data_array<1>; +using key10 = data_array<10>; + +// Key size does not factor into header byte size (for first key only). +constexpr size_t header_size = 105; +constexpr auto links = header_size / link5::size; +static_assert(links == 21u); + +// Bucket count is one less than link count, due to header.size field. +constexpr auto buckets = bc::sub1(links); +static_assert(buckets == 20u); + +struct slab0 { static constexpr size_t size = max_size_t; }; +struct record4 { static constexpr size_t size = 4; }; +using slab_table = hashmap_; +using record_table = hashmap_; + +// record hashmap +// ---------------------------------------------------------------------------- + +BOOST_AUTO_TEST_CASE(hashmap__record_construct__empty__expected) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + const record_table instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(body_store.buffer().empty()); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__record_construct__non_empty__expected) +{ + constexpr auto body_size = 12345u; + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + body_store.buffer().resize(body_size); + const record_table instance{ head_store, body_store, buckets }; + BOOST_REQUIRE_EQUAL(body_store.buffer().size(), body_size); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__record_getter__terminal__false) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + record_table instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + ////BOOST_REQUIRE(!instance.getter_(link5::terminal)); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__record_getter__empty__exhausted) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + record_table instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + ////BOOST_REQUIRE(instance.getter_(0)->is_exhausted()); + ////BOOST_REQUIRE(instance.getter_(19)->is_exhausted()); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__record_getter__empty__false) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + record_table instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + ////BOOST_REQUIRE(!instance.getter_(key10{ 0x00 })); + ////BOOST_REQUIRE(!instance.getter_(key10{ 0x42 })); + BOOST_REQUIRE(!instance.get_fault()); +} + +// slab hashmap +// ---------------------------------------------------------------------------- + +BOOST_AUTO_TEST_CASE(hashmap__slab_construct__empty__expected) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + const slab_table instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(body_store.buffer().empty()); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__slab_construct__non_empty__expected_enabled) +{ + constexpr auto body_size = 12345u; + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + body_store.buffer().resize(body_size); + const slab_table instance{ head_store, body_store, buckets }; + BOOST_REQUIRE_EQUAL(body_store.buffer().size(), body_size); + BOOST_REQUIRE(instance.enabled()); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__slab_getter__terminal__false) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + slab_table instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + ////BOOST_REQUIRE(!instance.getter_(link5::terminal)); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__slab_getter__empty__exhausted) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + slab_table instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + ////BOOST_REQUIRE(instance.getter_(0)->is_exhausted()); + ////BOOST_REQUIRE(instance.getter_(19)->is_exhausted()); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__slab_getter__empty__false) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + slab_table instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + ////BOOST_REQUIRE(!instance.getter_(key10{ 0x00 })); + ////BOOST_REQUIRE(!instance.getter_(key10{ 0x42 })); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__enabled__non_empty_slab_zero_buckets__false) +{ + constexpr auto body_size = 12345u; + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + body_store.buffer().resize(body_size); + const slab_table instance{ head_store, body_store, 0 }; + BOOST_REQUIRE(!instance.enabled()); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__enabled__empty_slab_one_bucket__false) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + slab_table instance{ head_store, body_store, 1 }; + BOOST_REQUIRE(!instance.enabled()); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__enabled__empty_slab_nonzero_buckets__true) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + slab_table instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.enabled()); + BOOST_REQUIRE(!instance.get_fault()); +} + +// get/put +// ---------------------------------------------------------------------------- + +class little_record +{ +public: + // record bytes or zero for slab (for template). + static constexpr size_t size = sizeof(uint32_t); + + // record count or bytes count for slab (for allocate). + static constexpr link5 count() NOEXCEPT { return 1; } + + bool from_data(database::reader& source) NOEXCEPT + { + value = source.read_little_endian(); + return source; + } + + bool to_data(database::finalizer& sink) const NOEXCEPT + { + sink.write_little_endian(value); + return sink; + } + + uint32_t value{ 0 }; +}; + +class big_record +{ +public: + static constexpr size_t size = sizeof(uint32_t); + static constexpr link5 count() NOEXCEPT { return 1; } + + bool from_data(database::reader& source) NOEXCEPT + { + value = source.read_big_endian(); + return source; + } + + bool to_data(database::finalizer& sink) const NOEXCEPT + { + sink.write_big_endian(value); + return sink; + } + + uint32_t value{ 0 }; +}; + +BOOST_AUTO_TEST_CASE(hashmap__record_get__terminal__invalid) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + const hashmap instance{ head_store, body_store, buckets }; + + little_record record{}; + BOOST_REQUIRE(!instance.get(link5::terminal, record)); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__record_get__empty__invalid) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + const hashmap instance{ head_store, body_store, buckets }; + + little_record record{}; + BOOST_REQUIRE(!instance.get(0, record)); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__record_get__populated__valid) +{ + data_chunk head_file; + data_chunk body_file + { + 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, + 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, + 0x01, 0x02, 0x03, 0x04 + }; + test::chunk_storage head_store{ head_file }; + test::chunk_storage body_store{ body_file }; + const hashmap instance{ head_store, body_store, buckets }; + + little_record record{}; + BOOST_REQUIRE(instance.get(0, record)); + BOOST_REQUIRE_EQUAL(record.value, 0x04030201_u32); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__record_put__multiple__expected) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + hashmap instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + + constexpr key1 key1_big{ 0x41 }; + constexpr key1 key1_little{ 0x42 }; + + link5 link{}; + BOOST_REQUIRE(instance.put_link(link, key1_big, big_record{ 0xa1b2c3d4_u32 })); + BOOST_REQUIRE(!link.is_terminal()); + BOOST_REQUIRE_EQUAL(link, 0u); + + link = instance.put_link(key1_little, little_record{ 0xa1b2c3d4_u32 }); + BOOST_REQUIRE(!link.is_terminal()); + BOOST_REQUIRE_EQUAL(link, 1u); + + big_record record1{}; + BOOST_REQUIRE(instance.get(0, record1)); + BOOST_REQUIRE_EQUAL(record1.value, 0xa1b2c3d4_u32); + + little_record record2{}; + BOOST_REQUIRE(instance.get(1, record2)); + BOOST_REQUIRE_EQUAL(record2.value, 0xa1b2c3d4_u32); + + // This expecatation relies on the fact of no hash table conflict between 0x41 and 0x42. + const data_chunk expected_file + { + 0xff, 0xff, 0xff, 0xff, 0xff, + 0x41, + 0xa1, 0xb2, 0xc3, 0xd4, + + 0xff, 0xff, 0xff, 0xff, 0xff, + 0x42, + 0xd4, 0xc3, 0xb2, 0xa1 + }; + BOOST_REQUIRE_EQUAL(body_store.buffer(), expected_file); + BOOST_REQUIRE(!instance.get_fault()); +} + +class little_slab +{ +public: + static constexpr size_t size = max_size_t; + static constexpr link5 count() NOEXCEPT + { + return link5::size + array_count + sizeof(uint32_t); + } + + bool from_data(database::reader& source) NOEXCEPT + { + value = source.read_little_endian(); + return source; + } + + bool to_data(database::finalizer& sink) const NOEXCEPT + { + sink.write_little_endian(value); + return sink; + } + + uint32_t value{ 0 }; +}; + +class big_slab +{ +public: + static constexpr size_t size = max_size_t; + static constexpr link5 count() NOEXCEPT + { + return link5::size + array_count + sizeof(uint32_t); + } + + bool from_data(database::reader& source) NOEXCEPT + { + value = source.read_big_endian(); + return source; + } + + bool to_data(database::finalizer& sink) const NOEXCEPT + { + sink.write_big_endian(value); + return sink; + } + + uint32_t value{ 0 }; +}; + +BOOST_AUTO_TEST_CASE(hashmap__slab_put__multiple__expected) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + + hashmap instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + + constexpr key1 key_big{ 0x41 }; + constexpr key1 key_little{ 0x42 }; + + link5 link{}; + BOOST_REQUIRE(instance.put_link(link, key_big, big_slab{ 0xa1b2c3d4_u32 })); + BOOST_REQUIRE(!link.is_terminal()); + BOOST_REQUIRE_EQUAL(link, 0u); + + link = instance.put_link(key_little, little_slab{ 0xa1b2c3d4_u32 }); + BOOST_REQUIRE(!link.is_terminal()); + BOOST_REQUIRE_EQUAL(link, big_slab::count()); + + big_slab slab1{}; + BOOST_REQUIRE(instance.get(0, slab1)); + BOOST_REQUIRE_EQUAL(slab1.value, 0xa1b2c3d4_u32); + + little_slab slab2{}; + BOOST_REQUIRE(instance.get(big_slab::count(), slab2)); + BOOST_REQUIRE_EQUAL(slab2.value, 0xa1b2c3d4_u32); + + // This expecatation relies on the fact of no hash table conflict between 0x41 and 0x42. + const data_chunk expected_file + { + 0xff, 0xff, 0xff, 0xff, 0xff, + 0x41, + 0xa1, 0xb2, 0xc3, 0xd4, + + 0xff, 0xff, 0xff, 0xff, 0xff, + 0x42, + 0xd4, 0xc3, 0xb2, 0xa1 + }; + BOOST_REQUIRE_EQUAL(body_store.buffer(), expected_file); + BOOST_REQUIRE(!instance.get_fault()); +} + +// advertises 32 but reads/writes 64 +class record_excess +{ +public: + static constexpr size_t size = sizeof(uint32_t); + static constexpr link5 count() NOEXCEPT { return 1; } + + bool from_data(database::reader& source) NOEXCEPT + { + value = source.read_big_endian(); + return source; + } + + bool to_data(database::finalizer& sink) const NOEXCEPT + { + sink.write_big_endian(value); + return sink; + } + + uint64_t value{ 0 }; +}; + +BOOST_AUTO_TEST_CASE(hashmap__record_get__excess__false) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + hashmap instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + + constexpr key1 key{ 0x41 }; + BOOST_REQUIRE(!instance.put_link(key, big_record{ 0xa1b2c3d4_u32 }).is_terminal()); + + record_excess record{}; + BOOST_REQUIRE(!instance.get(0, record)); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__record_get_key__excess__expected) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + hashmap instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + + constexpr key1 key{ 0x41 }; + BOOST_REQUIRE(!instance.put_link(key, big_record{ 0xa1b2c3d4_u32 }).is_terminal()); + BOOST_REQUIRE_EQUAL(instance.get_key(0), key); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__record_put__excess__false) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + hashmap instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + + constexpr key1 key{ 0x41 }; + BOOST_REQUIRE(instance.put_link(key, record_excess{ 0xa1b2c3d4_u32 }).is_terminal()); + BOOST_REQUIRE(!instance.get_fault()); +} + +// advertises 32 but reads/writes 64 +class slab_excess +{ +public: + static constexpr size_t size = max_size_t; + static constexpr link5 count() NOEXCEPT { return sizeof(uint32_t); } + + bool from_data(database::reader& source) NOEXCEPT + { + value = source.read_big_endian(); + return source; + } + + bool to_data(database::finalizer& sink) const NOEXCEPT + { + sink.write_big_endian(value); + return sink; + } + + uint64_t value{ 0 }; +}; + +// advertises 32 but reads 65 (file is 64)/writes 64 +class file_excess +{ +public: + static constexpr size_t size = max_size_t; + static constexpr link5 count() NOEXCEPT { return sizeof(uint32_t); } + + bool from_data(database::reader& source) NOEXCEPT + { + value = source.read_big_endian(); + source.read_byte(); + return source; + } + + bool to_data(database::finalizer& sink) const NOEXCEPT + { + sink.write_big_endian(value); + return sink; + } + + uint64_t value{ 0 }; +}; + +BOOST_AUTO_TEST_CASE(hashmap__slab_get__excess__true) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + hashmap instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + + constexpr key1 key{ 0x41 }; + BOOST_REQUIRE(!instance.put_link(key, big_slab{ 0xa1b2c3d4_u32 }).is_terminal()); + BOOST_REQUIRE(!instance.put_link(key, big_slab{ 0xa1b2c3d4_u32 }).is_terminal()); + + // Excess read allowed to eof here (reader has only knowledge of size). + slab_excess slab{}; + BOOST_REQUIRE(instance.get(0, slab)); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__slab_get_key__excess__expected) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + hashmap instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + + constexpr key1 key{ 0x41 }; + BOOST_REQUIRE(!instance.put_link(key, big_slab{ 0xa1b2c3d4_u32 }).is_terminal()); + BOOST_REQUIRE(!instance.put_link(key, big_slab{ 0xa1b2c3d4_u32 }).is_terminal()); + BOOST_REQUIRE_EQUAL(instance.get_key(0), key); + BOOST_REQUIRE_EQUAL(instance.get_key(10), key); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__slab_get__file_excess__false) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + hashmap instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + + constexpr key1 key{ 0x41 }; + BOOST_REQUIRE(!instance.put_link(key, big_slab{ 0xa1b2c3d4_u32 }).is_terminal()); + + // Excess read disallowed to here (past eof). + slab_excess slab{}; + BOOST_REQUIRE(!instance.get(0, slab)); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__slab_put__excess__false) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + hashmap instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + + constexpr key1 key{ 0x41 }; + BOOST_REQUIRE(instance.put_link(key, slab_excess{ 0xa1b2c3d4_u32 }).is_terminal()); +} + +BOOST_AUTO_TEST_CASE(hashmap__record_top__default__terminal) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + hashmap instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + BOOST_REQUIRE(instance.top(0).is_terminal()); + BOOST_REQUIRE(instance.top(19).is_terminal()); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__record_top__past_end__terminal) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + hashmap instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + BOOST_REQUIRE(instance.top(20).is_terminal()); + BOOST_REQUIRE(instance.top(21).is_terminal()); +} + +BOOST_AUTO_TEST_CASE(hashmap__record_top__existing__expected) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + hashmap instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + + BOOST_REQUIRE(!instance.put_link({ 0x41 }, big_record{ 0xa1b2c3d4_u32 }).is_terminal()); + BOOST_REQUIRE(!instance.put_link({ 0x42 }, big_record{ 0xa2b2c3d4_u32 }).is_terminal()); + BOOST_REQUIRE(!instance.put_link({ 0x43 }, big_record{ 0xa3b2c3d4_u32 }).is_terminal()); + BOOST_REQUIRE(!instance.top(0).is_terminal()); + BOOST_REQUIRE(instance.top(1).is_terminal()); + BOOST_REQUIRE(instance.top(2).is_terminal()); + BOOST_REQUIRE(instance.top(3).is_terminal()); + BOOST_REQUIRE(instance.top(4).is_terminal()); + BOOST_REQUIRE(instance.top(5).is_terminal()); + BOOST_REQUIRE(instance.top(6).is_terminal()); + BOOST_REQUIRE(instance.top(7).is_terminal()); + BOOST_REQUIRE(instance.top(8).is_terminal()); + BOOST_REQUIRE(instance.top(9).is_terminal()); + BOOST_REQUIRE(instance.top(10).is_terminal()); + BOOST_REQUIRE(instance.top(11).is_terminal()); + BOOST_REQUIRE(instance.top(12).is_terminal()); + BOOST_REQUIRE(instance.top(13).is_terminal()); + BOOST_REQUIRE(instance.top(14).is_terminal()); + BOOST_REQUIRE(instance.top(15).is_terminal()); + BOOST_REQUIRE(instance.top(16).is_terminal()); + BOOST_REQUIRE(instance.top(17).is_terminal()); + BOOST_REQUIRE(!instance.top(18).is_terminal()); + BOOST_REQUIRE(!instance.top(19).is_terminal()); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__record_exists__exists__true) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + hashmap instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + + constexpr key1 key{ 0x41 }; + BOOST_REQUIRE(!instance.exists(key)); + BOOST_REQUIRE(!instance.put_link(key, big_record{ 0xa1b2c3d4_u32 }).is_terminal()); + BOOST_REQUIRE(instance.exists(key)); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__slab_exists__exists__true) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + hashmap instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + + constexpr key1 key{ 0x41 }; + BOOST_REQUIRE(!instance.exists(key)); + BOOST_REQUIRE(!instance.put_link(key, big_slab{ 0xa1b2c3d4_u32 }).is_terminal()); + BOOST_REQUIRE(instance.exists(key)); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__record_first__exists__true) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + hashmap instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + + constexpr key1 key{ 0x41 }; + BOOST_REQUIRE(instance.first(key).is_terminal()); + const auto link = instance.put_link(key, big_record{ 0xa1b2c3d4_u32 }); + BOOST_REQUIRE(!link.is_terminal()); + BOOST_REQUIRE_EQUAL(instance.first(key), link); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__slab_first__exists__true) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + hashmap instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + + constexpr key1 key{ 0x41 }; + BOOST_REQUIRE(instance.first(key).is_terminal()); + const auto link = instance.put_link(key, big_slab{ 0xa1b2c3d4_u32 }); + BOOST_REQUIRE(!link.is_terminal()); + BOOST_REQUIRE_EQUAL(instance.first(key), link); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__record_it__exists__non_terminal) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + hashmap instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + + constexpr key1 key{ 0x41 }; + BOOST_REQUIRE(instance.it(key).self().is_terminal()); + BOOST_REQUIRE(!instance.put_link(key, big_record{ 0xa1b2c3d4_u32 }).is_terminal()); + BOOST_REQUIRE(!instance.it(key).self().is_terminal()); + + big_record record{}; + BOOST_REQUIRE(instance.get(instance.it(key).self(), record)); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__record_it__multiple__iterated) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + hashmap instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + + constexpr key1 key_a{ 0xaa }; + constexpr key1 key_b{ 0xbb }; + constexpr key1 key_c{ 0xcc }; + + BOOST_REQUIRE(!instance.put_link(key_a, big_record{ 0x000000a1_u32 }).is_terminal()); + BOOST_REQUIRE(!instance.put_link(key_a, big_record{ 0x000000a2_u32 }).is_terminal()); + BOOST_REQUIRE(!instance.put_link(key_a, big_record{ 0x000000a3_u32 }).is_terminal()); + BOOST_REQUIRE(!instance.put_link(key_b, big_record{ 0x000000b1_u32 }).is_terminal()); + BOOST_REQUIRE(!instance.put_link(key_b, big_record{ 0x000000b2_u32 }).is_terminal()); + BOOST_REQUIRE(!instance.put_link(key_b, big_record{ 0x000000b3_u32 }).is_terminal()); + BOOST_REQUIRE(!instance.put_link(key_c, big_record{ 0x000000c1_u32 }).is_terminal()); + BOOST_REQUIRE(!instance.put_link(key_c, big_record{ 0x000000c2_u32 }).is_terminal()); + BOOST_REQUIRE(!instance.put_link(key_c, big_record{ 0x000000c3_u32 }).is_terminal()); + + auto it_a = instance.it(key_a); + + big_record record{}; + BOOST_REQUIRE(instance.get(it_a.self(), record)); + BOOST_REQUIRE_EQUAL(record.value, 0x000000a3_u32); + BOOST_REQUIRE(it_a.advance()); + BOOST_REQUIRE(instance.get(it_a.self(), record)); + BOOST_REQUIRE_EQUAL(record.value, 0x000000a2_u32); + BOOST_REQUIRE(it_a.advance()); + BOOST_REQUIRE(instance.get(it_a.self(), record)); + BOOST_REQUIRE_EQUAL(record.value, 0x000000a1_u32); + BOOST_REQUIRE(!it_a.advance()); + BOOST_REQUIRE(!instance.get(it_a.self(), record)); + + auto it_b = instance.it(key_b); + + BOOST_REQUIRE(instance.get(it_b.self(), record)); + BOOST_REQUIRE_EQUAL(record.value, 0x000000b3_u32); + BOOST_REQUIRE(it_b.advance()); + BOOST_REQUIRE(instance.get(it_b.self(), record)); + BOOST_REQUIRE_EQUAL(record.value, 0x000000b2_u32); + BOOST_REQUIRE(it_b.advance()); + BOOST_REQUIRE(instance.get(it_b.self(), record)); + BOOST_REQUIRE_EQUAL(record.value, 0x000000b1_u32); + BOOST_REQUIRE(!it_b.advance()); + BOOST_REQUIRE(!instance.get(it_b.self(), record)); + + auto it_c = instance.it(key_c); + + BOOST_REQUIRE(instance.get(it_c.self(), record)); + BOOST_REQUIRE_EQUAL(record.value, 0x000000c3_u32); + BOOST_REQUIRE(it_c.advance()); + BOOST_REQUIRE(instance.get(it_c.self(), record)); + BOOST_REQUIRE_EQUAL(record.value, 0x000000c2_u32); + BOOST_REQUIRE(it_c.advance()); + BOOST_REQUIRE(instance.get(it_c.self(), record)); + BOOST_REQUIRE_EQUAL(record.value, 0x000000c1_u32); + BOOST_REQUIRE(!it_c.advance()); + BOOST_REQUIRE(!instance.get(it_c.self(), record)); + BOOST_REQUIRE(!instance.get_fault()); + + // [0000000000] + //[b] 0500000000 + // ffffffffff + // ffffffffff + //[a] 0200000000 + // ffffffffff + // ffffffffff + // ffffffffff + // ffffffffff + // ffffffffff + // ffffffffff + // ffffffffff + // ffffffffff + // ffffffffff + // ffffffffff + // ffffffffff + // ffffffffff + // ffffffffff + //[c] 0800000000 + // ffffffffff + // ffffffffff + //================== + //[0] ffffffffff + // aa + // 000000a1 + // + //[1] 0000000000 + // aa + // 000000a2 + // + //[2] 0100000000 + // aa + // 000000a3 + // + //[3] ffffffffff + // bb + // 000000b1 + // + //[4] 0300000000 + // bb + // 000000b2 + // + //[5] 0400000000 + // bb + // 000000b3 + // + //[6] ffffffffff + // cc + // 000000c1 + // + //[7] 0600000000 + // cc + // 000000c2 + // + //[8] 0700000000 + // cc + // 000000c3 +} + +// mutiphase commit. +// ---------------------------------------------------------------------------- + +class flex_record +{ +public: + static constexpr size_t size = sizeof(uint32_t); + static constexpr link5 count() NOEXCEPT { return 1; } + + template + bool to_data(Sinker& sink) const NOEXCEPT + { + sink.write_little_endian(value); + return sink; + } + + uint32_t value{ 0 }; +}; + +class flex_slab +{ +public: + static constexpr size_t size = max_size_t; + static constexpr link5 count() NOEXCEPT + { + return link5::size + array_count + sizeof(uint32_t); + } + + template + bool to_data(Sinker& sink) const NOEXCEPT + { + sink.write_little_endian(value); + return sink; + } + + uint32_t value{ 0 }; +}; + +BOOST_AUTO_TEST_CASE(hashmap__set_commit__record__expected) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + hashmap instance{ head_store, body_store, 2 }; + BOOST_REQUIRE(instance.create()); + + constexpr auto size = link5::size + array_count + flex_record::size; + const auto link = instance.set_link(flex_record{ 0x01020304_u32 }); + BOOST_REQUIRE(!link.is_terminal()); + BOOST_REQUIRE_EQUAL(link, 0u); + BOOST_REQUIRE_EQUAL(body_store.buffer().size(), size); + BOOST_REQUIRE_EQUAL(head_store.buffer(), base16_chunk("0000000000ffffffffffffffffffff")); + BOOST_REQUIRE_EQUAL(body_store.buffer(), base16_chunk("00000000000000000000000000000004030201")); + + constexpr key10 key1{ 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a }; + BOOST_REQUIRE(instance.commit(link, key1)); + BOOST_REQUIRE_EQUAL(head_store.buffer(), base16_chunk("00000000000000000000ffffffffff")); + BOOST_REQUIRE_EQUAL(body_store.buffer(), base16_chunk("ffffffffff0102030405060708090a04030201")); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__allocate_set_commit__record__expected) +{ + data_chunk head_file; + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + hashmap instance{ head_store, body_store, 2 }; + BOOST_REQUIRE(instance.create()); + + constexpr auto size = link5::size + array_count + flex_record::size; + const auto link = instance.allocate(1); + BOOST_REQUIRE_EQUAL(link, 0u); + BOOST_REQUIRE_EQUAL(body_store.buffer().size(), size); + + BOOST_REQUIRE(instance.set(link, flex_record{ 0x01020304_u32 })); + BOOST_REQUIRE_EQUAL(head_store.buffer(), base16_chunk("0000000000ffffffffffffffffffff")); + BOOST_REQUIRE_EQUAL(body_store.buffer(), base16_chunk("00000000000000000000000000000004030201")); + + constexpr key10 key1{ 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a }; + BOOST_REQUIRE(instance.commit(link, key1)); + BOOST_REQUIRE_EQUAL(head_store.buffer(), base16_chunk("00000000000000000000ffffffffff")); + BOOST_REQUIRE_EQUAL(body_store.buffer(), base16_chunk("ffffffffff0102030405060708090a04030201")); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__allocate_put1__record__expected) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + hashmap instance{ head_store, body_store, 2 }; + BOOST_REQUIRE(instance.create()); + + constexpr auto size = link5::size + array_count + sizeof(uint32_t); + const auto link = instance.allocate(1); + BOOST_REQUIRE_EQUAL(link, 0u); + BOOST_REQUIRE_EQUAL(body_store.buffer().size(), size); + + constexpr key10 key1{ 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a }; + BOOST_REQUIRE(instance.put(link, key1, flex_record{ 0x01020304_u32 })); + BOOST_REQUIRE_EQUAL(head_store.buffer(), base16_chunk("00000000000000000000ffffffffff")); + BOOST_REQUIRE_EQUAL(body_store.buffer(), base16_chunk("ffffffffff0102030405060708090a04030201")); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__allocate_put2__record__expected) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + hashmap instance{ head_store, body_store, 2 }; + BOOST_REQUIRE(instance.create()); + + constexpr key10 key1{ 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a }; + BOOST_REQUIRE(instance.put(key1, flex_record{ 0x01020304_u32 })); + BOOST_REQUIRE_EQUAL(head_store.buffer(), base16_chunk("00000000000000000000ffffffffff")); + BOOST_REQUIRE_EQUAL(body_store.buffer(), base16_chunk("ffffffffff0102030405060708090a04030201")); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__set_commit_link__slab__expected) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + hashmap instance{ head_store, body_store, 2 }; + BOOST_REQUIRE(instance.create()); + + constexpr auto size = link5::size + array_count + sizeof(uint32_t); + link5 link{}; + BOOST_REQUIRE(instance.set_link(link, flex_slab{ 0x01020304_u32 })); + BOOST_REQUIRE(!link.is_terminal()); + BOOST_REQUIRE_EQUAL(link, 0u); + BOOST_REQUIRE_EQUAL(body_store.buffer().size(), size); + BOOST_REQUIRE_EQUAL(head_store.buffer(), base16_chunk("0000000000ffffffffffffffffffff")); + BOOST_REQUIRE_EQUAL(body_store.buffer(), base16_chunk("00000000000000000000000000000004030201")); + + constexpr key10 key1{ 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a }; + BOOST_REQUIRE(!instance.commit_link(link, key1).is_terminal()); + BOOST_REQUIRE_EQUAL(head_store.buffer(), base16_chunk("00000000000000000000ffffffffff")); + BOOST_REQUIRE_EQUAL(body_store.buffer(), base16_chunk("ffffffffff0102030405060708090a04030201")); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__allocate_set_commit__slab__expected) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + hashmap instance{ head_store, body_store, 2 }; + BOOST_REQUIRE(instance.create()); + + constexpr auto size = link5::size + array_count + sizeof(uint32_t); + const auto link = instance.allocate(size); + BOOST_REQUIRE_EQUAL(link, 0u); + BOOST_REQUIRE_EQUAL(body_store.buffer().size(), size); + + BOOST_REQUIRE(instance.set(link, flex_slab{ 0x01020304_u32 })); + BOOST_REQUIRE_EQUAL(head_store.buffer(), base16_chunk("0000000000ffffffffffffffffffff")); + BOOST_REQUIRE_EQUAL(body_store.buffer(), base16_chunk("00000000000000000000000000000004030201")); + + constexpr key10 key1{ 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a }; + BOOST_REQUIRE(instance.commit(link, key1)); + BOOST_REQUIRE_EQUAL(head_store.buffer(), base16_chunk("00000000000000000000ffffffffff")); + BOOST_REQUIRE_EQUAL(body_store.buffer(), base16_chunk("ffffffffff0102030405060708090a04030201")); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__allocate_put1__slab__expected) +{ + data_chunk head_file; + data_chunk body_file; + test::chunk_storage head_store{ head_file }; + test::chunk_storage body_store{ body_file }; + hashmap instance{ head_store, body_store, 2 }; + BOOST_REQUIRE(instance.create()); + + constexpr auto size = link5::size + array_count + sizeof(uint32_t); + const auto link = instance.allocate(size); + BOOST_REQUIRE_EQUAL(link, 0u); + BOOST_REQUIRE_EQUAL(body_file.size(), size); + + constexpr key10 key1{ 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a }; + BOOST_REQUIRE(instance.put(link, key1, flex_slab{ 0x01020304_u32 })); + BOOST_REQUIRE_EQUAL(head_file, base16_chunk("00000000000000000000ffffffffff")); + BOOST_REQUIRE_EQUAL(body_file, base16_chunk("ffffffffff0102030405060708090a04030201")); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__allocate_put2__slab__expected) +{ + data_chunk head_file; + data_chunk body_file; + test::chunk_storage head_store{ head_file }; + test::chunk_storage body_store{ body_file }; + hashmap instance{ head_store, body_store, 2 }; + BOOST_REQUIRE(instance.create()); + + constexpr key10 key1{ 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a }; + BOOST_REQUIRE(instance.put(key1, flex_slab{ 0x01020304_u32 })); + BOOST_REQUIRE_EQUAL(head_file, base16_chunk("00000000000000000000ffffffffff")); + BOOST_REQUIRE_EQUAL(body_file, base16_chunk("ffffffffff0102030405060708090a04030201")); + BOOST_REQUIRE(!instance.get_fault()); +} + +// record create/close/backup/restore/verify +// ---------------------------------------------------------------------------- + +BOOST_AUTO_TEST_CASE(hashmap__record_verify__empty_files__expected) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + record_table instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(!instance.verify()); + BOOST_REQUIRE(instance.create()); + BOOST_REQUIRE(instance.verify()); + BOOST_REQUIRE_EQUAL(head_store.buffer().size(), header_size); + BOOST_REQUIRE(body_store.buffer().empty()); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__record_create__non_empty_head_file__failure) +{ + data_chunk head_file{ 0x42 }; + data_chunk body_file; + test::chunk_storage head_store{ head_file }; + test::chunk_storage body_store{ body_file }; + record_table instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(!instance.verify()); + BOOST_REQUIRE(!instance.create()); + BOOST_REQUIRE_EQUAL(head_file.size(), 1u); + BOOST_REQUIRE(body_file.empty()); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__record_create__non_empty_body_file__body_zeroed) +{ + data_chunk head_file; + data_chunk body_file{ 0x42 }; + test::chunk_storage head_store{ head_file }; + test::chunk_storage body_store{ body_file }; + record_table instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(!instance.verify()); + BOOST_REQUIRE(instance.create()); + BOOST_REQUIRE(instance.verify()); + BOOST_REQUIRE_EQUAL(head_file.size(), header_size); + BOOST_REQUIRE(body_file.empty()); +} + +BOOST_AUTO_TEST_CASE(hashmap__record_body_count__create__zero) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + record_table instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + BOOST_REQUIRE_EQUAL(head_store.buffer(), base16_chunk("0000000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__record_body_count__empty_close__zero) +{ + auto head_file = base16_chunk("1234567890"); + data_chunk body_file; + test::chunk_storage head_store{ head_file }; + test::chunk_storage body_store{ body_file }; + record_table instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.close()); + BOOST_REQUIRE_EQUAL(head_file, base16_chunk("0000000000")); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__record_body_count__two_close__two) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + record_table instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + body_store.buffer() = base16_chunk("1122334455667788990011223344556677889911223344556677889900112233445566778899"); + BOOST_REQUIRE(instance.close()); + BOOST_REQUIRE_EQUAL(head_store.buffer(), base16_chunk("0200000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__record_body_count__two_backup__two) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + record_table instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + body_store.buffer() = base16_chunk("1122334455667788990011223344556677889911223344556677889900112233445566778899"); + BOOST_REQUIRE(instance.backup()); + BOOST_REQUIRE_EQUAL(head_store.buffer(), base16_chunk("0200000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__record_body_count__empty_restore__truncates) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + record_table instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + body_store.buffer() = base16_chunk("1234567812345678"); + BOOST_REQUIRE(instance.restore()); + BOOST_REQUIRE(body_store.buffer().empty()); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__record_body_count__non_empty_restore__truncates) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + record_table instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + head_store.buffer() = base16_chunk("0100000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); + body_store.buffer() = base16_chunk("1122334455667788990011223344556677889911223344556677889900112233445566778899"); + BOOST_REQUIRE(instance.restore()); + BOOST_REQUIRE_EQUAL(body_store.buffer(), base16_chunk("11223344556677889900112233445566778899")); + BOOST_REQUIRE(!instance.get_fault()); +} + +// slab create/close/backup/restore/verify +// ---------------------------------------------------------------------------- + +BOOST_AUTO_TEST_CASE(hashmap__slab_verify__empty_files__expected) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + slab_table instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(!instance.verify()); + BOOST_REQUIRE(instance.create()); + BOOST_REQUIRE(instance.verify()); + BOOST_REQUIRE_EQUAL(head_store.buffer().size(), header_size); + BOOST_REQUIRE(body_store.buffer().empty()); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__slab_create__non_empty_head_file__failure) +{ + data_chunk head_file{ 0x42 }; + data_chunk body_file; + test::chunk_storage head_store{ head_file }; + test::chunk_storage body_store{ body_file }; + slab_table instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(!instance.verify()); + BOOST_REQUIRE(!instance.create()); + BOOST_REQUIRE_EQUAL(head_file.size(), 1u); + BOOST_REQUIRE(body_file.empty()); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__slab_create__non_empty_body_file__body_zeroed) +{ + data_chunk head_file; + data_chunk body_file{ 0x42 }; + test::chunk_storage head_store{ head_file }; + test::chunk_storage body_store{ body_file }; + slab_table instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(!instance.verify()); + BOOST_REQUIRE(instance.create()); + BOOST_REQUIRE(instance.verify()); + BOOST_REQUIRE_EQUAL(head_file.size(), header_size); + BOOST_REQUIRE(body_file.empty()); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__slab_body_count__create__zero) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + slab_table instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + BOOST_REQUIRE_EQUAL(head_store.buffer(), base16_chunk("0000000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__slab_body_count__empty_close__zero) +{ + auto head_file = base16_chunk("1234567890"); + data_chunk body_file; + test::chunk_storage head_store{ head_file }; + test::chunk_storage body_store{ body_file }; + slab_table instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.close()); + BOOST_REQUIRE_EQUAL(head_file, base16_chunk("0000000000")); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__slab_body_count__two_close__two) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + slab_table instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + body_store.buffer() = base16_chunk("1234"); + BOOST_REQUIRE(instance.close()); + BOOST_REQUIRE_EQUAL(head_store.buffer(), base16_chunk("0200000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__slab_body_count__two_backup__two) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + slab_table instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + body_store.buffer() = base16_chunk("1234"); + BOOST_REQUIRE(instance.backup()); + BOOST_REQUIRE_EQUAL(head_store.buffer(), base16_chunk("0200000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__slab_body_count__empty_restore__truncates) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + slab_table instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + body_store.buffer() = base16_chunk("1234567812345678"); + BOOST_REQUIRE(instance.restore()); + BOOST_REQUIRE(body_store.buffer().empty()); + BOOST_REQUIRE(!instance.get_fault()); +} + +BOOST_AUTO_TEST_CASE(hashmap__slab_body_count__non_empty_restore__truncates) +{ + test::chunk_storage head_store{}; + test::chunk_storage body_store{}; + slab_table instance{ head_store, body_store, buckets }; + BOOST_REQUIRE(instance.create()); + head_store.buffer() = base16_chunk("0300000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); + body_store.buffer() = base16_chunk("1234567812345678"); + BOOST_REQUIRE(instance.restore()); + BOOST_REQUIRE_EQUAL(body_store.buffer(), base16_chunk("123456")); + BOOST_REQUIRE(!instance.get_fault()); +} + +////std::cout << head_file << std::endl << std::endl; +////std::cout << body_file << std::endl << std::endl; + +BOOST_AUTO_TEST_SUITE_END() diff --git a/test/primitives/head2.cpp b/test/primitives/head2.cpp new file mode 100644 index 000000000..18766f128 --- /dev/null +++ b/test/primitives/head2.cpp @@ -0,0 +1,132 @@ +/** + * Copyright (c) 2011-2025 libbitcoin developers (see AUTHORS) + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "../test.hpp" +#include "../mocks/chunk_storage.hpp" + +BOOST_AUTO_TEST_SUITE(head2_tests) + +using namespace system; + +constexpr auto link_size = 5_size; +constexpr auto head_size = 105_size; + +// Key size does not factor into head byte size (for search key only). +constexpr auto links = head_size / link_size; +static_assert(links == 21u); + +// Bucket count is one less than link count, due to head.size field. +constexpr auto buckets = sub1(links); +static_assert(buckets == 20u); + +using link = linkage; +using test_header = head2; + +class nullptr_storage + : public test::chunk_storage +{ +public: + using chunk_storage::chunk_storage; + + memory_ptr get(size_t size) const NOEXCEPT override + { + return is_zero(size) ? chunk_storage::get(size) : nullptr; + } +}; + +BOOST_AUTO_TEST_CASE(head__create__size__expected) +{ + data_chunk data; + test::chunk_storage store{ data }; + test_header head{ store, buckets }; + BOOST_REQUIRE(head.create()); + BOOST_REQUIRE_EQUAL(data.size(), head_size); +} + +BOOST_AUTO_TEST_CASE(head__verify__uncreated__false) +{ + data_chunk data; + test::chunk_storage store{ data }; + test_header head{ store, buckets }; + ////BOOST_REQUIRE(head.create()); + BOOST_REQUIRE(!head.verify()); +} + +BOOST_AUTO_TEST_CASE(head__verify__created__false) +{ + data_chunk data; + test::chunk_storage store{ data }; + test_header head{ store, buckets }; + BOOST_REQUIRE(head.create()); + BOOST_REQUIRE(head.verify()); +} + +BOOST_AUTO_TEST_CASE(head__get_body_count__created__zero) +{ + data_chunk data; + test::chunk_storage store{ data }; + test_header head{ store, buckets }; + BOOST_REQUIRE(head.create()); + + link count{}; + BOOST_REQUIRE(head.get_body_count(count)); + BOOST_REQUIRE_EQUAL(count, zero); +} + +BOOST_AUTO_TEST_CASE(head__set_body_count__get__expected) +{ + data_chunk data; + test::chunk_storage store{ data }; + test_header head{ store, buckets }; + BOOST_REQUIRE(head.create()); + + constexpr auto expected = 42u; + BOOST_REQUIRE(head.set_body_count(expected)); + + link count{}; + BOOST_REQUIRE(head.get_body_count(count)); + BOOST_REQUIRE_EQUAL(count, expected); +} + +BOOST_AUTO_TEST_CASE(head__top__link__terminal) +{ + test::chunk_storage store; + test_header head{ store, buckets }; + BOOST_REQUIRE(head.create()); + BOOST_REQUIRE(head.top(9).is_terminal()); +} + +BOOST_AUTO_TEST_CASE(head__top__nullptr__terminal) +{ + nullptr_storage store; + test_header head{ store, buckets }; + BOOST_REQUIRE(head.create()); + BOOST_REQUIRE(head.top(9).is_terminal()); +} + +BOOST_AUTO_TEST_CASE(head__top__key__terminal) +{ + test::chunk_storage store; + test_header head{ store, buckets }; + + // create() allocates and fills buckets with terminal. + BOOST_REQUIRE(head.create()); + BOOST_REQUIRE(head.top(zero).is_terminal()); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/test/tables/indexes/spend.cpp b/test/tables/archives/spend.cpp similarity index 100% rename from test/tables/indexes/spend.cpp rename to test/tables/archives/spend.cpp diff --git a/test/tables/caches/buffer.cpp b/test/tables/caches/prevouts.cpp similarity index 100% rename from test/tables/caches/buffer.cpp rename to test/tables/caches/prevouts.cpp diff --git a/test/tables/indexes/address.cpp b/test/tables/optional/address.cpp similarity index 100% rename from test/tables/indexes/address.cpp rename to test/tables/optional/address.cpp diff --git a/test/tables/caches/bootstrap.cpp b/test/tables/optional/bootstrap.cpp similarity index 100% rename from test/tables/caches/bootstrap.cpp rename to test/tables/optional/bootstrap.cpp diff --git a/test/tables/caches/neutrino.cpp b/test/tables/optional/neutrino.cpp similarity index 100% rename from test/tables/caches/neutrino.cpp rename to test/tables/optional/neutrino.cpp