Optimize duplicate point detection during archival.

Author: evoskuil

include/bitcoin/database/impl/query/archive_write.ipp

@@ -105,6 +105,9 @@ code CLASS::set_code(const tx_link& tx_fk, const transaction& tx) NOEXCEPT
     // ========================================================================
     const auto scope = store_.get_transactor();
 
+    // If dirty we must guard against duplicates.
+    const auto dirty = store_.is_dirty();
+
     // Allocate contiguously and store inputs.
     input_link in_fk{};
     if (!store_.input.put_link(in_fk, table::input::put_ref{ {}, tx }))
@@ -158,31 +161,43 @@ code CLASS::set_code(const tx_link& tx_fk, const transaction& tx) NOEXCEPT
         if (!store_.point.expand(ins_fk + inputs))
             return error::tx_point_allocate;
 
-        // Collect duplicates to store in duplicate table.
-        std::vector<chain::point> twins{};
-        auto ptr = store_.point.get_memory();
-
-        // This must be set after tx.set and before tx.commit, since searchable and
-        // produces an association to tx.link, and is also an integral part of tx.
-        for (const auto& in: *ins)
+        // Must be set after tx.set and before tx.commit, since searchable and
+        // produces association to tx.link, and is also an integral part of tx.
+        if (dirty)
         {
-            bool duplicate{};
-            if (!store_.point.put(duplicate, ptr, ins_fk++, in->point(),
-                table::point::record{}))
-                return error::tx_point_put;
-
-            if (duplicate)
-                twins.push_back(in->point());
+            // Collect duplicates to store in duplicate table.
+            std::vector<chain::point> twins{};
+            auto ptr = store_.point.get_memory();
+            for (const auto& in: *ins)
+            {
+                bool duplicate{};
+                if (!store_.point.put(duplicate, ptr, ins_fk++, in->point(),
+                    table::point::record{}))
+                    return error::tx_point_put;
+
+                if (duplicate)
+                    twins.push_back(in->point());
+            }
+
+            ptr.reset();
+
+            // As few duplicates are expected, duplicate domain is only 2^16.
+            // Return of tx_duplicate_put implies link domain has overflowed.
+            for (const auto& twin: twins)
+                if (!store_.duplicate.exists(twin))
+                    if (!store_.duplicate.put(twin, table::duplicate::record{}))
+                        return error::tx_duplicate_put;
         }
+        else
+        {
+            auto ptr = store_.point.get_memory();
+            for (const auto& in: *ins)
+                if (!store_.point.put(ptr, ins_fk++, in->point(),
+                    table::point::record{}))
+                    return error::tx_point_put;
 
-        ptr.reset();
-
-        // As few duplicates are expected, the duplicate domain is only 2^16.
-        // Return of tx_duplicate_put implies that the link domain has overflowed.
-        for (const auto& twin: twins)
-            if (!store_.duplicate.exists(twin))
-                if (!store_.duplicate.put(twin, table::duplicate::record{}))
-                    return error::tx_duplicate_put;
+            ptr.reset();
+        }
     }
 
     // Commit address index records (hashmap).

include/bitcoin/database/impl/store.ipp

@@ -24,6 +24,7 @@
 #include <bitcoin/database/boost.hpp>
 #include <bitcoin/database/define.hpp>
 #include <bitcoin/database/file/file.hpp>
+#include <bitcoin/database/tables/schema.hpp>
 
 // TODO: evaluate performance benefits of concurrency.
 
@@ -542,7 +543,7 @@ code CLASS::reload(const event_handler& handler) NOEXCEPT
     }
 
     code ec{ error::success };
-    const auto reload = [&handler](code& ec, auto& storage,
+    const auto reload = [&handler, this](code& ec, auto& storage,
         table_t table) NOEXCEPT
     {
         if (!ec)
@@ -552,6 +553,7 @@ code CLASS::reload(const event_handler& handler) NOEXCEPT
             {
                 handler(event_t::load_file, table);
                 ec = storage.reload();
+                this->dirty_ = true;
             }
         }
     };
@@ -763,6 +765,8 @@ code CLASS::open_load(const event_handler& handler) NOEXCEPT
     load(ec, filter_tx_head_, table_t::filter_tx_head);
     load(ec, filter_tx_body_, table_t::filter_tx_body);
 
+    // create, open, and restore each invoke open_load.
+    dirty_ = header_body_.size() > schema::header::minrow;
     return ec;
 }
 
@@ -1149,6 +1153,12 @@ const typename CLASS::transactor CLASS::get_transactor() NOEXCEPT
     return transactor{ transactor_mutex_ };
 }
 
+TEMPLATE
+bool CLASS::is_dirty() const NOEXCEPT
+{
+    return dirty_;
+}
+
 TEMPLATE
 code CLASS::get_fault() const NOEXCEPT
 {

include/bitcoin/database/store.hpp

@@ -90,6 +90,9 @@ class store
     /// Get a transactor object.
     const transactor get_transactor() NOEXCEPT;
 
+    /// Determine if the store is non-empty/initialized.
+    bool is_dirty() const NOEXCEPT;
+
     /// Get first fault code or error::success.
     code get_fault() const NOEXCEPT;
 
@@ -229,6 +232,7 @@ class store
     flush_lock flush_lock_;
     interprocess_lock process_lock_;
     std::shared_timed_mutex transactor_mutex_{};
+    bool dirty_{ true };
 
 private:
     using path = std::filesystem::path;

test/store.cpp

@@ -20,7 +20,7 @@
 #include "mocks/blocks.hpp"
 #include "mocks/map_store.hpp"
 
-// these are the slow tests (mmap)
+// these include the slow tests (mmap)
 
 BOOST_FIXTURE_TEST_SUITE(store_tests, test::directory_setup_fixture)
 
@@ -45,6 +45,68 @@ BOOST_AUTO_TEST_CASE(store__construct__default_configuration__referenced)
     BOOST_REQUIRE_EQUAL(&instance.configuration(), &configuration);
 }
 
+BOOST_AUTO_TEST_CASE(store__is_dirty__uninitialized__true)
+{
+    const settings configuration{};
+    store<map> instance{ configuration };
+    BOOST_REQUIRE(instance.is_dirty());
+}
+
+BOOST_AUTO_TEST_CASE(store__is_dirty__initialized__true)
+{
+    settings configuration{};
+    configuration.path = TEST_DIRECTORY;
+    store<map> instance{ configuration };
+    query<store<map>> query_{ instance };
+    BOOST_REQUIRE(!instance.create(events));
+    BOOST_REQUIRE(query_.initialize(test::genesis));
+    BOOST_REQUIRE(!instance.is_dirty());
+    BOOST_REQUIRE(!instance.close(events));
+}
+
+BOOST_AUTO_TEST_CASE(store__is_dirty__open__false)
+{
+    settings configuration{};
+    configuration.path = TEST_DIRECTORY;
+    store<map> instance{ configuration };
+    query<store<map>> query_{ instance };
+    BOOST_REQUIRE(!instance.create(events));
+    BOOST_REQUIRE(query_.initialize(test::genesis));
+    BOOST_REQUIRE(!instance.is_dirty());
+    BOOST_REQUIRE(!instance.close(events));
+}
+
+BOOST_AUTO_TEST_CASE(store__is_dirty__open_add_header__false)
+{
+    settings configuration{};
+    configuration.path = TEST_DIRECTORY;
+    store<map> instance{ configuration };
+    query<store<map>> query_{ instance };
+    BOOST_REQUIRE(!instance.create(events));
+    BOOST_REQUIRE(query_.initialize(test::genesis));
+    BOOST_REQUIRE(query_.set(system::chain::header{}, context{}, false));
+    BOOST_REQUIRE(!instance.is_dirty());
+    BOOST_REQUIRE(!instance.close(events));
+}
+
+BOOST_AUTO_TEST_CASE(store__is_dirty__open_with_two_headers__true)
+{
+    settings configuration{};
+    configuration.path = TEST_DIRECTORY;
+
+    store<map> instance1{ configuration };
+    query<store<map>> query1_{ instance1 };
+    BOOST_REQUIRE(!instance1.create(events));
+    BOOST_REQUIRE(query1_.initialize(test::genesis));
+    BOOST_REQUIRE(query1_.set(system::chain::header{}, context{}, false));
+    BOOST_REQUIRE(!instance1.close(events));
+
+    store<map> instance2{ configuration };
+    BOOST_REQUIRE(!instance1.open(events));
+    BOOST_REQUIRE(instance2.is_dirty());
+    BOOST_REQUIRE(!instance1.close(events));
+}
+
 BOOST_AUTO_TEST_CASE(store__paths__default_configuration__expected)
 {
     const settings configuration{};