Skip to content

Commit bcf47fc

Browse files
evoskuilevoskuil
authored
Merge pull request libbitcoin#699 from evoskuil/master
Optimize duplicate point detection during archival.
2 parents 2aef566 + 0fe3db6 commit bcf47fc

File tree

6 files changed

+137
-46
lines changed

6 files changed

+137
-46
lines changed

‎include/bitcoin/database/impl/primitives/hashmap.ipp‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -555,16 +555,12 @@ bool CLASS::write(Link& previous, const memory_ptr& ptr, const Link& link,
555555
return false;
556556

557557
// Commit element to search (terminal is a valid bucket index).
558-
bool collision{};
559-
if (!head_.push(collision, link, next, key))
558+
bool search{};
559+
if (!head_.push(search, link, next, key))
560560
return false;
561561

562-
// If filter collision set previous stack head for conflict resolution.
563-
if (collision)
564-
previous = next;
565-
else
566-
previous = Link::terminal;
567-
562+
// If collision set previous stack head for conflict resolution search.
563+
previous = search ? Link{ next } : Link{};
568564
return true;
569565
}
570566

‎include/bitcoin/database/impl/query/archive_write.ipp‎

Lines changed: 50 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -91,10 +91,11 @@ TEMPLATE
9191
code CLASS::set_code(const tx_link& tx_fk, const transaction& tx) NOEXCEPT
9292
{
9393
// This is the only multitable write query (except initialize/genesis).
94-
using namespace system;
94+
9595
if (tx.is_empty())
9696
return error::tx_empty;
9797

98+
using namespace system;
9899
using ix = linkage<schema::index>;
99100
const auto& ins = tx.inputs_ptr();
100101
const auto& ous = tx.outputs_ptr();
@@ -105,24 +106,31 @@ code CLASS::set_code(const tx_link& tx_fk, const transaction& tx) NOEXCEPT
105106
// ========================================================================
106107
const auto scope = store_.get_transactor();
107108

109+
// If dirty we must guard against duplicates.
110+
const auto dirty = store_.is_dirty();
111+
108112
// Allocate contiguously and store inputs.
109113
input_link in_fk{};
110-
if (!store_.input.put_link(in_fk, table::input::put_ref{ {}, tx }))
114+
if (!store_.input.put_link(in_fk,
115+
table::input::put_ref{ {}, tx }))
111116
return error::tx_input_put;
112117

113118
// Allocate contiguously and store outputs.
114119
output_link out_fk{};
115-
if (!store_.output.put_link(out_fk, table::output::put_ref{ {}, tx_fk, tx }))
120+
if (!store_.output.put_link(out_fk,
121+
table::output::put_ref{ {}, tx_fk, tx }))
116122
return error::tx_output_put;
117123

118124
// Allocate and contiguously store input links.
119125
ins_link ins_fk{};
120-
if (!store_.ins.put_link(ins_fk, table::ins::put_ref{ {}, in_fk, tx_fk, tx }))
126+
if (!store_.ins.put_link(ins_fk,
127+
table::ins::put_ref{ {}, in_fk, tx_fk, tx }))
121128
return error::tx_ins_put;
122129

123130
// Allocate and contiguously store output links.
124131
outs_link outs_fk{};
125-
if (!store_.outs.put_link(outs_fk, table::outs::put_ref{ {}, out_fk, tx }))
132+
if (!store_.outs.put_link(outs_fk,
133+
table::outs::put_ref{ {}, out_fk, tx }))
126134
return error::tx_outs_put;
127135

128136
// Create tx record.
@@ -148,41 +156,54 @@ code CLASS::set_code(const tx_link& tx_fk, const transaction& tx) NOEXCEPT
148156
return error::tx_point_allocate;
149157

150158
for (const auto& in: *ins)
151-
if (!store_.point.put(ins_fk++, in->point(), table::point::record{}))
159+
if (!store_.point.put(ins_fk++, in->point(),
160+
table::point::record{}))
152161
return error::tx_null_point_put;
153162
}
154163
else
155164
{
156-
// Expand synchronizes keys with ins_fk, entries dropped into same offset.
165+
// Expand synchronizes keys with ins_fk, entries set into same offset.
157166
// Allocate contiguous points (at sequential keys matching ins_fk).
158167
if (!store_.point.expand(ins_fk + inputs))
159168
return error::tx_point_allocate;
160169

161-
// Collect duplicates to store in duplicate table.
162-
std::vector<chain::point> twins{};
163-
auto ptr = store_.point.get_memory();
164-
165-
// This must be set after tx.set and before tx.commit, since searchable and
166-
// produces an association to tx.link, and is also an integral part of tx.
167-
for (const auto& in: *ins)
170+
// Must be set after tx.set and before tx.commit, since searchable and
171+
// produces association to tx.link, and is also an integral part of tx.
172+
if (dirty)
168173
{
169-
bool duplicate{};
170-
if (!store_.point.put(duplicate, ptr, ins_fk++, in->point(),
171-
table::point::record{}))
172-
return error::tx_point_put;
173-
174-
if (duplicate)
175-
twins.push_back(in->point());
174+
// Collect duplicates to store in duplicate table.
175+
std::vector<chain::point> twins{};
176+
auto ptr = store_.point.get_memory();
177+
for (const auto& in: *ins)
178+
{
179+
bool duplicate{};
180+
if (!store_.point.put(duplicate, ptr, ins_fk++, in->point(),
181+
table::point::record{}))
182+
return error::tx_point_put;
183+
184+
if (duplicate)
185+
twins.push_back(in->point());
186+
}
187+
188+
ptr.reset();
189+
190+
// As few duplicates are expected, duplicate domain is only 2^16.
191+
// Return of tx_duplicate_put implies link domain has overflowed.
192+
for (const auto& twin: twins)
193+
if (!store_.duplicate.exists(twin))
194+
if (!store_.duplicate.put(twin, table::duplicate::record{}))
195+
return error::tx_duplicate_put;
176196
}
197+
else
198+
{
199+
auto ptr = store_.point.get_memory();
200+
for (const auto& in: *ins)
201+
if (!store_.point.put(ptr, ins_fk++, in->point(),
202+
table::point::record{}))
203+
return error::tx_point_put;
177204

178-
ptr.reset();
179-
180-
// As few duplicates are expected, the duplicate domain is only 2^16.
181-
// Return of tx_duplicate_put implies that the link domain has overflowed.
182-
for (const auto& twin: twins)
183-
if (!store_.duplicate.exists(twin))
184-
if (!store_.duplicate.put(twin, table::duplicate::record{}))
185-
return error::tx_duplicate_put;
205+
ptr.reset();
206+
}
186207
}
187208

188209
// Commit address index records (hashmap).
@@ -361,8 +382,6 @@ code CLASS::set_code(const block& block, const header_link& key,
361382

362383
code ec{};
363384
auto fk = tx_fks;
364-
365-
// Each tx is set under a distinct transactor.
366385
for (const auto& tx: *block.transactions_ptr())
367386
if ((ec = set_code(fk++, *tx)))
368387
return ec;

‎include/bitcoin/database/impl/query/optional.ipp‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,9 @@ inline code CLASS::parallel_address_transform(std::atomic_bool& cancel,
4545
std::atomic_bool fail{};
4646
std::vector<outpoint> outpoints(links.size());
4747
std::transform(parallel, links.begin(), links.end(), outpoints.begin(),
48-
[&functor, &fail](const auto& link) NOEXCEPT
48+
[&functor, &cancel, &fail](const auto& link) NOEXCEPT
4949
{
50-
return functor(link, fail);
50+
return functor(link, cancel, fail);
5151
});
5252

5353
out.clear();
@@ -96,7 +96,7 @@ code CLASS::get_address_outputs_turbo(std::atomic_bool& cancel, outpoints& out,
9696
return ec;
9797

9898
return parallel_address_transform(cancel, out, links,
99-
[this, &cancel](const auto& link, auto& fail) NOEXCEPT
99+
[this](const auto& link, auto& cancel, auto& fail) NOEXCEPT
100100
{
101101
if (cancel || fail) return outpoint{};
102102
auto outpoint = get_spent(link);
@@ -139,7 +139,7 @@ code CLASS::get_confirmed_unspent_outputs_turbo(std::atomic_bool& cancel,
139139
return ec;
140140

141141
return parallel_address_transform(cancel, out, links,
142-
[this, &cancel](const auto& link, auto& fail) NOEXCEPT
142+
[this](const auto& link, auto& cancel, auto& fail) NOEXCEPT
143143
{
144144
if (cancel || fail || !is_confirmed_unspent(link))
145145
return outpoint{};
@@ -185,7 +185,7 @@ code CLASS::get_minimum_unspent_outputs_turbo(std::atomic_bool& cancel,
185185
return ec;
186186

187187
return parallel_address_transform(cancel, out, links,
188-
[this, &cancel, minimum](const auto& link, auto& fail) NOEXCEPT
188+
[this, minimum](const auto& link, auto& cancel, auto& fail) NOEXCEPT
189189
{
190190
if (cancel || fail || !is_confirmed_unspent(link))
191191
return outpoint{};

‎include/bitcoin/database/impl/store.ipp‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
#include <bitcoin/database/boost.hpp>
2525
#include <bitcoin/database/define.hpp>
2626
#include <bitcoin/database/file/file.hpp>
27+
#include <bitcoin/database/tables/schema.hpp>
2728

2829
// TODO: evaluate performance benefits of concurrency.
2930

@@ -542,7 +543,7 @@ code CLASS::reload(const event_handler& handler) NOEXCEPT
542543
}
543544

544545
code ec{ error::success };
545-
const auto reload = [&handler](code& ec, auto& storage,
546+
const auto reload = [&handler, this](code& ec, auto& storage,
546547
table_t table) NOEXCEPT
547548
{
548549
if (!ec)
@@ -552,6 +553,7 @@ code CLASS::reload(const event_handler& handler) NOEXCEPT
552553
{
553554
handler(event_t::load_file, table);
554555
ec = storage.reload();
556+
this->dirty_ = true;
555557
}
556558
}
557559
};
@@ -763,6 +765,8 @@ code CLASS::open_load(const event_handler& handler) NOEXCEPT
763765
load(ec, filter_tx_head_, table_t::filter_tx_head);
764766
load(ec, filter_tx_body_, table_t::filter_tx_body);
765767

768+
// create, open, and restore each invoke open_load.
769+
dirty_ = header_body_.size() > schema::header::minrow;
766770
return ec;
767771
}
768772

@@ -1149,6 +1153,12 @@ const typename CLASS::transactor CLASS::get_transactor() NOEXCEPT
11491153
return transactor{ transactor_mutex_ };
11501154
}
11511155

1156+
TEMPLATE
1157+
bool CLASS::is_dirty() const NOEXCEPT
1158+
{
1159+
return dirty_;
1160+
}
1161+
11521162
TEMPLATE
11531163
code CLASS::get_fault() const NOEXCEPT
11541164
{

‎include/bitcoin/database/store.hpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,9 @@ class store
9090
/// Get a transactor object.
9191
const transactor get_transactor() NOEXCEPT;
9292

93+
/// Determine if the store is non-empty/initialized.
94+
bool is_dirty() const NOEXCEPT;
95+
9396
/// Get first fault code or error::success.
9497
code get_fault() const NOEXCEPT;
9598

@@ -229,6 +232,7 @@ class store
229232
flush_lock flush_lock_;
230233
interprocess_lock process_lock_;
231234
std::shared_timed_mutex transactor_mutex_{};
235+
bool dirty_{ true };
232236

233237
private:
234238
using path = std::filesystem::path;

‎test/store.cpp‎

Lines changed: 63 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
#include "mocks/blocks.hpp"
2121
#include "mocks/map_store.hpp"
2222

23-
// these are the slow tests (mmap)
23+
// these include the slow tests (mmap)
2424

2525
BOOST_FIXTURE_TEST_SUITE(store_tests, test::directory_setup_fixture)
2626

@@ -45,6 +45,68 @@ BOOST_AUTO_TEST_CASE(store__construct__default_configuration__referenced)
4545
BOOST_REQUIRE_EQUAL(&instance.configuration(), &configuration);
4646
}
4747

48+
BOOST_AUTO_TEST_CASE(store__is_dirty__uninitialized__true)
49+
{
50+
const settings configuration{};
51+
store<map> instance{ configuration };
52+
BOOST_REQUIRE(instance.is_dirty());
53+
}
54+
55+
BOOST_AUTO_TEST_CASE(store__is_dirty__initialized__true)
56+
{
57+
settings configuration{};
58+
configuration.path = TEST_DIRECTORY;
59+
store<map> instance{ configuration };
60+
query<store<map>> query_{ instance };
61+
BOOST_REQUIRE(!instance.create(events));
62+
BOOST_REQUIRE(query_.initialize(test::genesis));
63+
BOOST_REQUIRE(!instance.is_dirty());
64+
BOOST_REQUIRE(!instance.close(events));
65+
}
66+
67+
BOOST_AUTO_TEST_CASE(store__is_dirty__open__false)
68+
{
69+
settings configuration{};
70+
configuration.path = TEST_DIRECTORY;
71+
store<map> instance{ configuration };
72+
query<store<map>> query_{ instance };
73+
BOOST_REQUIRE(!instance.create(events));
74+
BOOST_REQUIRE(query_.initialize(test::genesis));
75+
BOOST_REQUIRE(!instance.is_dirty());
76+
BOOST_REQUIRE(!instance.close(events));
77+
}
78+
79+
BOOST_AUTO_TEST_CASE(store__is_dirty__open_add_header__false)
80+
{
81+
settings configuration{};
82+
configuration.path = TEST_DIRECTORY;
83+
store<map> instance{ configuration };
84+
query<store<map>> query_{ instance };
85+
BOOST_REQUIRE(!instance.create(events));
86+
BOOST_REQUIRE(query_.initialize(test::genesis));
87+
BOOST_REQUIRE(query_.set(system::chain::header{}, context{}, false));
88+
BOOST_REQUIRE(!instance.is_dirty());
89+
BOOST_REQUIRE(!instance.close(events));
90+
}
91+
92+
BOOST_AUTO_TEST_CASE(store__is_dirty__open_with_two_headers__true)
93+
{
94+
settings configuration{};
95+
configuration.path = TEST_DIRECTORY;
96+
97+
store<map> instance1{ configuration };
98+
query<store<map>> query1_{ instance1 };
99+
BOOST_REQUIRE(!instance1.create(events));
100+
BOOST_REQUIRE(query1_.initialize(test::genesis));
101+
BOOST_REQUIRE(query1_.set(system::chain::header{}, context{}, false));
102+
BOOST_REQUIRE(!instance1.close(events));
103+
104+
store<map> instance2{ configuration };
105+
BOOST_REQUIRE(!instance1.open(events));
106+
BOOST_REQUIRE(instance2.is_dirty());
107+
BOOST_REQUIRE(!instance1.close(events));
108+
}
109+
48110
BOOST_AUTO_TEST_CASE(store__paths__default_configuration__expected)
49111
{
50112
const settings configuration{};

0 commit comments

Comments
 (0)