Merge pull request #747 from evoskuil/master

Hack in point search diagnostics (temporary).

Author: evoskuil

console/executor_scans.cpp

@@ -241,15 +241,6 @@ void executor::scan_collisions() const
         return map;
     };
 
-    constexpr auto hash = [](const auto& key)
-    {
-        constexpr auto length = array_count<decltype(key)>;
-        constexpr auto size = std::min(length, sizeof(size_t));
-        size_t value{};
-        std::copy_n(key.begin(), size, system::byte_cast(value).begin());
-        return value;
-    };
-
     logger(BN_OPERATION_INTERRUPT);
 
     // header & txs (txs is a proxy for validated_bk)
@@ -264,8 +255,11 @@ void executor::scan_collisions() const
     while (!cancel_ && (++index < header_records))
     {
         const header_link link{ possible_narrow_cast<hint>(index) };
-        ++header.at(hash(query_.get_header_key(link.value)) % header_buckets);
-        ++txs.at(hash((header_link::bytes)link) % header_buckets);
+        const auto key = query_.get_header_key(link.value);
+        ++header.at(database::keys::hash(key) % header_buckets);
+        ++txs.at(database::keys::hash(
+            link.operator data_array<header_link::size>()) %
+            header_buckets);
 
         if (is_zero(index % block_frequency))
             logger(format("header/txs" BN_READ_ROW) % index %
@@ -317,8 +311,10 @@ void executor::scan_collisions() const
     while (!cancel_ && (++index < tx_records))
     {
         const tx_link link{ possible_narrow_cast<tx_link::integer>(index) };
-        ++tx.at(hash(query_.get_tx_key(link.value)) % tx_buckets);
-        ++strong_tx.at(hash((tx_link::bytes)link) % tx_buckets);
+        const auto key = query_.get_tx_key(link.value);
+        ++tx.at(database::keys::hash(key) % tx_buckets);
+        ++strong_tx.at(database::keys::hash(
+            link.operator data_array<tx_link::size>()) % tx_buckets);
 
         if (is_zero(index % tx_frequency))
             logger(format("tx & strong_tx" BN_READ_ROW) % index %
@@ -364,8 +360,8 @@ void executor::scan_collisions() const
     auto total = zero;
     index = max_size_t;
     start = logger::now();
-    const auto spend_buckets = query_.point_buckets();
-    std_vector<size_t> spend(spend_buckets, empty);
+    const auto point_buckets = query_.point_buckets();
+    std_vector<size_t> spend(point_buckets, empty);
     while (!cancel_ && (++index < query_.header_records()))
     {
         const header_link link{ possible_narrow_cast<hint>(index) };
@@ -375,11 +371,12 @@ void executor::scan_collisions() const
             const auto points = query_.to_points(transaction);
             for (const auto& point: points)
             {
+                const auto key = query_.get_point(point);
+                ++spend.at(database::keys::hash(key) % point_buckets);
                 ++total;
-                ++spend.at(hash(query_.get_point_key(point)) % spend_buckets);
 
-                if (is_zero(index % put_frequency))
-                    logger(format("spend" BN_READ_ROW) % total %
+                if (is_zero(total % put_frequency))
+                    logger(format("point" BN_READ_ROW) % total %
                         duration_cast<seconds>(logger::now() - start).count());
             }
         }
@@ -390,18 +387,18 @@ void executor::scan_collisions() const
 
     // ........................................................................
 
-    const auto spend_count = count(spend);
+    const auto point_count = count(spend);
     span = duration_cast<seconds>(logger::now() - start);
-    logger(format("spend: %1% in %2%s buckets %3% filled %4% rate %5%") %
-        total % span.count() % spend_buckets % spend_count %
-        (to_double(spend_count) / spend_buckets));
+    logger(format("point: %1% in %2%s buckets %3% filled %4% rate %5%") %
+        total % span.count() % point_buckets % point_count %
+        (to_double(point_count) / point_buckets));
 
     for (const auto& entry: dump(spend))
-        logger(format("spend: %1% frequency: %2%") %
+        logger(format("point: %1% frequency: %2%") %
             entry.first % entry.second);
 
-    spend.clear();
-    spend.shrink_to_fit();
+    ////point.clear();
+    ////point.shrink_to_fit();
 }
 
 } // namespace node

console/executor_test_reader.cpp

@@ -40,6 +40,14 @@ using namespace std::chrono;
 using namespace std::placeholders;
 
 // arbitrary testing (const).
+void executor::read_test(bool) const
+{
+    logger(format("Point table body searches: %1% / (%2% + %1%)") %
+        store_.point.positive_search_count() %
+        store_.point.negative_search_count());
+}
+
+#if defined(UNDEFINED)
 
 void executor::read_test(bool dump) const
 {
@@ -272,8 +280,6 @@ void executor::read_test(bool dump) const
     }
 }
 
-#if defined(UNDEFINED)
-
 void executor::read_test(bool) const
 {
     database::header_link link{ 350'017_u32 };

src/parser.cpp

@@ -71,6 +71,12 @@ parser::parser(system::chain::selection context) NOEXCEPT
         service::node_witness;
 
     // database (archive)
+    // Bits cannot be set to the word boundary (32 or 64) as this
+    // consumes the sentinel into the value domain. The bucket must
+    // be able to hold a sentinel that is larger than the bucket count.
+    // This presents a waste problem in the case of 32, since it will
+    // cause an overflow into 5 byte links and 8 byte buckets. In this
+    // case modulo bucketization will become more efficient.
 
     configured.database.header_bits = 20;
     configured.database.header_size = 21'000'000;

src/protocols/protocol_block_in_31800.cpp

@@ -295,7 +295,7 @@ bool protocol_block_in_31800::handle_receive_block(const code& ec,
         return true;
     }
 
-    const auto& link = it->link;
+    const auto link = it->link;
     const auto height = it->context.height;
 
     // Check block.
@@ -366,7 +366,9 @@ bool protocol_block_in_31800::handle_receive_block(const code& ec,
         << "] from [" << authority() << "].");
 
     notify(ec, chase::checked, height);
-    fire(events::block_archived, height);
+    ////fire(events::block_archived, height);
+    fire(events::block_archived, archive().positive_search_count());
+    fire(events::block_buffered, archive().negative_search_count());
     count(block->serialized_size(true));
     map_->erase(it);
     if (is_idle())