Merge pull request #667 from evoskuil/master

Implement random access header and sequential body file and map.

Author: evoskuil

include/bitcoin/database/error.hpp

@@ -68,6 +68,7 @@ enum error_t : uint8_t
     mremap_failure,
     munmap_failure,
     madvise_failure,
+    sysconf_failure,
     ftruncate_failure,
     fsync_failure,
 

include/bitcoin/database/file/utilities.hpp

@@ -72,8 +72,9 @@ BCD_API bool copy_directory(const path& from, const path& to) NOEXCEPT;
 BCD_API code copy_directory_ex(const path& from, const path& to) NOEXCEPT;
 
 /// File descriptor functions (for memory mapping).
-BCD_API int open(const path& filename) NOEXCEPT;
-BCD_API code open_ex(int& file_descriptor, const path& filename) NOEXCEPT;
+BCD_API int open(const path& filename, bool random=true) NOEXCEPT;
+BCD_API code open_ex(int& file_descriptor, const path& filename,
+    bool random=true) NOEXCEPT;
 BCD_API bool close(int file_descriptor) NOEXCEPT;
 BCD_API code close_ex(int file_descriptor) NOEXCEPT;
 BCD_API bool size(size_t& out, int file_descriptor) NOEXCEPT;

include/bitcoin/database/impl/store.ipp

@@ -31,6 +31,9 @@
 namespace libbitcoin {
 namespace database {
 
+constexpr auto random = true;
+constexpr auto sequential = false;
+
 BC_PUSH_WARNING(NO_THROW_IN_NOEXCEPT)
 
 // public
@@ -134,85 +137,85 @@ CLASS::store(const settings& config) NOEXCEPT
     // Archive.
     // ------------------------------------------------------------------------
 
-    header_head_(head(config.path / schema::dir::heads, schema::archive::header)),
-    header_body_(body(config.path, schema::archive::header), config.header_size, config.header_rate),
+    header_head_(head(config.path / schema::dir::heads, schema::archive::header), 1, 0, random),
+    header_body_(body(config.path, schema::archive::header), config.header_size, config.header_rate, sequential),
     header(header_head_, header_body_, config.header_buckets),
 
-    input_head_(head(config.path / schema::dir::heads, schema::archive::input)),
-    input_body_(body(config.path, schema::archive::input), config.input_size, config.input_rate),
+    input_head_(head(config.path / schema::dir::heads, schema::archive::input), 1, 0, random),
+    input_body_(body(config.path, schema::archive::input), config.input_size, config.input_rate, sequential),
     input(input_head_, input_body_),
 
-    output_head_(head(config.path / schema::dir::heads, schema::archive::output)),
-    output_body_(body(config.path, schema::archive::output), config.output_size, config.output_rate),
+    output_head_(head(config.path / schema::dir::heads, schema::archive::output), 1, 0, random),
+    output_body_(body(config.path, schema::archive::output), config.output_size, config.output_rate, sequential),
     output(output_head_, output_body_),
 
-    point_head_(head(config.path / schema::dir::heads, schema::archive::point)),
-    point_body_(body(config.path, schema::archive::point), config.point_size, config.point_rate),
+    point_head_(head(config.path / schema::dir::heads, schema::archive::point), 1, 0, random),
+    point_body_(body(config.path, schema::archive::point), config.point_size, config.point_rate, sequential),
     point(point_head_, point_body_, config.point_buckets),
 
-    ins_head_(head(config.path / schema::dir::heads, schema::archive::ins)),
-    ins_body_(body(config.path, schema::archive::ins), config.ins_size, config.ins_rate),
+    ins_head_(head(config.path / schema::dir::heads, schema::archive::ins), 1, 0, random),
+    ins_body_(body(config.path, schema::archive::ins), config.ins_size, config.ins_rate, sequential),
     ins(ins_head_, ins_body_),
 
-    outs_head_(head(config.path / schema::dir::heads, schema::archive::outs)),
-    outs_body_(body(config.path, schema::archive::outs), config.outs_size, config.outs_rate),
+    outs_head_(head(config.path / schema::dir::heads, schema::archive::outs), 1, 0, random),
+    outs_body_(body(config.path, schema::archive::outs), config.outs_size, config.outs_rate, sequential),
     outs(outs_head_, outs_body_),
 
-    tx_head_(head(config.path / schema::dir::heads, schema::archive::tx)),
-    tx_body_(body(config.path, schema::archive::tx), config.tx_size, config.tx_rate),
+    tx_head_(head(config.path / schema::dir::heads, schema::archive::tx), 1, 0, random),
+    tx_body_(body(config.path, schema::archive::tx), config.tx_size, config.tx_rate, sequential),
     tx(tx_head_, tx_body_, config.tx_buckets),
 
-    txs_head_(head(config.path / schema::dir::heads, schema::archive::txs)),
-    txs_body_(body(config.path, schema::archive::txs), config.txs_size, config.txs_rate),
+    txs_head_(head(config.path / schema::dir::heads, schema::archive::txs), 1, 0, random),
+    txs_body_(body(config.path, schema::archive::txs), config.txs_size, config.txs_rate, sequential),
     txs(txs_head_, txs_body_, config.txs_buckets),
 
     // Indexes.
     // ------------------------------------------------------------------------
 
-    candidate_head_(head(config.path / schema::dir::heads, schema::indexes::candidate)),
-    candidate_body_(body(config.path, schema::indexes::candidate), config.candidate_size, config.candidate_rate),
+    candidate_head_(head(config.path / schema::dir::heads, schema::indexes::candidate), 1, 0, random),
+    candidate_body_(body(config.path, schema::indexes::candidate), config.candidate_size, config.candidate_rate, sequential),
     candidate(candidate_head_, candidate_body_),
 
-    confirmed_head_(head(config.path / schema::dir::heads, schema::indexes::confirmed)),
-    confirmed_body_(body(config.path, schema::indexes::confirmed), config.confirmed_size, config.confirmed_rate),
+    confirmed_head_(head(config.path / schema::dir::heads, schema::indexes::confirmed), 1, 0, random),
+    confirmed_body_(body(config.path, schema::indexes::confirmed), config.confirmed_size, config.confirmed_rate, sequential),
     confirmed(confirmed_head_, confirmed_body_),
 
-    strong_tx_head_(head(config.path / schema::dir::heads, schema::indexes::strong_tx)),
-    strong_tx_body_(body(config.path, schema::indexes::strong_tx), config.strong_tx_size, config.strong_tx_rate),
+    strong_tx_head_(head(config.path / schema::dir::heads, schema::indexes::strong_tx), 1, 0, random),
+    strong_tx_body_(body(config.path, schema::indexes::strong_tx), config.strong_tx_size, config.strong_tx_rate, sequential),
     strong_tx(strong_tx_head_, strong_tx_body_, config.strong_tx_buckets),
 
     // Caches.
     // ------------------------------------------------------------------------
 
-    duplicate_head_(head(config.path / schema::dir::heads, schema::caches::duplicate)),
-    duplicate_body_(body(config.path, schema::caches::duplicate), config.duplicate_size, config.duplicate_rate),
+    duplicate_head_(head(config.path / schema::dir::heads, schema::caches::duplicate), 1, 0, random),
+    duplicate_body_(body(config.path, schema::caches::duplicate), config.duplicate_size, config.duplicate_rate, sequential),
     duplicate(duplicate_head_, duplicate_body_, config.duplicate_buckets),
 
-    prevout_head_(head(config.path / schema::dir::heads, schema::caches::prevout)),
-    prevout_body_(body(config.path, schema::caches::prevout), config.prevout_size, config.prevout_rate),
+    prevout_head_(head(config.path / schema::dir::heads, schema::caches::prevout), 1, 0, random),
+    prevout_body_(body(config.path, schema::caches::prevout), config.prevout_size, config.prevout_rate, sequential),
     prevout(prevout_head_, prevout_body_, config.prevout_buckets),
 
-    validated_bk_head_(head(config.path / schema::dir::heads, schema::caches::validated_bk)),
-    validated_bk_body_(body(config.path, schema::caches::validated_bk), config.validated_bk_size, config.validated_bk_rate),
+    validated_bk_head_(head(config.path / schema::dir::heads, schema::caches::validated_bk), 1, 0, random),
+    validated_bk_body_(body(config.path, schema::caches::validated_bk), config.validated_bk_size, config.validated_bk_rate, sequential),
     validated_bk(validated_bk_head_, validated_bk_body_, config.validated_bk_buckets),
 
-    validated_tx_head_(head(config.path / schema::dir::heads, schema::caches::validated_tx)),
-    validated_tx_body_(body(config.path, schema::caches::validated_tx), config.validated_tx_size, config.validated_tx_rate),
+    validated_tx_head_(head(config.path / schema::dir::heads, schema::caches::validated_tx), 1, 0, random),
+    validated_tx_body_(body(config.path, schema::caches::validated_tx), config.validated_tx_size, config.validated_tx_rate, sequential),
     validated_tx(validated_tx_head_, validated_tx_body_, config.validated_tx_buckets),
 
     // Optionals.
     // ------------------------------------------------------------------------
 
-    address_head_(head(config.path / schema::dir::heads, schema::optionals::address)),
-    address_body_(body(config.path, schema::optionals::address), config.address_size, config.address_rate),
+    address_head_(head(config.path / schema::dir::heads, schema::optionals::address), 1, 0, random),
+    address_body_(body(config.path, schema::optionals::address), config.address_size, config.address_rate, sequential),
     address(address_head_, address_body_, config.address_buckets),
 
-    filter_bk_head_(head(config.path / schema::dir::heads, schema::optionals::filter_bk)),
-    filter_bk_body_(body(config.path, schema::optionals::filter_bk), config.filter_bk_size, config.filter_bk_rate),
+    filter_bk_head_(head(config.path / schema::dir::heads, schema::optionals::filter_bk), 1, 0, random),
+    filter_bk_body_(body(config.path, schema::optionals::filter_bk), config.filter_bk_size, config.filter_bk_rate, sequential),
     filter_bk(filter_bk_head_, filter_bk_body_, config.filter_bk_buckets),
 
-    filter_tx_head_(head(config.path / schema::dir::heads, schema::optionals::filter_tx)),
-    filter_tx_body_(body(config.path, schema::optionals::filter_tx), config.filter_tx_size, config.filter_tx_rate),
+    filter_tx_head_(head(config.path / schema::dir::heads, schema::optionals::filter_tx), 1, 0, random),
+    filter_tx_body_(body(config.path, schema::optionals::filter_tx), config.filter_tx_size, config.filter_tx_rate, sequential),
     filter_tx(filter_tx_head_, filter_tx_body_, config.filter_tx_buckets),
 
     // Locks.

include/bitcoin/database/memory/map.hpp

@@ -43,7 +43,7 @@ class BCD_API map
     DELETE_COPY_MOVE(map);
 
     map(const std::filesystem::path& filename, size_t minimum=1,
-        size_t expansion=0) NOEXCEPT;
+        size_t expansion=0, bool random=true) NOEXCEPT;
 
     /// Destruct for debug assertion only.
     virtual ~map() NOEXCEPT;
@@ -140,6 +140,7 @@ class BCD_API map
     const std::filesystem::path filename_;
     const size_t minimum_;
     const size_t expansion_;
+    const bool random_;
 
     // Protected by remap_mutex.
     // requires remap_mutex_ exclusive lock for write.

src/error.cpp

@@ -61,6 +61,7 @@ DEFINE_ERROR_T_MESSAGE_MAP(error)
     { mremap_failure, "mremap failure" },
     { munmap_failure, "munmap failure" },
     { madvise_failure, "madvise failure" },
+    { sysconf_failure, "sysconf failure" },
     { ftruncate_failure, "ftruncate failure" },
     { fsync_failure, "fsync failure" },
 

src/file/utilities.cpp

@@ -242,29 +242,62 @@ code copy_directory_ex(const path& from, const path& to) NOEXCEPT
 
 // File descriptor functions required for memory mapping.
 
-int open(const path& filename) NOEXCEPT
+#if defined(HAVE_MSC) || !defined(HAVE_APPLE)
+    #define MSC_OR_NOAPPLE(parameter) parameter
+#else
+    #define MSC_OR_NOAPPLE(parameter)
+#endif
+
+int open(const path& filename, bool MSC_OR_NOAPPLE(random)) NOEXCEPT
 {
     const auto path = system::to_extended_path(filename);
     int file_descriptor{};
 
-    // _wsopen_s and open set errno on failure.
-    // _wsopen_s and wstring do not throw (but are unannotated).
 #if defined(HAVE_MSC)
-    // sets file_descriptor = -1 on error.
-    _wsopen_s(&file_descriptor, path.c_str(),
-        O_RDWR | _O_BINARY | _O_RANDOM, _SH_DENYWR, _S_IREAD | _S_IWRITE);
+    // _wsopen_s and wstring do not throw (but are unannotated).
+    // sets file_descriptor = -1 and errno on error.
+    const auto access = (random ? _O_RANDOM : _O_SEQUENTIAL);
+    ::_wsopen_s(&file_descriptor, path.c_str(),
+        O_RDWR | _O_BINARY | access, _SH_DENYWR, _S_IREAD | _S_IWRITE);
 #else
-    // returns -1 on failure.
-    file_descriptor = ::open(path.c_str(),
-        O_RDWR, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
-#endif
+    // open sets errno on failure.
+    file_descriptor = ::open(path.c_str(), O_RDWR, S_IRUSR | S_IWUSR);
+
+#if !defined(HAVE_APPLE)
+    if (file_descriptor != -1)
+    {
+        // _O_RANDOM equivalent, posix_fadvise returns error on failure.
+        const auto advice = random ? POSIX_FADV_RANDOM : POSIX_FADV_SEQUENTIAL;
+        const auto result = ::posix_fadvise(file_descriptor, 0, 0, advice);
+        if (!is_zero(result))
+        {
+            close(file_descriptor);
+            file_descriptor = -1;
+            errno = result;
+        }
+        else
+        {
+            // _SH_DENYWR equivalent.
+            const struct flock lock{ F_WRLCK, SEEK_SET, 0, 0 };
+            if (::fcntl(file_descriptor, F_SETLK, &lock) == -1)
+            {
+                const auto last = errno;
+                close(file_descriptor);
+                file_descriptor = -1;
+                errno = last;
+            }
+        }
+    }
+#endif // !HAVE_APPLE
+#endif // HAVE_MSC
+
     return file_descriptor;
 }
 
-code open_ex(int& file_descriptor, const path& filename) NOEXCEPT
+code open_ex(int& file_descriptor, const path& filename, bool random) NOEXCEPT
 {
     system::error::clear_errno();
-    file_descriptor = open(filename);
+    file_descriptor = open(filename, random);
     return system::error::get_errno();
 }
 

src/memory/map.cpp

@@ -24,6 +24,7 @@
     #include <sys/mman.h>
     #include <sys/stat.h>
     #include <sys/types.h>
+    #include <unistd.h>
 #endif
 #include <algorithm>
 #include <chrono>
@@ -43,8 +44,12 @@ BC_PUSH_WARNING(NO_THROW_IN_NOEXCEPT)
 
 using namespace system;
 
-map::map(const path& filename, size_t minimum, size_t expansion) NOEXCEPT
-  : filename_(filename), minimum_(minimum), expansion_(expansion)
+map::map(const path& filename, size_t minimum, size_t expansion,
+    bool random) NOEXCEPT
+  : filename_(filename),
+    minimum_(minimum),
+    expansion_(expansion),
+    random_(random)
 {
 }
 
@@ -69,7 +74,8 @@ code map::open() NOEXCEPT
     if (opened_ != file::invalid)
         return error::open_open;
 
-    if (const auto ec = file::open_ex(opened_, filename_))
+    // Windows doesn't use madvise, instead infers map access from file open.
+    if (const auto ec = file::open_ex(opened_, filename_, random_))
         return ec;
 
     return file::size_ex(logical_, opened_);
@@ -583,43 +589,41 @@ bool map::finalize_(size_t size) NOEXCEPT
         return false;
     }
 
-    // TODO: also, madvise only the size increase.
-    // TODO: madvise with large length value fails on linux (and zero is noop).
-    // TODO: Random may not be best since writes are sequential (for bodies).
-    // TODO: Heads are truly read/write random so could benefit.
-    // TODO: this fails with large and/or unaligned size. Align size to page,
-    // TODO: rounded up. Iterate over calls at 1GB (1_u64 << 30), sample:
-    ////// Get page size (usually 4KB)
-    ////const size_t page_size = sysconf(_SC_PAGESIZE);
-    ////if (page_size == static_cast<size_t>(-1))
-    ////{
-    ////    set_first_code(error::sysconf_failure);
-    ////    unmap_();
-    ////    return false;
-    ////}
-    ////
-    ////// Align size up to page boundary
-    ////const size_t aligned_size = (size + page_size - 1) & ~(page_size - 1);
-    ////
-    ////// Use 1GB chunks to avoid large-length issues
-    ////constexpr size_t chunk_size = 1ULL << 30; // 1GB
-    ////uint8_t* ptr = memory_map_;
-    ////for (size_t offset = 0; offset < aligned_size; offset += chunk_size)
-    ////{
-    ////    size_t len = std::min(chunk_size, aligned_size - offset);
-    ////    if (::madvise(ptr + offset, len, MADV_SEQUENTIAL) == fail)
-    ////    {
-    ////        set_first_code(error::madvise_failure);
-    ////        unmap_();
-    ////        return false;
-    ////    }
-    ////}
-    ////if (::madvise(memory_map_, size, MADV_RANDOM) == fail)
-    ////{
-    ////    set_first_code(error::madvise_failure);
-    ////    unmap_();
-    ////    return false;
-    ////}
+#if !defined(HAVE_MSC)
+    // Get page size (usually 4KB).
+    const int page_size = ::sysconf(_SC_PAGESIZE);
+    const auto page = possible_narrow_sign_cast<size_t>(page_size);
+
+    // If not one bit then page size is not a power of two as required.
+    if (page_size == fail || !is_one(ones_count(page)))
+    {
+        set_first_code(error::sysconf_failure);
+        unmap_();
+        return false;
+    }
+    
+    // Align size up to page boundary.
+    const auto max = sub1(page);
+    const auto align = bit_and(ceilinged_add(size, max), bit_not(max));
+
+    // Use 1GB chunks to avoid large-length issues.
+    constexpr auto chunk = power2(30u);
+    const auto advice = random_ ? MADV_RANDOM : MADV_SEQUENTIAL;
+
+    for (auto offset = zero; offset < align; offset += chunk)
+    {
+        BC_PUSH_WARNING(NO_POINTER_ARITHMETIC)
+        const auto start = memory_map_ + offset;
+        BC_POP_WARNING()
+
+        if (::madvise(start, std::min(chunk, align - offset), advice) == fail)
+        {
+            set_first_code(error::madvise_failure);
+            unmap_();
+            return false;
+        }
+    }
+#endif
 
     loaded_ = true;
     capacity_ = size;