Implement random access header and sequential body file and map.

Author: evoskuil

include/bitcoin/database/file/utilities.hpp

@@ -72,8 +72,9 @@ BCD_API bool copy_directory(const path& from, const path& to) NOEXCEPT;
 BCD_API code copy_directory_ex(const path& from, const path& to) NOEXCEPT;
 
 /// File descriptor functions (for memory mapping).
-BCD_API int open(const path& filename) NOEXCEPT;
-BCD_API code open_ex(int& file_descriptor, const path& filename) NOEXCEPT;
+BCD_API int open(const path& filename, bool random=true) NOEXCEPT;
+BCD_API code open_ex(int& file_descriptor, const path& filename,
+    bool random=true) NOEXCEPT;
 BCD_API bool close(int file_descriptor) NOEXCEPT;
 BCD_API code close_ex(int file_descriptor) NOEXCEPT;
 BCD_API bool size(size_t& out, int file_descriptor) NOEXCEPT;

include/bitcoin/database/impl/store.ipp

@@ -31,6 +31,9 @@
 namespace libbitcoin {
 namespace database {
 
+constexpr auto random = true;
+constexpr auto sequential = false;
+
 BC_PUSH_WARNING(NO_THROW_IN_NOEXCEPT)
 
 // public
@@ -134,85 +137,85 @@ CLASS::store(const settings& config) NOEXCEPT
     // Archive.
     // ------------------------------------------------------------------------
 
-    header_head_(head(config.path / schema::dir::heads, schema::archive::header)),
-    header_body_(body(config.path, schema::archive::header), config.header_size, config.header_rate),
+    header_head_(head(config.path / schema::dir::heads, schema::archive::header), 1, 0, random),
+    header_body_(body(config.path, schema::archive::header), config.header_size, config.header_rate, sequential),
     header(header_head_, header_body_, config.header_buckets),
 
-    input_head_(head(config.path / schema::dir::heads, schema::archive::input)),
-    input_body_(body(config.path, schema::archive::input), config.input_size, config.input_rate),
+    input_head_(head(config.path / schema::dir::heads, schema::archive::input), 1, 0, random),
+    input_body_(body(config.path, schema::archive::input), config.input_size, config.input_rate, sequential),
     input(input_head_, input_body_),
 
-    output_head_(head(config.path / schema::dir::heads, schema::archive::output)),
-    output_body_(body(config.path, schema::archive::output), config.output_size, config.output_rate),
+    output_head_(head(config.path / schema::dir::heads, schema::archive::output), 1, 0, random),
+    output_body_(body(config.path, schema::archive::output), config.output_size, config.output_rate, sequential),
     output(output_head_, output_body_),
 
-    point_head_(head(config.path / schema::dir::heads, schema::archive::point)),
-    point_body_(body(config.path, schema::archive::point), config.point_size, config.point_rate),
+    point_head_(head(config.path / schema::dir::heads, schema::archive::point), 1, 0, random),
+    point_body_(body(config.path, schema::archive::point), config.point_size, config.point_rate, sequential),
     point(point_head_, point_body_, config.point_buckets),
 
-    ins_head_(head(config.path / schema::dir::heads, schema::archive::ins)),
-    ins_body_(body(config.path, schema::archive::ins), config.ins_size, config.ins_rate),
+    ins_head_(head(config.path / schema::dir::heads, schema::archive::ins), 1, 0, random),
+    ins_body_(body(config.path, schema::archive::ins), config.ins_size, config.ins_rate, sequential),
     ins(ins_head_, ins_body_),
 
-    outs_head_(head(config.path / schema::dir::heads, schema::archive::outs)),
-    outs_body_(body(config.path, schema::archive::outs), config.outs_size, config.outs_rate),
+    outs_head_(head(config.path / schema::dir::heads, schema::archive::outs), 1, 0, random),
+    outs_body_(body(config.path, schema::archive::outs), config.outs_size, config.outs_rate, sequential),
     outs(outs_head_, outs_body_),
 
-    tx_head_(head(config.path / schema::dir::heads, schema::archive::tx)),
-    tx_body_(body(config.path, schema::archive::tx), config.tx_size, config.tx_rate),
+    tx_head_(head(config.path / schema::dir::heads, schema::archive::tx), 1, 0, random),
+    tx_body_(body(config.path, schema::archive::tx), config.tx_size, config.tx_rate, sequential),
     tx(tx_head_, tx_body_, config.tx_buckets),
 
-    txs_head_(head(config.path / schema::dir::heads, schema::archive::txs)),
-    txs_body_(body(config.path, schema::archive::txs), config.txs_size, config.txs_rate),
+    txs_head_(head(config.path / schema::dir::heads, schema::archive::txs), 1, 0, random),
+    txs_body_(body(config.path, schema::archive::txs), config.txs_size, config.txs_rate, sequential),
     txs(txs_head_, txs_body_, config.txs_buckets),
 
     // Indexes.
     // ------------------------------------------------------------------------
 
-    candidate_head_(head(config.path / schema::dir::heads, schema::indexes::candidate)),
-    candidate_body_(body(config.path, schema::indexes::candidate), config.candidate_size, config.candidate_rate),
+    candidate_head_(head(config.path / schema::dir::heads, schema::indexes::candidate), 1, 0, random),
+    candidate_body_(body(config.path, schema::indexes::candidate), config.candidate_size, config.candidate_rate, sequential),
     candidate(candidate_head_, candidate_body_),
 
-    confirmed_head_(head(config.path / schema::dir::heads, schema::indexes::confirmed)),
-    confirmed_body_(body(config.path, schema::indexes::confirmed), config.confirmed_size, config.confirmed_rate),
+    confirmed_head_(head(config.path / schema::dir::heads, schema::indexes::confirmed), 1, 0, random),
+    confirmed_body_(body(config.path, schema::indexes::confirmed), config.confirmed_size, config.confirmed_rate, sequential),
     confirmed(confirmed_head_, confirmed_body_),
 
-    strong_tx_head_(head(config.path / schema::dir::heads, schema::indexes::strong_tx)),
-    strong_tx_body_(body(config.path, schema::indexes::strong_tx), config.strong_tx_size, config.strong_tx_rate),
+    strong_tx_head_(head(config.path / schema::dir::heads, schema::indexes::strong_tx), 1, 0, random),
+    strong_tx_body_(body(config.path, schema::indexes::strong_tx), config.strong_tx_size, config.strong_tx_rate, sequential),
     strong_tx(strong_tx_head_, strong_tx_body_, config.strong_tx_buckets),
 
     // Caches.
     // ------------------------------------------------------------------------
 
-    duplicate_head_(head(config.path / schema::dir::heads, schema::caches::duplicate)),
-    duplicate_body_(body(config.path, schema::caches::duplicate), config.duplicate_size, config.duplicate_rate),
+    duplicate_head_(head(config.path / schema::dir::heads, schema::caches::duplicate), 1, 0, random),
+    duplicate_body_(body(config.path, schema::caches::duplicate), config.duplicate_size, config.duplicate_rate, sequential),
     duplicate(duplicate_head_, duplicate_body_, config.duplicate_buckets),
 
-    prevout_head_(head(config.path / schema::dir::heads, schema::caches::prevout)),
-    prevout_body_(body(config.path, schema::caches::prevout), config.prevout_size, config.prevout_rate),
+    prevout_head_(head(config.path / schema::dir::heads, schema::caches::prevout), 1, 0, random),
+    prevout_body_(body(config.path, schema::caches::prevout), config.prevout_size, config.prevout_rate, sequential),
     prevout(prevout_head_, prevout_body_, config.prevout_buckets),
 
-    validated_bk_head_(head(config.path / schema::dir::heads, schema::caches::validated_bk)),
-    validated_bk_body_(body(config.path, schema::caches::validated_bk), config.validated_bk_size, config.validated_bk_rate),
+    validated_bk_head_(head(config.path / schema::dir::heads, schema::caches::validated_bk), 1, 0, random),
+    validated_bk_body_(body(config.path, schema::caches::validated_bk), config.validated_bk_size, config.validated_bk_rate, sequential),
     validated_bk(validated_bk_head_, validated_bk_body_, config.validated_bk_buckets),
 
-    validated_tx_head_(head(config.path / schema::dir::heads, schema::caches::validated_tx)),
-    validated_tx_body_(body(config.path, schema::caches::validated_tx), config.validated_tx_size, config.validated_tx_rate),
+    validated_tx_head_(head(config.path / schema::dir::heads, schema::caches::validated_tx), 1, 0, random),
+    validated_tx_body_(body(config.path, schema::caches::validated_tx), config.validated_tx_size, config.validated_tx_rate, sequential),
     validated_tx(validated_tx_head_, validated_tx_body_, config.validated_tx_buckets),
 
     // Optionals.
     // ------------------------------------------------------------------------
 
-    address_head_(head(config.path / schema::dir::heads, schema::optionals::address)),
-    address_body_(body(config.path, schema::optionals::address), config.address_size, config.address_rate),
+    address_head_(head(config.path / schema::dir::heads, schema::optionals::address), 1, 0, random),
+    address_body_(body(config.path, schema::optionals::address), config.address_size, config.address_rate, sequential),
     address(address_head_, address_body_, config.address_buckets),
 
-    filter_bk_head_(head(config.path / schema::dir::heads, schema::optionals::filter_bk)),
-    filter_bk_body_(body(config.path, schema::optionals::filter_bk), config.filter_bk_size, config.filter_bk_rate),
+    filter_bk_head_(head(config.path / schema::dir::heads, schema::optionals::filter_bk), 1, 0, random),
+    filter_bk_body_(body(config.path, schema::optionals::filter_bk), config.filter_bk_size, config.filter_bk_rate, sequential),
     filter_bk(filter_bk_head_, filter_bk_body_, config.filter_bk_buckets),
 
-    filter_tx_head_(head(config.path / schema::dir::heads, schema::optionals::filter_tx)),
-    filter_tx_body_(body(config.path, schema::optionals::filter_tx), config.filter_tx_size, config.filter_tx_rate),
+    filter_tx_head_(head(config.path / schema::dir::heads, schema::optionals::filter_tx), 1, 0, random),
+    filter_tx_body_(body(config.path, schema::optionals::filter_tx), config.filter_tx_size, config.filter_tx_rate, sequential),
     filter_tx(filter_tx_head_, filter_tx_body_, config.filter_tx_buckets),
 
     // Locks.

include/bitcoin/database/memory/map.hpp

@@ -43,7 +43,7 @@ class BCD_API map
     DELETE_COPY_MOVE(map);
 
     map(const std::filesystem::path& filename, size_t minimum=1,
-        size_t expansion=0) NOEXCEPT;
+        size_t expansion=0, bool random=true) NOEXCEPT;
 
     /// Destruct for debug assertion only.
     virtual ~map() NOEXCEPT;
@@ -136,10 +136,14 @@ class BCD_API map
     bool resize_(size_t size) NOEXCEPT;
     bool finalize_(size_t size) NOEXCEPT;
 
+    ////void log(auto kind) NOEXCEPT;
+    ////void log(auto kind, auto extra) NOEXCEPT;
+
     // Constants.
     const std::filesystem::path filename_;
     const size_t minimum_;
     const size_t expansion_;
+    const bool random_;
 
     // Protected by remap_mutex.
     // requires remap_mutex_ exclusive lock for write.

src/file/utilities.cpp

@@ -242,29 +242,62 @@ code copy_directory_ex(const path& from, const path& to) NOEXCEPT
 
 // File descriptor functions required for memory mapping.
 
-int open(const path& filename) NOEXCEPT
+#if defined(HAVE_MSC) || !defined(HAVE_APPLE)
+    #define MSC_OR_NOAPPLE(parameter) parameter
+#else
+    #define MSC_OR_NOAPPLE(parameter)
+#endif
+
+int open(const path& filename, bool MSC_OR_NOAPPLE(random)) NOEXCEPT
 {
     const auto path = system::to_extended_path(filename);
     int file_descriptor{};
 
-    // _wsopen_s and open set errno on failure.
-    // _wsopen_s and wstring do not throw (but are unannotated).
 #if defined(HAVE_MSC)
-    // sets file_descriptor = -1 on error.
-    _wsopen_s(&file_descriptor, path.c_str(),
-        O_RDWR | _O_BINARY | _O_RANDOM, _SH_DENYWR, _S_IREAD | _S_IWRITE);
+    // _wsopen_s and wstring do not throw (but are unannotated).
+    // sets file_descriptor = -1 and errno on error.
+    const auto access = (random ? _O_RANDOM : _O_SEQUENTIAL);
+    ::_wsopen_s(&file_descriptor, path.c_str(),
+        O_RDWR | _O_BINARY | access, _SH_DENYWR, _S_IREAD | _S_IWRITE);
 #else
-    // returns -1 on failure.
-    file_descriptor = ::open(path.c_str(),
-        O_RDWR, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
-#endif
+    // open sets errno on failure.
+    file_descriptor = ::open(path.c_str(), O_RDWR, S_IRUSR | S_IWUSR);
+
+#if !defined(HAVE_APPLE)
+    if (file_descriptor != -1)
+    {
+        // _O_RANDOM equivalent, posix_fadvise returns error on failure.
+        const auto advice = random ? POSIX_FADV_RANDOM : POSIX_FADV_SEQUENTIAL;
+        const auto result = ::posix_fadvise(file_descriptor, 0, 0, advice);
+        if (!is_zero(result))
+        {
+            close(file_descriptor);
+            file_descriptor = -1;
+            errno = result;
+        }
+        else
+        {
+            // _SH_DENYWR equivalent.
+            const struct flock lock{ F_WRLCK, SEEK_SET, 0, 0 };
+            if (::fcntl(file_descriptor, F_SETLK, &lock) == -1)
+            {
+                const auto last = errno;
+                close(file_descriptor);
+                file_descriptor = -1;
+                errno = last;
+            }
+        }
+    }
+#endif // !HAVE_APPLE
+#endif // HAVE_MSC
+
     return file_descriptor;
 }
 
-code open_ex(int& file_descriptor, const path& filename) NOEXCEPT
+code open_ex(int& file_descriptor, const path& filename, bool random) NOEXCEPT
 {
     system::error::clear_errno();
-    file_descriptor = open(filename);
+    file_descriptor = open(filename, random);
     return system::error::get_errno();
 }