diff --git a/include/bitcoin/database/error.hpp b/include/bitcoin/database/error.hpp index bf1126a4..acac774d 100644 --- a/include/bitcoin/database/error.hpp +++ b/include/bitcoin/database/error.hpp @@ -68,6 +68,7 @@ enum error_t : uint8_t mremap_failure, munmap_failure, madvise_failure, + sysconf_failure, ftruncate_failure, fsync_failure, diff --git a/include/bitcoin/database/file/utilities.hpp b/include/bitcoin/database/file/utilities.hpp index c0ae36c9..2c823753 100644 --- a/include/bitcoin/database/file/utilities.hpp +++ b/include/bitcoin/database/file/utilities.hpp @@ -72,8 +72,9 @@ BCD_API bool copy_directory(const path& from, const path& to) NOEXCEPT; BCD_API code copy_directory_ex(const path& from, const path& to) NOEXCEPT; /// File descriptor functions (for memory mapping). -BCD_API int open(const path& filename) NOEXCEPT; -BCD_API code open_ex(int& file_descriptor, const path& filename) NOEXCEPT; +BCD_API int open(const path& filename, bool random=true) NOEXCEPT; +BCD_API code open_ex(int& file_descriptor, const path& filename, + bool random=true) NOEXCEPT; BCD_API bool close(int file_descriptor) NOEXCEPT; BCD_API code close_ex(int file_descriptor) NOEXCEPT; BCD_API bool size(size_t& out, int file_descriptor) NOEXCEPT; diff --git a/include/bitcoin/database/impl/store.ipp b/include/bitcoin/database/impl/store.ipp index 0310a5ee..2b337efd 100644 --- a/include/bitcoin/database/impl/store.ipp +++ b/include/bitcoin/database/impl/store.ipp @@ -31,6 +31,9 @@ namespace libbitcoin { namespace database { +constexpr auto random = true; +constexpr auto sequential = false; + BC_PUSH_WARNING(NO_THROW_IN_NOEXCEPT) // public @@ -134,85 +137,85 @@ CLASS::store(const settings& config) NOEXCEPT // Archive. // ------------------------------------------------------------------------ - header_head_(head(config.path / schema::dir::heads, schema::archive::header)), - header_body_(body(config.path, schema::archive::header), config.header_size, config.header_rate), + header_head_(head(config.path / schema::dir::heads, schema::archive::header), 1, 0, random), + header_body_(body(config.path, schema::archive::header), config.header_size, config.header_rate, sequential), header(header_head_, header_body_, config.header_buckets), - input_head_(head(config.path / schema::dir::heads, schema::archive::input)), - input_body_(body(config.path, schema::archive::input), config.input_size, config.input_rate), + input_head_(head(config.path / schema::dir::heads, schema::archive::input), 1, 0, random), + input_body_(body(config.path, schema::archive::input), config.input_size, config.input_rate, sequential), input(input_head_, input_body_), - output_head_(head(config.path / schema::dir::heads, schema::archive::output)), - output_body_(body(config.path, schema::archive::output), config.output_size, config.output_rate), + output_head_(head(config.path / schema::dir::heads, schema::archive::output), 1, 0, random), + output_body_(body(config.path, schema::archive::output), config.output_size, config.output_rate, sequential), output(output_head_, output_body_), - point_head_(head(config.path / schema::dir::heads, schema::archive::point)), - point_body_(body(config.path, schema::archive::point), config.point_size, config.point_rate), + point_head_(head(config.path / schema::dir::heads, schema::archive::point), 1, 0, random), + point_body_(body(config.path, schema::archive::point), config.point_size, config.point_rate, sequential), point(point_head_, point_body_, config.point_buckets), - ins_head_(head(config.path / schema::dir::heads, schema::archive::ins)), - ins_body_(body(config.path, schema::archive::ins), config.ins_size, config.ins_rate), + ins_head_(head(config.path / schema::dir::heads, schema::archive::ins), 1, 0, random), + ins_body_(body(config.path, schema::archive::ins), config.ins_size, config.ins_rate, sequential), ins(ins_head_, ins_body_), - outs_head_(head(config.path / schema::dir::heads, schema::archive::outs)), - outs_body_(body(config.path, schema::archive::outs), config.outs_size, config.outs_rate), + outs_head_(head(config.path / schema::dir::heads, schema::archive::outs), 1, 0, random), + outs_body_(body(config.path, schema::archive::outs), config.outs_size, config.outs_rate, sequential), outs(outs_head_, outs_body_), - tx_head_(head(config.path / schema::dir::heads, schema::archive::tx)), - tx_body_(body(config.path, schema::archive::tx), config.tx_size, config.tx_rate), + tx_head_(head(config.path / schema::dir::heads, schema::archive::tx), 1, 0, random), + tx_body_(body(config.path, schema::archive::tx), config.tx_size, config.tx_rate, sequential), tx(tx_head_, tx_body_, config.tx_buckets), - txs_head_(head(config.path / schema::dir::heads, schema::archive::txs)), - txs_body_(body(config.path, schema::archive::txs), config.txs_size, config.txs_rate), + txs_head_(head(config.path / schema::dir::heads, schema::archive::txs), 1, 0, random), + txs_body_(body(config.path, schema::archive::txs), config.txs_size, config.txs_rate, sequential), txs(txs_head_, txs_body_, config.txs_buckets), // Indexes. // ------------------------------------------------------------------------ - candidate_head_(head(config.path / schema::dir::heads, schema::indexes::candidate)), - candidate_body_(body(config.path, schema::indexes::candidate), config.candidate_size, config.candidate_rate), + candidate_head_(head(config.path / schema::dir::heads, schema::indexes::candidate), 1, 0, random), + candidate_body_(body(config.path, schema::indexes::candidate), config.candidate_size, config.candidate_rate, sequential), candidate(candidate_head_, candidate_body_), - confirmed_head_(head(config.path / schema::dir::heads, schema::indexes::confirmed)), - confirmed_body_(body(config.path, schema::indexes::confirmed), config.confirmed_size, config.confirmed_rate), + confirmed_head_(head(config.path / schema::dir::heads, schema::indexes::confirmed), 1, 0, random), + confirmed_body_(body(config.path, schema::indexes::confirmed), config.confirmed_size, config.confirmed_rate, sequential), confirmed(confirmed_head_, confirmed_body_), - strong_tx_head_(head(config.path / schema::dir::heads, schema::indexes::strong_tx)), - strong_tx_body_(body(config.path, schema::indexes::strong_tx), config.strong_tx_size, config.strong_tx_rate), + strong_tx_head_(head(config.path / schema::dir::heads, schema::indexes::strong_tx), 1, 0, random), + strong_tx_body_(body(config.path, schema::indexes::strong_tx), config.strong_tx_size, config.strong_tx_rate, sequential), strong_tx(strong_tx_head_, strong_tx_body_, config.strong_tx_buckets), // Caches. // ------------------------------------------------------------------------ - duplicate_head_(head(config.path / schema::dir::heads, schema::caches::duplicate)), - duplicate_body_(body(config.path, schema::caches::duplicate), config.duplicate_size, config.duplicate_rate), + duplicate_head_(head(config.path / schema::dir::heads, schema::caches::duplicate), 1, 0, random), + duplicate_body_(body(config.path, schema::caches::duplicate), config.duplicate_size, config.duplicate_rate, sequential), duplicate(duplicate_head_, duplicate_body_, config.duplicate_buckets), - prevout_head_(head(config.path / schema::dir::heads, schema::caches::prevout)), - prevout_body_(body(config.path, schema::caches::prevout), config.prevout_size, config.prevout_rate), + prevout_head_(head(config.path / schema::dir::heads, schema::caches::prevout), 1, 0, random), + prevout_body_(body(config.path, schema::caches::prevout), config.prevout_size, config.prevout_rate, sequential), prevout(prevout_head_, prevout_body_, config.prevout_buckets), - validated_bk_head_(head(config.path / schema::dir::heads, schema::caches::validated_bk)), - validated_bk_body_(body(config.path, schema::caches::validated_bk), config.validated_bk_size, config.validated_bk_rate), + validated_bk_head_(head(config.path / schema::dir::heads, schema::caches::validated_bk), 1, 0, random), + validated_bk_body_(body(config.path, schema::caches::validated_bk), config.validated_bk_size, config.validated_bk_rate, sequential), validated_bk(validated_bk_head_, validated_bk_body_, config.validated_bk_buckets), - validated_tx_head_(head(config.path / schema::dir::heads, schema::caches::validated_tx)), - validated_tx_body_(body(config.path, schema::caches::validated_tx), config.validated_tx_size, config.validated_tx_rate), + validated_tx_head_(head(config.path / schema::dir::heads, schema::caches::validated_tx), 1, 0, random), + validated_tx_body_(body(config.path, schema::caches::validated_tx), config.validated_tx_size, config.validated_tx_rate, sequential), validated_tx(validated_tx_head_, validated_tx_body_, config.validated_tx_buckets), // Optionals. // ------------------------------------------------------------------------ - address_head_(head(config.path / schema::dir::heads, schema::optionals::address)), - address_body_(body(config.path, schema::optionals::address), config.address_size, config.address_rate), + address_head_(head(config.path / schema::dir::heads, schema::optionals::address), 1, 0, random), + address_body_(body(config.path, schema::optionals::address), config.address_size, config.address_rate, sequential), address(address_head_, address_body_, config.address_buckets), - filter_bk_head_(head(config.path / schema::dir::heads, schema::optionals::filter_bk)), - filter_bk_body_(body(config.path, schema::optionals::filter_bk), config.filter_bk_size, config.filter_bk_rate), + filter_bk_head_(head(config.path / schema::dir::heads, schema::optionals::filter_bk), 1, 0, random), + filter_bk_body_(body(config.path, schema::optionals::filter_bk), config.filter_bk_size, config.filter_bk_rate, sequential), filter_bk(filter_bk_head_, filter_bk_body_, config.filter_bk_buckets), - filter_tx_head_(head(config.path / schema::dir::heads, schema::optionals::filter_tx)), - filter_tx_body_(body(config.path, schema::optionals::filter_tx), config.filter_tx_size, config.filter_tx_rate), + filter_tx_head_(head(config.path / schema::dir::heads, schema::optionals::filter_tx), 1, 0, random), + filter_tx_body_(body(config.path, schema::optionals::filter_tx), config.filter_tx_size, config.filter_tx_rate, sequential), filter_tx(filter_tx_head_, filter_tx_body_, config.filter_tx_buckets), // Locks. diff --git a/include/bitcoin/database/memory/map.hpp b/include/bitcoin/database/memory/map.hpp index aa70b27a..41d63d75 100644 --- a/include/bitcoin/database/memory/map.hpp +++ b/include/bitcoin/database/memory/map.hpp @@ -43,7 +43,7 @@ class BCD_API map DELETE_COPY_MOVE(map); map(const std::filesystem::path& filename, size_t minimum=1, - size_t expansion=0) NOEXCEPT; + size_t expansion=0, bool random=true) NOEXCEPT; /// Destruct for debug assertion only. virtual ~map() NOEXCEPT; @@ -140,6 +140,7 @@ class BCD_API map const std::filesystem::path filename_; const size_t minimum_; const size_t expansion_; + const bool random_; // Protected by remap_mutex. // requires remap_mutex_ exclusive lock for write. diff --git a/src/error.cpp b/src/error.cpp index 488bf3a1..307a2bab 100644 --- a/src/error.cpp +++ b/src/error.cpp @@ -61,6 +61,7 @@ DEFINE_ERROR_T_MESSAGE_MAP(error) { mremap_failure, "mremap failure" }, { munmap_failure, "munmap failure" }, { madvise_failure, "madvise failure" }, + { sysconf_failure, "sysconf failure" }, { ftruncate_failure, "ftruncate failure" }, { fsync_failure, "fsync failure" }, diff --git a/src/file/utilities.cpp b/src/file/utilities.cpp index 91d51a85..793b328f 100644 --- a/src/file/utilities.cpp +++ b/src/file/utilities.cpp @@ -242,29 +242,62 @@ code copy_directory_ex(const path& from, const path& to) NOEXCEPT // File descriptor functions required for memory mapping. -int open(const path& filename) NOEXCEPT +#if defined(HAVE_MSC) || !defined(HAVE_APPLE) + #define MSC_OR_NOAPPLE(parameter) parameter +#else + #define MSC_OR_NOAPPLE(parameter) +#endif + +int open(const path& filename, bool MSC_OR_NOAPPLE(random)) NOEXCEPT { const auto path = system::to_extended_path(filename); int file_descriptor{}; - // _wsopen_s and open set errno on failure. - // _wsopen_s and wstring do not throw (but are unannotated). #if defined(HAVE_MSC) - // sets file_descriptor = -1 on error. - _wsopen_s(&file_descriptor, path.c_str(), - O_RDWR | _O_BINARY | _O_RANDOM, _SH_DENYWR, _S_IREAD | _S_IWRITE); + // _wsopen_s and wstring do not throw (but are unannotated). + // sets file_descriptor = -1 and errno on error. + const auto access = (random ? _O_RANDOM : _O_SEQUENTIAL); + ::_wsopen_s(&file_descriptor, path.c_str(), + O_RDWR | _O_BINARY | access, _SH_DENYWR, _S_IREAD | _S_IWRITE); #else - // returns -1 on failure. - file_descriptor = ::open(path.c_str(), - O_RDWR, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); -#endif + // open sets errno on failure. + file_descriptor = ::open(path.c_str(), O_RDWR, S_IRUSR | S_IWUSR); + +#if !defined(HAVE_APPLE) + if (file_descriptor != -1) + { + // _O_RANDOM equivalent, posix_fadvise returns error on failure. + const auto advice = random ? POSIX_FADV_RANDOM : POSIX_FADV_SEQUENTIAL; + const auto result = ::posix_fadvise(file_descriptor, 0, 0, advice); + if (!is_zero(result)) + { + close(file_descriptor); + file_descriptor = -1; + errno = result; + } + else + { + // _SH_DENYWR equivalent. + const struct flock lock{ F_WRLCK, SEEK_SET, 0, 0 }; + if (::fcntl(file_descriptor, F_SETLK, &lock) == -1) + { + const auto last = errno; + close(file_descriptor); + file_descriptor = -1; + errno = last; + } + } + } +#endif // !HAVE_APPLE +#endif // HAVE_MSC + return file_descriptor; } -code open_ex(int& file_descriptor, const path& filename) NOEXCEPT +code open_ex(int& file_descriptor, const path& filename, bool random) NOEXCEPT { system::error::clear_errno(); - file_descriptor = open(filename); + file_descriptor = open(filename, random); return system::error::get_errno(); } diff --git a/src/memory/map.cpp b/src/memory/map.cpp index ab569bd6..38e83a1e 100644 --- a/src/memory/map.cpp +++ b/src/memory/map.cpp @@ -24,6 +24,7 @@ #include #include #include + #include #endif #include #include @@ -43,8 +44,12 @@ BC_PUSH_WARNING(NO_THROW_IN_NOEXCEPT) using namespace system; -map::map(const path& filename, size_t minimum, size_t expansion) NOEXCEPT - : filename_(filename), minimum_(minimum), expansion_(expansion) +map::map(const path& filename, size_t minimum, size_t expansion, + bool random) NOEXCEPT + : filename_(filename), + minimum_(minimum), + expansion_(expansion), + random_(random) { } @@ -69,7 +74,8 @@ code map::open() NOEXCEPT if (opened_ != file::invalid) return error::open_open; - if (const auto ec = file::open_ex(opened_, filename_)) + // Windows doesn't use madvise, instead infers map access from file open. + if (const auto ec = file::open_ex(opened_, filename_, random_)) return ec; return file::size_ex(logical_, opened_); @@ -583,43 +589,41 @@ bool map::finalize_(size_t size) NOEXCEPT return false; } - // TODO: also, madvise only the size increase. - // TODO: madvise with large length value fails on linux (and zero is noop). - // TODO: Random may not be best since writes are sequential (for bodies). - // TODO: Heads are truly read/write random so could benefit. - // TODO: this fails with large and/or unaligned size. Align size to page, - // TODO: rounded up. Iterate over calls at 1GB (1_u64 << 30), sample: - ////// Get page size (usually 4KB) - ////const size_t page_size = sysconf(_SC_PAGESIZE); - ////if (page_size == static_cast(-1)) - ////{ - //// set_first_code(error::sysconf_failure); - //// unmap_(); - //// return false; - ////} - //// - ////// Align size up to page boundary - ////const size_t aligned_size = (size + page_size - 1) & ~(page_size - 1); - //// - ////// Use 1GB chunks to avoid large-length issues - ////constexpr size_t chunk_size = 1ULL << 30; // 1GB - ////uint8_t* ptr = memory_map_; - ////for (size_t offset = 0; offset < aligned_size; offset += chunk_size) - ////{ - //// size_t len = std::min(chunk_size, aligned_size - offset); - //// if (::madvise(ptr + offset, len, MADV_SEQUENTIAL) == fail) - //// { - //// set_first_code(error::madvise_failure); - //// unmap_(); - //// return false; - //// } - ////} - ////if (::madvise(memory_map_, size, MADV_RANDOM) == fail) - ////{ - //// set_first_code(error::madvise_failure); - //// unmap_(); - //// return false; - ////} +#if !defined(HAVE_MSC) + // Get page size (usually 4KB). + const int page_size = ::sysconf(_SC_PAGESIZE); + const auto page = possible_narrow_sign_cast(page_size); + + // If not one bit then page size is not a power of two as required. + if (page_size == fail || !is_one(ones_count(page))) + { + set_first_code(error::sysconf_failure); + unmap_(); + return false; + } + + // Align size up to page boundary. + const auto max = sub1(page); + const auto align = bit_and(ceilinged_add(size, max), bit_not(max)); + + // Use 1GB chunks to avoid large-length issues. + constexpr auto chunk = power2(30u); + const auto advice = random_ ? MADV_RANDOM : MADV_SEQUENTIAL; + + for (auto offset = zero; offset < align; offset += chunk) + { + BC_PUSH_WARNING(NO_POINTER_ARITHMETIC) + const auto start = memory_map_ + offset; + BC_POP_WARNING() + + if (::madvise(start, std::min(chunk, align - offset), advice) == fail) + { + set_first_code(error::madvise_failure); + unmap_(); + return false; + } + } +#endif loaded_ = true; capacity_ = size; diff --git a/src/memory/mman-win32/mman.cpp b/src/memory/mman-win32/mman.cpp index 36d8301b..50df0040 100644 --- a/src/memory/mman-win32/mman.cpp +++ b/src/memory/mman-win32/mman.cpp @@ -35,12 +35,13 @@ static DWORD protect_page(int prot) noexcept if ((prot & PROT_EXEC) != 0) { - return ((prot & PROT_WRITE) != 0) ? PAGE_EXECUTE_READWRITE : - PAGE_EXECUTE_READ; + return ((prot & PROT_WRITE) != 0) ? + PAGE_EXECUTE_READWRITE : PAGE_EXECUTE_READ; } else { - return ((prot & PROT_WRITE) != 0) ? PAGE_READWRITE : PAGE_READONLY; + return ((prot & PROT_WRITE) != 0) ? + PAGE_READWRITE : PAGE_READONLY; } } @@ -79,38 +80,37 @@ void* mmap(void* /*addr*/, size_t len, int prot, int flags, int fd, const auto file_lo = large ? (DWORD)((off) & MAXDWORD) : (DWORD)off; const auto file_hi = large ? (DWORD)((off >> 32) & MAXDWORD) : (DWORD)0; - if (len == 0 || (flags & MAP_FIXED) != 0 || prot == PROT_EXEC) + // Disallow fixed, executable, and or anonymous mapping. + if (len == 0 || + (prot == PROT_EXEC) || + (flags & MAP_FIXED) != 0 || + (flags & MAP_ANONYMOUS) != 0) { errno = EINVAL; return MAP_FAILED; } // Never call CloseHandle on the return value of this function. - const auto handle = ((flags & MAP_ANONYMOUS) == 0) ? - (HANDLE)_get_osfhandle(fd) : INVALID_HANDLE_VALUE; - - if ((flags & MAP_ANONYMOUS) == 0 && handle == INVALID_HANDLE_VALUE) + const auto handle = (HANDLE)_get_osfhandle(fd); + if (handle == INVALID_HANDLE_VALUE) { errno = EBADF; return MAP_FAILED; } - const auto mapping = CreateFileMappingW(handle, NULL, protect, max_hi, - max_lo, NULL); - + const auto mapping = CreateFileMappingW(handle, NULL, protect, max_hi, max_lo, NULL); if (mapping == NULL) { errno = last_error(EPERM); return MAP_FAILED; } - const auto map = MapViewOfFile(mapping, access, file_hi, file_lo, len); - // "to fully close a file mapping object, an application must unmap all // mapped views of the file mapping object by calling UnmapViewOfFile and // close the file mapping object handle by calling CloseHandle. These // functions can be called in any order." - so we close the handle here and // retain only the map. + const auto map = MapViewOfFile(mapping, access, file_hi, file_lo, len); if (map == NULL || CloseHandle(mapping) == FALSE) { errno = last_error(EPERM); @@ -156,7 +156,6 @@ int mprotect(void* addr, size_t len, int prot) noexcept { DWORD old_protect = 0; const auto new_protect = protect_page(prot); - if (VirtualProtect(addr, len, new_protect, &old_protect) == FALSE) { errno = last_error(EPERM); @@ -271,4 +270,10 @@ int ftruncate(int fd, oft__ size) noexcept return 0; } +// stub +int sysconf(int) noexcept +{ + return {}; +} + #endif // _WIN32 diff --git a/src/memory/mman-win32/mman.hpp b/src/memory/mman-win32/mman.hpp index abef2a1f..b34839b3 100644 --- a/src/memory/mman-win32/mman.hpp +++ b/src/memory/mman-win32/mman.hpp @@ -30,9 +30,6 @@ typedef size_t oft__; #define MS_SYNC 2 #define MS_INVALIDATE 4 -// Flags for madvise (stub). -#define MADV_RANDOM 0 - void* mmap(void* addr, size_t len, int prot, int flags, int fd, oft__ off) noexcept; void* mremap_(void* addr, size_t old_size, size_t new_size, int prot, int flags, int fd) noexcept; diff --git a/test/error.cpp b/test/error.cpp index 876a7687..cce073c9 100644 --- a/test/error.cpp +++ b/test/error.cpp @@ -203,6 +203,15 @@ BOOST_AUTO_TEST_CASE(error_t__code__madvise_failure__true_exected_message) BOOST_REQUIRE_EQUAL(ec.message(), "madvise failure"); } +BOOST_AUTO_TEST_CASE(error_t__code__sysconf_failure__true_exected_message) +{ + constexpr auto value = error::sysconf_failure; + const auto ec = code(value); + BOOST_REQUIRE(ec); + BOOST_REQUIRE(ec == value); + BOOST_REQUIRE_EQUAL(ec.message(), "sysconf failure"); +} + BOOST_AUTO_TEST_CASE(error_t__code__ftruncate_failure__true_exected_message) { constexpr auto value = error::ftruncate_failure; diff --git a/test/mocks/chunk_storage.cpp b/test/mocks/chunk_storage.cpp index 1b4fd027..e17f95d7 100644 --- a/test/mocks/chunk_storage.cpp +++ b/test/mocks/chunk_storage.cpp @@ -40,7 +40,7 @@ chunk_storage::chunk_storage(system::data_chunk& reference) NOEXCEPT } chunk_storage::chunk_storage(const std::filesystem::path& filename, - size_t, size_t) NOEXCEPT + size_t, size_t, bool) NOEXCEPT : buffer_{ local_ }, path_{ filename }, logical_{} { } diff --git a/test/mocks/chunk_storage.hpp b/test/mocks/chunk_storage.hpp index 29b81752..00f2d6e9 100644 --- a/test/mocks/chunk_storage.hpp +++ b/test/mocks/chunk_storage.hpp @@ -32,7 +32,7 @@ class chunk_storage chunk_storage() NOEXCEPT; chunk_storage(system::data_chunk& reference) NOEXCEPT; chunk_storage(const std::filesystem::path& filename, size_t minimum=1, - size_t expansion=0) NOEXCEPT; + size_t expansion=0, bool random=true) NOEXCEPT; // test side door. system::data_chunk& buffer() NOEXCEPT;