diff --git a/include/engine/data_watchdog.hpp b/include/engine/data_watchdog.hpp index a4583398a25..8ba21dd5bbf 100644 --- a/include/engine/data_watchdog.hpp +++ b/include/engine/data_watchdog.hpp @@ -6,7 +6,6 @@ #include "engine/datafacade_factory.hpp" #include "storage/shared_datatype.hpp" -#include "storage/shared_memory.hpp" #include "storage/shared_monitor.hpp" #include @@ -59,8 +58,8 @@ class DataWatchdogImpl( std::make_shared( - std::vector{ - static_region.shm_key, updatable_region.shm_key})); + std::vector{static_region.proj_id, + updatable_region.proj_id})); } } @@ -112,18 +111,18 @@ class DataWatchdogImpl swap_lock(factory_mutex); facade_factory = DataFacadeFactory( std::make_shared( - std::vector{ - static_region.shm_key, updatable_region.shm_key})); + std::vector{static_region.proj_id, + updatable_region.proj_id})); } + + util::Log() << "updated facade to regions " << (int)static_region.proj_id << " and " + << (int)updatable_region.proj_id << " with timestamps " + << static_region.timestamp << " and " << updatable_region.timestamp; } util::Log() << "DataWatchdog thread stopped"; diff --git a/include/engine/datafacade/shared_memory_allocator.hpp b/include/engine/datafacade/shared_memory_allocator.hpp index 71170316c6d..f91e9e86a78 100644 --- a/include/engine/datafacade/shared_memory_allocator.hpp +++ b/include/engine/datafacade/shared_memory_allocator.hpp @@ -19,8 +19,7 @@ namespace osrm::engine::datafacade class SharedMemoryAllocator final : public ContiguousBlockAllocator { public: - explicit SharedMemoryAllocator( - const std::vector &shm_keys); + explicit SharedMemoryAllocator(const std::vector &proj_ids); ~SharedMemoryAllocator() override final; // interface to give access to the datafacades diff --git a/include/storage/shared_datatype.hpp b/include/storage/shared_datatype.hpp index 7890b8b6956..a0b97252bc1 100644 --- a/include/storage/shared_datatype.hpp +++ b/include/storage/shared_datatype.hpp @@ -2,15 +2,12 @@ #define SHARED_DATA_TYPE_HPP #include "storage/block.hpp" -#include "storage/io_fwd.hpp" #include "util/exception.hpp" -#include "util/exception_utils.hpp" #include #include #include -#include #include namespace osrm::storage @@ -67,10 +64,7 @@ class BaseDataLayout return GetBlock(name).byte_size; } - inline bool HasBlock(const std::string &name) const - { - return blocks.find(name) != blocks.end(); - } + inline bool HasBlock(const std::string &name) const { return blocks.contains(name); } // Depending on the name prefix this function either lists all blocks with the same prefix // or all entries in the sub-directory. @@ -192,13 +186,19 @@ class TarDataLayout final : public BaseDataLayout } }; +// The second parameter passed to ftok(). See: man 3 ftok +// It should actually be an int, but for compatibility with earlier versions of OSRM it +// is an uint16. It should't matter since, according to the man page, only the lowest 8 +// bits are used. +using ProjID = uint16_t; + struct SharedRegion { static constexpr const int MAX_NAME_LENGTH = 254; SharedRegion() : name{0}, timestamp{0} {} - SharedRegion(const std::string &name_, std::uint64_t timestamp, std::uint16_t shm_key) - : name{0}, timestamp{timestamp}, shm_key{shm_key} + SharedRegion(const std::string &name_, std::uint64_t timestamp, ProjID proj_id) + : name{0}, timestamp{timestamp}, proj_id{proj_id} { std::copy_n(name_.begin(), std::min(MAX_NAME_LENGTH, name_.size()), name); } @@ -207,7 +207,7 @@ struct SharedRegion char name[MAX_NAME_LENGTH + 1]; std::uint64_t timestamp; - std::uint16_t shm_key = 0; + ProjID proj_id = 0; }; // Keeps a list of all shared regions in a fixed-sized struct @@ -216,7 +216,6 @@ struct SharedRegionRegister { using RegionID = std::uint16_t; static constexpr const RegionID INVALID_REGION_ID = std::numeric_limits::max(); - using ShmKey = decltype(SharedRegion::shm_key); // Returns the key of the region with the given name RegionID Find(const std::string &name) const @@ -238,7 +237,7 @@ struct SharedRegionRegister } } - RegionID Register(const std::string &name, ShmKey key) + RegionID Register(const std::string &name, ProjID proj_id) { auto iter = std::find_if( regions.begin(), regions.end(), [&](const auto ®ion) { return region.IsEmpty(); }); @@ -250,7 +249,7 @@ struct SharedRegionRegister else { constexpr std::uint32_t INITIAL_TIMESTAMP = 1; - *iter = SharedRegion{name, INITIAL_TIMESTAMP, key}; + *iter = SharedRegion{name, INITIAL_TIMESTAMP, proj_id}; RegionID key = std::distance(regions.begin(), iter); return key; } @@ -271,7 +270,7 @@ struct SharedRegionRegister auto &GetRegion(const RegionID key) { return regions[key]; } - ShmKey ReserveKey() + ProjID ReserveKey() { auto free_key_iter = std::find(shm_key_in_use.begin(), shm_key_in_use.end(), false); if (free_key_iter == shm_key_in_use.end()) @@ -283,7 +282,7 @@ struct SharedRegionRegister return std::distance(shm_key_in_use.begin(), free_key_iter); } - void ReleaseKey(ShmKey key) { shm_key_in_use[key] = false; } + void ReleaseKey(ProjID proj_id) { shm_key_in_use[proj_id] = false; } static constexpr const std::size_t MAX_SHARED_REGIONS = 512; static_assert(MAX_SHARED_REGIONS < std::numeric_limits::max(), diff --git a/include/storage/shared_memory.hpp b/include/storage/shared_memory.hpp index 457a688a9ff..9284491ed4b 100644 --- a/include/storage/shared_memory.hpp +++ b/include/storage/shared_memory.hpp @@ -1,10 +1,9 @@ #ifndef SHARED_MEMORY_HPP #define SHARED_MEMORY_HPP -#include "util/exception.hpp" -#include "util/exception_utils.hpp" -#include "util/log.hpp" +#include "storage/shared_datatype.hpp" +#include #include #ifndef _WIN32 #include @@ -18,312 +17,61 @@ #endif #include -#include -#include #include -#include -#include - -#include "storage/shared_memory_ownership.hpp" namespace osrm::storage { -// Returns directory for OSRM lock files (SHM_LOCK_DIR env var or system temp) -inline std::filesystem::path getLockDir() -{ - if (const char *lock_dir = std::getenv("SHM_LOCK_DIR")) - { - std::filesystem::path dir(lock_dir); - if (!std::filesystem::exists(dir)) - { - throw util::exception("SHM_LOCK_DIR directory does not exist: " + dir.string() + - SOURCE_REF); - } - return dir; - } - return std::filesystem::temp_directory_path(); -} - -struct OSRMLockFile -{ - template std::filesystem::path operator()(const IdentifierT &id) - { - return getLockDir() / ("osrm-" + std::to_string(id) + ".lock"); - } -}; - -#ifndef _WIN32 -class SharedMemory -{ - public: - void *Ptr() const { return region.get_address(); } - std::size_t Size() const { return region.get_size(); } - - SharedMemory(const SharedMemory &) = delete; - SharedMemory &operator=(const SharedMemory &) = delete; - - template - SharedMemory(const std::filesystem::path &lock_file, - const IdentifierT id, - const uint64_t size = 0) - : key(lock_file.string().c_str(), id) - { - // open only - if (0 == size) - { - shm = boost::interprocess::xsi_shared_memory(boost::interprocess::open_only, key); - - util::Log(logDEBUG) << "opening " << shm.get_shmid() << " from id " << (int)id; - - region = boost::interprocess::mapped_region(shm, boost::interprocess::read_only); - } - // open or create - else - { - shm = boost::interprocess::xsi_shared_memory( - boost::interprocess::open_or_create, key, size); - util::Log(logDEBUG) << "opening/creating " << shm.get_shmid() << " from id " << id - << " with size " << size; -#ifdef __linux__ - if (-1 == shmctl(shm.get_shmid(), SHM_LOCK, nullptr)) - { - if (ENOMEM == errno) - { - util::Log(logWARNING) << "could not lock shared memory to RAM"; - } - } -#endif - region = boost::interprocess::mapped_region(shm, boost::interprocess::read_write); - } - } - - template static bool RegionExists(const IdentifierT id) - { - bool result = true; - try - { - OSRMLockFile lock_file; - boost::interprocess::xsi_key key(lock_file(id).string().c_str(), id); - result = RegionExists(key); - } - catch (...) - { - result = false; - } - return result; - } - - template static bool Remove(const IdentifierT id) - { - OSRMLockFile lock_file; - boost::interprocess::xsi_key key(lock_file(id).string().c_str(), id); - return Remove(key); - } - -#ifdef __linux__ - void WaitForDetach() - { - auto shmid = shm.get_shmid(); - ::shmid_ds xsi_ds; - const auto errorToMessage = [](int error) -> std::string - { - switch (error) - { - case EPERM: - return "EPERM"; - break; - case EACCES: - return "ACCESS"; - break; - case EINVAL: - return "EINVAL"; - break; - case EFAULT: - return "EFAULT"; - break; - default: - return "Unknown Error " + std::to_string(error); - break; - } - }; - - do - { - // On OSX this returns EINVAL for whatever reason, hence we need to disable it - int ret = ::shmctl(shmid, IPC_STAT, &xsi_ds); - if (ret < 0) - { - auto error_code = errno; - throw util::exception("shmctl encountered an error: " + errorToMessage(error_code) + - SOURCE_REF); - } - BOOST_ASSERT(ret >= 0); - - std::this_thread::sleep_for(std::chrono::microseconds(100)); - } while (xsi_ds.shm_nattch > 1); - } -#else - void WaitForDetach() - { - util::Log(logDEBUG) - << "Shared memory support for non-Linux systems does not wait for clients to " - "dettach. Going to sleep for 50ms."; - std::this_thread::sleep_for(std::chrono::milliseconds(50)); - } -#endif - - private: - static bool RegionExists(const boost::interprocess::xsi_key &key) - { - bool result = true; - try - { - boost::interprocess::xsi_shared_memory shm(boost::interprocess::open_only, key); - } - catch (const boost::interprocess::interprocess_exception &e) - { - if (e.get_error_code() != boost::interprocess::not_found_error) - { - throw; - } - result = false; - } - - return result; - } - - static bool Remove(const boost::interprocess::xsi_key &key) - { - boost::interprocess::xsi_shared_memory xsi(boost::interprocess::open_only, key); - util::Log(logDEBUG) << "deallocating prev memory " << xsi.get_shmid(); - return boost::interprocess::xsi_shared_memory::remove(xsi.get_shmid()); - } - - boost::interprocess::xsi_key key; - boost::interprocess::xsi_shared_memory shm; - boost::interprocess::mapped_region region; -}; -#else -// Windows - specific code class SharedMemory { SharedMemory(const SharedMemory &) = delete; SharedMemory &operator=(const SharedMemory &) = delete; public: + SharedMemory(const ProjID proj_id, const uint64_t size = 0); + void *Ptr() const { return region.get_address(); } std::size_t Size() const { return region.get_size(); } - SharedMemory(const std::filesystem::path &lock_file, const int id, const uint64_t size = 0) - { - sprintf(key, "%s.%d", "osrm.lock", id); - if (0 == size) - { // read_only - shm = boost::interprocess::shared_memory_object( - boost::interprocess::open_only, key, boost::interprocess::read_only); - region = boost::interprocess::mapped_region(shm, boost::interprocess::read_only); - } - else - { // writeable pointer - shm = boost::interprocess::shared_memory_object( - boost::interprocess::open_or_create, key, boost::interprocess::read_write); - shm.truncate(size); - region = boost::interprocess::mapped_region(shm, boost::interprocess::read_write); - - util::Log(logDEBUG) << "writeable memory allocated " << size << " bytes"; - } - } - - static bool RegionExists(const int id) - { - bool result = true; - try - { - char k[500]; - build_key(id, k); - result = RegionExists(k); - } - catch (...) - { - result = false; - } - return result; - } - - static bool Remove(const int id) - { - char k[500]; - build_key(id, k); - return Remove(k); - } - - void WaitForDetach() - { - // FIXME this needs an implementation for Windows - util::Log(logDEBUG) << "Shared memory support for Windows does not wait for clients to " - "dettach. Going to sleep for 50ms."; - std::this_thread::sleep_for(std::chrono::milliseconds(50)); - } - private: - static void build_key(int id, char *key) { sprintf(key, "%s.%d", "osrm.lock", id); } - - static bool RegionExists(const char *key) - { - bool result = true; - try - { - boost::interprocess::shared_memory_object shm( - boost::interprocess::open_only, key, boost::interprocess::read_write); - } - catch (...) - { - result = false; - } - return result; - } - - static bool Remove(char *key) - { - util::Log(logDEBUG) << "deallocating prev memory for key " << key; - return boost::interprocess::shared_memory_object::remove(key); - } - - char key[500]; - boost::interprocess::shared_memory_object shm; boost::interprocess::mapped_region region; }; -#endif -template -std::unique_ptr makeSharedMemory(const IdentifierT &id, const uint64_t size = 0) -{ - static_assert(sizeof(id) == sizeof(std::uint16_t), "Key type is not 16 bits"); - try - { - LockFileT lock_file; - if (!std::filesystem::exists(lock_file(id))) - { - if (0 == size) - { - throw util::exception("lock file does not exist, exiting" + SOURCE_REF); - } - else - { - std::ofstream ofs(lock_file(id)); - } - } - return std::make_unique(lock_file(id), id, size); - } - catch (const boost::interprocess::interprocess_exception &e) - { - util::Log(logERROR) << "Error while attempting to allocate shared memory: " << e.what() - << ", code " << e.get_error_code(); - throw util::exception(e.what() + SOURCE_REF); - } -} +/** + * @brief Returns the directory to use for OSRM lock files + * + * Returns the contents of the environment variable SHM_LOCK_DIR if set else the system + * temp directory. + * + * @return std::filesystem::path The directory to usew for lock files + */ +std::filesystem::path getLockDir(); + +std::unique_ptr makeSharedMemory(const ProjID proj_id, const uint64_t size = 0); + +/** + * @brief Tests if a shared memory region exists + * + * @param key A ProjID + * @return bool Returns true if the region exists + */ +bool RegionExists(const ProjID proj_id); +/** + * @brief Destroys the shared memory region + * + * @param key A valid ProjID + * @return bool returns false on error. + */ +bool Remove(const ProjID proj_id); +/** + * @brief Waits for all processes to detach from the shared memory region + * + * @param key A ProjID + * @param timeout Timeout in ms + */ +void WaitForDetach(const ProjID proj_id, int timeout); + } // namespace osrm::storage #endif // SHARED_MEMORY_HPP diff --git a/include/storage/shared_monitor.hpp b/include/storage/shared_monitor.hpp index ad33864ec32..ff7e0239f41 100644 --- a/include/storage/shared_monitor.hpp +++ b/include/storage/shared_monitor.hpp @@ -1,8 +1,8 @@ #ifndef SHARED_MONITOR_HPP #define SHARED_MONITOR_HPP -#include "storage/shared_datatype.hpp" - +#include "util/exception.hpp" +#include "util/exception_utils.hpp" #include #include #include @@ -197,7 +197,7 @@ template struct SharedMonitor auto get_semaphore(std::size_t index) { return reinterpret_cast( - buffer + index * sizeof(bi::interprocess_semaphore)); + buffer + (index * sizeof(bi::interprocess_semaphore))); } void invalidate_semaphore(void *semaphore) const diff --git a/src/engine/datafacade/shared_memory_allocator.cpp b/src/engine/datafacade/shared_memory_allocator.cpp index 97c98b80dff..a4bb8b92f55 100644 --- a/src/engine/datafacade/shared_memory_allocator.cpp +++ b/src/engine/datafacade/shared_memory_allocator.cpp @@ -9,16 +9,15 @@ namespace osrm::engine::datafacade { -SharedMemoryAllocator::SharedMemoryAllocator( - const std::vector &shm_keys) +SharedMemoryAllocator::SharedMemoryAllocator(const std::vector &proj_ids) { std::vector regions; - for (const auto shm_key : shm_keys) + for (const auto proj_id : proj_ids) { - util::Log(logDEBUG) << "Loading new data for region " << (int)shm_key; - BOOST_ASSERT(storage::SharedMemory::RegionExists(shm_key)); - auto mem = storage::makeSharedMemory(shm_key); + util::Log(logDEBUG) << "Loading new data for region " << (int)proj_id; + BOOST_ASSERT(storage::RegionExists(proj_id)); + auto mem = storage::makeSharedMemory(proj_id); storage::io::BufferReader reader(reinterpret_cast(mem->Ptr()), mem->Size()); std::unique_ptr layout = diff --git a/src/storage/shared_memory.cpp b/src/storage/shared_memory.cpp new file mode 100644 index 00000000000..bded14abe30 --- /dev/null +++ b/src/storage/shared_memory.cpp @@ -0,0 +1,232 @@ +#include "storage/shared_memory.hpp" +#include "storage/shared_datatype.hpp" +#include +#include +#include +#include + +namespace osrm::storage +{ +using namespace boost::interprocess; + +struct OSRMLockFile +{ + OSRMLockFile(const ProjID proj_id) + { + std::filesystem::path filename = + std::filesystem::path("osrm-" + std::to_string(proj_id) + ".lock"); + lock_file = getLockDir() / filename; + } + const std::filesystem::path to_path() { return lock_file; } + operator const std::filesystem::path::value_type *() const noexcept + { + return lock_file.c_str(); + } + + private: + std::filesystem::path lock_file; +}; + +#ifndef _WIN32 + +SharedMemory::SharedMemory(const ProjID proj_id, const uint64_t size) +{ + OSRMLockFile lock_file(proj_id); + xsi_key xsi_key(lock_file, proj_id); + // open only + if (size == 0) + { + xsi_shared_memory xsi_shm(open_only, xsi_key); + + util::Log(logDEBUG) << "opening " << xsi_shm.get_shmid() << " from id " << (int)proj_id; + + region = mapped_region(xsi_shm, read_only); + } + // open or create + else + { + xsi_shared_memory xsi_shm(open_or_create, xsi_key, size); + util::Log(logDEBUG) << "opening/creating " << xsi_shm.get_shmid() << " from id " << proj_id + << " with size " << size; +#ifdef __linux__ + if (-1 == shmctl(xsi_shm.get_shmid(), SHM_LOCK, nullptr)) + { + if (ENOMEM == errno) + { + util::Log(logWARNING) << "could not lock shared memory to RAM"; + } + } +#endif + region = mapped_region(xsi_shm, read_write); + } +} + +bool RegionExists(const ProjID proj_id) +{ + try + { + OSRMLockFile lock_file(proj_id); + xsi_key xsi_key(lock_file, proj_id); + xsi_shared_memory xsi(open_only, xsi_key); + return true; + } + catch (...) + { + } + return false; +} + +bool Remove(const ProjID proj_id) +{ + OSRMLockFile lock_file(proj_id); + xsi_key xsi_key(lock_file, proj_id); + xsi_shared_memory xsi(open_only, xsi_key); + util::Log(logDEBUG) << "deallocating prev memory " << xsi.get_shmid(); + return xsi_shared_memory::remove(xsi.get_shmid()); +} + +void WaitForDetach(const ProjID proj_id, int timeout) +{ + try + { + OSRMLockFile lock_file(proj_id); + xsi_key xsi_key(lock_file, proj_id); + xsi_shared_memory xsi(open_only, xsi_key); + while (true) + { + ::shmid_ds xsi_ds; + if (::shmctl(xsi.get_shmid(), IPC_STAT, &xsi_ds) < 0) + { + if (errno != EIDRM) + throw util::exception("Error while waiting for clients to detach: " + + std::string(strerror(errno)) + " at " + SOURCE_REF); + break; + } + if (xsi_ds.shm_nattch == 0) + break; + if (--timeout < 0) + break; + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + } + catch (interprocess_exception &) + { + // already detached + } +} + +#else + +class OSRMShmName +{ + public: + OSRMShmName(const ProjID proj_id) { name = "osrm-" + std::to_string(proj_id); } + operator const char *() const noexcept { return name.c_str(); } + + private: + std::string name; +}; + +SharedMemory::SharedMemory(const ProjID proj_id, const uint64_t size) +{ + OSRMShmName name(proj_id); + if (size == 0) + { // read_only + shared_memory_object shm_obj(shared_memory_object(open_only, name, read_only)); + region = mapped_region(shm_obj, read_only); + } + else + { // writeable pointer + shared_memory_object shm_obj(shared_memory_object(open_or_create, name, read_write)); + shm_obj.truncate(size); + region = mapped_region(shm_obj, read_write); + + util::Log(logDEBUG) << "writeable memory allocated " << size << " bytes"; + } +} + +bool RegionExists(const ProjID proj_id) +{ + OSRMShmName name(proj_id); + try + { + shared_memory_object shm_obj(open_only, name, read_write); + return true; + } + catch (...) + { + } + return false; +} + +bool Remove(const ProjID proj_id) +{ + OSRMShmName name(proj_id); + util::Log(logDEBUG) << "deallocating prev memory for name " << name; + return shared_memory_object::remove(name); +} + +void WaitForDetach(const ProjID proj_id, int timeout) +{ + OSRMShmName name(proj_id); + try + { + while (true) + { + shared_memory_object shm_obj(open_only, name, read_write); + if (--timeout < 0) + break; + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + } + catch (interprocess_exception &) + { + // already detached + } +} + +#endif + +std::unique_ptr makeSharedMemory(const ProjID proj_id, const uint64_t size) +{ + OSRMLockFile lock_file(proj_id); + try + { + if (!std::filesystem::exists(lock_file.to_path())) + { + if (size == 0) + { + throw util::exception("Lock file does not exist, exiting. " + SOURCE_REF); + } + else + { + std::ofstream ofs(lock_file.to_path()); + } + } + return std::make_unique(proj_id, size); + } + catch (const interprocess_exception &e) + { + util::Log(logERROR) << "Error while attempting to allocate shared memory: " << e.what() + << ", code: " << e.get_error_code() + << ", lock file: " << lock_file.to_path(); + throw util::exception(e.what() + SOURCE_REF); + } +} + +std::filesystem::path getLockDir() +{ + if (const char *lock_dir = std::getenv("SHM_LOCK_DIR")) + { + std::filesystem::path dir(lock_dir); + if (!std::filesystem::exists(dir)) + { + throw util::exception("SHM_LOCK_DIR directory does not exist: " + dir.string() + + SOURCE_REF); + } + return dir; + } + return std::filesystem::temp_directory_path(); +} + +} // namespace osrm::storage diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index eb88957181d..674b3fbb15a 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -3,7 +3,6 @@ #include "storage/io.hpp" #include "storage/shared_datatype.hpp" #include "storage/shared_memory.hpp" -#include "storage/shared_memory_ownership.hpp" #include "storage/shared_monitor.hpp" #include "storage/view_factory.hpp" @@ -14,8 +13,6 @@ #include "partitioner/files.hpp" #include "util/exception.hpp" -#include "util/exception_utils.hpp" -#include "util/fingerprint.hpp" #include "util/log.hpp" #ifdef __linux__ @@ -30,7 +27,6 @@ #include #include #include -#include #include namespace osrm::storage @@ -43,23 +39,24 @@ struct RegionHandle { std::unique_ptr memory; char *data_ptr; - std::uint16_t shm_key; + ProjID proj_id; }; RegionHandle setupRegion(SharedRegionRegister &shared_register, const storage::BaseDataLayout &layout) { // This is safe because we have an exclusive lock for all osrm-datastore processes. - auto shm_key = shared_register.ReserveKey(); + ProjID proj_id = shared_register.ReserveKey(); // ensure that the shared memory region we want to write to is really removed - // this is only needef for failure recovery because we actually wait for all clients + // this is only needed for failure recovery because we actually wait for all clients // to detach at the end of the function - if (storage::SharedMemory::RegionExists(shm_key)) + if (storage::RegionExists(proj_id)) { - util::Log(logWARNING) << "Old shared memory region " << (int)shm_key << " still exists."; - util::UnbufferedLog() << "Retrying removal... "; - storage::SharedMemory::Remove(shm_key); + util::Log(logWARNING) << "Old shared memory region " << (int)proj_id << " still exists."; + util::UnbufferedLog() << "Retrying removal of shared memory region " << (int)proj_id; + storage::Remove(proj_id); + storage::WaitForDetach(proj_id, 5000); util::UnbufferedLog() << "ok."; } @@ -71,14 +68,14 @@ RegionHandle setupRegion(SharedRegionRegister &shared_register, auto regions_size = encoded_static_layout.size() + layout.GetSizeOfLayout(); util::Log() << "Data layout has a size of " << encoded_static_layout.size() << " bytes"; util::Log() << "Allocating shared memory of " << regions_size << " bytes"; - auto memory = makeSharedMemory(shm_key, regions_size); + auto memory = makeSharedMemory(proj_id, regions_size); // Copy memory static_layout to shared memory and populate data char *shared_memory_ptr = static_cast(memory->Ptr()); auto data_ptr = std::copy_n(encoded_static_layout.data(), encoded_static_layout.size(), shared_memory_ptr); - return RegionHandle{std::move(memory), data_ptr, shm_key}; + return RegionHandle{std::move(memory), data_ptr, proj_id}; } bool swapData(Monitor &monitor, @@ -102,7 +99,7 @@ bool swapData(Monitor &monitor, for (auto &pair : handles) { - SharedMemory::Remove(pair.second.shm_key); + Remove(pair.second.proj_id); } return false; } @@ -117,43 +114,46 @@ bool swapData(Monitor &monitor, auto region_id = shared_register.Find(pair.first); if (region_id == SharedRegionRegister::INVALID_REGION_ID) { - region_id = shared_register.Register(pair.first, pair.second.shm_key); + region_id = shared_register.Register(pair.first, pair.second.proj_id); } else { auto &shared_region = shared_register.GetRegion(region_id); old_handles.push_back(RegionHandle{ - makeSharedMemory(shared_region.shm_key), nullptr, shared_region.shm_key}); + makeSharedMemory(shared_region.proj_id), nullptr, shared_region.proj_id}); - shared_region.shm_key = pair.second.shm_key; + shared_region.proj_id = pair.second.proj_id; shared_region.timestamp++; } } } - util::Log() << "All data loaded. Notify all client about new data in:"; + util::Log() << "All data loaded. Notify all clients about new data in:"; for (const auto &pair : handles) { - util::Log() << pair.first << "\t" << static_cast(pair.second.shm_key); + util::Log() << pair.first << "\t" << static_cast(pair.second.proj_id); } monitor.notify_all(); for (auto &old_handle : old_handles) { util::UnbufferedLog() << "Marking old shared memory region " - << static_cast(old_handle.shm_key) << " for removal... "; + << static_cast(old_handle.proj_id) << " for removal... "; // SHMCTL(2): Mark the segment to be destroyed. The segment will actually be destroyed // only after the last process detaches it. - storage::SharedMemory::Remove(old_handle.shm_key); - util::UnbufferedLog() << "ok."; - - util::UnbufferedLog() << "Waiting for clients to detach... "; - old_handle.memory->WaitForDetach(); - util::UnbufferedLog() << " ok."; - - shared_register.ReleaseKey(old_handle.shm_key); + if (storage::Remove(old_handle.proj_id)) + { + util::UnbufferedLog() << "waiting for clients to detach..."; + storage::WaitForDetach(old_handle.proj_id, 5000); + util::UnbufferedLog() << "ok."; + } + else + { + util::UnbufferedLog() << "failed."; + } + shared_register.ReleaseKey(old_handle.proj_id); } util::Log() << "All clients switched."; @@ -188,7 +188,7 @@ int Storage::Run(int max_wait, const std::string &dataset_name, bool only_metric util::LogPolicy::GetInstance().Unmute(); - std::filesystem::path lock_path = getLockDir() / "osrm-datastore.lock"; + std::filesystem::path lock_path = storage::getLockDir() / "osrm-datastore.lock"; if (!std::filesystem::exists(lock_path)) { std::ofstream ofs(lock_path); @@ -236,7 +236,7 @@ int Storage::Run(int max_wait, const std::string &dataset_name, bool only_metric throw util::exception("Cannot update the metric to a dataset that does not exist yet."); } auto static_region = shared_register.GetRegion(region_id); - auto static_memory = makeSharedMemory(static_region.shm_key); + auto static_memory = makeSharedMemory(static_region.proj_id); std::unique_ptr static_layout = std::make_unique(); @@ -247,7 +247,7 @@ int Storage::Run(int max_wait, const std::string &dataset_name, bool only_metric auto *data_ptr = reinterpret_cast(static_memory->Ptr()) + layout_size; regions.push_back({data_ptr, std::move(static_layout)}); - readonly_handles.push_back({std::move(static_memory), data_ptr, static_region.shm_key}); + readonly_handles.push_back({std::move(static_memory), data_ptr, static_region.proj_id}); } else { diff --git a/src/tools/store.cpp b/src/tools/store.cpp index 6cd555f04b9..47e9fbd912d 100644 --- a/src/tools/store.cpp +++ b/src/tools/store.cpp @@ -21,11 +21,12 @@ using namespace osrm; void removeLocks() { storage::SharedMonitor::remove(); } -void deleteRegion(const storage::SharedRegionRegister::ShmKey key) +void deleteRegion(const storage::ProjID proj_id) { - if (storage::SharedMemory::RegionExists(key) && !storage::SharedMemory::Remove(key)) + if (storage::RegionExists(proj_id) && !storage::Remove(proj_id)) { - util::Log(logWARNING) << "could not delete shared memory region " << static_cast(key); + util::Log(logWARNING) << "could not delete shared memory region " + << static_cast(proj_id); } } @@ -44,14 +45,14 @@ void listRegions(bool show_blocks) { auto id = shared_register.Find(name); auto region = shared_register.GetRegion(id); - auto shm = osrm::storage::makeSharedMemory(region.shm_key); - osrm::util::Log() << name << "\t" << static_cast(region.shm_key) << "\t" + auto shm = osrm::storage::makeSharedMemory(region.proj_id); + osrm::util::Log() << name << "\t" << static_cast(region.proj_id) << "\t" << region.timestamp << "\t" << shm->Size(); if (show_blocks) { using namespace storage; - auto memory = makeSharedMemory(region.shm_key); + auto memory = makeSharedMemory(region.proj_id); io::BufferReader reader(reinterpret_cast(memory->Ptr()), memory->Size()); std::unique_ptr layout = std::make_unique();