add: Single Writer Multiple Reader (SWMR) support (#68)

Author: radistmorse

.github/run_examples.sh

@@ -15,6 +15,11 @@ fi
 
 for f in "${examples_dir}"/*_bin
 do
+  if [[ "${f}" == *"swmr_"* ]]
+  then
+    continue
+  fi
+
   echo "-- ${f}"
   if [[ "${f}" == *"parallel_"* ]]
   then
@@ -23,3 +28,12 @@ do
     "${f}"
   fi
 done
+
+for f in "${examples_dir}"/swmr_*_bin
+do
+  [ -f "${f}" ] || continue
+  echo "-- ${f}"
+  "${f}" &
+done
+
+wait

include/highfive/H5File.hpp

@@ -40,6 +40,10 @@ class File: public Object, public NodeTraits<File>, public AnnotateTraits<File>
         Debug = 0x10u,
         /// Open flag: Create non existing file
         Create = 0x20u,
+        /// Open flag: Open in SWMR read
+        ReadSWMR = 0x40u,
+        /// Open flag: Open in SWMR write
+        WriteSWMR = 0x80u,
         /// Derived open flag: common write mode (=ReadWrite|Create|Truncate)
         Overwrite = Truncate,
         /// Derived open flag: Opens RW or exclusively creates
@@ -54,6 +58,8 @@ class File: public Object, public NodeTraits<File>, public AnnotateTraits<File>
     constexpr static AccessMode Create = AccessMode::Create;
     constexpr static AccessMode Overwrite = AccessMode::Overwrite;
     constexpr static AccessMode OpenOrCreate = AccessMode::OpenOrCreate;
+    constexpr static AccessMode ReadSWMR = AccessMode::ReadSWMR;
+    constexpr static AccessMode WriteSWMR = AccessMode::WriteSWMR;
 
     ///
     /// \brief File
@@ -122,6 +128,11 @@ class File: public Object, public NodeTraits<File>, public AnnotateTraits<File>
     ///
     void flush();
 
+#if H5_VERSION_GE(1, 10, 0)
+    /// \brief Switches file to SWMR write mode
+    void startSWMRWrite();
+#endif
+
     /// \brief Get the list of properties for creation of this file
     FileCreateProps getCreatePropertyList() const {
         return details::get_plist<FileCreateProps>(*this, H5Fget_create_plist);

include/highfive/bits/H5File_misc.hpp

@@ -34,6 +34,15 @@ inline unsigned convert_open_flag(File::AccessMode openFlags) {
         res_open |= H5F_ACC_TRUNC;
     if (any(openFlags & File::Excl))
         res_open |= H5F_ACC_EXCL;
+#if H5_VERSION_GE(1, 10, 0)
+    if (any(openFlags & File::ReadSWMR))
+        res_open |= H5F_ACC_SWMR_READ | H5F_ACC_RDONLY;
+    if (any(openFlags & File::WriteSWMR))
+        res_open |= H5F_ACC_SWMR_WRITE | H5F_ACC_RDWR;
+#else
+    if (any(openFlags & (File::ReadSWMR | File::WriteSWMR)))
+        throw FileException("Your HDF5 library is too old for SWMR mode, you need at least 1.10");
+#endif
     return res_open;
 }
 }  // namespace
@@ -52,6 +61,9 @@ inline File::File(const std::string& filename,
 
     unsigned createMode = openFlags & (H5F_ACC_TRUNC | H5F_ACC_EXCL);
     unsigned openMode = openFlags & (H5F_ACC_RDWR | H5F_ACC_RDONLY);
+#if H5_VERSION_GE(1, 10, 0)
+    openMode |= openFlags & (H5F_ACC_SWMR_READ | H5F_ACC_SWMR_WRITE);
+#endif
     bool mustCreate = createMode > 0;
     bool openOrCreate = (openFlags & H5F_ACC_CREAT) > 0;
 
@@ -127,6 +139,12 @@ inline void File::flush() {
     detail::h5f_flush(_hid, H5F_SCOPE_GLOBAL);
 }
 
+#if H5_VERSION_GE(1, 10, 0)
+inline void File::startSWMRWrite() {
+    detail::h5f_start_swmr_write(_hid);
+}
+#endif
+
 inline size_t File::getFileSize() const {
     hsize_t sizeValue = 0;
     detail::h5f_get_filesize(_hid, &sizeValue);

include/highfive/bits/h5f_wrapper.hpp

@@ -54,5 +54,16 @@ inline hssize_t h5f_get_freespace(hid_t file_id) {
     return free_space;
 }
 
+#if H5_VERSION_GE(1, 10, 0)
+inline herr_t h5f_start_swmr_write(hid_t file_id) {
+    herr_t err = H5Fstart_swmr_write(file_id);
+    if (err < 0) {
+        HDF5ErrMapper::ToException<FileException>(std::string("Failed to start SWMR write"));
+    }
+
+    return err;
+}
+#endif
+
 }  // namespace detail
 }  // namespace HighFive

src/examples/CMakeLists.txt

@@ -54,6 +54,11 @@ set(half_float_examples
   ${CMAKE_CURRENT_SOURCE_DIR}/create_dataset_half_float.cpp
 )
 
+set(swmr_examples
+  ${CMAKE_CURRENT_SOURCE_DIR}/swmr_read.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/swmr_write.cpp
+)
+
 function(compile_example example_source)
   get_filename_component(example_filename ${example_source} NAME)
   string(REPLACE ".cpp" "_bin" example_name ${example_filename})
@@ -98,6 +103,12 @@ if(HDF5_IS_PARALLEL)
   endforeach()
 endif()
 
+if(HDF5_VERSION VERSION_GREATER_EQUAL 1.10.2)
+  foreach(example_source ${swmr_examples})
+    compile_example(${example_source})
+  endforeach()
+endif()
+
 add_library(HighFiveHlHdf5Dependency INTERFACE)
 find_package(HDF5 QUIET COMPONENTS HL NAMES HDF5_HL)
 if(${HDF5_HL_FOUND})

src/examples/swmr_read.cpp

@@ -0,0 +1,76 @@
+/*
+ *  Copyright (c), 2025, George Sedov
+ *
+ *  Distributed under the Boost Software License, Version 1.0.
+ *    (See accompanying file LICENSE_1_0.txt or copy at
+ *          http://www.boost.org/LICENSE_1_0.txt)
+ *
+ */
+#include <iostream>
+#include <string>
+#include <thread>
+#include <vector>
+
+#include <highfive/highfive.hpp>
+
+const std::string file_name("swmr_read_write.h5");
+const std::string dataset_name("array");
+
+/**
+ * This is the SWMR reader.
+ * It should be used in conjunction with SWMR writer (see swmr_write example)
+ */
+int main(void) {
+    using namespace HighFive;
+
+    // give time for the writer to create the file
+    std::this_thread::sleep_for(std::chrono::milliseconds(10));
+
+    // Open file for SWMR read
+    File file(file_name, File::ReadSWMR);
+
+    std::cout << "Started the SWMR read" << std::endl;
+
+    auto dataset = file.getDataSet(dataset_name);
+    auto dims = dataset.getDimensions();
+    auto old_dims = std::vector<size_t>{0ul};
+    size_t max_dim = 10;
+
+    size_t fail_count = 0;
+    while (true) {
+        // refresh is needed for SWMR read
+        dataset.refresh();
+
+        dims = dataset.getDimensions();
+        // if dimensions changed, it means new data was written to a file
+        if (dims[0] != old_dims[0]) {
+            std::vector<size_t> slice{dims[0] - old_dims[0]};
+            auto values = dataset.select(old_dims, slice).read<std::vector<int>>();
+            for (const auto& v: values) {
+                std::cout << v << " ";
+            }
+            std::cout << std::flush;
+            old_dims = dims;
+            fail_count = 0;
+        } else {
+            fail_count++;
+        }
+
+        // there is no way to know that the writer has stopped
+        // we know that our example writer writes exactly 10 values
+        if (dims[0] >= max_dim) {
+            break;
+        }
+
+        // our example writer should add a value every 100 ms
+        // longer delay means something went wrong
+        if (fail_count >= 10) {
+            throw std::runtime_error("SWMR reader timed out.");
+        }
+
+        std::this_thread::sleep_for(std::chrono::milliseconds(50));
+    }
+
+    std::cout << std::endl;
+    return 0;
+}

src/examples/swmr_write.cpp

@@ -0,0 +1,65 @@
+/*
+ *  Copyright (c), 2025, George Sedov
+ *
+ *  Distributed under the Boost Software License, Version 1.0.
+ *    (See accompanying file LICENSE_1_0.txt or copy at
+ *          http://www.boost.org/LICENSE_1_0.txt)
+ *
+ */
+#include <iostream>
+#include <string>
+#include <thread>
+#include <vector>
+
+#include <highfive/highfive.hpp>
+
+const std::string file_name("swmr_read_write.h5");
+const std::string dataset_name("array");
+
+/**
+ * This is the SWMR writer.
+ * It should be used in conjunction with SWMR reader (see swmr_read example)
+ */
+int main(void) {
+    using namespace HighFive;
+
+    // Create a new file
+    // For SWMR we need to force the latest header, which is passed in AccessProps
+    FileAccessProps fapl;
+    fapl.add(FileVersionBounds(H5F_LIBVER_LATEST, H5F_LIBVER_LATEST));
+    File file(file_name, File::Truncate, fapl);
+
+    // To make sense for SWMR, the dataset should be extendable, and hence - chunkable
+    DataSetCreateProps props;
+    props.add(Chunking({1}));
+    auto dataset =
+        file.createDataSet<int>(dataset_name, DataSpace({0ul}, {DataSpace::UNLIMITED}), props);
+
+    // Start the SWMR write
+    // you are not allowed to create new data headers (i.e. DataSets, Groups, and Attributes) after
+    // that, you should also make sure all the Groups and Attributes are closed (i.e. the objects
+    // representing them are out of scope or destroyed) before calling this function
+    // see HDF5 SWMR tutorial for details
+    file.startSWMRWrite();
+
+    // If you want to open an already-existing file for SWMR write, use
+    // File file(file_name, File::WriteSWMR);
+    // auto dataset = file.getDataSet(dataset_name);
+
+    std::cout << "Started the SWMR write" << std::endl;
+
+    // Let's write to file.
+    for (int i = 0; i < 10; i++) {
+        // resize the dataset to fit the new element
+        dataset.resize({static_cast<size_t>(i + 1)});
+        // select the dataset slice and write the number to it
+        dataset.select({static_cast<size_t>(i)}, {1ul}).write(i);
+        // in SWMR mode you need to explicitly flush the DataSet
+        dataset.flush();
+
+        // give time for the reader to react
+        std::this_thread::sleep_for(std::chrono::milliseconds(100));
+    }
+
+    return 0;
+}