Introduce more flexible storeChunk() syntax, use to add ADIOS2 memory selection

CMakeLists.txt

@@ -400,6 +400,7 @@ set(CORE_SOURCE
         src/Format.cpp
         src/Iteration.cpp
         src/IterationEncoding.cpp
+        src/LoadStoreChunk.cpp
         src/Mesh.cpp
         src/ParticlePatches.cpp
         src/ParticleSpecies.cpp
@@ -411,6 +412,7 @@ set(CORE_SOURCE
         src/version.cpp
         src/auxiliary/Date.cpp
         src/auxiliary/Filesystem.cpp
+        src/auxiliary/Future.cpp
         src/auxiliary/JSON.cpp
         src/auxiliary/JSONMatcher.cpp
         src/auxiliary/Memory.cpp

include/openPMD/Dataset.hpp

@@ -34,6 +34,12 @@ namespace openPMD
 using Extent = std::vector<std::uint64_t>;
 using Offset = std::vector<std::uint64_t>;
 
+struct MemorySelection
+{
+    Offset offset;
+    Extent extent;
+};
+
 class Dataset
 {
     friend class RecordComponent;

include/openPMD/Datatype.hpp

@@ -294,7 +294,8 @@ template <typename T>
 inline constexpr Datatype determineDatatype(T &&val)
 {
     (void)val; // don't need this, it only has a name for Doxygen
-    using T_stripped = std::remove_cv_t<std::remove_reference_t<T>>;
+    using T_stripped =
+        std::remove_extent_t<std::remove_cv_t<std::remove_reference_t<T>>>;
     if constexpr (auxiliary::IsPointer_v<T_stripped>)
     {
         return determineDatatype<auxiliary::IsPointer_t<T_stripped>>();

include/openPMD/IO/ADIOS/ADIOS2File.hpp

@@ -20,6 +20,7 @@
  */
 #pragma once
 
+#include "openPMD/Dataset.hpp"
 #include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp"
 #include "openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp"
 #include "openPMD/IO/ADIOS/ADIOS2PreloadVariables.hpp"
@@ -112,6 +113,7 @@ struct BufferedUniquePtrPut
     std::string name;
     Offset offset;
     Extent extent;
+    std::optional<MemorySelection> memorySelection;
     UniquePtrWithLambda<void> data;
     Datatype dtype = Datatype::UNDEFINED;
 

include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp

@@ -21,6 +21,7 @@
  */
 #pragma once
 
+#include "openPMD/Dataset.hpp"
 #include "openPMD/Error.hpp"
 #include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp"
 #include "openPMD/IO/ADIOS/ADIOS2FilePosition.hpp"
@@ -509,6 +510,7 @@ class ADIOS2IOHandlerImpl
     adios2::Variable<T> verifyDataset(
         Offset const &offset,
         Extent const &extent,
+        std::optional<MemorySelection> const &memorySelection,
         adios2::IO &IO,
         adios2::Engine &engine,
         std::string const &varName,
@@ -622,13 +624,26 @@ class ADIOS2IOHandlerImpl
         var.SetSelection(
             {adios2::Dims(offset.begin(), offset.end()),
              adios2::Dims(extent.begin(), extent.end())});
+
+        if (memorySelection.has_value())
+        {
+            var.SetMemorySelection(
+                {adios2::Dims(
+                     memorySelection->offset.begin(),
+                     memorySelection->offset.end()),
+                 adios2::Dims(
+                     memorySelection->extent.begin(),
+                     memorySelection->extent.end())});
+        }
+
         return var;
     }
 
     struct
     {
         bool noGroupBased = false;
         bool blosc2bp5 = false;
+        bool memorySelection = false;
     } printedWarningsAlready;
 }; // ADIOS2IOHandlerImpl
 
@@ -942,7 +957,7 @@ class ADIOS2IOHandler : public AbstractIOHandler
         try
         {
             auto params = internal::defaultParsedFlushParams;
-            this->flush(params);
+            this->flush_impl(params);
         }
         catch (std::exception const &ex)
         {
@@ -990,6 +1005,6 @@ class ADIOS2IOHandler : public AbstractIOHandler
         return true;
     }
 
-    std::future<void> flush(internal::ParsedFlushParams &) override;
+    std::future<void> flush_impl(internal::ParsedFlushParams &) override;
 }; // ADIOS2IOHandler
 } // namespace openPMD

include/openPMD/IO/ADIOS/macros.hpp

@@ -41,6 +41,29 @@
 #define openPMD_HAVE_ADIOS2_BP5 0
 #endif
 
+namespace openPMD
+{
+namespace detail
+{
+    template <typename Variable, typename SFINAE = void>
+    struct CanTheMemorySelectionBeReset
+    {
+        static constexpr bool value = false;
+    };
+
+    template <typename Variable>
+    struct CanTheMemorySelectionBeReset<
+        Variable,
+        decltype(std::declval<Variable>().SetMemorySelection())>
+    {
+        static constexpr bool value = true;
+    };
+} // namespace detail
+
+constexpr bool CanTheMemorySelectionBeReset =
+    detail::CanTheMemorySelectionBeReset<adios2::Variable<int>>::value;
+} // namespace openPMD
+
 #else
 
 #define openPMD_HAS_ADIOS_2_8 0

include/openPMD/IO/AbstractIOHandler.hpp

@@ -265,13 +265,15 @@ class AbstractIOHandler
      * backends that decide to implement this operation asynchronously.
      */
     std::future<void> flush(internal::FlushParams const &);
+    std::shared_ptr<unsigned long long> m_flushCounter =
+        std::make_shared<unsigned long long>(0);
 
     /** Process operations in queue according to FIFO.
      *
      * @return  Future indicating the completion state of the operation for
      * backends that decide to implement this operation asynchronously.
      */
-    virtual std::future<void> flush(internal::ParsedFlushParams &) = 0;
+    std::future<void> flush(internal::ParsedFlushParams &);
 
     /** The currently used backend */
     virtual std::string backendName() const = 0;
@@ -315,6 +317,9 @@ class AbstractIOHandler
     IterationEncoding m_encoding = IterationEncoding::groupBased;
     OpenpmdStandard m_standard = auxiliary::parseStandard(getStandardDefault());
     bool m_verify_homogeneous_extents = true;
+
+protected:
+    virtual std::future<void> flush_impl(internal::ParsedFlushParams &) = 0;
 }; // AbstractIOHandler
 
 } // namespace openPMD

include/openPMD/IO/DummyIOHandler.hpp

@@ -44,7 +44,7 @@ class DummyIOHandler : public AbstractIOHandler
     /** No-op consistent with the IOHandler interface to enable library use
      * without IO.
      */
-    std::future<void> flush(internal::ParsedFlushParams &) override;
+    std::future<void> flush_impl(internal::ParsedFlushParams &) override;
     std::string backendName() const override;
 }; // DummyIOHandler
 } // namespace openPMD

include/openPMD/IO/HDF5/HDF5IOHandler.hpp

@@ -46,7 +46,7 @@ class HDF5IOHandler : public AbstractIOHandler
         return "HDF5";
     }
 
-    std::future<void> flush(internal::ParsedFlushParams &) override;
+    std::future<void> flush_impl(internal::ParsedFlushParams &) override;
 
 private:
     std::unique_ptr<HDF5IOHandlerImpl> m_impl;

include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp

@@ -56,7 +56,7 @@ class ParallelHDF5IOHandler : public AbstractIOHandler
         return "MPI_HDF5";
     }
 
-    std::future<void> flush(internal::ParsedFlushParams &) override;
+    std::future<void> flush_impl(internal::ParsedFlushParams &) override;
 
 private:
     std::unique_ptr<ParallelHDF5IOHandlerImpl> m_impl;

include/openPMD/IO/IOTask.hpp

@@ -495,6 +495,7 @@ struct OPENPMDAPI_EXPORT
 
     Extent extent = {};
     Offset offset = {};
+    std::optional<MemorySelection> memorySelection = std::nullopt;
     Datatype dtype = Datatype::UNDEFINED;
     auxiliary::WriteBuffer data;
 };

include/openPMD/IO/JSON/JSONIOHandler.hpp

@@ -59,7 +59,7 @@ class JSONIOHandler : public AbstractIOHandler
         return "JSON";
     }
 
-    std::future<void> flush(internal::ParsedFlushParams &) override;
+    std::future<void> flush_impl(internal::ParsedFlushParams &) override;
 
 private:
     JSONIOHandlerImpl m_impl;