diff --git a/CMakeLists.txt b/CMakeLists.txt index 03637b1938..b3a0cc30c2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -787,6 +787,15 @@ if(openPMD_BUILD_TESTING) test/Files_SerialIO/close_and_reopen_test.cpp test/Files_SerialIO/filebased_write_test.cpp ) + elseif(${test_name} STREQUAL "ParallelIO" AND openPMD_HAVE_MPI) + list(APPEND ${out_list} + test/Files_ParallelIO/read_variablebased_randomaccess.cpp + test/Files_ParallelIO/iterate_nonstreaming_series.cpp + ) + elseif(${test_name} STREQUAL "Core") + list(APPEND ${out_list} + test/Files_Core/automatic_variable_encoding.cpp + ) endif() endmacro() diff --git a/docs/source/usage/concepts.rst b/docs/source/usage/concepts.rst index 91253f8945..e9957e5f4b 100644 --- a/docs/source/usage/concepts.rst +++ b/docs/source/usage/concepts.rst @@ -48,7 +48,7 @@ openPMD-api implements various file-formats (backends) and encoding strategies f **Iteration encoding:** The openPMD-api can encode iterations in different ways. The method ``Series::setIterationEncoding()`` (C++) or ``Series.set_iteration_encoding()`` (Python) may be used in writing for selecting one of the following encodings explicitly: -* **group-based iteration encoding:** This encoding is the default. +* **group-based iteration encoding:** This encoding is the default for HDF5 and JSON. In ADIOS2, variable-based encoding is preferred when possible due to better performance characteristics, see below. It creates a separate group in the hierarchy of the openPMD standard for each iteration. As an example, all data pertaining to iteration 0 may be found in group ``/data/0``, for iteration 100 in ``/data/100``. * **file-based iteration encoding:** A unique file on the filesystem is created for each iteration. @@ -57,6 +57,7 @@ The method ``Series::setIterationEncoding()`` (C++) or ``Series.set_iteration_en A padding may be specified by ``"series_%06T.json"`` to create files ``series_000000.json``, ``series_000100.json`` and ``series_000200.json``. The inner group layout of each file is identical to that of the group-based encoding. * **variable-based iteration encoding:** This experimental encoding uses a feature of some backends (i.e., ADIOS2) to maintain datasets and attributes in several versions (i.e., iterations are stored inside *variables*). + When creating an ADIOS2 Series with steps (e.g. via ``series.writeIterations()`` / ``series.write_iterations()``), this encoding will be picked as a default instead of group-based encoding due to bad performance characteristics of group-based encoding in ADIOS2. No iteration-specific groups are created and the corresponding layer is dropped from the openPMD hierarchy. In backends that do not support this feature, a series created with this encoding can only contain one iteration. diff --git a/docs/source/usage/workflow.rst b/docs/source/usage/workflow.rst index 03fa4ce9d4..a69a498cb4 100644 --- a/docs/source/usage/workflow.rst +++ b/docs/source/usage/workflow.rst @@ -76,10 +76,14 @@ The openPMD-api distinguishes between a number of different access modes: 3. In streaming backends, random-access is not possible. When using such a backend, the access mode will be coerced automatically to *linear read mode*. Use of Series::readIterations() is mandatory for access. - 4. Reading a variable-based Series is only fully supported with *linear access mode*. - If using *random-access read mode*, the dataset will be considered to only have one single step. - If the dataset only has one single step, this is guaranteed to work as expected. - Otherwise, it is undefined which step's data is returned. + 4. *Random-access read mode* for a variable-based Series is currently experimental. + There is currently only very restricted support for metadata definitions that change across steps: + + 1. Modifiable attributes (except ``/data/snapshot``) can currently not be read. Attributes such as ``/data/time`` that naturally change their value across Iterations will hence not be really well-usable; the last Iteration's value will currently leak into all other Iterations. + 2. There is no support for datasets that do not exist in all Iterations. The internal Iteration layouts should be homogeneous. + If you need this feature, please contact the openPMD-api developers; implementing this is currently not a priority. + Datasets that do not exist in all steps will be skipped at read time (with an error). + 3. Datasets with changing extents are supported. * **Read/Write mode**: Creates a new Series if not existing, otherwise opens an existing Series for reading and writing. New datasets and iterations will be inserted as needed. diff --git a/examples/5_write_parallel.py b/examples/5_write_parallel.py index 8574c1d66e..64ae652348 100644 --- a/examples/5_write_parallel.py +++ b/examples/5_write_parallel.py @@ -56,8 +56,7 @@ # in streaming setups, e.g. an iteration cannot be opened again once # it has been closed. # `Series.iterations` can be directly accessed in random-access workflows. - series.iterations[1].open() - mymesh = series.iterations[1]. \ + mymesh = series.write_iterations()[1]. \ meshes["mymesh"] # example 1D domain decomposition in first index @@ -92,7 +91,7 @@ # The iteration can be closed in order to help free up resources. # The iteration's content will be flushed automatically. # An iteration once closed cannot (yet) be reopened. - series.iterations[1].close() + series.write_iterations()[1].close() if 0 == comm.rank: print("Dataset content has been fully written to disk") diff --git a/include/openPMD/Datatype.hpp b/include/openPMD/Datatype.hpp index c18b7cd82b..24f77218a0 100644 --- a/include/openPMD/Datatype.hpp +++ b/include/openPMD/Datatype.hpp @@ -513,7 +513,7 @@ inline bool isFloatingPoint(Datatype d) * @param d Datatype to test * @return true if complex floating point, otherwise false */ -inline bool isComplexFloatingPoint(Datatype d) +constexpr inline bool isComplexFloatingPoint(Datatype d) { using DT = Datatype; @@ -554,7 +554,7 @@ inline bool isFloatingPoint() * @return true if complex floating point, otherwise false */ template -inline bool isComplexFloatingPoint() +constexpr inline bool isComplexFloatingPoint() { Datatype dtype = determineDatatype(); diff --git a/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp b/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp index 2741a91fb0..5f9839769d 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp @@ -97,6 +97,8 @@ namespace adios_defaults constexpr const_str str_usesteps = "usesteps"; constexpr const_str str_flushtarget = "preferred_flush_target"; constexpr const_str str_usesstepsAttribute = "__openPMD_internal/useSteps"; + constexpr const_str str_useModifiableAttributes = + "__openPMD_internal/useModifiableAttributes"; constexpr const_str str_adios2Schema = "__openPMD_internal/openPMD2_adios2_schema"; constexpr const_str str_isBoolean = "__is_boolean__"; diff --git a/include/openPMD/IO/ADIOS/ADIOS2File.hpp b/include/openPMD/IO/ADIOS/ADIOS2File.hpp index 5d519a85ad..147e79d2ee 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2File.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2File.hpp @@ -79,7 +79,8 @@ struct DatasetReader BufferedGet &bp, adios2::IO &IO, adios2::Engine &engine, - std::string const &fileName); + std::string const &fileName, + std::optional stepSelection); static constexpr char const *errorMsg = "ADIOS2: readDataset()"; }; @@ -412,6 +413,8 @@ class ADIOS2File StreamStatus streamStatus = StreamStatus::OutsideOfStep; size_t currentStep(); + void setStepSelection(std::optional); + [[nodiscard]] std::optional stepSelection() const; private: ADIOS2IOHandlerImpl *m_impl; @@ -420,8 +423,11 @@ class ADIOS2File /* * Not all engines support the CurrentStep() call, so we have to * implement this manually. + * Note: We don't use a std::optional here since the currentStep + * is always being counted. */ size_t m_currentStep = 0; + bool useStepSelection = false; /* * ADIOS2 does not give direct access to its internal attribute and diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index fa300da472..355d1aa87f 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -23,6 +23,7 @@ #include "openPMD/Error.hpp" #include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" #include "openPMD/IO/ADIOS/ADIOS2FilePosition.hpp" +#include "openPMD/IO/ADIOS/macros.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/AbstractIOHandlerImpl.hpp" #include "openPMD/IO/AbstractIOHandlerImplCommon.hpp" @@ -190,6 +191,9 @@ class ADIOS2IOHandlerImpl void readAttribute(Writable *, Parameter &) override; + void readAttributeAllsteps( + Writable *, Parameter &) override; + void listPaths(Writable *, Parameter &) override; void @@ -431,7 +435,8 @@ class ADIOS2IOHandlerImpl Offset const &offset, Extent const &extent, adios2::IO &IO, - std::string const &varName) + std::string const &varName, + std::optional stepSelection) { { auto requiredType = adios2::GetType(); @@ -458,6 +463,10 @@ class ADIOS2IOHandlerImpl throw std::runtime_error( "[ADIOS2] Internal error: Failed opening ADIOS2 variable."); } + if (stepSelection.has_value()) + { + var.SetStepSelection({*stepSelection, 1}); + } // TODO leave this check to ADIOS? adios2::Dims shape = var.Shape(); auto actualDim = shape.size(); @@ -533,11 +542,8 @@ namespace detail struct AttributeReader { template - static Datatype call( - ADIOS2IOHandlerImpl &, - adios2::IO &IO, - std::string name, - Attribute::resource &resource); + static Datatype + call(adios2::IO &IO, std::string name, Attribute::resource &resource); template static Datatype call(Params &&...); @@ -562,7 +568,8 @@ namespace detail ADIOS2IOHandlerImpl *impl, InvalidatableFile const &, std::string const &varName, - Parameter ¶meters); + Parameter ¶meters, + std::optional stepSelection); static constexpr char const *errorMsg = "ADIOS2: openDataset()"; }; @@ -854,6 +861,11 @@ class ADIOS2IOHandler : public AbstractIOHandler return "ADIOS2"; } + bool fullSupportForVariableBasedEncoding() const override + { + return true; + } + std::future flush(internal::ParsedFlushParams &) override; }; // ADIOS2IOHandler } // namespace openPMD diff --git a/include/openPMD/IO/AbstractIOHandler.hpp b/include/openPMD/IO/AbstractIOHandler.hpp index e8e55457eb..291bc405a2 100644 --- a/include/openPMD/IO/AbstractIOHandler.hpp +++ b/include/openPMD/IO/AbstractIOHandler.hpp @@ -250,6 +250,7 @@ class AbstractIOHandler /** The currently used backend */ virtual std::string backendName() const = 0; + virtual bool fullSupportForVariableBasedEncoding() const; std::string directory; /* diff --git a/include/openPMD/IO/AbstractIOHandlerImpl.hpp b/include/openPMD/IO/AbstractIOHandlerImpl.hpp index 10b0fd0c97..a7a5746b19 100644 --- a/include/openPMD/IO/AbstractIOHandlerImpl.hpp +++ b/include/openPMD/IO/AbstractIOHandlerImpl.hpp @@ -72,6 +72,13 @@ class AbstractIOHandlerImpl * IO actions up to the point of closing a step must be performed now. * * The advance mode is determined by parameters.mode. + * parameters.mode has type std::variant: + * + * 1. AdvanceMode is for processing steps sequentially. In this case, a step + * is either begun or closed. + * 2. StepSelection is for random-accessing steps. A target step number is + * specified. + * * The return status code shall be stored as parameters.status. */ virtual void advance(Writable *, Parameter ¶meters) @@ -360,6 +367,24 @@ class AbstractIOHandlerImpl */ virtual void readAttribute(Writable *, Parameter &) = 0; + /** Collective task to read modifiable attributes over steps. + * + * Has a default implementation for backends that do not support steps; + * here, the task is relayed to normal READ_ATT. + * This task is key for implementing the preparsing logic needed in + * random-access read mode for variable-encoded ADIOS2 files. + * adios2::Mode::ReadRandomAccess does not support modifiable attributes, + * so this task will instead quickly open the file's metadata in + * adios2::Mode::Read, go through all its steps and register the attribute + * values. Expensive and collective operation, run only once at startup. + * Absolutely necessary for reading /data/snapshot. + * Necessary (but not yet used) for having correct values in attributes + * such as /data/time. + * In future: Let this task preparse the entirety of all modifiable + * attributes. + */ + virtual void readAttributeAllsteps( + Writable *, Parameter &); /** List all paths/sub-groups inside a group, non-recursively. * * The operation should fail if the Writable was not marked written. diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index 5589db6923..fafe9c669b 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -26,13 +26,14 @@ #include "openPMD/Streaming.hpp" #include "openPMD/auxiliary/Export.hpp" #include "openPMD/auxiliary/Memory.hpp" +#include "openPMD/auxiliary/TypeTraits.hpp" #include "openPMD/auxiliary/Variant.hpp" #include "openPMD/backend/Attribute.hpp" #include "openPMD/backend/ParsePreference.hpp" #include -#include #include +#include #include #include #include @@ -72,6 +73,7 @@ OPENPMDAPI_EXPORT_ENUM_CLASS(Operation){ DELETE_ATT, WRITE_ATT, READ_ATT, + READ_ATT_ALLSTEPS, LIST_ATTS, ADVANCE, @@ -602,6 +604,38 @@ struct OPENPMDAPI_EXPORT Parameter std::make_shared(); }; +template <> +struct OPENPMDAPI_EXPORT Parameter + : public AbstractParameter +{ + Parameter() = default; + Parameter(Parameter &&) = default; + Parameter(Parameter const &) = default; + Parameter &operator=(Parameter &&) = default; + Parameter &operator=(Parameter const &) = default; + + std::unique_ptr to_heap() && override + { + return std::unique_ptr( + new Parameter(std::move(*this))); + } + + std::string name = ""; + std::shared_ptr dtype = std::make_shared(); + + struct to_vector_type + { + template + using type = std::vector; + }; + // std::variant, std::vector, ...> + // for all T_i in openPMD::Datatype. + using result_type = typename auxiliary::detail:: + map_variant::type; + + std::shared_ptr resource = std::make_shared(); +}; + template <> struct OPENPMDAPI_EXPORT Parameter : public AbstractParameter @@ -638,10 +672,23 @@ struct OPENPMDAPI_EXPORT Parameter new Parameter(std::move(*this))); } - //! input parameter - AdvanceMode mode; + struct StepSelection + { + std::optional step; + }; + + // input parameters + /** + * AdvanceMode: Is one of BeginStep/EndStep. Used during writing and in + * linear read mode to step sequentially through steps. + * StepSelection: Used in random-access read mode, jump to the specified + * step. Can be nullopt in order to reset the backend to read + * step-agnostically, e.g. for reading global datasets such as + * /rankTable. + */ + std::variant mode; bool isThisStepMandatory = false; - //! output parameter + // output parameter std::shared_ptr status = std::make_shared(AdvanceStatus::OK); }; diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index 8c39c0a518..43ee1084bb 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -52,6 +52,32 @@ namespace internal propagated to the backend */ }; + namespace BeginStepTypes + { + struct DontBeginStep + {}; + struct BeginStepSequentially + {}; + struct BeginStepRandomAccess + { + size_t step; + }; + } // namespace BeginStepTypes + + using BeginStep = std::variant< + BeginStepTypes::DontBeginStep, + BeginStepTypes::BeginStepSequentially, + BeginStepTypes::BeginStepRandomAccess>; + + namespace BeginStepTypes + { + template + constexpr auto make(Args &&...args) -> BeginStep + { + return BeginStep{T{std::forward(args)...}}; + } + } // namespace BeginStepTypes + struct DeferredParseAccess { /** @@ -69,7 +95,7 @@ namespace internal * (Group- and variable-based parsing shares the same code logic.) */ bool fileBased = false; - bool beginStep = false; + BeginStep beginStep = BeginStepTypes::DontBeginStep{}; }; class IterationData : public AttributableData @@ -305,7 +331,8 @@ class Iteration : public Attributable std::string const &filePath, std::string const &groupPath, bool beginStep); - void readGorVBased(std::string const &groupPath, bool beginStep); + void readGorVBased( + std::string const &groupPath, internal::BeginStep const &beginStep); void read_impl(std::string const &groupPath); void readMeshes(std::string const &meshesPath); void readParticles(std::string const &particlesPath); diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index c387c803f7..0a568ed559 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -68,6 +68,13 @@ class Series; namespace internal { + /* Just a more self-documenting boolean used for + * m_iterationEncodingSetExplicitly */ + enum class default_or_explicit : bool + { + default_, + explicit_ + }; /** * @brief Data members for Series. Pinned at one memory location. * @@ -118,6 +125,10 @@ namespace internal * snapshot attribute. */ std::set m_currentlyActiveIterations; + /** + * For reading: In which IO step do I need to look for an Iteration? + */ + std::unordered_map m_snapshotToStep; /** * This map contains the filenames of those Iterations which were found * on the file system upon opening the Series for reading in file-based @@ -179,6 +190,17 @@ namespace internal * The iteration encoding used in this series. */ IterationEncoding m_iterationEncoding{}; + /* + * ADIOS2 should use variable-based encoding as default rather than + * group-based encoding as much as possible. + * Since this cannot be decided at construction time, groupBased + * encoding is selected first, and re-decided later. + * However, when group-based encoding is selected by the user explcitly, + * that selection should not be changed again. + * Hence, remember that here. + */ + default_or_explicit m_iterationEncodingSetExplicitly = + default_or_explicit::default_; /** * Detected IO format (backend). */ @@ -965,16 +987,38 @@ OPENPMD_private */ void flushStep(bool doFlush); + /* + * setIterationEncoding() should only be called by users of our public API, + * but never internally. We need to distinguish if the iteration encoding + * was selected explicitly or implicitly, see + * m_iterationEncodingSetExplicitly for further details. + */ + Series &setIterationEncoding_internal( + IterationEncoding iterationEncoding, internal::default_or_explicit); + /* * Returns the current content of the /data/snapshot attribute. * (We could also add this to the public API some time) */ - std::optional> currentSnapshot() const; + std::optional> currentSnapshot(); AbstractIOHandler *runDeferredInitialization(); AbstractIOHandler *IOHandler(); AbstractIOHandler const *IOHandler() const; + + /* adios2::Mode::ReadRandomAccess does not support reading modifiable + * attributes. However, we need the values of /data/snapshot as a modifiable + * attribute, so this function quickly opens the file in adios2::Mode::Read + * and retrieves the changings values over time. + * Return std::nullopt if /data/snapshot is not present. + */ + std::optional>> + preparseSnapshots(); + /* Should adios2::Variable::SetStepSelection() be used for accessing + * steps? + */ + [[nodiscard]] bool randomAccessSteps() const; }; // Series using SnapshotWorkflow = Series::SnapshotWorkflow; diff --git a/include/openPMD/auxiliary/Mpi.hpp b/include/openPMD/auxiliary/Mpi.hpp index f8eefe0cc5..658523f7d9 100644 --- a/include/openPMD/auxiliary/Mpi.hpp +++ b/include/openPMD/auxiliary/Mpi.hpp @@ -47,10 +47,38 @@ namespace { return MPI_CHAR; } + else if constexpr (std::is_same_v) + { + return MPI_SIGNED_CHAR; + } + else if constexpr (std::is_same_v) + { + return MPI_UNSIGNED_CHAR; + } + else if constexpr (std::is_same_v) + { + return MPI_SHORT; + } + else if constexpr (std::is_same_v) + { + return MPI_UNSIGNED_SHORT; + } else if constexpr (std::is_same_v) { return MPI_UNSIGNED; } + else if constexpr (std::is_same_v) + { + return MPI_INT; + } + else if constexpr (std::is_same_v) + { + return MPI_LONG; + } + else if constexpr (std::is_same_v) + { + return MPI_LONG_LONG; + } else if constexpr (std::is_same_v) { return MPI_UNSIGNED_LONG; @@ -59,6 +87,18 @@ namespace { return MPI_UNSIGNED_LONG_LONG; } + else if constexpr (std::is_same_v) + { + return MPI_FLOAT; + } + else if constexpr (std::is_same_v) + { + return MPI_DOUBLE; + } + else if constexpr (std::is_same_v) + { + return MPI_LONG_DOUBLE; + } else { static_assert( diff --git a/include/openPMD/backend/Attribute.hpp b/include/openPMD/backend/Attribute.hpp index a183b7818a..e2ba7e1dd5 100644 --- a/include/openPMD/backend/Attribute.hpp +++ b/include/openPMD/backend/Attribute.hpp @@ -310,6 +310,25 @@ namespace detail #else } #endif + + template + auto doConvertOptional(T const *pv) -> std::optional + { + auto eitherValueOrError = doConvert(pv); + return std::visit( + [](auto &containedValue) -> std::optional { + using Res = std::decay_t; + if constexpr (std::is_same_v) + { + return std::nullopt; + } + else + { + return {std::move(containedValue)}; + } + }, + eitherValueOrError); + } } // namespace detail template @@ -339,25 +358,12 @@ U Attribute::get() const template std::optional Attribute::getOptional() const { - auto eitherValueOrError = std::visit( - [](auto &&containedValue) -> std::variant { - using containedType = std::decay_t; - return detail::doConvert(&containedValue); - }, - Variant::getResource()); return std::visit( [](auto &&containedValue) -> std::optional { - using T = std::decay_t; - if constexpr (std::is_same_v) - { - return std::nullopt; - } - else - { - return {std::move(containedValue)}; - } + using containedType = std::decay_t; + return detail::doConvertOptional(&containedValue); }, - std::move(eitherValueOrError)); + Variant::getResource()); } } // namespace openPMD diff --git a/include/openPMD/benchmark/mpi/MPIBenchmark.hpp b/include/openPMD/benchmark/mpi/MPIBenchmark.hpp index 3d6d78c7e4..b88e80adc3 100644 --- a/include/openPMD/benchmark/mpi/MPIBenchmark.hpp +++ b/include/openPMD/benchmark/mpi/MPIBenchmark.hpp @@ -325,12 +325,17 @@ MPIBenchmark::BenchmarkExecution::writeBenchmark( m_benchmark->communicator, jsonConfig); + typename Clock::duration ignore{}; + for (Series::IterationIndex_t i = 0; i < iterations; i++) { + auto ignore_start = Clock::now(); auto writeData = datasetFiller->produceData(); + auto ignore_end = Clock::now(); + ignore += ignore_end - ignore_start; - MeshRecordComponent id = - series.iterations[i].meshes["id"][MeshRecordComponent::SCALAR]; + MeshRecordComponent id = series.writeIterations()[i] + .meshes["id"][MeshRecordComponent::SCALAR]; Datatype datatype = determineDatatype(writeData); Dataset dataset = Dataset(datatype, m_benchmark->totalExtent); @@ -342,18 +347,11 @@ MPIBenchmark::BenchmarkExecution::writeBenchmark( id.storeChunk(writeData, offset, extent); series.flush(); } - + series.close(); MPI_Barrier(m_benchmark->communicator); auto end = Clock::now(); - // deduct the time needed for data generation - for (Series::IterationIndex_t i = 0; i < iterations; i++) - { - datasetFiller->produceData(); - } - auto deduct = Clock::now(); - - return end - start - (deduct - end); + return (end - start) - ignore; } template @@ -378,7 +376,7 @@ MPIBenchmark::BenchmarkExecution::readBenchmark( for (Series::IterationIndex_t i = 0; i < iterations; i++) { MeshRecordComponent id = - series.iterations[i].meshes["id"][MeshRecordComponent::SCALAR]; + series.snapshots()[i].meshes["id"][MeshRecordComponent::SCALAR]; auto chunk_data = id.loadChunk(offset, extent); series.flush(); diff --git a/src/IO/ADIOS/ADIOS2File.cpp b/src/IO/ADIOS/ADIOS2File.cpp index c6f4747aff..f313869d1f 100644 --- a/src/IO/ADIOS/ADIOS2File.cpp +++ b/src/IO/ADIOS/ADIOS2File.cpp @@ -21,12 +21,14 @@ #include "openPMD/IO/ADIOS/ADIOS2File.hpp" #include "openPMD/Error.hpp" +#include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" #include "openPMD/IO/ADIOS/ADIOS2IOHandler.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IterationEncoding.hpp" #include "openPMD/auxiliary/Environment.hpp" #include "openPMD/auxiliary/StringManip.hpp" +#include #include #if openPMD_USE_VERIFY @@ -58,10 +60,11 @@ void DatasetReader::call( detail::BufferedGet &bp, adios2::IO &IO, adios2::Engine &engine, - std::string const &fileName) + std::string const &fileName, + std::optional stepSelection) { - adios2::Variable var = - impl->verifyDataset(bp.param.offset, bp.param.extent, IO, bp.name); + adios2::Variable var = impl->verifyDataset( + bp.param.offset, bp.param.extent, IO, bp.name, stepSelection); if (!var) { throw std::runtime_error( @@ -90,7 +93,11 @@ void WriteDataset::call(ADIOS2File &ba, detail::BufferedPut &bp) auto ptr = static_cast(arg.get()); adios2::Variable var = ba.m_impl->verifyDataset( - bp.param.offset, bp.param.extent, ba.m_IO, bp.name); + bp.param.offset, + bp.param.extent, + ba.m_IO, + bp.name, + std::nullopt); ba.getEngine().Put(var, ptr); } @@ -133,7 +140,13 @@ void WriteDataset::call(Params &&...) void BufferedGet::run(ADIOS2File &ba) { switchAdios2VariableType( - param.dtype, ba.m_impl, *this, ba.m_IO, ba.getEngine(), ba.m_file); + param.dtype, + ba.m_impl, + *this, + ba.m_IO, + ba.getEngine(), + ba.m_file, + ba.stepSelection()); } void BufferedPut::run(ADIOS2File &ba) @@ -148,7 +161,11 @@ struct RunUniquePtrPut { auto ptr = static_cast(bufferedPut.data.get()); adios2::Variable var = ba.m_impl->verifyDataset( - bufferedPut.offset, bufferedPut.extent, ba.m_IO, bufferedPut.name); + bufferedPut.offset, + bufferedPut.extent, + ba.m_IO, + bufferedPut.name, + std::nullopt); ba.getEngine().Put(var, ptr); } @@ -337,6 +354,38 @@ size_t ADIOS2File::currentStep() } } +void ADIOS2File::setStepSelection(std::optional step) +{ + if (streamStatus != StreamStatus::ReadWithoutStream) + { + throw error::Internal( + "ADIOS2 backend: Cannot only use random-access step selections " + "when reading without streaming mode."); + } + if (!step.has_value()) + { + m_currentStep = 0; + useStepSelection = false; + } + else + { + m_currentStep = *step; + useStepSelection = true; + } +} + +std::optional ADIOS2File::stepSelection() const +{ + if (useStepSelection) + { + return {m_currentStep}; + } + else + { + return std::nullopt; + } +} + void ADIOS2File::configure_IO_Read() { bool upfrontParsing = supportsUpfrontParsing( @@ -432,14 +481,10 @@ void ADIOS2File::configure_IO() { switch (m_impl->m_handler->m_encoding) { - /* - * For variable-based encoding, this does not matter as it is - * new and requires >= v2.9 features anyway. - */ case IterationEncoding::variableBased: + case IterationEncoding::groupBased: m_impl->m_useGroupTable = UseGroupTable::Yes; break; - case IterationEncoding::groupBased: case IterationEncoding::fileBased: m_impl->m_useGroupTable = UseGroupTable::No; break; @@ -454,6 +499,12 @@ void ADIOS2File::configure_IO() ? ADIOS2IOHandlerImpl::ModifiableAttributes::Yes : ADIOS2IOHandlerImpl::ModifiableAttributes::No; } + m_IO.DefineAttribute( + adios_defaults::str_useModifiableAttributes, + m_impl->m_modifiableAttributes == + ADIOS2IOHandlerImpl::ModifiableAttributes::No + ? 0 + : 1); } // set engine type diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 5bd3803446..67a18f7c11 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -27,20 +27,25 @@ #include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" #include "openPMD/IO/ADIOS/ADIOS2FilePosition.hpp" #include "openPMD/IO/ADIOS/ADIOS2IOHandler.hpp" +#include "openPMD/IO/IOTask.hpp" #include "openPMD/IterationEncoding.hpp" +#include "openPMD/Streaming.hpp" +#include "openPMD/ThrowError.hpp" #include "openPMD/auxiliary/Environment.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/JSON_internal.hpp" #include "openPMD/auxiliary/Mpi.hpp" #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/auxiliary/TypeTraits.hpp" +#include "openPMD/auxiliary/Variant.hpp" #include -#include // std::tolower #include +#include #include #include #include +#include #include #include #include @@ -85,6 +90,17 @@ std::optional joinedDimension(adios2::Dims const &dims) #if openPMD_HAVE_MPI +#define MPI_CHECK(cmd) \ + do \ + { \ + int error = cmd; \ + if (error != MPI_SUCCESS) \ + { \ + std::cerr << "<" << __FILE__ << ">:" << __LINE__; \ + throw std::runtime_error(std::string("[MPI] Error")); \ + } \ + } while (false); + ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl( AbstractIOHandler *handler, MPI_Comm communicator, @@ -642,13 +658,9 @@ void ADIOS2IOHandlerImpl::createFile( m_writeAttributesFromThisRank && m_handler->m_encoding == IterationEncoding::groupBased) { - // For a peaceful phase-out of group-based encoding in ADIOS2, - // print this warning only in the new layout (with group table) - if (m_useGroupTable.value_or(UseGroupTable::No) == - UseGroupTable::Yes && - (m_engineType == "bp5" || - (m_engineType == "file" || m_engineType == "filestream" || - m_engineType == "bp"))) + if (m_engineType == "bp5" || + (m_engineType == "file" || m_engineType == "filestream" || + m_engineType == "bp")) { std::cerr << warningADIOS2NoGroupbasedEncoding << std::endl; printedWarningsAlready.noGroupBased = true; @@ -995,7 +1007,12 @@ void ADIOS2IOHandlerImpl::openDataset( *parameters.dtype = detail::fromADIOS2Type(fileData.m_IO.VariableType(varName)); switchAdios2VariableType( - *parameters.dtype, this, file, varName, parameters); + *parameters.dtype, + this, + file, + varName, + parameters, + fileData.stepSelection()); writable->written = true; } @@ -1080,7 +1097,7 @@ namespace detail auto &IO = ba.m_IO; auto &engine = ba.getEngine(); adios2::Variable variable = impl->verifyDataset( - params.offset, params.extent, IO, varName); + params.offset, params.extent, IO, varName, std::nullopt); adios2::Dims offset(params.offset.begin(), params.offset.end()); adios2::Dims extent(params.extent.begin(), params.extent.end()); variable.SetSelection({std::move(offset), std::move(extent)}); @@ -1216,10 +1233,352 @@ void ADIOS2IOHandlerImpl::readAttribute( } Datatype ret = switchType( - type, *this, ba.m_IO, name, *parameters.resource); + type, ba.m_IO, name, *parameters.resource); *parameters.dtype = ret; } +namespace +{ + /* Used by both readAttribute() and readAttributeAllsteps() tasks. + Functor fun will be called with the value of the retrieved attribute; + both functions use different logic for processing the retrieved values. + */ + template + Datatype + genericReadAttribute(Functor &&fun, adios2::IO &IO, std::string const &name) + { + /* + * If we store an attribute of boolean type, we store an additional + * attribute prefixed with '__is_boolean__' to indicate this information + * that would otherwise be lost. Check whether this has been done. + */ + using rep = detail::AttributeTypes::rep; + + if constexpr (std::is_same::value) + { + auto attr = IO.InquireAttribute(name); + if (!attr) + { + throw std::runtime_error( + "[ADIOS2] Internal error: Failed reading attribute '" + + name + "'."); + } + + std::string metaAttr; + metaAttr = adios_defaults::str_isBoolean + name; + /* + * In verbose mode, attributeInfo will yield a warning if not + * finding the requested attribute. Since we expect the attribute + * not to be present in many cases (i.e. when it is actually not + * a boolean), let's tell attributeInfo to be quiet. + */ + auto type = detail::attributeInfo( + IO, + metaAttr, + /* verbose = */ false); + + if (type == determineDatatype()) + { + auto meta = IO.InquireAttribute(metaAttr); + if (meta.Data().size() == 1 && meta.Data()[0] == 1) + { + std::forward(fun)( + detail::bool_repr::fromRep(attr.Data()[0])); + return determineDatatype(); + } + } + std::forward(fun)(attr.Data()[0]); + } + else if constexpr (detail::IsUnsupportedComplex_v) + { + throw std::runtime_error( + "[ADIOS2] Internal error: no support for long double complex " + "attribute types"); + } + else if constexpr (auxiliary::IsVector_v) + { + auto attr = IO.InquireAttribute(name); + if (!attr) + { + throw std::runtime_error( + "[ADIOS2] Internal error: Failed reading attribute '" + + name + "'."); + } + std::forward(fun)(attr.Data()); + } + else if constexpr (auxiliary::IsArray_v) + { + auto attr = IO.InquireAttribute(name); + if (!attr) + { + throw std::runtime_error( + "[ADIOS2] Internal error: Failed reading attribute '" + + name + "'."); + } + auto data = attr.Data(); + T res; + for (size_t i = 0; i < data.size(); i++) + { + res[i] = data[i]; + } + std::forward(fun)(std::move(res)); + } + else if constexpr (std::is_same_v) + { + throw std::runtime_error( + "Observed boolean attribute. ADIOS2 does not have these?"); + } + else + { + auto attr = IO.InquireAttribute(name); + if (!attr) + { + throw std::runtime_error( + "[ADIOS2] Internal error: Failed reading attribute '" + + name + "'."); + } + std::forward(fun)(attr.Data()[0]); + } + + return determineDatatype(); + } + + struct ReadAttributeAllsteps + { + template + static void call( + adios2::IO &IO, + adios2::Engine &engine, + std::string const &name, + adios2::StepStatus status, + Parameter::result_type + &put_result_here) + { + std::vector res; + res.reserve(engine.Steps()); + while (status == adios2::StepStatus::OK) + { + genericReadAttribute( + [&res](auto &&val) { + using type = std::remove_reference_t; + if constexpr (std::is_same_v) + { + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + "ADIOS2", + "[ReadAttributeAllsteps] No support for " + "Boolean attributes."); + } + else + { + res.emplace_back(static_cast(val)); + } + }, + IO, + name); + engine.EndStep(); + status = engine.BeginStep(); + } + switch (status) + { + case adios2::StepStatus::OK: + throw error::Internal("Control flow error."); + case adios2::StepStatus::NotReady: + case adios2::StepStatus::OtherError: + throw error::ReadError( + error::AffectedObject::File, + error::Reason::CannotRead, + "ADIOS2", + "Unexpected step status while preparsing snapshots."); + case adios2::StepStatus::EndOfStream: + break; + } + put_result_here = std::move(res); + } + + static constexpr char const *errorMsg = "ReadAttributeAllsteps"; + }; + +#if openPMD_HAVE_MPI + struct DistributeToAllRanks + { + template + static void call( + Parameter::result_type + &put_result_here_in, + MPI_Comm comm, + int rank) + { + if (rank != 0) + { + put_result_here_in = std::vector{}; + } + std::vector &put_result_here = + std::get>(put_result_here_in); + size_t num_items = put_result_here.size(); + MPI_CHECK(MPI_Bcast( + &num_items, 1, auxiliary::openPMD_MPI_type(), 0, comm)); + if constexpr ( + std::is_same_v || + std::is_same_v> || + std::is_same_v || + std::is_same_v> || + auxiliary::IsArray_v || isComplexFloatingPoint()) + { + throw error::OperationUnsupportedInBackend( + "ADIOS2", + "[readAttributeAllsteps] No support for attributes of type " + "std::string, bool, std::complex or std::array in " + "parallel."); + } + else if constexpr ( + // auxiliary::IsArray_v || + auxiliary::IsVector_v) + { + std::vector sizes; + sizes.reserve(num_items); + if (rank == 0) + { + std::transform( + put_result_here.begin(), + put_result_here.end(), + std::back_inserter(sizes), + [](T const &arr) { return arr.size(); }); + } + sizes.resize(num_items); + MPI_CHECK(MPI_Bcast( + sizes.data(), + num_items, + auxiliary::openPMD_MPI_type(), + 0, + comm)); + size_t total_flat_size = + std::accumulate(sizes.begin(), sizes.end(), size_t(0)); + using flat_type = typename T::value_type; + std::vector flat_vector; + flat_vector.reserve(total_flat_size); + if (rank == 0) + { + for (auto const &arr : put_result_here) + { + for (auto val : arr) + { + flat_vector.push_back(val); + } + } + } + flat_vector.resize(total_flat_size); + MPI_CHECK(MPI_Bcast( + flat_vector.data(), + total_flat_size, + auxiliary::openPMD_MPI_type(), + 0, + comm)); + if (rank != 0) + { + size_t offset = 0; + put_result_here.reserve(num_items); + for (size_t current_extent : sizes) + { + put_result_here.emplace_back( + flat_vector.begin() + offset, + flat_vector.begin() + offset + current_extent); + offset += current_extent; + } + } + } + else + { + std::vector receive; + if (rank != 0) + { + receive.resize(num_items); + } + MPI_CHECK(MPI_Bcast( + rank == 0 ? put_result_here.data() : receive.data(), + num_items, + auxiliary::openPMD_MPI_type(), + 0, + comm)); + if (rank != 0) + { + put_result_here = std::move(receive); + } + } + } + static constexpr char const *errorMsg = "DistributeToAllRanks"; + }; +#endif + + void warn_ignored_modifiable_attributes(adios2::IO &IO) + { + auto modifiable_flag = IO.InquireAttribute( + adios_defaults::str_useModifiableAttributes); + auto print_warning = [](std::string const ¬e) { + std::cerr << "Warning: " << note << R"( +Random-access for variable-encoding in ADIOS2 is currently +experimental. Support for modifiable attributes is currently not implemented +yet, meaning that attributes such as /data/time will show useless values. +Use Access::READ_LINEAR to retrieve those values if needed. +The following modifiable attributes have been found: +)"; + }; + if (!modifiable_flag) + { + print_warning("File might be using modifiable attributes."); + } + else if (modifiable_flag.Data().at(0) != 0) + { + print_warning("File uses modifiable attributes."); + } + } +} // namespace + +void ADIOS2IOHandlerImpl::readAttributeAllsteps( + Writable *writable, Parameter ¶m) +{ + auto file = refreshFileFromParent(writable, /* preferParentFile = */ false); + auto pos = setAndGetFilePosition(writable); + auto name = nameOfAttribute(writable, param.name); + + auto read_from_file_in_serial = [&]() { + adios2::ADIOS adios; + auto IO = adios.DeclareIO("PreparseSnapshots"); + // @todo check engine type + IO.SetEngine(realEngineType()); + IO.SetParameter("StreamReader", "ON"); // this be for BP4 + auto engine = IO.Open(fullPath(*file), adios2::Mode::Read); + auto status = engine.BeginStep(); + warn_ignored_modifiable_attributes(IO); + auto type = detail::attributeInfo(IO, name, /* verbose = */ true); + switchType( + type, IO, engine, name, status, *param.resource); + engine.Close(); + return type; + }; +#if openPMD_HAVE_MPI + if (!m_communicator.has_value()) + { + read_from_file_in_serial(); + return; + } + int rank, size; + MPI_Comm_rank(*m_communicator, &rank); + MPI_Comm_size(*m_communicator, &size); + Datatype type; + if (rank == 0) + { + type = read_from_file_in_serial(); + } + MPI_CHECK(MPI_Bcast(&type, 1, MPI_INT, 0, *m_communicator)); + switchType( + type, *param.resource, *m_communicator, rank); +#else + read_from_file_in_serial(); +#endif +} + void ADIOS2IOHandlerImpl::listPaths( Writable *writable, Parameter ¶meters) { @@ -1425,7 +1784,14 @@ void ADIOS2IOHandlerImpl::advance( { auto file = m_files.at(writable); auto &ba = getFileData(file, IfFileNotOpen::ThrowError); - *parameters.status = ba.advance(parameters.mode); + std::visit( + auxiliary::overloaded{ + [&](AdvanceMode mode) { *parameters.status = ba.advance(mode); }, + [&](Parameter::StepSelection step) { + ba.setStepSelection(step.step); + *parameters.status = AdvanceStatus::RANDOMACCESS; + }}, + parameters.mode); } void ADIOS2IOHandlerImpl::closePath( @@ -1745,105 +2111,14 @@ namespace detail { template Datatype AttributeReader::call( - ADIOS2IOHandlerImpl &impl, - adios2::IO &IO, - std::string name, - Attribute::resource &resource) + adios2::IO &IO, std::string name, Attribute::resource &resource) { - (void)impl; - /* - * If we store an attribute of boolean type, we store an additional - * attribute prefixed with '__is_boolean__' to indicate this information - * that would otherwise be lost. Check whether this has been done. - */ - using rep = AttributeTypes::rep; - - if constexpr (std::is_same::value) - { - auto attr = IO.InquireAttribute(name); - if (!attr) - { - throw std::runtime_error( - "[ADIOS2] Internal error: Failed reading attribute '" + - name + "'."); - } - - std::string metaAttr; - metaAttr = adios_defaults::str_isBoolean + name; - /* - * In verbose mode, attributeInfo will yield a warning if not - * finding the requested attribute. Since we expect the attribute - * not to be present in many cases (i.e. when it is actually not - * a boolean), let's tell attributeInfo to be quiet. - */ - auto type = attributeInfo( - IO, - metaAttr, - /* verbose = */ false); - - if (type == determineDatatype()) - { - auto meta = IO.InquireAttribute(metaAttr); - if (meta.Data().size() == 1 && meta.Data()[0] == 1) - { - resource = bool_repr::fromRep(attr.Data()[0]); - return determineDatatype(); - } - } - resource = attr.Data()[0]; - } - else if constexpr (IsUnsupportedComplex_v) - { - throw std::runtime_error( - "[ADIOS2] Internal error: no support for long double complex " - "attribute types"); - } - else if constexpr (auxiliary::IsVector_v) - { - auto attr = IO.InquireAttribute(name); - if (!attr) - { - throw std::runtime_error( - "[ADIOS2] Internal error: Failed reading attribute '" + - name + "'."); - } - resource = attr.Data(); - } - else if constexpr (auxiliary::IsArray_v) - { - auto attr = IO.InquireAttribute(name); - if (!attr) - { - throw std::runtime_error( - "[ADIOS2] Internal error: Failed reading attribute '" + - name + "'."); - } - auto data = attr.Data(); - T res; - for (size_t i = 0; i < data.size(); i++) - { - res[i] = data[i]; - } - resource = res; - } - else if constexpr (std::is_same_v) - { - throw std::runtime_error( - "Observed boolean attribute. ADIOS2 does not have these?"); - } - else - { - auto attr = IO.InquireAttribute(name); - if (!attr) - { - throw std::runtime_error( - "[ADIOS2] Internal error: Failed reading attribute '" + - name + "'."); - } - resource = attr.Data()[0]; - } - - return determineDatatype(); + return genericReadAttribute( + [&resource](auto &&value) { + resource = static_cast(value); + }, + IO, + name); } template @@ -2004,12 +2279,42 @@ namespace detail ADIOS2IOHandlerImpl *impl, InvalidatableFile const &file, const std::string &varName, - Parameter ¶meters) + Parameter ¶meters, + std::optional stepSelection) { auto &fileData = impl->getFileData( file, ADIOS2IOHandlerImpl::IfFileNotOpen::ThrowError); auto &IO = fileData.m_IO; adios2::Variable var = IO.InquireVariable(varName); + + if (fileData.stepSelection().has_value()) + { + auto file_steps = fileData.getEngine().Steps(); + auto var_steps = var.Steps(); + if (file_steps != var_steps) + { + throw error::ReadError( + error::AffectedObject::Dataset, + error::Reason::UnexpectedContent, + "ADIOS2", + &R"( +The opened file contains different data per step. +When using variable-based encoding, such files must be opened in linear read +mode, since random-access mode cannot easily associate variable steps +to iterations under these circumstances (yet). +If random-access read mode is required, file-based iteration encoding is more +useful for such data in ADIOS2. You may use the openpmd-pipe command line tool +for converting from variable-based to file-based iteration encoding. +ERROR: Variable ')"[1] + varName + + "' has " + std::to_string(var_steps) + + " step(s), but the file has " + + std::to_string(file_steps) + " step(s)."); + } + } + if (stepSelection.has_value()) + { + var.SetStepSelection({*stepSelection, 1}); + } if (!var) { throw std::runtime_error( diff --git a/src/IO/AbstractIOHandler.cpp b/src/IO/AbstractIOHandler.cpp index 8f0bd4524e..c8d3412fe2 100644 --- a/src/IO/AbstractIOHandler.cpp +++ b/src/IO/AbstractIOHandler.cpp @@ -116,4 +116,9 @@ std::future AbstractIOHandler::flush(internal::FlushParams const ¶ms) json::warnGlobalUnusedOptions(parsedParams.backendConfig); return future; } + +bool AbstractIOHandler::fullSupportForVariableBasedEncoding() const +{ + return false; +} } // namespace openPMD diff --git a/src/IO/AbstractIOHandlerImpl.cpp b/src/IO/AbstractIOHandlerImpl.cpp index 7675cc3e07..95fb7dfa09 100644 --- a/src/IO/AbstractIOHandlerImpl.cpp +++ b/src/IO/AbstractIOHandlerImpl.cpp @@ -21,13 +21,18 @@ #include "openPMD/IO/AbstractIOHandlerImpl.hpp" +#include "openPMD/IO/IOTask.hpp" +#include "openPMD/Streaming.hpp" #include "openPMD/auxiliary/Environment.hpp" +#include "openPMD/auxiliary/Variant.hpp" +#include "openPMD/backend/Attribute.hpp" #include "openPMD/backend/Writable.hpp" #include #include #include #include +#include namespace openPMD { @@ -314,7 +319,10 @@ std::future AbstractIOHandlerImpl::flush() i.writable->parent, "->", i.writable, - "] READ_DATASET"); + "] READ_DATASET, offset=", + [¶meter]() { return vec_as_string(parameter.offset); }, + ", extent=", + [¶meter]() { return vec_as_string(parameter.extent); }); readDataset(i.writable, parameter); break; } @@ -344,6 +352,20 @@ std::future AbstractIOHandlerImpl::flush() readAttribute(i.writable, parameter); break; } + case O::READ_ATT_ALLSTEPS: { + auto ¶meter = + deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] READ_ATT_ALLSTEPS: ", + parameter.name); + readAttributeAllsteps(i.writable, parameter); + break; + } case O::LIST_PATHS: { auto ¶meter = deref_dynamic_cast>( i.parameter.get()); @@ -383,15 +405,30 @@ std::future AbstractIOHandlerImpl::flush() i.writable, "] ADVANCE ", [&]() { - switch (parameter.mode) - { + return std::visit( + auxiliary::overloaded{ + [](AdvanceMode mode) -> std::string { + switch (mode) + { - case AdvanceMode::BEGINSTEP: - return "BEGINSTEP"; - case AdvanceMode::ENDSTEP: - return "ENDSTEP"; - } - throw std::runtime_error("Unreachable!"); + case AdvanceMode::BEGINSTEP: + return "BEGINSTEP"; + case AdvanceMode::ENDSTEP: + return "ENDSTEP"; + } + throw std::runtime_error("Unreachable!"); + }, + [](Parameter::StepSelection + step) { + std::stringstream s; + s << "RANDOMACCESS '" + << (step.step.has_value() + ? std::to_string(*step.step) + : std::string("RESET")) + << "'"; + return s.str(); + }}, + parameter.mode); }()); advance(i.writable, parameter); break; @@ -485,6 +522,21 @@ std::future AbstractIOHandlerImpl::flush() return std::future(); } +void AbstractIOHandlerImpl::readAttributeAllsteps( + Writable *w, Parameter ¶m) +{ + using result_type = Parameter::result_type; + Parameter param_internal; + param_internal.name = param.name; + param_internal.dtype = param.dtype; + readAttribute(w, param_internal); + *param.resource = std::visit( + [](auto &val) -> result_type { + return result_type{std::vector{std::move(val)}}; + }, + *param_internal.resource); +} + void AbstractIOHandlerImpl::setWritten( Writable *w, Parameter const ¶m) { diff --git a/src/IO/IOTask.cpp b/src/IO/IOTask.cpp index 47b0bea4ca..0692547745 100644 --- a/src/IO/IOTask.cpp +++ b/src/IO/IOTask.cpp @@ -149,10 +149,23 @@ namespace internal case Operation::AVAILABLE_CHUNKS: return "AVAILABLE_CHUNKS"; break; - default: - return "unknown"; + case Operation::CHECK_FILE: + return "CHECK_FILE"; + break; + case Operation::READ_ATT_ALLSTEPS: + return "READ_ATT_ALLSTEPS"; + break; + case Operation::DEREGISTER: + return "DEREGISTER"; + break; + case Operation::TOUCH: + return "TOUCH"; + break; + case Operation::SET_WRITTEN: + return "SET_WRITTEN"; break; } + return "unknown"; } } // namespace internal diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 931a1f9e3d..c991c2b79b 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -29,6 +29,7 @@ #include "openPMD/auxiliary/DerefDynamicCast.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/StringManip.hpp" +#include "openPMD/auxiliary/Variant.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Writable.hpp" @@ -39,6 +40,7 @@ #include #include #include +#include namespace openPMD { @@ -440,15 +442,28 @@ void Iteration::readFileBased( read_impl(groupPath); } -void Iteration::readGorVBased(std::string const &groupPath, bool doBeginStep) +void Iteration::readGorVBased( + std::string const &groupPath, internal::BeginStep const &doBeginStep) { - if (doBeginStep) - { - /* - * beginStep() must take care to open files - */ - beginStep(/* reread = */ false); - } + std::visit( + auxiliary::overloaded{ + [](internal::BeginStepTypes::DontBeginStep const &) {}, + [&](internal::BeginStepTypes::BeginStepSequentially const &) { + /* + * beginStep() must take care to open files + */ + beginStep(/* reread = */ false); + }, + [&](internal::BeginStepTypes::BeginStepRandomAccess const + &randomAccess) { + Parameter param; + param.mode = Parameter::StepSelection{ + randomAccess.step}; + IOHandler()->enqueue( + IOTask(&retrieveSeries().writable(), std::move(param))); + }, + }, + doBeginStep); read_impl(groupPath); } @@ -920,7 +935,9 @@ void Iteration::runDeferredParseAccess() deferred.iteration, filename, deferred.path, - deferred.beginStep); + std::holds_alternative< + internal::BeginStepTypes::BeginStepSequentially>( + deferred.beginStep)); } else { diff --git a/src/Series.cpp b/src/Series.cpp index 792be0555f..2fe870a4a9 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -38,6 +38,7 @@ #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/auxiliary/Variant.hpp" #include "openPMD/backend/Attributable.hpp" +#include "openPMD/backend/Attribute.hpp" #include "openPMD/snapshots/ContainerImpls.hpp" #include "openPMD/snapshots/IteratorTraits.hpp" #include "openPMD/snapshots/RandomAccessIterator.hpp" @@ -57,6 +58,7 @@ #include #include #include +#include #include #include @@ -254,6 +256,14 @@ chunk_assignment::RankMeta Series::rankTable([[maybe_unused]] bool collective) rankTable.m_bufferedRead = chunk_assignment::RankMeta{}; return {}; } + if (iterationEncoding() == IterationEncoding::variableBased && + IOHandler()->m_backendAccess == Access::READ_RANDOM_ACCESS) + { + Parameter advance; + advance.mode = + Parameter::StepSelection{std::nullopt}; + IOHandler()->enqueue(IOTask(this, std::move(advance))); + } Parameter openDataset; openDataset.name = "rankTable"; IOHandler()->enqueue(IOTask(&rankTable.m_attributable, openDataset)); @@ -579,45 +589,7 @@ IterationEncoding Series::iterationEncoding() const Series &Series::setIterationEncoding(IterationEncoding ie) { - auto &series = get(); - if (series.m_deferred_initialization) - { - runDeferredInitialization(); - } - if (written()) - throw std::runtime_error( - "A files iterationEncoding can not (yet) be changed after it has " - "been written."); - - series.m_iterationEncoding = ie; - switch (ie) - { - case IterationEncoding::fileBased: - setIterationFormat(series.m_name); - setAttribute("iterationEncoding", std::string("fileBased")); - // This checks that the name contains the expansion pattern - // (e.g. %T) and parses it - if (series.m_filenamePadding < 0) - { - if (!reparseExpansionPattern(series.m_name)) - { - throw error::WrongAPIUsage( - "For fileBased formats the iteration expansion pattern " - "%T must " - "be included in the file name"); - } - } - break; - case IterationEncoding::groupBased: - setIterationFormat(BASEPATH); - setAttribute("iterationEncoding", std::string("groupBased")); - break; - case IterationEncoding::variableBased: - setIterationFormat(auxiliary::replace_first(basePath(), "/%T/", "")); - setAttribute("iterationEncoding", std::string("variableBased")); - break; - } - IOHandler()->setIterationEncoding(ie); + setIterationEncoding_internal(ie, internal::default_or_explicit::explicit_); return *this; } @@ -1167,8 +1139,10 @@ Given file pattern: ')END" * allow setting attributes in that case */ setWritten(false, Attributable::EnqueueAsynchronously::No); - initDefaults(input->iterationEncoding); - setIterationEncoding(input->iterationEncoding); + initDefaults(series.m_iterationEncoding); + setIterationEncoding_internal( + series.m_iterationEncoding, + series.m_iterationEncodingSetExplicitly); setWritten(true, Attributable::EnqueueAsynchronously::No); } @@ -1184,12 +1158,14 @@ Given file pattern: ')END" } case Access::CREATE: { initDefaults(input->iterationEncoding); - setIterationEncoding(input->iterationEncoding); + setIterationEncoding_internal( + input->iterationEncoding, series.m_iterationEncodingSetExplicitly); break; } case Access::APPEND: { initDefaults(input->iterationEncoding); - setIterationEncoding(input->iterationEncoding); + setIterationEncoding_internal( + input->iterationEncoding, series.m_iterationEncodingSetExplicitly); if (input->iterationEncoding != IterationEncoding::fileBased) { break; @@ -1446,6 +1422,13 @@ void Series::flushGorVBased( break; case IO::HasBeenOpened: // continue below + if (randomAccessSteps() && !series.m_snapshotToStep.empty()) + { + Parameter param; + param.mode = Parameter::StepSelection{ + series.m_snapshotToStep.at(it->first)}; + IOHandler()->enqueue(IOTask(this, std::move(param))); + } it->second.flush(flushParams); break; } @@ -1848,7 +1831,8 @@ void Series::readOneIterationFileBased(std::string const &filePath) "Unknown iterationEncoding: " + encoding); auto old_written = written(); setWritten(false, Attributable::EnqueueAsynchronously::No); - setIterationEncoding(encoding_out); + setIterationEncoding_internal( + encoding_out, internal::default_or_explicit::explicit_); setWritten(old_written, Attributable::EnqueueAsynchronously::Yes); } else @@ -1949,18 +1933,9 @@ auto Series::readGorVBased( else if (encoding == "variableBased") { series.m_iterationEncoding = IterationEncoding::variableBased; - if (IOHandler()->m_frontendAccess == Access::READ_ONLY) - { - std::cerr << R"( -The opened Series uses variable-based encoding, but is being accessed by -READ_ONLY mode which operates in random-access manner. -Random-access is (currently) unsupported by variable-based encoding -and some iterations may not be found by this access mode. -Consider using Access::READ_LINEAR and Series::readIterations().)" - << std::endl; - } - else if (IOHandler()->m_frontendAccess == Access::READ_WRITE) + if (IOHandler()->m_frontendAccess == Access::READ_WRITE) { + // @todo hmm see what happens throw error::WrongAPIUsage(R"( The opened Series uses variable-based encoding, but is being accessed by READ_WRITE mode which does not (yet) support variable-based encoding. @@ -2057,11 +2032,11 @@ creating new iterations. /* * Return error if one is caught. */ - auto readSingleIteration = - [&series, &pOpen, this]( - IterationIndex_t index, - std::string const &path, - bool beginStep) -> std::optional { + auto readSingleIteration = [&series, &pOpen, this]( + IterationIndex_t index, + std::string const &path, + internal::BeginStep const &beginStep) + -> std::optional { if (series.iterations.contains(index)) { // maybe re-read @@ -2133,7 +2108,12 @@ creating new iterations. } if (auto err = internal::withRWAccess( IOHandler()->m_seriesStatus, - [&]() { return readSingleIteration(index, it, false); }); + [&]() { + return readSingleIteration( + index, + it, + internal::BeginStepTypes::DontBeginStep{}); + }); err) { std::cerr << "Cannot read iteration " << index @@ -2164,10 +2144,29 @@ creating new iterations. case IterationEncoding::variableBased: { if (!currentSteps.has_value() || currentSteps.value().empty()) { - currentSteps = std::vector{ - read_only_this_single_iteration.has_value() - ? *read_only_this_single_iteration - : 0}; + if (!read_only_this_single_iteration.has_value()) + { + Parameter ld; + Parameter lp; + IOHandler()->enqueue(IOTask(&iterations, ld)); + IOHandler()->enqueue(IOTask(&iterations, lp)); + IOHandler()->flush(internal::defaultFlushParams); + if (ld.datasets->empty() && lp.paths->empty()) + { + return {}; // no iterations, just global attributes + } + else + { + // there is data, defaulting to calling this Iteration idx 0 + // when no further info is available + currentSteps = std::vector{0}; + } + } + else + { + currentSteps = std::vector{ + *read_only_this_single_iteration}; + } } else if (read_only_this_single_iteration.has_value()) { @@ -2190,10 +2189,18 @@ creating new iterations. * Variable-based iteration encoding relies on steps, so parsing * must happen after opening the first step. */ + using namespace internal; + BeginStep beginStep = randomAccessSteps() + ? (series.m_snapshotToStep.empty() + ? BeginStepTypes::make() + : BeginStepTypes::make< + BeginStepTypes::BeginStepRandomAccess>( + series.m_snapshotToStep.at(it))) + : BeginStepTypes::make(); if (auto err = internal::withRWAccess( IOHandler()->m_seriesStatus, - [&readSingleIteration, it]() { - return readSingleIteration(it, "", true); + [&readSingleIteration, it, &beginStep]() { + return readSingleIteration(it, "", beginStep); }); err) { @@ -2630,6 +2637,59 @@ void Series::flushStep(bool doFlush) series.m_wroteAtLeastOneIOStep = true; } +Series &Series::setIterationEncoding_internal( + IterationEncoding ie, internal::default_or_explicit doe) +{ + auto &series = get(); + switch (doe) + { + case internal::default_or_explicit::default_: + case internal::default_or_explicit::explicit_: + // mark this option as set explicitly by the user + series.m_iterationEncodingSetExplicitly = doe; + break; + } + if (series.m_deferred_initialization) + { + runDeferredInitialization(); + } + if (written()) + throw std::runtime_error( + "A files iterationEncoding can not (yet) be changed after it has " + "been written."); + + series.m_iterationEncoding = ie; + switch (ie) + { + case IterationEncoding::fileBased: + setIterationFormat(series.m_name); + setAttribute("iterationEncoding", std::string("fileBased")); + // This checks that the name contains the expansion pattern + // (e.g. %T) and parses it + if (series.m_filenamePadding < 0) + { + if (!reparseExpansionPattern(series.m_name)) + { + throw error::WrongAPIUsage( + "For fileBased formats the iteration expansion pattern " + "%T must " + "be included in the file name"); + } + } + break; + case IterationEncoding::groupBased: + setIterationFormat(BASEPATH); + setAttribute("iterationEncoding", std::string("groupBased")); + break; + case IterationEncoding::variableBased: + setIterationFormat(auxiliary::replace_first(basePath(), "/%T/", "")); + setAttribute("iterationEncoding", std::string("variableBased")); + break; + } + IOHandler()->setIterationEncoding(ie); + return *this; +} + auto Series::openIterationIfDirty(IterationIndex_t index, Iteration &iteration) -> IterationOpened { @@ -2924,6 +2984,8 @@ void Series::parseJsonOptions(TracingJSON &options, ParsedInput &input) options, "iteration_encoding", iterationEncoding); if (!iterationEncoding.empty()) { + series.m_iterationEncodingSetExplicitly = + internal::default_or_explicit::explicit_; auto it = ieDescriptors.find(iterationEncoding); if (it != ieDescriptors.end()) { @@ -3175,6 +3237,38 @@ Series::snapshots(std::optional const snapshot_workflow) } } + /* + * ADIOS2 should use variable-based encoding as a default when applicable, + * since group-based encoding has severe limitations in ADIOS2. + * The below logic checks if variable-based encoding should be used. + */ + + if ( + // 1. No encoding has been explicitly selected by the user. + // Flag set by Series::setIterationEncoding(). + series.m_iterationEncodingSetExplicitly == + internal::default_or_explicit::default_ && + // 2. Iteration encoding was recognized as groupBased by init() + // procedures (and not file-based). + series.m_iterationEncoding == IterationEncoding::groupBased && + // 3. The IO workflow will be synchronous, necessary for writing + // variable-based data (but not for reading!). + usedSnapshotWorkflow == SnapshotWorkflow::Synchronous && + // 4. The chosen access type is write-only, otherwise the encoding is + // determined by the previous file content. + access::writeOnly(access) && + // 5. The backend is ADIOS2 in a recent enough version to support + // modifiable attributes (v2.9). + IOHandler()->fullSupportForVariableBasedEncoding() && + // 6. The Series must not yet be written, otherwise we're too late + // for this + !this->written()) + { + setIterationEncoding_internal( + IterationEncoding::variableBased, + internal::default_or_explicit::default_); + } + switch (usedSnapshotWorkflow) { case SnapshotWorkflow::RandomAccess: { @@ -3223,8 +3317,7 @@ void Series::close() m_attri.reset(); } -auto Series::currentSnapshot() const - -> std::optional> +auto Series::currentSnapshot() -> std::optional> { using vec_t = std::vector; auto &series = get(); @@ -3235,7 +3328,32 @@ auto Series::currentSnapshot() const * `/data/snapshot`. This makes it possible to retrieve it from * `series.iterations`. */ - if (series.iterations.containsAttribute("snapshot")) + if (!series.iterations.containsAttribute("snapshot")) + { + return std::nullopt; + } + else if (randomAccessSteps()) + { + auto preparsed = preparseSnapshots(); + if (!preparsed.has_value()) + { + return std::nullopt; + } + std::set res; + for (size_t step = 0; step < preparsed->size(); ++step) + { + for (IterationIndex_t iteration : (*preparsed)[step]) + { + res.emplace(iteration); + // If an Iteration is found in multiple steps, this logic will + // prefer the Iteration from the later step. + series.m_snapshotToStep[iteration] = step; + } + } + std::cout.flush(); + return vec_t{res.begin(), res.end()}; + } + else { auto const &attribute = series.iterations.getAttribute("snapshot"); auto res = attribute.getOptional(); @@ -3257,10 +3375,6 @@ auto Series::currentSnapshot() const s.str()); } } - else - { - return std::optional>{}; - } } AbstractIOHandler *Series::runDeferredInitialization() @@ -3294,6 +3408,85 @@ AbstractIOHandler const *Series::IOHandler() const return res; } +auto Series::preparseSnapshots() + -> std::optional>> +{ + using vec_t = std::vector; + auto &series = get(); + if (!series.m_snapshotToStep.empty()) + { + throw error::Internal( + "Control flow error: This is an expensive operation and should be " + "called only once upon initialization."); + } + auto io_handler = IOHandler(); + if (!series.iterations.containsAttribute("snapshot")) + { + return std::nullopt; + } + Parameter readAttr; + readAttr.name = "snapshot"; + io_handler->enqueue(IOTask(&series.iterations, readAttr)); + io_handler->flush(internal::defaultFlushParams); + auto wrong_datatype = [&]() { + std::stringstream s; + s << "Unexpected datatype for '/data/snapshot': " << *readAttr.dtype + << " (expected a vector of integer)."; + return error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, + s.str()); + }; + return std::visit( + [&](auto &vec) -> std::vector { + using containedType = + typename std::decay_t::value_type; + if constexpr (std::is_same_v) + { + // need to keep the vector overload away from the + // below implementation + throw wrong_datatype(); + } + else + { + std::vector res; + res.reserve(vec.size()); + for (auto const &val : vec) + { + auto converted = + detail::doConvertOptional(&val); + if (!converted.has_value()) + { + throw wrong_datatype(); + } + res.emplace_back(*converted); + } + return res; + } + }, + *readAttr.resource); +} + +bool Series::randomAccessSteps() const +{ + auto randomAccess = [](Access access) { + switch (access) + { + case Access::READ_RANDOM_ACCESS: + case Access::READ_WRITE: + return true; + case Access::READ_LINEAR: + case Access::CREATE: + case Access::APPEND: + return false; + } + return false; + }; + return iterationEncoding() == IterationEncoding::variableBased && + randomAccess(IOHandler()->m_backendAccess); +} + namespace { CleanedFilename cleanFilename( diff --git a/test/CoreTest.cpp b/test/CoreTest.cpp index 3f88a18864..c5d98a73c5 100644 --- a/test/CoreTest.cpp +++ b/test/CoreTest.cpp @@ -1,11 +1,12 @@ // expose private and protected members for invasive testing -#include "openPMD/UnitDimension.hpp" #if openPMD_USE_INVASIVE_TESTS #define OPENPMD_private public: #define OPENPMD_protected public: #endif #include "openPMD/openPMD.hpp" +#include "Files_Core/CoreTests.hpp" + #include "openPMD/IO/ADIOS/macros.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/JSON.hpp" @@ -1499,6 +1500,11 @@ TEST_CASE("unavailable_backend", "[core]") #endif } +TEST_CASE("automatic_variable_encoding", "[adios2]") +{ + automatic_variable_encoding::automatic_variable_encoding(); +} + TEST_CASE("unique_ptr", "[core]") { auto stdptr = std::make_unique(5); diff --git a/test/Files_Core/CoreTests.hpp b/test/Files_Core/CoreTests.hpp new file mode 100644 index 0000000000..27cb896706 --- /dev/null +++ b/test/Files_Core/CoreTests.hpp @@ -0,0 +1,6 @@ +#include + +namespace automatic_variable_encoding +{ +auto automatic_variable_encoding() -> void; +} diff --git a/test/Files_Core/automatic_variable_encoding.cpp b/test/Files_Core/automatic_variable_encoding.cpp new file mode 100644 index 0000000000..172d7162d1 --- /dev/null +++ b/test/Files_Core/automatic_variable_encoding.cpp @@ -0,0 +1,339 @@ +#include "CoreTests.hpp" +#include + +#include + +namespace automatic_variable_encoding +{ +auto automatic_variable_encoding() -> void +{ +#if openPMD_HAVE_ADIOS2 && openPMD_HAS_ADIOS_2_9 + using namespace openPMD; + + size_t filename_counter = 0; + auto filename = [&filename_counter]() { + std::stringstream res; + res << "../samples/automatic_variable_encoding/test_no_" + << filename_counter << ".bp5"; + return res.str(); + }; + auto next_filename = [&filename_counter, &filename]() { + ++filename_counter; + return filename(); + }; + auto require_encoding = [&filename](IterationEncoding ie) { + Series read(filename(), openPMD::Access::READ_RANDOM_ACCESS); + REQUIRE(read.iterationEncoding() == ie); + }; + + // TESTS + + { + Series write(next_filename(), Access::CREATE); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.flush(); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.flush(); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + // explicitly set variable encoding + + { + Series write( + next_filename(), + Access::CREATE, + R"(iteration_encoding = "variable_based")"); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write( + next_filename(), + Access::CREATE, + R"(iteration_encoding = "variable_based")"); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write( + next_filename(), + Access::CREATE, + R"(iteration_encoding = "variable_based")"); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write( + next_filename(), + Access::CREATE, + R"(iteration_encoding = "variable_based")"); + write.flush(); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write( + next_filename(), + Access::CREATE, + R"(iteration_encoding = "variable_based")"); + write.flush(); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + // explicitly set variable encoding + + { + Series write( + next_filename(), + Access::CREATE, + R"(iteration_encoding = "group_based")"); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write( + next_filename(), + Access::CREATE, + R"(iteration_encoding = "group_based")"); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write( + next_filename(), + Access::CREATE, + R"(iteration_encoding = "group_based")"); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write( + next_filename(), + Access::CREATE, + R"(iteration_encoding = "group_based")"); + write.flush(); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write( + next_filename(), + Access::CREATE, + R"(iteration_encoding = "group_based")"); + write.flush(); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + // explicitly use API call to set variable encoding + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::variableBased); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::variableBased); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::variableBased); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::variableBased); + write.flush(); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::variableBased); + write.flush(); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + // explicitly use API call to set group encoding + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::groupBased); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::groupBased); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::groupBased); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::groupBased); + write.flush(); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::groupBased); + write.flush(); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + // explicitly use API call to set group encoding + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::variableBased); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::variableBased); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::variableBased); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::variableBased); + write.flush(); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::variableBased); + write.flush(); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + // explicitly use API call to set group encoding a bit late + + { + Series write(next_filename(), Access::CREATE); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.setIterationEncoding(IterationEncoding::groupBased); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.snapshots(SnapshotWorkflow::Synchronous); + write.setIterationEncoding(IterationEncoding::groupBased); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + // explicitly use API call to set variable encoding a bit late + + { + Series write(next_filename(), Access::CREATE); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.setIterationEncoding(IterationEncoding::variableBased); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.snapshots(SnapshotWorkflow::Synchronous); + write.setIterationEncoding(IterationEncoding::variableBased); + write.close(); + require_encoding(IterationEncoding::variableBased); + } +#endif +} +} // namespace automatic_variable_encoding diff --git a/test/Files_ParallelIO/ParallelIOTests.hpp b/test/Files_ParallelIO/ParallelIOTests.hpp new file mode 100644 index 0000000000..0cb247700e --- /dev/null +++ b/test/Files_ParallelIO/ParallelIOTests.hpp @@ -0,0 +1,78 @@ +#include + +#if openPMD_HAVE_MPI + +using namespace openPMD; + +struct BackendSelection +{ + std::string backendName; + std::string extension; + + [[nodiscard]] inline std::string jsonBaseConfig() const + { + return R"({"backend": ")" + backendName + "\"}"; + } +}; + +inline std::vector testedBackends() +{ + auto variants = getVariants(); + std::map extensions{ + {"adios2", "bp"}, {"hdf5", "h5"}}; + std::vector res; + for (auto const &pair : variants) + { + if (pair.second) + { + auto lookup = extensions.find(pair.first); + if (lookup != extensions.end()) + { + std::string extension = lookup->second; + res.push_back({pair.first, std::move(extension)}); + } + } + } + return res; +} + +inline std::vector getBackends() +{ + // first component: backend file ending + // second component: whether to test 128 bit values + std::vector res; +#if openPMD_HAVE_ADIOS2 + res.emplace_back("bp"); +#endif +#if openPMD_HAVE_HDF5 + res.emplace_back("h5"); +#endif + return res; +} + +inline auto const backends = getBackends(); + +inline std::vector testedFileExtensions() +{ + auto allExtensions = getFileExtensions(); + auto newEnd = std::remove_if( + allExtensions.begin(), allExtensions.end(), [](std::string const &ext) { + // sst and ssc need a receiver for testing + // bp4 is already tested via bp + return ext == "sst" || ext == "ssc" || ext == "bp4" || + ext == "toml" || ext == "json"; + }); + return {allExtensions.begin(), newEnd}; +} + +namespace read_variablebased_randomaccess +{ +auto read_variablebased_randomaccess() -> void; +} + +namespace iterate_nonstreaming_series +{ +auto iterate_nonstreaming_series() -> void; +} + +#endif diff --git a/test/Files_ParallelIO/iterate_nonstreaming_series.cpp b/test/Files_ParallelIO/iterate_nonstreaming_series.cpp new file mode 100644 index 0000000000..acce61a162 --- /dev/null +++ b/test/Files_ParallelIO/iterate_nonstreaming_series.cpp @@ -0,0 +1,186 @@ +#include "ParallelIOTests.hpp" + +#include "openPMD/IO/ADIOS/macros.hpp" + +#include +#include + +namespace iterate_nonstreaming_series +{ + +static auto run_test( + std::string const &file, + bool variableBasedLayout, + std::string const &jsonConfig) -> void +{ + int mpi_size, mpi_rank; + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + + constexpr size_t base_extent = 10; + size_t const extent = base_extent * size_t(mpi_size); + + { + Series writeSeries(file, Access::CREATE, MPI_COMM_WORLD, jsonConfig); + if (variableBasedLayout) + { + writeSeries.setIterationEncoding(IterationEncoding::variableBased); + } + // use conventional API to write iterations + auto iterations = writeSeries.iterations; + for (size_t i = 0; i < 10; ++i) + { + auto iteration = iterations[i]; + auto E_x = iteration.meshes["E"]["x"]; + E_x.resetDataset( + openPMD::Dataset(openPMD::Datatype::INT, {2, extent})); + int value = variableBasedLayout ? 0 : i; + std::vector data(extent, value); + E_x.storeChunk( + data, {0, base_extent * size_t(mpi_rank)}, {1, base_extent}); + bool taskSupportedByBackend = true; + DynamicMemoryView memoryView; + { + auto currentBuffer = memoryView.currentBuffer(); + REQUIRE(currentBuffer.data() == nullptr); + REQUIRE(currentBuffer.size() == 0); + } + memoryView = E_x.storeChunk( + {1, base_extent * size_t(mpi_rank)}, + {1, base_extent}, + /* + * Hijack the functor that is called for buffer creation. + * This allows us to check if the backend has explicit support + * for buffer creation or if the fallback implementation is + * used. + */ + [&taskSupportedByBackend](size_t size) { + taskSupportedByBackend = false; + return std::shared_ptr{ + new int[size], [](auto *ptr) { delete[] ptr; }}; + }); + if (writeSeries.backend() == "ADIOS2") + { + // that backend must support span creation + REQUIRE(taskSupportedByBackend); + } + auto span = memoryView.currentBuffer(); + for (size_t j = 0; j < span.size(); ++j) + { + span[j] = j; + } + + /* + * This is to test whether defaults are correctly written for + * scalar record components since there previously was a bug. + */ + auto scalarMesh = + iteration + .meshes["i_energyDensity"][MeshRecordComponent::SCALAR]; + scalarMesh.resetDataset( + Dataset(Datatype::INT, {5 * size_t(mpi_size)})); + auto scalarSpan = + scalarMesh.storeChunk({5 * size_t(mpi_rank)}, {5}) + .currentBuffer(); + for (size_t j = 0; j < scalarSpan.size(); ++j) + { + scalarSpan[j] = j; + } + // we encourage manually closing iterations, but it should not + // matter so let's do the switcharoo for this test + if (i % 2 == 0) + { + writeSeries.flush(); + } + else + { + iteration.close(); + } + } + } + + for (auto access : {Access::READ_LINEAR, Access::READ_ONLY}) + { + Series readSeries( + file, + access, + MPI_COMM_WORLD, + json::merge(jsonConfig, R"({"defer_iteration_parsing": true})")); + + size_t last_iteration_index = 0; + // conventionally written Series must be readable with streaming-aware + // API! + for (auto iteration : readSeries.readIterations()) + { + // ReadIterations takes care of Iteration::open()ing iterations + auto E_x = iteration.meshes["E"]["x"]; + REQUIRE(E_x.getDimensionality() == 2); + REQUIRE(E_x.getExtent()[0] == 2); + REQUIRE(E_x.getExtent()[1] == extent); + auto chunk = E_x.loadChunk({0, 0}, {1, extent}); + auto chunk2 = E_x.loadChunk({1, 0}, {1, extent}); + // we encourage manually closing iterations, but it should not + // matter so let's do the switcharoo for this test + if (last_iteration_index % 2 == 0) + { + readSeries.flush(); + } + else + { + iteration.close(); + } + + int value = variableBasedLayout ? 0 : iteration.iterationIndex; + for (size_t i = 0; i < extent; ++i) + { + REQUIRE(chunk.get()[i] == value); + REQUIRE(chunk2.get()[i] == int(i % base_extent)); + } + last_iteration_index = iteration.iterationIndex; + } + REQUIRE(last_iteration_index == 9); + } +} + +auto iterate_nonstreaming_series() -> void +{ + for (auto const &backend : testedBackends()) + { + run_test( + "../samples/iterate_nonstreaming_series/parallel_filebased_%T." + + backend.extension, + false, + backend.jsonBaseConfig()); + run_test( + "../samples/iterate_nonstreaming_series/parallel_groupbased." + + backend.extension, + false, + backend.jsonBaseConfig()); +#if openPMD_HAVE_ADIOS2 && openPMD_HAVE_ADIOS2_BP5 + if (backend.extension == "bp") + { + run_test( + "../samples/iterate_nonstreaming_series/" + "parallel_filebased_bp5_%T." + + backend.extension, + false, + json::merge( + backend.jsonBaseConfig(), "adios2.engine.type = \"bp5\"")); + run_test( + "../samples/iterate_nonstreaming_series/" + "parallel_groupbased_bp5." + + backend.extension, + false, + json::merge( + backend.jsonBaseConfig(), "adios2.engine.type = \"bp5\"")); + } +#endif + } +#if openPMD_HAVE_ADIOS2 && openPMD_HAS_ADIOS_2_9 + run_test( + "../samples/iterate_nonstreaming_series/parallel_variablebased.bp", + true, + R"({"backend": "adios2"})"); +#endif +} +} // namespace iterate_nonstreaming_series diff --git a/test/Files_ParallelIO/read_variablebased_randomaccess.cpp b/test/Files_ParallelIO/read_variablebased_randomaccess.cpp new file mode 100644 index 0000000000..39f5c88c56 --- /dev/null +++ b/test/Files_ParallelIO/read_variablebased_randomaccess.cpp @@ -0,0 +1,128 @@ +#include "ParallelIOTests.hpp" + +#include "openPMD/IO/ADIOS/macros.hpp" +#include "openPMD/openPMD.hpp" + +#include + +#include + +#if openPMD_HAVE_ADIOS2 && openPMD_HAVE_MPI && openPMD_HAS_ADIOS_2_9 +#include +#include + +namespace read_variablebased_randomaccess +{ +static void create_file_in_serial() +{ + { + adios2::ADIOS adios; + auto IO = adios.DeclareIO("IO"); + IO.SetEngine("bp5"); + auto engine = IO.Open( + "../samples/read_variablebased_randomaccess.bp", + adios2::Mode::Write); + + auto variable = + IO.DefineVariable("/data/meshes/theta", {10}, {0}, {10}); + + for (size_t step = 0; step < 5; ++step) + { + engine.BeginStep(); + + // write default openPMD attributes + IO.DefineAttribute("/basePath", std::string("/data/%T/")); + IO.DefineAttribute( + "/date", std::string("2021-02-22 11:14:00 +0000")); + IO.DefineAttribute( + "/iterationEncoding", std::string("variableBased")); + IO.DefineAttribute("/iterationFormat", std::string("/data/%T/")); + IO.DefineAttribute("/meshesPath", std::string("meshes/")); + IO.DefineAttribute("/openPMD", std::string("1.1.0")); + IO.DefineAttribute("/openPMDextension", uint32_t(0)); + IO.DefineAttribute("/software", std::string("openPMD-api")); + IO.DefineAttribute("/softwareVersion", std::string("0.17.0-dev")); + + std::vector current_snapshots(10); + std::iota( + current_snapshots.begin(), + current_snapshots.end(), + current_snapshots.size() * step); + IO.DefineAttribute( + "/data/snapshot", + current_snapshots.data(), + current_snapshots.size(), + "", + "/", + /* allowModification = */ true); + + IO.DefineAttribute("/data/dt", double(1)); + IO.DefineAttribute( + "/data/meshes/theta/axisLabels", + std::vector{"x"}.data(), + 1); + IO.DefineAttribute( + "/data/meshes/theta/dataOrder", std::string("C")); + IO.DefineAttribute( + "/data/meshes/theta/geometry", std::string("cartesian")); + IO.DefineAttribute( + "/data/meshes/theta/gridGlobalOffset", double(0)); + IO.DefineAttribute("/data/meshes/theta/gridSpacing", double(1)); + IO.DefineAttribute("/data/meshes/theta/gridUnitSI", double(1)); + IO.DefineAttribute("/data/meshes/theta/position", double(0)); + IO.DefineAttribute("/data/meshes/theta/timeOffset", double(0)); + IO.DefineAttribute( + "/data/meshes/theta/unitDimension", + std::vector(7, 0).data(), + 7); + IO.DefineAttribute("/data/meshes/theta/unitSI", double(1)); + IO.DefineAttribute("/data/time", double(0)); + IO.DefineAttribute("/data/timeUnitSI", double(1)); + + IO.DefineAttribute( + "__openPMD_internal/openPMD2_adios2_schema", 0); + IO.DefineAttribute("__openPMD_internal/useSteps", 0); + + std::vector data{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + engine.Put(variable, data.data()); + + engine.EndStep(); + } + + engine.Close(); + } +} + +auto read_variablebased_randomaccess() -> void +{ + int rank; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + if (rank == 0) + { + create_file_in_serial(); + } + MPI_Barrier(MPI_COMM_WORLD); + + { + openPMD::Series read( + "../samples/read_variablebased_randomaccess.bp", + openPMD::Access::READ_ONLY, + MPI_COMM_WORLD, + "adios2.engine.type = \"bp5\""); + auto data = + read.iterations[0].meshes["theta"].loadChunk({0}, {10}); + read.flush(); + for (size_t i = 0; i < 10; ++i) + { + REQUIRE(data.get()[i] == int(i)); + } + } +} +} // namespace read_variablebased_randomaccess +#else +namespace read_variablebased_randomaccess +{ +void read_variablebased_randomaccess() +{} +} // namespace read_variablebased_randomaccess +#endif diff --git a/test/Files_SerialIO/close_and_reopen_test.cpp b/test/Files_SerialIO/close_and_reopen_test.cpp index 56be4a39f6..8fad892635 100644 --- a/test/Files_SerialIO/close_and_reopen_test.cpp +++ b/test/Files_SerialIO/close_and_reopen_test.cpp @@ -172,7 +172,10 @@ auto run_test_groupbased( { std::string filename = "../samples/close_iteration_reopen/groupbased." + ext; - Series series(filename, Access::CREATE, write_cfg); + Series series( + filename, + Access::CREATE, + json::merge(write_cfg, R"({"iteration_encoding": "group_based"})")); { auto it = writeIterations(series)[0]; auto E_x = it.meshes["E"]["x"]; diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 2d4cc6554e..9ade0c8f44 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -1,6 +1,8 @@ /* Running this test in parallel with MPI requires MPI::Init. * To guarantee a correct call to Init, launch the tests manually. */ +#include "Files_ParallelIO/ParallelIOTests.hpp" + #include "openPMD/IO/ADIOS/macros.hpp" #include "openPMD/IO/Access.hpp" #include "openPMD/auxiliary/Environment.hpp" @@ -8,7 +10,12 @@ #include "openPMD/openPMD.hpp" #include -#if openPMD_HAVE_MPI +#if !openPMD_HAVE_MPI +TEST_CASE("none", "[parallel]") +{} + +#else + #include #if openPMD_HAVE_ADIOS2 @@ -32,42 +39,6 @@ using namespace openPMD; -std::vector getBackends() -{ - // first component: backend file ending - // second component: whether to test 128 bit values - std::vector res; -#if openPMD_HAVE_ADIOS2 - res.emplace_back("bp"); -#endif -#if openPMD_HAVE_HDF5 - res.emplace_back("h5"); -#endif - return res; -} - -auto const backends = getBackends(); - -std::vector testedFileExtensions() -{ - auto allExtensions = getFileExtensions(); - auto newEnd = std::remove_if( - allExtensions.begin(), allExtensions.end(), [](std::string const &ext) { - // sst and ssc need a receiver for testing - // bp4 is already tested via bp - return ext == "sst" || ext == "ssc" || ext == "bp4" || - ext == "toml" || ext == "json"; - }); - return {allExtensions.begin(), newEnd}; -} - -#else - -TEST_CASE("none", "[parallel]") -{} -#endif - -#if openPMD_HAVE_MPI TEST_CASE("parallel_multi_series_test", "[parallel]") { std::list allSeries; @@ -86,7 +57,8 @@ TEST_CASE("parallel_multi_series_test", "[parallel]") .append(".") .append(file_ending), Access::CREATE, - MPI_COMM_WORLD); + MPI_COMM_WORLD, + R"(iteration_encoding = "variable_based")"); allSeries.back().iterations[sn].setAttribute("wululu", sn); allSeries.back().flush(); } @@ -139,7 +111,7 @@ void write_test_zero_extent( for (int step = 0; step <= max_step; step += 20) { - Iteration it = o.iterations[step]; + Iteration it = o.writeIterations()[step]; it.setAttribute("yolo", "yo"); if (rank != 0 || declareFromAll) @@ -496,7 +468,7 @@ void extendDataset(std::string const &ext, std::string const &jsonConfig) // array record component -> array record component // should work - auto E_x = write.iterations[0].meshes["E"]["x"]; + auto E_x = write.writeIterations()[0].meshes["E"]["x"]; E_x.resetDataset(ds1); E_x.storeChunk(data1, {mpi_rank, 0}, {1, 25}); write.flush(); @@ -702,7 +674,7 @@ void write_4D_test(std::string const &file_ending) std::string name = "../samples/parallel_write_4d." + file_ending; Series o = Series(name, Access::CREATE, MPI_COMM_WORLD); - auto it = o.iterations[1]; + auto it = o.writeIterations()[1]; auto E_x = it.meshes["E"]["x"]; // every rank out of mpi_size MPI ranks contributes two writes: @@ -737,7 +709,7 @@ void write_makeconst_some(std::string const &file_ending) std::cout << name << std::endl; Series o = Series(name, Access::CREATE, MPI_COMM_WORLD); - auto it = o.iterations[1]; + auto it = o.writeIterations()[1]; // I would have expected we need this, since the first call that writes // data below (makeConstant) is not executed in MPI collective manner // it.open(); @@ -1167,7 +1139,7 @@ TEST_CASE("independent_write_with_collective_flush", "[parallel]") int size, rank; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); - auto iteration = write.iterations[0]; + auto iteration = write.writeIterations()[0]; auto E_x = iteration.meshes["E"]["x"]; E_x.resetDataset({Datatype::DOUBLE, {10}}); write.flush(); @@ -1211,7 +1183,10 @@ void adios2_streaming(bool variableBasedLayout) Series writeSeries( "../samples/adios2_stream.sst", Access::CREATE, - "adios2.engine.type = \"sst\""); + variableBasedLayout + ? "adios2.engine.type = \"sst\"" + : "adios2.engine.type = \"sst\"\niteration_encoding = " + "\"group_based\""); if (variableBasedLayout) { writeSeries.setIterationEncoding(IterationEncoding::variableBased); @@ -2197,4 +2172,14 @@ TEST_CASE("adios2_flush_via_step") } #endif +TEST_CASE("read_variablebased_randomaccess") +{ + read_variablebased_randomaccess::read_variablebased_randomaccess(); +} + +TEST_CASE("iterate_nonstreaming_series", "[serial][adios2]") +{ + iterate_nonstreaming_series::iterate_nonstreaming_series(); +} + #endif // openPMD_HAVE_ADIOS2 && openPMD_HAVE_MPI diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index ab8c2ed15c..0b050384fb 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -122,7 +122,7 @@ namespace detail template void writeChar(Series &series, std::string const &component_name) { - auto component = series.iterations[0].meshes["E"][component_name]; + auto component = series.writeIterations()[0].meshes["E"][component_name]; std::vector data(10); component.resetDataset({determineDatatype(), {10}}); component.storeChunk(data, {0}, {10}); @@ -279,7 +279,7 @@ TEST_CASE("multi_series_test", "[serial]") .append(".") .append(file_ending), Access::CREATE); - allSeries.back().iterations[sn].setAttribute("wululu", sn); + allSeries.back().writeIterations()[sn].setAttribute("wululu", sn); allSeries.back().flush(); } } @@ -636,12 +636,13 @@ void close_and_copy_attributable_test(std::string const &file_ending) { if (iteration_ptr) { - *iteration_ptr = series.iterations[i]; + *iteration_ptr = series.writeIterations()[i]; } else { // use copy constructor - iteration_ptr = std::make_unique(series.iterations[i]); + iteration_ptr = + std::make_unique(series.writeIterations()[i]); } Record electronPositions = iteration_ptr->particles["e"]["position"]; // TODO set this automatically to zero if not provided @@ -773,17 +774,20 @@ inline void empty_dataset_test(std::string const &file_ending) "../samples/empty_datasets." + file_ending, Access::CREATE); auto makeEmpty_dim_7_int = - series.iterations[1].meshes["rho"]["makeEmpty_dim_7_int"]; + series.writeIterations()[1].meshes["rho"]["makeEmpty_dim_7_int"]; auto makeEmpty_dim_7_long = - series.iterations[1].meshes["rho"]["makeEmpty_dim_7_bool"]; + series.writeIterations()[1].meshes["rho"]["makeEmpty_dim_7_bool"]; auto makeEmpty_dim_7_int_alt = - series.iterations[1].meshes["rho"]["makeEmpty_dim_7_int_alt"]; + series.writeIterations()[1] + .meshes["rho"]["makeEmpty_dim_7_int_alt"]; auto makeEmpty_dim_7_long_alt = - series.iterations[1].meshes["rho"]["makeEmpty_dim_7_bool_alt"]; + series.writeIterations()[1] + .meshes["rho"]["makeEmpty_dim_7_bool_alt"]; auto makeEmpty_resetDataset_dim3 = - series.iterations[1].meshes["rho"]["makeEmpty_resetDataset_dim3"]; + series.writeIterations()[1] + .meshes["rho"]["makeEmpty_resetDataset_dim3"]; auto makeEmpty_resetDataset_dim3_notallzero = - series.iterations[1] + series.writeIterations()[1] .meshes["rho"]["makeEmpty_resetDataset_dim3_notallzero"]; makeEmpty_dim_7_int.makeEmpty(7); makeEmpty_dim_7_long.makeEmpty(7); @@ -888,17 +892,18 @@ inline void constant_scalar(std::string const &file_ending) Series s = Series("../samples/constant_scalar." + file_ending, Access::CREATE); s.setOpenPMD("2.0.0"); - auto rho = s.iterations[1].meshes["rho"][MeshRecordComponent::SCALAR]; - REQUIRE(s.iterations[1].meshes["rho"].scalar()); + auto rho = + s.writeIterations()[1].meshes["rho"][MeshRecordComponent::SCALAR]; + REQUIRE(s.writeIterations()[1].meshes["rho"].scalar()); rho.resetDataset(Dataset(Datatype::CHAR, {1, 2, 3})); rho.makeConstant(static_cast('a')); REQUIRE(rho.constant()); // mixed constant/non-constant - auto E_x = s.iterations[1].meshes["E"]["x"]; + auto E_x = s.writeIterations()[1].meshes["E"]["x"]; E_x.resetDataset(Dataset(Datatype::FLOAT, {1, 2, 3})); E_x.makeConstant(static_cast(13.37)); - auto E_y = s.iterations[1].meshes["E"]["y"]; + auto E_y = s.writeIterations()[1].meshes["E"]["y"]; E_y.resetDataset(Dataset(Datatype::UINT, {1, 2, 3})); UniquePtrWithLambda E( new unsigned int[6], [](unsigned int const *p) { delete[] p; }); @@ -907,7 +912,7 @@ inline void constant_scalar(std::string const &file_ending) E_y.storeChunk(std::move(E), {0, 0, 0}, {1, 2, 3}); // store a number of predefined attributes in E - Mesh &E_mesh = s.iterations[1].meshes["E"]; + Mesh &E_mesh = s.writeIterations()[1].meshes["E"]; // test that these can be defined successively E_mesh.setGridUnitDimension({{{UnitDimension::L, 1}}, {}, {}}); E_mesh.setGridUnitDimension( @@ -932,21 +937,21 @@ inline void constant_scalar(std::string const &file_ending) std::vector{gridUnitSI, gridUnitSI, gridUnitSI}); // constant scalar - auto pos = - s.iterations[1].particles["e"]["position"][RecordComponent::SCALAR]; + auto pos = s.writeIterations()[1] + .particles["e"]["position"][RecordComponent::SCALAR]; pos.resetDataset(Dataset(Datatype::DOUBLE, {3, 2, 1})); pos.makeConstant(static_cast(42.)); auto posOff = - s.iterations[1] + s.writeIterations()[1] .particles["e"]["positionOffset"][RecordComponent::SCALAR]; posOff.resetDataset(Dataset(Datatype::INT, {3, 2, 1})); posOff.makeConstant(static_cast(-42)); // mixed constant/non-constant - auto vel_x = s.iterations[1].particles["e"]["velocity"]["x"]; + auto vel_x = s.writeIterations()[1].particles["e"]["velocity"]["x"]; vel_x.resetDataset(Dataset(Datatype::SHORT, {3, 2, 1})); vel_x.makeConstant(static_cast(-1)); - auto vel_y = s.iterations[1].particles["e"]["velocity"]["y"]; + auto vel_y = s.writeIterations()[1].particles["e"]["velocity"]["y"]; vel_y.resetDataset(Dataset(Datatype::ULONGLONG, {3, 2, 1})); UniquePtrWithLambda vel( new unsigned long long[6], @@ -1135,7 +1140,7 @@ TEST_CASE("flush_without_position_positionOffset", "[serial]") Series s = Series( "../samples/flush_without_position_positionOffset." + file_ending, Access::CREATE); - ParticleSpecies e = s.iterations[0].particles["e"]; + ParticleSpecies e = s.writeIterations()[0].particles["e"]; RecordComponent weighting = e["weighting"][RecordComponent::SCALAR]; weighting.resetDataset(Dataset(Datatype::FLOAT, Extent{2, 2})); weighting.storeChunk( @@ -1396,7 +1401,7 @@ inline void dtype_test( // should be possible to parse without error upon opening // the series for reading { - auto E = s.iterations[0].meshes["E"]; + auto E = s.writeIterations()[0].meshes["E"]; E.setGridSpacing(std::vector{1.0, 1.0}); auto E_x = E["x"]; E_x.makeEmpty(1); @@ -1611,7 +1616,7 @@ inline void write_test( Series o = Series("../samples/serial_write." + backend, Access::CREATE, jsonCfg); - ParticleSpecies &e_1 = o.iterations[1].particles["e"]; + ParticleSpecies &e_1 = o.writeIterations()[1].particles["e"]; std::vector position_global(4); double pos{0.}; @@ -1646,7 +1651,7 @@ inline void write_test( e_1["positionOffset"]["x"].storeChunk(positionOffset_local_1, {i}, {1}); } - ParticleSpecies &e_2 = o.iterations[2].particles["e"]; + ParticleSpecies &e_2 = o.writeIterations()[2].particles["e"]; std::generate(position_global.begin(), position_global.end(), [&pos] { return pos++; @@ -1677,7 +1682,7 @@ inline void write_test( o.flush(); - ParticleSpecies &e_3 = o.iterations[3].particles["e"]; + ParticleSpecies &e_3 = o.writeIterations()[3].particles["e"]; std::generate(position_global.begin(), position_global.end(), [&pos] { return pos++; @@ -1786,7 +1791,8 @@ void test_complex(const std::string &backend) o.setAttribute( "longDoublesYouSay", std::complex(5.5, -4.55)); - auto Cflt = o.iterations[0].meshes["Cflt"][RecordComponent::SCALAR]; + auto Cflt = + o.writeIterations()[0].meshes["Cflt"][RecordComponent::SCALAR]; std::vector> cfloats(3); cfloats.at(0) = {1., 2.}; cfloats.at(1) = {-3., 4.}; @@ -1794,7 +1800,8 @@ void test_complex(const std::string &backend) Cflt.resetDataset(Dataset(Datatype::CFLOAT, {cfloats.size()})); Cflt.storeChunk(cfloats, {0}); - auto Cdbl = o.iterations[0].meshes["Cdbl"][RecordComponent::SCALAR]; + auto Cdbl = + o.writeIterations()[0].meshes["Cdbl"][RecordComponent::SCALAR]; std::vector> cdoubles(3); cdoubles.at(0) = {2., 1.}; cdoubles.at(1) = {-4., 3.}; @@ -1806,7 +1813,7 @@ void test_complex(const std::string &backend) if (o.backend() != "ADIOS2") { auto Cldbl = - o.iterations[0].meshes["Cldbl"][RecordComponent::SCALAR]; + o.writeIterations()[0].meshes["Cldbl"][RecordComponent::SCALAR]; cldoubles.at(0) = {3., 2.}; cldoubles.at(1) = {-5., 4.}; cldoubles.at(2) = {7., -6.}; @@ -2416,7 +2423,10 @@ TEST_CASE("sample_write_thetaMode", "[serial][thetaMode]") inline void bool_test(const std::string &backend) { { - Series o = Series("../samples/serial_bool." + backend, Access::CREATE); + Series o = Series( + "../samples/serial_bool." + backend, + Access::CREATE, + R"({"iteration_encoding": "variable_based"})"); o.setAttribute("Bool attribute true", true); o.setAttribute("Bool attribute false", false); @@ -2452,7 +2462,7 @@ inline void patch_test(const std::string &backend) { Series o = Series("../samples/serial_patch." + backend, Access::CREATE); - auto e = o.iterations[1].particles["e"]; + auto e = o.writeIterations()[1].particles["e"]; uint64_t const num_particles = 1u; auto dset_d = Dataset(Datatype::DOUBLE, {num_particles}); @@ -4636,7 +4646,12 @@ TEST_CASE("adios2_engines_and_file_endings") Series write( name, Access::CREATE, - json::merge("backend = \"adios2\"", jsonCfg)); + json::merge( + R"( + backend = "adios2" + iteration_encoding = "variable_based" + )", + jsonCfg)); } if (directory) { @@ -4701,7 +4716,11 @@ TEST_CASE("adios2_engines_and_file_endings") auto filesystemname = filesystemExt.empty() ? name : basename + filesystemExt; { - Series write(name, Access::CREATE, jsonCfg); + Series write( + name, + Access::CREATE, + json::merge( + R"(iteration_encoding = "variable_based")", jsonCfg)); write.close(); } if (directory) @@ -5219,7 +5238,14 @@ void serial_iterator(std::string const &file) { constexpr Extent::value_type extent = 1000; { - Series writeSeries(file, Access::CREATE); + Series writeSeries( + file, + Access::CREATE +#ifndef _WIN32 + , + R"({"rank_table": "posix_hostname"})" +#endif + ); auto iterations = writeSeries.writeIterations(); for (size_t i = 0; i < 10; ++i) { @@ -5250,6 +5276,18 @@ void serial_iterator(std::string const &file) } last_iteration_index = iteration.iterationIndex; } +#ifndef _WIN32 + if (readSeries.iterationEncoding() != IterationEncoding::fileBased) + { + auto rank_table = readSeries.rankTable(true); + for (auto const &[rank, host] : rank_table) + { + std::cout << "POST Rank '" << rank << "' written from host '" + << host << "'\n"; + } + REQUIRE(rank_table.size() == 1); + } +#endif REQUIRE(last_iteration_index == 9); REQUIRE(numberOfIterations == 10); } @@ -5293,8 +5331,7 @@ void variableBasedSingleIteration(std::string const &file) } { - Series readSeries(file, Access::READ_LINEAR); - readSeries.parseBase(); + Series readSeries(file, Access::READ_RANDOM_ACCESS); auto E_x = readSeries.iterations[0].meshes["E"]["x"]; REQUIRE(E_x.getDimensionality() == 1); @@ -5846,32 +5883,39 @@ void variableBasedSeries(std::string const &file) auto testRead = [&file, &extent]( std::string const &parseMode, - bool supportsModifiableAttributes) { + bool supportsModifiableAttributes, + Access access = Access::READ_LINEAR) { /* * Need linear read mode to access more than a single iteration in * variable-based iteration encoding. */ - Series readSeries(file, Access::READ_LINEAR, parseMode); + Series readSeries(file, access, parseMode); bool is_adios2 = readSeries.backend() == "ADIOS2"; size_t last_iteration_index = 0; - REQUIRE(!readSeries.containsAttribute("some_global")); + if (access == Access::READ_LINEAR) + { + REQUIRE(!readSeries.containsAttribute("some_global")); + } readSeries.parseBase(); REQUIRE( readSeries.getAttribute("some_global").get() == "attribute"); for (auto iteration : readSeries.readIterations()) { - if (iteration.iterationIndex > 2) - { - REQUIRE( - iteration.getAttribute("iteration_is_larger_than_two") - .get() == "it truly is"); - } - else + if (access == Access::READ_LINEAR) { - REQUIRE_FALSE(iteration.containsAttribute( - "iteration_is_larger_than_two")); + if (iteration.iterationIndex > 2) + { + REQUIRE( + iteration.getAttribute("iteration_is_larger_than_two") + .get() == "it truly is"); + } + else + { + REQUIRE_FALSE(iteration.containsAttribute( + "iteration_is_larger_than_two")); + } } // If modifiable attributes are unsupported, the attribute is @@ -5881,8 +5925,11 @@ void variableBasedSeries(std::string const &file) { REQUIRE( iteration.getAttribute("changing_value").get() == - (supportsModifiableAttributes ? iteration.iterationIndex - : 0)); + (supportsModifiableAttributes + ? (access == Access::READ_LINEAR + ? iteration.iterationIndex + : 9) + : 0)); } auto E_x = iteration.meshes["E"]["x"]; REQUIRE(E_x.getDimensionality() == 1); @@ -5900,6 +5947,13 @@ void variableBasedSeries(std::string const &file) Extent changingExtent(dimensionality, len); REQUIRE(E_y.getExtent() == changingExtent); + last_iteration_index = iteration.iterationIndex; + + if (access == Access::READ_RANDOM_ACCESS) + { + continue; + } + // this loop ensures that only the recordcomponent ["E"]["i"] is // present where i == iteration.iterationIndex for (uint64_t otherIteration = 0; otherIteration < 10; @@ -5943,8 +5997,6 @@ void variableBasedSeries(std::string const &file) REQUIRE( constantParticles.getAttribute("value").get() == iteration.iterationIndex); - - last_iteration_index = iteration.iterationIndex; } REQUIRE(last_iteration_index == (is_adios2 ? 9 : 0)); }; @@ -5964,6 +6016,14 @@ void variableBasedSeries(std::string const &file) testRead( "{\"defer_iteration_parsing\": false}", /*supportsModifiableAttributes = */ true); + testRead( + "{\"defer_iteration_parsing\": true}", + /*supportsModifiableAttributes = */ true, + Access::READ_RANDOM_ACCESS); + testRead( + "{\"defer_iteration_parsing\": false}", + /*supportsModifiableAttributes = */ true, + Access::READ_RANDOM_ACCESS); jsonConfig = "{}"; testWrite(jsonConfig); @@ -5973,6 +6033,14 @@ void variableBasedSeries(std::string const &file) testRead( "{\"defer_iteration_parsing\": false}", /*supportsModifiableAttributes = */ true); + testRead( + "{\"defer_iteration_parsing\": true}", + /*supportsModifiableAttributes = */ true, + Access::READ_RANDOM_ACCESS); + testRead( + "{\"defer_iteration_parsing\": false}", + /*supportsModifiableAttributes = */ true, + Access::READ_RANDOM_ACCESS); jsonConfig = R"( { @@ -5987,6 +6055,14 @@ void variableBasedSeries(std::string const &file) testRead( "{\"defer_iteration_parsing\": false}", /*supportsModifiableAttributes = */ false); + testRead( + "{\"defer_iteration_parsing\": true}", + /*supportsModifiableAttributes = */ false, + Access::READ_RANDOM_ACCESS); + testRead( + "{\"defer_iteration_parsing\": false}", + /*supportsModifiableAttributes = */ false, + Access::READ_RANDOM_ACCESS); } TEST_CASE("variableBasedSeries", "[serial][adios2]") @@ -6081,8 +6157,8 @@ TEST_CASE("automatically_deactivate_span", "[serial][adios2]") // automatically (de)activate span-based storeChunking { Series write("../samples/span_based.bp", Access::CREATE); - auto E_uncompressed = write.iterations[0].meshes["E"]["x"]; - auto E_compressed = write.iterations[0].meshes["E"]["y"]; + auto E_uncompressed = write.writeIterations()[0].meshes["E"]["x"]; + auto E_compressed = write.writeIterations()[0].meshes["E"]["y"]; Dataset ds{Datatype::INT, {10}}; @@ -6132,8 +6208,8 @@ TEST_CASE("automatically_deactivate_span", "[serial][adios2]") } })END"; Series write("../samples/span_based.bp", Access::CREATE, enable); - auto E_uncompressed = write.iterations[0].meshes["E"]["x"]; - auto E_compressed = write.iterations[0].meshes["E"]["y"]; + auto E_uncompressed = write.writeIterations()[0].meshes["E"]["x"]; + auto E_compressed = write.writeIterations()[0].meshes["E"]["y"]; Dataset ds{Datatype::INT, {10}}; @@ -6196,8 +6272,8 @@ TEST_CASE("automatically_deactivate_span", "[serial][adios2]") } })END"; Series write("../samples/span_based.bp", Access::CREATE, disable); - auto E_uncompressed = write.iterations[0].meshes["E"]["x"]; - auto E_compressed = write.iterations[0].meshes["E"]["y"]; + auto E_uncompressed = write.writeIterations()[0].meshes["E"]["x"]; + auto E_compressed = write.writeIterations()[0].meshes["E"]["y"]; Dataset ds{Datatype::INT, {10}}; @@ -6371,12 +6447,12 @@ TEST_CASE("iterate_nonstreaming_series", "[serial][adios2]") for (auto const &backend : testedBackends()) { iterate_nonstreaming_series( - "../samples/iterate_nonstreaming_series_filebased_%T." + + "../samples/iterate_nonstreaming_series/serial_filebased_%T." + backend.extension, false, backend.jsonBaseConfig()); iterate_nonstreaming_series( - "../samples/iterate_nonstreaming_series_groupbased." + + "../samples/iterate_nonstreaming_series/serial_groupbased." + backend.extension, false, backend.jsonBaseConfig()); @@ -6384,13 +6460,15 @@ TEST_CASE("iterate_nonstreaming_series", "[serial][adios2]") if (backend.extension == "bp") { iterate_nonstreaming_series( - "../samples/iterate_nonstreaming_series_filebased_bp5_%T." + + "../samples/iterate_nonstreaming_series/" + "serial_filebased_bp5_%T." + backend.extension, false, json::merge( backend.jsonBaseConfig(), "adios2.engine.type = \"bp5\"")); iterate_nonstreaming_series( - "../samples/iterate_nonstreaming_series_groupbased_bp5." + + "../samples/iterate_nonstreaming_series/" + "serial_groupbased_bp5." + backend.extension, false, json::merge( @@ -6400,7 +6478,7 @@ TEST_CASE("iterate_nonstreaming_series", "[serial][adios2]") } #if openPMD_HAVE_ADIOS2 iterate_nonstreaming_series( - "../samples/iterate_nonstreaming_series_variablebased.bp", + "../samples/iterate_nonstreaming_series/serial_variablebased.bp", true, R"({"backend": "adios2"})"); #endif @@ -6649,7 +6727,7 @@ void deferred_parsing(std::string const &extension) Series series(basename + "%06T." + extension, Access::CREATE); std::vector buffer(20); std::iota(buffer.begin(), buffer.end(), 0.f); - auto dataset = series.iterations[0].meshes["E"]["x"]; + auto dataset = series.writeIterations()[0].meshes["E"]["x"]; dataset.resetDataset({Datatype::FLOAT, {20}}); dataset.storeChunk(buffer, {0}, {20}); series.flush(); @@ -6994,7 +7072,7 @@ TEST_CASE("unfinished_iteration_test", "[serial]") unfinished_iteration_test( "bp", IterationEncoding::groupBased, - R"({"backend": "adios2"})", + R"({"backend": "adios2", "iteration_encoding": "group_based"})", /* test_linear_access = */ false); unfinished_iteration_test( "bp5",