From 420267732d876e0f04d0d62ab80377940915c359 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 7 Aug 2024 17:48:03 +0200 Subject: [PATCH 01/20] Make shape an optional attribute for constant components --- include/openPMD/backend/Attributable.hpp | 29 ++++++++ src/Iteration.cpp | 3 +- src/ParticleSpecies.cpp | 3 +- src/RecordComponent.cpp | 94 ++++++++++++++++-------- 4 files changed, 95 insertions(+), 34 deletions(-) diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index d34d5bb48f..40195dd8cc 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -20,6 +20,7 @@ */ #pragma once +#include "openPMD/Error.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/ThrowError.hpp" #include "openPMD/auxiliary/OutOfRangeMsg.hpp" @@ -29,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -105,6 +107,7 @@ namespace internal friend class openPMD::Attributable; using SharedData_t = std::shared_ptr; + using A_MAP = SharedData_t::element_type::A_MAP; public: AttributableData(); @@ -155,6 +158,32 @@ namespace internal std::shared_ptr(self, [](auto const *) {})); return res; } + + inline auto attributes() -> A_MAP & + { + return operator*().m_attributes; + } + [[nodiscard]] inline auto attributes() const -> A_MAP const & + { + return operator*().m_attributes; + } + [[nodiscard]] inline auto readAttribute(std::string const &name) const + -> Attribute const & + { + auto const &attr = attributes(); + if (auto it = attr.find(name); it != attr.end()) + { + return it->second; + } + else + { + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::NotFound, + std::nullopt, + "Not found: '" + name + "'."); + } + } }; template diff --git a/src/Iteration.cpp b/src/Iteration.cpp index d50d7d6a2f..1ece5b4aa2 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -678,8 +678,7 @@ void Iteration::readMeshes(std::string const &meshesPath) auto att_begin = aList.attributes->begin(); auto att_end = aList.attributes->end(); auto value = std::find(att_begin, att_end, "value"); - auto shape = std::find(att_begin, att_end, "shape"); - if (value != att_end && shape != att_end) + if (value != att_end) { MeshRecordComponent &mrc = m; IOHandler()->enqueue(IOTask(&mrc, pOpen)); diff --git a/src/ParticleSpecies.cpp b/src/ParticleSpecies.cpp index 99ec6dfe6d..1009ff89a3 100644 --- a/src/ParticleSpecies.cpp +++ b/src/ParticleSpecies.cpp @@ -76,8 +76,7 @@ void ParticleSpecies::read() auto att_begin = aList.attributes->begin(); auto att_end = aList.attributes->end(); auto value = std::find(att_begin, att_end, "value"); - auto shape = std::find(att_begin, att_end, "shape"); - if (value != att_end && shape != att_end) + if (value != att_end) { RecordComponent &rc = r; IOHandler()->enqueue(IOTask(&rc, pOpen)); diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index 6d11fbfa77..51b46ecfdc 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -118,7 +118,14 @@ RecordComponent &RecordComponent::resetDataset(Dataset d) throw std::runtime_error("Dataset extent must be at least 1D."); if (d.empty()) { - if (d.dtype != Datatype::UNDEFINED) + if (d.extent.empty()) + { + throw error::Internal( + "A zero-dimensional dataset is not to be considered empty, but " + "undefined. This is an internal safeguard against future " + "changes that might not consider this."); + } + else if (d.dtype != Datatype::UNDEFINED) { return makeEmpty(std::move(d)); } @@ -300,6 +307,13 @@ void RecordComponent::flush( { setUnitSI(1); } + auto constant_component_write_shape = [&]() { + auto extent = getExtent(); + return !extent.empty() && + std::none_of(extent.begin(), extent.end(), [](auto val) { + return val == Dataset::JOINED_DIMENSION; + }); + }; if (!written()) { if (constant()) @@ -319,16 +333,19 @@ void RecordComponent::flush( Operation::WRITE_ATT>::ChangesOverSteps::IfPossible; } IOHandler()->enqueue(IOTask(this, aWrite)); - aWrite.name = "shape"; - Attribute a(getExtent()); - aWrite.dtype = a.dtype; - aWrite.m_resource = a.getAny(); - if (isVBased) + if (constant_component_write_shape()) { - aWrite.changesOverSteps = Parameter< - Operation::WRITE_ATT>::ChangesOverSteps::IfPossible; + aWrite.name = "shape"; + Attribute a(getExtent()); + aWrite.dtype = a.dtype; + aWrite.m_resource = a.getAny(); + if (isVBased) + { + aWrite.changesOverSteps = Parameter< + Operation::WRITE_ATT>::ChangesOverSteps::IfPossible; + } + IOHandler()->enqueue(IOTask(this, aWrite)); } - IOHandler()->enqueue(IOTask(this, aWrite)); } else { @@ -343,6 +360,13 @@ void RecordComponent::flush( { if (constant()) { + if (!constant_component_write_shape()) + { + throw error::WrongAPIUsage( + "Extended constant component from a previous shape to " + "one that cannot be written (empty or with joined " + "dimension)."); + } bool isVBased = retrieveSeries().iterationEncoding() == IterationEncoding::variableBased; Parameter aWrite; @@ -407,28 +431,35 @@ namespace }; } // namespace +inline void breakpoint() +{} + void RecordComponent::readBase(bool require_unit_si) { using DT = Datatype; - // auto & rc = get(); - Parameter aRead; + auto &rc = get(); - if (constant() && !empty()) - { - aRead.name = "value"; - IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(internal::defaultFlushParams); + readAttributes(ReadMode::FullyReread); - Attribute a(Attribute::from_any, *aRead.m_resource); - DT dtype = *aRead.dtype; + auto read_constant = + [&]() // comment for forcing clang-format into putting a newline here + { + Attribute a = rc.readAttribute("value"); + DT dtype = a.dtype; setWritten(false, Attributable::EnqueueAsynchronously::No); switchNonVectorType(dtype, *this, a); setWritten(true, Attributable::EnqueueAsynchronously::No); - aRead.name = "shape"; - IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(internal::defaultFlushParams); - a = Attribute(Attribute::from_any, *aRead.m_resource); + if (!containsAttribute("shape")) + { + setWritten(false, Attributable::EnqueueAsynchronously::No); + resetDataset(Dataset(dtype, {})); + setWritten(true, Attributable::EnqueueAsynchronously::No); + + return; + } + + a = rc.attributes().at("shape"); Extent e; // uint64_t check @@ -438,7 +469,7 @@ void RecordComponent::readBase(bool require_unit_si) else { std::ostringstream oss; - oss << "Unexpected datatype (" << *aRead.dtype + oss << "Unexpected datatype (" << a.dtype << ") for attribute 'shape' (" << determineDatatype() << " aka uint64_t)"; throw error::ReadError( @@ -451,9 +482,13 @@ void RecordComponent::readBase(bool require_unit_si) setWritten(false, Attributable::EnqueueAsynchronously::No); resetDataset(Dataset(dtype, e)); setWritten(true, Attributable::EnqueueAsynchronously::No); - } + }; - readAttributes(ReadMode::FullyReread); + if (constant() && !empty()) + { + breakpoint(); + read_constant(); + } if (require_unit_si) { @@ -467,7 +502,8 @@ void RecordComponent::readBase(bool require_unit_si) "'" + myPath().openPMDPath() + "'."); } - if (!getAttribute("unitSI").getOptional().has_value()) + if (auto attr = getAttribute("unitSI"); + !attr.getOptional().has_value()) { throw error::ReadError( error::AffectedObject::Attribute, @@ -475,10 +511,8 @@ void RecordComponent::readBase(bool require_unit_si) {}, "Unexpected Attribute datatype for 'unitSI' (expected double, " "found " + - datatypeToString( - Attribute(Attribute::from_any, *aRead.m_resource) - .dtype) + - ") in '" + myPath().openPMDPath() + "'."); + datatypeToString(attr.dtype) + ") in '" + + myPath().openPMDPath() + "'."); } } } From a086f9f39f0c15b9059faa4896b30b961ae82574 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 11 Oct 2024 13:34:05 +0200 Subject: [PATCH 02/20] Adhere to the standard changes more closely --- include/openPMD/RecordComponent.hpp | 3 ++- .../openPMD/backend/MeshRecordComponent.hpp | 3 ++- src/Iteration.cpp | 6 +++++- src/Mesh.cpp | 17 +++++++++++------ src/ParticleSpecies.cpp | 2 ++ src/Record.cpp | 17 +++++++++++------ src/RecordComponent.cpp | 19 ++++++++++++++++++- src/backend/MeshRecordComponent.cpp | 6 ++++-- src/backend/PatchRecord.cpp | 6 +++--- 9 files changed, 58 insertions(+), 21 deletions(-) diff --git a/include/openPMD/RecordComponent.hpp b/include/openPMD/RecordComponent.hpp index f319e10cff..10993424b6 100644 --- a/include/openPMD/RecordComponent.hpp +++ b/include/openPMD/RecordComponent.hpp @@ -483,7 +483,8 @@ class RecordComponent : public BaseRecordComponent static constexpr char const *const SCALAR = "\vScalar"; protected: - void flush(std::string const &, internal::FlushParams const &); + void + flush(std::string const &, internal::FlushParams const &, bool is_scalar); void read(bool require_unit_si); private: diff --git a/include/openPMD/backend/MeshRecordComponent.hpp b/include/openPMD/backend/MeshRecordComponent.hpp index d05163d754..f8229635ba 100644 --- a/include/openPMD/backend/MeshRecordComponent.hpp +++ b/include/openPMD/backend/MeshRecordComponent.hpp @@ -47,7 +47,8 @@ class MeshRecordComponent : public RecordComponent MeshRecordComponent(); MeshRecordComponent(NoInit); void read(); - void flush(std::string const &, internal::FlushParams const &); + void + flush(std::string const &, internal::FlushParams const &, bool is_scalar); public: ~MeshRecordComponent() override = default; diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 1ece5b4aa2..d40cf1327b 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -675,10 +675,14 @@ void Iteration::readMeshes(std::string const &meshesPath) IOHandler()->enqueue(IOTask(&m, aList)); IOHandler()->flush(internal::defaultFlushParams); + // Find constant scalar meshes. shape generally required for meshes, + // shape also required for scalars. + // https://github.com/openPMD/openPMD-standard/pull/289 auto att_begin = aList.attributes->begin(); auto att_end = aList.attributes->end(); auto value = std::find(att_begin, att_end, "value"); - if (value != att_end) + auto shape = std::find(att_begin, att_end, "shape"); + if (value != att_end && shape != att_end) { MeshRecordComponent &mrc = m; IOHandler()->enqueue(IOTask(&mrc, pOpen)); diff --git a/src/Mesh.cpp b/src/Mesh.cpp index cfd1735898..fd1bdc5241 100644 --- a/src/Mesh.cpp +++ b/src/Mesh.cpp @@ -375,12 +375,14 @@ void Mesh::flush_impl( auto &m = get(); if (m.m_datasetDefined) { - T_RecordComponent::flush(SCALAR, flushParams); + T_RecordComponent::flush( + SCALAR, flushParams, /* is_scalar = */ true); } else { for (auto &comp : *this) - comp.second.flush(comp.first, flushParams); + comp.second.flush( + comp.first, flushParams, /* is_scalar = */ false); } } else @@ -390,7 +392,7 @@ void Mesh::flush_impl( if (scalar()) { MeshRecordComponent &mrc = *this; - mrc.flush(name, flushParams); + mrc.flush(name, flushParams, /* is_scalar = */ true); } else { @@ -400,7 +402,8 @@ void Mesh::flush_impl( for (auto &comp : *this) { comp.second.parent() = &this->writable(); - comp.second.flush(comp.first, flushParams); + comp.second.flush( + comp.first, flushParams, /* is_scalar = */ false); } } } @@ -408,12 +411,14 @@ void Mesh::flush_impl( { if (scalar()) { - T_RecordComponent::flush(name, flushParams); + T_RecordComponent::flush( + name, flushParams, /* is_scalar = */ true); } else { for (auto &comp : *this) - comp.second.flush(comp.first, flushParams); + comp.second.flush( + comp.first, flushParams, /* is_scalar = */ false); } } if (!containsAttribute("gridUnitSI")) diff --git a/src/ParticleSpecies.cpp b/src/ParticleSpecies.cpp index 1009ff89a3..edd7c291bd 100644 --- a/src/ParticleSpecies.cpp +++ b/src/ParticleSpecies.cpp @@ -76,6 +76,8 @@ void ParticleSpecies::read() auto att_begin = aList.attributes->begin(); auto att_end = aList.attributes->end(); auto value = std::find(att_begin, att_end, "value"); + // @todo see this comment: + // https://github.com/openPMD/openPMD-standard/pull/289#issuecomment-2407263974 if (value != att_end) { RecordComponent &rc = r; diff --git a/src/Record.cpp b/src/Record.cpp index c6baedec7c..c0f4395189 100644 --- a/src/Record.cpp +++ b/src/Record.cpp @@ -60,12 +60,14 @@ void Record::flush_impl( { if (scalar()) { - T_RecordComponent::flush(SCALAR, flushParams); + T_RecordComponent::flush( + SCALAR, flushParams, /* is_scalar = */ true); } else { for (auto &comp : *this) - comp.second.flush(comp.first, flushParams); + comp.second.flush( + comp.first, flushParams, /* is_scalar = */ false); } } else @@ -75,7 +77,7 @@ void Record::flush_impl( if (scalar()) { RecordComponent &rc = *this; - rc.flush(name, flushParams); + rc.flush(name, flushParams, /* is_scalar = */ true); } else { @@ -85,7 +87,8 @@ void Record::flush_impl( for (auto &comp : *this) { comp.second.parent() = getWritable(this); - comp.second.flush(comp.first, flushParams); + comp.second.flush( + comp.first, flushParams, /* is_scalar = */ false); } } } @@ -94,12 +97,14 @@ void Record::flush_impl( if (scalar()) { - T_RecordComponent::flush(name, flushParams); + T_RecordComponent::flush( + name, flushParams, /* is_scalar = */ true); } else { for (auto &comp : *this) - comp.second.flush(comp.first, flushParams); + comp.second.flush( + comp.first, flushParams, /* is_scalar = */ false); } } diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index 51b46ecfdc..a99cc9dfef 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -260,7 +260,9 @@ bool RecordComponent::empty() const } void RecordComponent::flush( - std::string const &name, internal::FlushParams const &flushParams) + std::string const &name, + internal::FlushParams const &flushParams, + bool is_scalar) { if (!dirtyRecursive()) { @@ -308,6 +310,21 @@ void RecordComponent::flush( setUnitSI(1); } auto constant_component_write_shape = [&]() { + if (is_scalar) + { + // Must write shape in any case: + // 1. Non-scalar constant components can be distinguished from + // normal components by checking if the backend reports a + // group or a dataset. This does not work for scalar constant + // components, so the parser needs to check if the attributes + // value and shape are there. If they're not, the group is + // not considered as a constant component. + // 2. Scalar constant components are required to write the shape + // by standard anyway since the standard requires that at + // least one component in a record have a shape. For scalars, + // there is only one component, so it must have a shape. + return true; + } auto extent = getExtent(); return !extent.empty() && std::none_of(extent.begin(), extent.end(), [](auto val) { diff --git a/src/backend/MeshRecordComponent.cpp b/src/backend/MeshRecordComponent.cpp index 947db30f7c..0891bc5685 100644 --- a/src/backend/MeshRecordComponent.cpp +++ b/src/backend/MeshRecordComponent.cpp @@ -70,7 +70,9 @@ void MeshRecordComponent::read() } void MeshRecordComponent::flush( - std::string const &name, internal::FlushParams const ¶ms) + std::string const &name, + internal::FlushParams const ¶ms, + bool is_scalar) { if (!dirtyRecursive()) { @@ -81,7 +83,7 @@ void MeshRecordComponent::flush( { setPosition(std::vector{0}); } - RecordComponent::flush(name, params); + RecordComponent::flush(name, params, is_scalar); } template diff --git a/src/backend/PatchRecord.cpp b/src/backend/PatchRecord.cpp index e674828181..4304976972 100644 --- a/src/backend/PatchRecord.cpp +++ b/src/backend/PatchRecord.cpp @@ -45,17 +45,17 @@ void PatchRecord::flush_impl( { return; } - if (!this->datasetDefined()) + if (!this->scalar()) { if (IOHandler()->m_frontendAccess != Access::READ_ONLY) Container::flush( path, flushParams); // warning (clang-tidy-10): // bugprone-parent-virtual-call for (auto &comp : *this) - comp.second.flush(comp.first, flushParams); + comp.second.flush(comp.first, flushParams, /* is_scalar = */ false); } else - T_RecordComponent::flush(path, flushParams); + T_RecordComponent::flush(path, flushParams, /* is_scalar = */ true); if (flushParams.flushLevel != FlushLevel::SkeletonOnly) { setDirty(false); From dd9ed341bc0417213d8fdd09040e59a880edaf52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 4 Apr 2025 10:54:41 +0200 Subject: [PATCH 03/20] Remove is_scalar distinction from flushing again Both reasons for making the distinction now no longer valid --- include/openPMD/RecordComponent.hpp | 3 +-- .../openPMD/backend/MeshRecordComponent.hpp | 3 +-- src/Mesh.cpp | 17 ++++++----------- src/Record.cpp | 17 ++++++----------- src/RecordComponent.cpp | 19 +------------------ src/backend/MeshRecordComponent.cpp | 6 ++---- src/backend/PatchRecord.cpp | 4 ++-- 7 files changed, 19 insertions(+), 50 deletions(-) diff --git a/include/openPMD/RecordComponent.hpp b/include/openPMD/RecordComponent.hpp index 10993424b6..f319e10cff 100644 --- a/include/openPMD/RecordComponent.hpp +++ b/include/openPMD/RecordComponent.hpp @@ -483,8 +483,7 @@ class RecordComponent : public BaseRecordComponent static constexpr char const *const SCALAR = "\vScalar"; protected: - void - flush(std::string const &, internal::FlushParams const &, bool is_scalar); + void flush(std::string const &, internal::FlushParams const &); void read(bool require_unit_si); private: diff --git a/include/openPMD/backend/MeshRecordComponent.hpp b/include/openPMD/backend/MeshRecordComponent.hpp index f8229635ba..d05163d754 100644 --- a/include/openPMD/backend/MeshRecordComponent.hpp +++ b/include/openPMD/backend/MeshRecordComponent.hpp @@ -47,8 +47,7 @@ class MeshRecordComponent : public RecordComponent MeshRecordComponent(); MeshRecordComponent(NoInit); void read(); - void - flush(std::string const &, internal::FlushParams const &, bool is_scalar); + void flush(std::string const &, internal::FlushParams const &); public: ~MeshRecordComponent() override = default; diff --git a/src/Mesh.cpp b/src/Mesh.cpp index fd1bdc5241..cfd1735898 100644 --- a/src/Mesh.cpp +++ b/src/Mesh.cpp @@ -375,14 +375,12 @@ void Mesh::flush_impl( auto &m = get(); if (m.m_datasetDefined) { - T_RecordComponent::flush( - SCALAR, flushParams, /* is_scalar = */ true); + T_RecordComponent::flush(SCALAR, flushParams); } else { for (auto &comp : *this) - comp.second.flush( - comp.first, flushParams, /* is_scalar = */ false); + comp.second.flush(comp.first, flushParams); } } else @@ -392,7 +390,7 @@ void Mesh::flush_impl( if (scalar()) { MeshRecordComponent &mrc = *this; - mrc.flush(name, flushParams, /* is_scalar = */ true); + mrc.flush(name, flushParams); } else { @@ -402,8 +400,7 @@ void Mesh::flush_impl( for (auto &comp : *this) { comp.second.parent() = &this->writable(); - comp.second.flush( - comp.first, flushParams, /* is_scalar = */ false); + comp.second.flush(comp.first, flushParams); } } } @@ -411,14 +408,12 @@ void Mesh::flush_impl( { if (scalar()) { - T_RecordComponent::flush( - name, flushParams, /* is_scalar = */ true); + T_RecordComponent::flush(name, flushParams); } else { for (auto &comp : *this) - comp.second.flush( - comp.first, flushParams, /* is_scalar = */ false); + comp.second.flush(comp.first, flushParams); } } if (!containsAttribute("gridUnitSI")) diff --git a/src/Record.cpp b/src/Record.cpp index c0f4395189..c6baedec7c 100644 --- a/src/Record.cpp +++ b/src/Record.cpp @@ -60,14 +60,12 @@ void Record::flush_impl( { if (scalar()) { - T_RecordComponent::flush( - SCALAR, flushParams, /* is_scalar = */ true); + T_RecordComponent::flush(SCALAR, flushParams); } else { for (auto &comp : *this) - comp.second.flush( - comp.first, flushParams, /* is_scalar = */ false); + comp.second.flush(comp.first, flushParams); } } else @@ -77,7 +75,7 @@ void Record::flush_impl( if (scalar()) { RecordComponent &rc = *this; - rc.flush(name, flushParams, /* is_scalar = */ true); + rc.flush(name, flushParams); } else { @@ -87,8 +85,7 @@ void Record::flush_impl( for (auto &comp : *this) { comp.second.parent() = getWritable(this); - comp.second.flush( - comp.first, flushParams, /* is_scalar = */ false); + comp.second.flush(comp.first, flushParams); } } } @@ -97,14 +94,12 @@ void Record::flush_impl( if (scalar()) { - T_RecordComponent::flush( - name, flushParams, /* is_scalar = */ true); + T_RecordComponent::flush(name, flushParams); } else { for (auto &comp : *this) - comp.second.flush( - comp.first, flushParams, /* is_scalar = */ false); + comp.second.flush(comp.first, flushParams); } } diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index a99cc9dfef..51b46ecfdc 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -260,9 +260,7 @@ bool RecordComponent::empty() const } void RecordComponent::flush( - std::string const &name, - internal::FlushParams const &flushParams, - bool is_scalar) + std::string const &name, internal::FlushParams const &flushParams) { if (!dirtyRecursive()) { @@ -310,21 +308,6 @@ void RecordComponent::flush( setUnitSI(1); } auto constant_component_write_shape = [&]() { - if (is_scalar) - { - // Must write shape in any case: - // 1. Non-scalar constant components can be distinguished from - // normal components by checking if the backend reports a - // group or a dataset. This does not work for scalar constant - // components, so the parser needs to check if the attributes - // value and shape are there. If they're not, the group is - // not considered as a constant component. - // 2. Scalar constant components are required to write the shape - // by standard anyway since the standard requires that at - // least one component in a record have a shape. For scalars, - // there is only one component, so it must have a shape. - return true; - } auto extent = getExtent(); return !extent.empty() && std::none_of(extent.begin(), extent.end(), [](auto val) { diff --git a/src/backend/MeshRecordComponent.cpp b/src/backend/MeshRecordComponent.cpp index 0891bc5685..947db30f7c 100644 --- a/src/backend/MeshRecordComponent.cpp +++ b/src/backend/MeshRecordComponent.cpp @@ -70,9 +70,7 @@ void MeshRecordComponent::read() } void MeshRecordComponent::flush( - std::string const &name, - internal::FlushParams const ¶ms, - bool is_scalar) + std::string const &name, internal::FlushParams const ¶ms) { if (!dirtyRecursive()) { @@ -83,7 +81,7 @@ void MeshRecordComponent::flush( { setPosition(std::vector{0}); } - RecordComponent::flush(name, params, is_scalar); + RecordComponent::flush(name, params); } template diff --git a/src/backend/PatchRecord.cpp b/src/backend/PatchRecord.cpp index 4304976972..87546ca177 100644 --- a/src/backend/PatchRecord.cpp +++ b/src/backend/PatchRecord.cpp @@ -52,10 +52,10 @@ void PatchRecord::flush_impl( path, flushParams); // warning (clang-tidy-10): // bugprone-parent-virtual-call for (auto &comp : *this) - comp.second.flush(comp.first, flushParams, /* is_scalar = */ false); + comp.second.flush(comp.first, flushParams); } else - T_RecordComponent::flush(path, flushParams, /* is_scalar = */ true); + T_RecordComponent::flush(path, flushParams); if (flushParams.flushLevel != FlushLevel::SkeletonOnly) { setDirty(false); From 1dadf066ac9e27eb5f993f36d26fc9c877be8555 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 4 Apr 2025 11:09:24 +0200 Subject: [PATCH 04/20] Cleanup --- src/ParticleSpecies.cpp | 2 -- src/RecordComponent.cpp | 10 ++-------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/src/ParticleSpecies.cpp b/src/ParticleSpecies.cpp index edd7c291bd..1009ff89a3 100644 --- a/src/ParticleSpecies.cpp +++ b/src/ParticleSpecies.cpp @@ -76,8 +76,6 @@ void ParticleSpecies::read() auto att_begin = aList.attributes->begin(); auto att_end = aList.attributes->end(); auto value = std::find(att_begin, att_end, "value"); - // @todo see this comment: - // https://github.com/openPMD/openPMD-standard/pull/289#issuecomment-2407263974 if (value != att_end) { RecordComponent &rc = r; diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index 51b46ecfdc..9184f015a2 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -122,7 +122,7 @@ RecordComponent &RecordComponent::resetDataset(Dataset d) { throw error::Internal( "A zero-dimensional dataset is not to be considered empty, but " - "undefined. This is an internal safeguard against future " + "undefined. This error is an internal safeguard against future " "changes that might not consider this."); } else if (d.dtype != Datatype::UNDEFINED) @@ -431,9 +431,6 @@ namespace }; } // namespace -inline void breakpoint() -{} - void RecordComponent::readBase(bool require_unit_si) { using DT = Datatype; @@ -441,9 +438,7 @@ void RecordComponent::readBase(bool require_unit_si) readAttributes(ReadMode::FullyReread); - auto read_constant = - [&]() // comment for forcing clang-format into putting a newline here - { + auto read_constant = [&]() { Attribute a = rc.readAttribute("value"); DT dtype = a.dtype; setWritten(false, Attributable::EnqueueAsynchronously::No); @@ -486,7 +481,6 @@ void RecordComponent::readBase(bool require_unit_si) if (constant() && !empty()) { - breakpoint(); read_constant(); } From 790a5b4d839acd31ebebd05dd8222e68d1d7035e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 4 Apr 2025 12:29:13 +0200 Subject: [PATCH 05/20] Homogenize extents --- include/openPMD/Record.hpp | 3 +- include/openPMD/RecordComponent.hpp | 11 ++++ include/openPMD/backend/Attributable.hpp | 2 + src/Mesh.cpp | 7 +++ src/ParticleSpecies.cpp | 12 +++- src/Record.cpp | 12 +++- src/RecordComponent.cpp | 78 +++++++++++++++++++++++- 7 files changed, 120 insertions(+), 5 deletions(-) diff --git a/include/openPMD/Record.hpp b/include/openPMD/Record.hpp index 791c4c15f8..246b458b5f 100644 --- a/include/openPMD/Record.hpp +++ b/include/openPMD/Record.hpp @@ -53,7 +53,8 @@ class Record : public BaseRecord void flush_impl(std::string const &, internal::FlushParams const &) override; - void read(); + + [[nodiscard]] internal::HomogenizeExtents read(); }; // Record template diff --git a/include/openPMD/RecordComponent.hpp b/include/openPMD/RecordComponent.hpp index f319e10cff..4eda75f327 100644 --- a/include/openPMD/RecordComponent.hpp +++ b/include/openPMD/RecordComponent.hpp @@ -105,6 +105,17 @@ namespace internal }; template class BaseRecordData; + + struct HomogenizeExtents + { + std::deque without_extent; + std::optional retrieved_extent; + + void check_extent(Attributable const &callsite, RecordComponent &); + auto merge(Attributable const &callsite, HomogenizeExtents) + -> HomogenizeExtents &; + void homogenize(Attributable const &callsite) &&; + }; } // namespace internal template diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index 40195dd8cc..8c519a9e12 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -56,6 +56,7 @@ namespace internal { class IterationData; class SeriesData; + struct HomogenizeExtents; class SharedAttributableData { @@ -242,6 +243,7 @@ class Attributable friend class StatefulSnapshotsContainer; friend class internal::AttributableData; friend class Snapshots; + friend struct internal::HomogenizeExtents; protected: // tag for internal constructor diff --git a/src/Mesh.cpp b/src/Mesh.cpp index cfd1735898..809a418868 100644 --- a/src/Mesh.cpp +++ b/src/Mesh.cpp @@ -21,6 +21,7 @@ #include "openPMD/Mesh.hpp" #include "openPMD/Error.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/RecordComponent.hpp" #include "openPMD/Series.hpp" #include "openPMD/ThrowError.hpp" #include "openPMD/UnitDimension.hpp" @@ -434,6 +435,7 @@ void Mesh::flush_impl( void Mesh::read() { + internal::HomogenizeExtents homogenizeExtents; internal::EraseStaleEntries map{*this}; using DT = Datatype; @@ -604,6 +606,7 @@ void Mesh::read() if (scalar()) { T_RecordComponent::read(); + homogenizeExtents.check_extent(*this, *this); } else { @@ -629,6 +632,7 @@ void Mesh::read() << err.what() << std::endl; map.forget(component); } + homogenizeExtents.check_extent(*this, rc); } Parameter dList; @@ -656,9 +660,12 @@ void Mesh::read() << err.what() << std::endl; map.forget(component); } + homogenizeExtents.check_extent(*this, rc); } } + std::move(homogenizeExtents).homogenize(*this); + readBase(); readAttributes(ReadMode::FullyReread); diff --git a/src/ParticleSpecies.cpp b/src/ParticleSpecies.cpp index 1009ff89a3..d1645deae9 100644 --- a/src/ParticleSpecies.cpp +++ b/src/ParticleSpecies.cpp @@ -19,6 +19,7 @@ * If not, see . */ #include "openPMD/ParticleSpecies.hpp" +#include "openPMD/RecordComponent.hpp" #include "openPMD/Series.hpp" #include "openPMD/auxiliary/DerefDynamicCast.hpp" #include "openPMD/backend/Writable.hpp" @@ -35,6 +36,7 @@ ParticleSpecies::ParticleSpecies() void ParticleSpecies::read() { + internal::HomogenizeExtents homogenizeExtents; /* obtain all non-scalar records */ Parameter pList; IOHandler()->enqueue(IOTask(this, pList)); @@ -83,9 +85,10 @@ void ParticleSpecies::read() IOHandler()->flush(internal::defaultFlushParams); rc.get().m_isConstant = true; } + internal::HomogenizeExtents recordExtents; try { - r.read(); + recordExtents = r.read(); } catch (error::ReadError const &err) { @@ -95,6 +98,7 @@ void ParticleSpecies::read() map.forget(record_name); } + homogenizeExtents.merge(*this, std::move(recordExtents)); } } @@ -114,6 +118,7 @@ void ParticleSpecies::read() Parameter dOpen; for (auto const &record_name : *dList.datasets) { + internal::HomogenizeExtents recordExtents; try { Record &r = map[record_name]; @@ -126,7 +131,7 @@ void ParticleSpecies::read() rc.setWritten(false, Attributable::EnqueueAsynchronously::No); rc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); rc.setWritten(true, Attributable::EnqueueAsynchronously::No); - r.read(); + recordExtents = r.read(); } catch (error::ReadError const &err) { @@ -138,8 +143,11 @@ void ParticleSpecies::read() //(*this)[record_name].erase(RecordComponent::SCALAR); // this->erase(record_name); } + homogenizeExtents.merge(*this, std::move(recordExtents)); } + std::move(homogenizeExtents).homogenize(*this); + readAttributes(ReadMode::FullyReread); } diff --git a/src/Record.cpp b/src/Record.cpp index c6baedec7c..d9a1eebb42 100644 --- a/src/Record.cpp +++ b/src/Record.cpp @@ -19,7 +19,9 @@ * If not, see . */ #include "openPMD/Record.hpp" +#include "openPMD/Error.hpp" #include "openPMD/RecordComponent.hpp" +#include "openPMD/ThrowError.hpp" #include "openPMD/UnitDimension.hpp" #include "openPMD/backend/BaseRecord.hpp" @@ -107,8 +109,12 @@ void Record::flush_impl( } } -void Record::read() +auto Record::read() -> internal::HomogenizeExtents { + internal::HomogenizeExtents res; + auto check_extent = [&](RecordComponent &rc) { + res.check_extent(*this, rc); + }; if (scalar()) { /* using operator[] will incorrectly update parent */ @@ -122,6 +128,7 @@ void Record::read() "due to read error:\n" << err.what() << std::endl; } + check_extent(*this); } else { @@ -147,6 +154,7 @@ void Record::read() << err.what() << std::endl; this->container().erase(component); } + check_extent(rc); } Parameter dList; @@ -174,12 +182,14 @@ void Record::read() << err.what() << std::endl; this->container().erase(component); } + check_extent(rc); } } readBase(); readAttributes(ReadMode::FullyReread); + return res; } template class BaseRecord; diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index 9184f015a2..314a9b0d6d 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -59,6 +59,82 @@ namespace internal a.setDirtyRecursive(true); m_chunks.push(std::move(task)); } + + void HomogenizeExtents::check_extent( + Attributable const &callsite, RecordComponent &rc) + { + auto extent = rc.getExtent(); + if (extent.empty()) + { + without_extent.emplace_back(rc); + } + else if (retrieved_extent.has_value()) + { + if (extent != *retrieved_extent) + { + throw error::ReadError( + error::AffectedObject::Group, + error::Reason::UnexpectedContent, + std::nullopt, + "Inconsistent extents found for Record '" + + callsite.myPath().openPMDPath() + "'."); + } + } + else + { + retrieved_extent = std::move(extent); + } + } + + auto HomogenizeExtents::merge( + Attributable const &callsite, HomogenizeExtents other) + -> HomogenizeExtents & + { + if (retrieved_extent.has_value() && other.retrieved_extent.has_value()) + { + if (*retrieved_extent != *other.retrieved_extent) + { + throw error::ReadError( + error::AffectedObject::Group, + error::Reason::UnexpectedContent, + std::nullopt, + "Inconsistent extents found for Record '" + + callsite.myPath().openPMDPath() + "'."); + } + } + else if (!retrieved_extent.has_value()) + { + retrieved_extent = std::move(other.retrieved_extent); + } + + for (auto &rc : other.without_extent) + { + this->without_extent.emplace_back(std::move(rc)); + } + return *this; + } + + void HomogenizeExtents::homogenize(Attributable const &callsite) && + { + if (!retrieved_extent.has_value()) + { + throw error::ReadError( + error::AffectedObject::Group, + error::Reason::UnexpectedContent, + std::nullopt, + "No extent found for any component contained in '" + + callsite.myPath().openPMDPath() + "'."); + } + auto &ext = *retrieved_extent; + for (auto &rc : without_extent) + { + rc.setWritten(false, Attributable::EnqueueAsynchronously::No); + rc.resetDataset(Dataset(Datatype::UNDEFINED, ext)); + rc.setWritten(true, Attributable::EnqueueAsynchronously::No); + } + without_extent.clear(); + } + } // namespace internal template @@ -178,7 +254,7 @@ Extent RecordComponent::getExtent() const } else { - return {1}; + return {}; } } From 5cc9e3b1d6f55905ae7814c005d705f1c09779fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 4 Apr 2025 14:33:30 +0200 Subject: [PATCH 06/20] Check for extent consistency, fill in undefined extents --- include/openPMD/RecordComponent.hpp | 3 ++ src/RecordComponent.cpp | 48 +++++++++++++++++++++-------- 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/include/openPMD/RecordComponent.hpp b/include/openPMD/RecordComponent.hpp index 4eda75f327..478a457b57 100644 --- a/include/openPMD/RecordComponent.hpp +++ b/include/openPMD/RecordComponent.hpp @@ -111,6 +111,9 @@ namespace internal std::deque without_extent; std::optional retrieved_extent; + static constexpr char const *env_var_check_dataset_consistency = + "OPENPMD_VERIFY_HOMOGENEOUS_EXTENTS"; + void check_extent(Attributable const &callsite, RecordComponent &); auto merge(Attributable const &callsite, HomogenizeExtents) -> HomogenizeExtents &; diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index 314a9b0d6d..2f79f74f20 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -24,7 +24,9 @@ #include "openPMD/Error.hpp" #include "openPMD/IO/Format.hpp" #include "openPMD/Series.hpp" +#include "openPMD/auxiliary/Environment.hpp" #include "openPMD/auxiliary/Memory.hpp" +#include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/BaseRecord.hpp" #include "openPMD/backend/Variant_internal.hpp" @@ -70,14 +72,21 @@ namespace internal } else if (retrieved_extent.has_value()) { - if (extent != *retrieved_extent) + if (extent != *retrieved_extent && + auxiliary::getEnvNum(env_var_check_dataset_consistency, 1) != 0) { + std::stringstream error_msg; + error_msg << "Inconsistent extents found for Record '" + << callsite.myPath().openPMDPath() << "': Component '" + << rc.myPath().openPMDPath() << "' has extent"; + auxiliary::write_vec_to_stream(error_msg, extent) << ", but "; + auxiliary::write_vec_to_stream(error_msg, *retrieved_extent) + << " was found previously."; throw error::ReadError( error::AffectedObject::Group, error::Reason::UnexpectedContent, std::nullopt, - "Inconsistent extents found for Record '" + - callsite.myPath().openPMDPath() + "'."); + error_msg.str()); } } else @@ -92,14 +101,22 @@ namespace internal { if (retrieved_extent.has_value() && other.retrieved_extent.has_value()) { - if (*retrieved_extent != *other.retrieved_extent) + if (*retrieved_extent != *other.retrieved_extent && + auxiliary::getEnvNum(env_var_check_dataset_consistency, 1) != 0) { + std::stringstream error_msg; + error_msg << "Inconsistent extents found for Record '" + << callsite.myPath().openPMDPath() << "': "; + auxiliary::write_vec_to_stream(error_msg, *retrieved_extent) + << " vs. "; + auxiliary::write_vec_to_stream( + error_msg, *other.retrieved_extent) + << "."; throw error::ReadError( error::AffectedObject::Group, error::Reason::UnexpectedContent, std::nullopt, - "Inconsistent extents found for Record '" + - callsite.myPath().openPMDPath() + "'."); + error_msg.str()); } } else if (!retrieved_extent.has_value()) @@ -118,12 +135,19 @@ namespace internal { if (!retrieved_extent.has_value()) { - throw error::ReadError( - error::AffectedObject::Group, - error::Reason::UnexpectedContent, - std::nullopt, - "No extent found for any component contained in '" + - callsite.myPath().openPMDPath() + "'."); + if (auxiliary::getEnvNum(env_var_check_dataset_consistency, 1) != 0) + { + throw error::ReadError( + error::AffectedObject::Group, + error::Reason::UnexpectedContent, + std::nullopt, + "No extent found for any component contained in '" + + callsite.myPath().openPMDPath() + "'."); + } + else + { + return; + } } auto &ext = *retrieved_extent; for (auto &rc : without_extent) From ed6859fd15e68b23d96991fb5a1c58a7565163c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 4 Apr 2025 15:11:45 +0200 Subject: [PATCH 07/20] Make homogenization configurable --- include/openPMD/IO/AbstractIOHandler.hpp | 1 + include/openPMD/RecordComponent.hpp | 5 +-- src/Mesh.cpp | 3 +- src/ParticleSpecies.cpp | 3 +- src/Record.cpp | 2 +- src/RecordComponent.cpp | 14 +++++--- src/Series.cpp | 42 +++++++++++++++++++++--- 7 files changed, 55 insertions(+), 15 deletions(-) diff --git a/include/openPMD/IO/AbstractIOHandler.hpp b/include/openPMD/IO/AbstractIOHandler.hpp index 29b3de8bff..681343c0df 100644 --- a/include/openPMD/IO/AbstractIOHandler.hpp +++ b/include/openPMD/IO/AbstractIOHandler.hpp @@ -314,6 +314,7 @@ class AbstractIOHandler internal::SeriesStatus m_seriesStatus = internal::SeriesStatus::Default; IterationEncoding m_encoding = IterationEncoding::groupBased; OpenpmdStandard m_standard = auxiliary::parseStandard(getStandardDefault()); + bool m_verify_homogeneous_extents = true; }; // AbstractIOHandler } // namespace openPMD diff --git a/include/openPMD/RecordComponent.hpp b/include/openPMD/RecordComponent.hpp index 478a457b57..610315dead 100644 --- a/include/openPMD/RecordComponent.hpp +++ b/include/openPMD/RecordComponent.hpp @@ -110,9 +110,10 @@ namespace internal { std::deque without_extent; std::optional retrieved_extent; + bool verify_homogeneous_extents = true; - static constexpr char const *env_var_check_dataset_consistency = - "OPENPMD_VERIFY_HOMOGENEOUS_EXTENTS"; + explicit HomogenizeExtents(); + HomogenizeExtents(bool verify_homogeneous_extents); void check_extent(Attributable const &callsite, RecordComponent &); auto merge(Attributable const &callsite, HomogenizeExtents) diff --git a/src/Mesh.cpp b/src/Mesh.cpp index 809a418868..02dc4de42b 100644 --- a/src/Mesh.cpp +++ b/src/Mesh.cpp @@ -435,7 +435,8 @@ void Mesh::flush_impl( void Mesh::read() { - internal::HomogenizeExtents homogenizeExtents; + internal::HomogenizeExtents homogenizeExtents( + IOHandler()->m_verify_homogeneous_extents); internal::EraseStaleEntries map{*this}; using DT = Datatype; diff --git a/src/ParticleSpecies.cpp b/src/ParticleSpecies.cpp index d1645deae9..b1a725b0f7 100644 --- a/src/ParticleSpecies.cpp +++ b/src/ParticleSpecies.cpp @@ -36,7 +36,8 @@ ParticleSpecies::ParticleSpecies() void ParticleSpecies::read() { - internal::HomogenizeExtents homogenizeExtents; + internal::HomogenizeExtents homogenizeExtents( + IOHandler()->m_verify_homogeneous_extents); /* obtain all non-scalar records */ Parameter pList; IOHandler()->enqueue(IOTask(this, pList)); diff --git a/src/Record.cpp b/src/Record.cpp index d9a1eebb42..c101d44f2e 100644 --- a/src/Record.cpp +++ b/src/Record.cpp @@ -111,7 +111,7 @@ void Record::flush_impl( auto Record::read() -> internal::HomogenizeExtents { - internal::HomogenizeExtents res; + internal::HomogenizeExtents res(IOHandler()->m_verify_homogeneous_extents); auto check_extent = [&](RecordComponent &rc) { res.check_extent(*this, rc); }; diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index 2f79f74f20..fb2bad36e3 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -62,6 +62,11 @@ namespace internal m_chunks.push(std::move(task)); } + HomogenizeExtents::HomogenizeExtents() = default; + HomogenizeExtents::HomogenizeExtents(bool verify_homogeneous_extents_in) + : verify_homogeneous_extents(verify_homogeneous_extents_in) + {} + void HomogenizeExtents::check_extent( Attributable const &callsite, RecordComponent &rc) { @@ -72,8 +77,7 @@ namespace internal } else if (retrieved_extent.has_value()) { - if (extent != *retrieved_extent && - auxiliary::getEnvNum(env_var_check_dataset_consistency, 1) != 0) + if (verify_homogeneous_extents && extent != *retrieved_extent) { std::stringstream error_msg; error_msg << "Inconsistent extents found for Record '" @@ -101,8 +105,8 @@ namespace internal { if (retrieved_extent.has_value() && other.retrieved_extent.has_value()) { - if (*retrieved_extent != *other.retrieved_extent && - auxiliary::getEnvNum(env_var_check_dataset_consistency, 1) != 0) + if (verify_homogeneous_extents && + *retrieved_extent != *other.retrieved_extent) { std::stringstream error_msg; error_msg << "Inconsistent extents found for Record '" @@ -135,7 +139,7 @@ namespace internal { if (!retrieved_extent.has_value()) { - if (auxiliary::getEnvNum(env_var_check_dataset_consistency, 1) != 0) + if (verify_homogeneous_extents) { throw error::ReadError( error::AffectedObject::Group, diff --git a/src/Series.cpp b/src/Series.cpp index b1a1b33d3a..bac5cd7d81 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -32,6 +32,7 @@ #include "openPMD/IterationEncoding.hpp" #include "openPMD/ThrowError.hpp" #include "openPMD/auxiliary/Date.hpp" +#include "openPMD/auxiliary/Environment.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/JSON_internal.hpp" #include "openPMD/auxiliary/Mpi.hpp" @@ -138,6 +139,8 @@ struct Series::ParsedInput std::string filenamePostfix; std::optional filenameExtension; int filenamePadding = -1; + // optional fields + bool verify_homogeneous_extents = true; }; // ParsedInput std::string Series::openPMD() const @@ -838,7 +841,17 @@ void Series::init( // Either an MPI_Comm or none, the template works for both options MPI_Communicator &&...comm) { - auto init_directly = [this, &comm..., at, &filepath]( + auto emplace_parse_config_options_into_iohandler = + [](AbstractIOHandler &ioHandler, ParsedInput &input) { + ioHandler.m_verify_homogeneous_extents = + input.verify_homogeneous_extents; + }; + + auto init_directly = [this, + &comm..., + at, + &filepath, + &emplace_parse_config_options_into_iohandler]( std::unique_ptr parsed_input, json::TracingJSON tracing_json) { auto io_handler = createIOHandler( @@ -850,12 +863,17 @@ void Series::init( comm..., tracing_json, filepath); + emplace_parse_config_options_into_iohandler(*io_handler, *parsed_input); initSeries(std::move(io_handler), std::move(parsed_input)); json::warnGlobalUnusedOptions(tracing_json); }; - auto init_deferred = [this, at, &filepath, &options, &comm...]( - std::string const &parsed_directory) { + auto init_deferred = [this, + at, + &filepath, + &options, + &emplace_parse_config_options_into_iohandler, + &comm...](std::string const &parsed_directory) { // Set a temporary IOHandler so that API calls which require a present // IOHandler don't fail writable().IOHandler = @@ -865,8 +883,12 @@ void Series::init( series.iterations.linkHierarchy(writable()); series.m_rankTable.m_attributable.linkHierarchy(writable()); series.m_deferred_initialization = - [called_this_already = false, filepath, options, at, comm...]( - Series &s) mutable { + [called_this_already = false, + filepath, + options, + at, + emplace_parse_config_options_into_iohandler, + comm...](Series &s) mutable { if (called_this_already) { throw std::runtime_error("Must be called one time only"); @@ -896,6 +918,8 @@ void Series::init( comm..., tracing_json, filepath); + emplace_parse_config_options_into_iohandler( + *io_handler, *parsed_input); auto res = io_handler.get(); s.initSeries(std::move(io_handler), std::move(parsed_input)); json::warnGlobalUnusedOptions(tracing_json); @@ -3008,6 +3032,14 @@ void Series::parseJsonOptions(TracingJSON &options, ParsedInput &input) auto &series = get(); getJsonOption( options, "defer_iteration_parsing", series.m_parseLazily); + input.verify_homogeneous_extents = + auxiliary::getEnvNum( + "OPENPMD_VERIFY_HOMOGENEOUS_EXTENTS", + input.verify_homogeneous_extents ? 1 : 0) != 0; + getJsonOption( + options, + "verify_homogeneous_extents", + input.verify_homogeneous_extents); internal::SeriesData::SourceSpecifiedViaJSON rankTableSource; if (getJsonOptionLowerCase(options, "rank_table", rankTableSource.value)) { From b1fc7fe750fc8bd1fab84cd2697961892b212a3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 4 Apr 2025 15:30:22 +0200 Subject: [PATCH 08/20] Fix tests with this --- test/SerialIOTest.cpp | 33 +++++++++++++++++------------ test/python/unittest/API/APITest.py | 10 ++++----- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 432dd864f3..5442247015 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -806,7 +806,9 @@ inline void empty_dataset_test(std::string const &file_ending) } { Series series( - "../samples/empty_datasets." + file_ending, Access::READ_ONLY); + "../samples/empty_datasets." + file_ending, + Access::READ_ONLY, + R"({"verify_homogeneous_extents": false})"); REQUIRE(series.iterations.contains(1)); REQUIRE(series.iterations.count(1) == 1); @@ -2718,7 +2720,8 @@ TEST_CASE("empty_alternate_fbpic", "[serial][hdf5]") { Series s = Series( "../samples/issue-sample/empty_alternate_fbpic_%T.h5", - Access::READ_ONLY); + Access::READ_ONLY, + R"({"verify_homogeneous_extents": false})"); REQUIRE(s.iterations.contains(50)); REQUIRE(s.iterations[50].particles.contains("electrons")); REQUIRE( @@ -5856,21 +5859,21 @@ void variableBasedSeries(std::string const &file) // this tests changing extents across iterations // ADIOS2 does not support changing the dimensionality // (older versions used to somewhat support it, but not really) - auto E_y = iteration.meshes["E"]["y"]; + auto B_y = iteration.meshes["B"]["y"]; unsigned dimensionality = 3; unsigned len = i + 1; Extent changingExtent(dimensionality, len); - E_y.resetDataset({openPMD::Datatype::INT, changingExtent}); + B_y.resetDataset({openPMD::Datatype::INT, changingExtent}); std::vector changingData( std::pow(len, dimensionality), dimensionality); - E_y.storeChunk( + B_y.storeChunk( changingData, Offset(dimensionality, 0), changingExtent); // this tests datasets that are present in one iteration, but not // in others - auto E_z = iteration.meshes["E"][std::to_string(i)]; - E_z.resetDataset({Datatype::INT, {1}}); - E_z.makeConstant(i); + auto rho_i = iteration.meshes["rho"][std::to_string(i)]; + rho_i.resetDataset({Datatype::INT, {1}}); + rho_i.makeConstant(i); // this tests attributes that are present in one iteration, but not // in others iteration.meshes["E"].setAttribute("attr_" + std::to_string(i), i); @@ -5984,11 +5987,11 @@ void variableBasedSeries(std::string const &file) REQUIRE(chunk2.get()[i] == int(index)); } - auto E_y = iteration.meshes["E"]["y"]; + auto B_y = iteration.meshes["B"]["y"]; unsigned dimensionality = 3; unsigned len = index + 1; Extent changingExtent(dimensionality, len); - REQUIRE(E_y.getExtent() == changingExtent); + REQUIRE(B_y.getExtent() == changingExtent); last_iteration_index = index; @@ -5999,7 +6002,7 @@ void variableBasedSeries(std::string const &file) { // component is present <=> (otherIteration == i) REQUIRE( - iteration.meshes["E"].contains( + iteration.meshes["rho"].contains( std::to_string(otherIteration)) == (otherIteration == index)); REQUIRE( @@ -6008,7 +6011,7 @@ void variableBasedSeries(std::string const &file) (otherIteration <= index)); } REQUIRE( - iteration.meshes["E"][std::to_string(index)] + iteration.meshes["rho"][std::to_string(index)] .getAttribute("value") .get() == int(index)); REQUIRE( @@ -6734,7 +6737,11 @@ void extendDataset(std::string const &ext, std::string const &jsonConfig) } { - Series read(filename, Access::READ_ONLY, jsonConfig); + Series read( + filename, + Access::READ_ONLY, + json::merge( + jsonConfig, R"({"verify_homogeneous_extents": false})")); auto E_x = read.iterations[0].meshes["E"]["x"]; REQUIRE(E_x.getExtent() == Extent{10, 5}); auto chunk = E_x.loadChunk({0, 0}, {10, 5}); diff --git a/test/python/unittest/API/APITest.py b/test/python/unittest/API/APITest.py index 24dfec4051..500752b9b1 100644 --- a/test/python/unittest/API/APITest.py +++ b/test/python/unittest/API/APITest.py @@ -1953,9 +1953,9 @@ def makeIteratorRoundTrip(self, backend, file_ending): E_x.reset_dataset(DS(np.dtype("int"), extent)) E_x.store_chunk(data, [0], extent) - E_y = it.meshes["E"]["y"] - E_y.reset_dataset(DS(np.dtype("int"), [2, 2])) - span = E_y.store_chunk().current_buffer() + B_y = it.meshes["B"]["y"] + B_y.reset_dataset(DS(np.dtype("int"), [2, 2])) + span = B_y.store_chunk().current_buffer() span[0, 0] = 0 span[0, 1] = 1 span[1, 0] = 2 @@ -1978,8 +1978,8 @@ def makeIteratorRoundTrip(self, backend, file_ending): lastIterationIndex = it.iteration_index E_x = it.meshes["E"]["x"] chunk = E_x.load_chunk([0], extent) - E_y = it.meshes["E"]["y"] - chunk2 = E_y.load_chunk([0, 0], [2, 2]) + B_y = it.meshes["B"]["y"] + chunk2 = B_y.load_chunk([0, 0], [2, 2]) it.close() for i in range(len(data)): From 5c567704c7bf9c0087e883405b9798e05dfc2e27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 4 Apr 2025 15:39:17 +0200 Subject: [PATCH 09/20] Add note on how to deactivate this verification --- src/RecordComponent.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index fb2bad36e3..26882f02d7 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -62,6 +62,12 @@ namespace internal m_chunks.push(std::move(task)); } + static constexpr char const *note_on_deactivating_this_check = R"( +Note: In order to ignore inconsistent / incomplete extent definitions, +set the environment variable OPENPMD_VERIFY_HOMOGENEOUS_EXTENTS=0 +or alternatively the JSON option {"verify_homogeneous_extents": false}. + )"; + HomogenizeExtents::HomogenizeExtents() = default; HomogenizeExtents::HomogenizeExtents(bool verify_homogeneous_extents_in) : verify_homogeneous_extents(verify_homogeneous_extents_in) @@ -85,7 +91,8 @@ namespace internal << rc.myPath().openPMDPath() << "' has extent"; auxiliary::write_vec_to_stream(error_msg, extent) << ", but "; auxiliary::write_vec_to_stream(error_msg, *retrieved_extent) - << " was found previously."; + << " was found previously." + << note_on_deactivating_this_check; throw error::ReadError( error::AffectedObject::Group, error::Reason::UnexpectedContent, @@ -115,7 +122,7 @@ namespace internal << " vs. "; auxiliary::write_vec_to_stream( error_msg, *other.retrieved_extent) - << "."; + << "." << note_on_deactivating_this_check; throw error::ReadError( error::AffectedObject::Group, error::Reason::UnexpectedContent, @@ -146,7 +153,8 @@ namespace internal error::Reason::UnexpectedContent, std::nullopt, "No extent found for any component contained in '" + - callsite.myPath().openPMDPath() + "'."); + callsite.myPath().openPMDPath() + "'." + + note_on_deactivating_this_check); } else { From 1c1f16dcb103b9af9d698e51bdccd3f4b91b2bbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 4 Apr 2025 18:14:32 +0200 Subject: [PATCH 10/20] WIP: Testing --- CMakeLists.txt | 1 + src/RecordComponent.cpp | 5 +- test/Files_SerialIO/SerialIOTests.hpp | 4 ++ .../components_without_extent.cpp | 48 +++++++++++++++++++ test/SerialIOTest.cpp | 5 ++ 5 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 test/Files_SerialIO/components_without_extent.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index d37b76b25b..cfdfbe19d9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -823,6 +823,7 @@ if(openPMD_BUILD_TESTING) test/Files_SerialIO/close_and_reopen_test.cpp test/Files_SerialIO/filebased_write_test.cpp test/Files_SerialIO/issue_1744_unique_ptrs_at_close_time.cpp + test/Files_SerialIO/components_without_extent.cpp ) elseif(${test_name} STREQUAL "ParallelIO" AND openPMD_HAVE_MPI) list(APPEND ${out_list} diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index 26882f02d7..b49e93b873 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -226,8 +226,9 @@ RecordComponent &RecordComponent::resetDataset(Dataset d) rc.m_hasBeenExtended = true; } - if (d.extent.empty()) - throw std::runtime_error("Dataset extent must be at least 1D."); + // @todo check this while flushing + // if (d.extent.empty()) + // throw std::runtime_error("Dataset extent must be at least 1D."); if (d.empty()) { if (d.extent.empty()) diff --git a/test/Files_SerialIO/SerialIOTests.hpp b/test/Files_SerialIO/SerialIOTests.hpp index f5e770681b..c534ded579 100644 --- a/test/Files_SerialIO/SerialIOTests.hpp +++ b/test/Files_SerialIO/SerialIOTests.hpp @@ -12,3 +12,7 @@ namespace issue_1744_unique_ptrs_at_close_time { auto issue_1744_unique_ptrs_at_close_time() -> void; } +namespace components_without_extent +{ +auto components_without_extent() -> void; +} diff --git a/test/Files_SerialIO/components_without_extent.cpp b/test/Files_SerialIO/components_without_extent.cpp new file mode 100644 index 0000000000..50a876c334 --- /dev/null +++ b/test/Files_SerialIO/components_without_extent.cpp @@ -0,0 +1,48 @@ +#include "SerialIOTests.hpp" + +#include "openPMD/openPMD.hpp" + +#include + +#include +#include + +namespace components_without_extent +{ +auto components_without_extent() -> void +{ + auto filepath = "../samples/components_without_extent.bp5"; + // write + { + openPMD::Series write(filepath, openPMD::Access::CREATE); + auto it0 = write.writeIterations()[0]; + auto e = it0.particles["e"]; + for (auto comp_id : {"x", "y", "z"}) + { + auto position_comp = e["position"][comp_id]; + position_comp.resetDataset({openPMD::Datatype::FLOAT, {5}}); + std::unique_ptr data{new float[5]}; + std::iota(data.get(), data.get() + 5, 0); + position_comp.storeChunk(std::move(data), {0}, {5}); + + auto offset_comp = e["positionOffset"][comp_id]; + offset_comp.resetDataset({openPMD::Datatype::INT, {}}); + offset_comp.makeConstant(0); + } + write.close(); + } + + // read + { + openPMD::Series read(filepath, openPMD::Access::READ_RANDOM_ACCESS); + auto e = read.snapshots()[0].particles["e"]; + for (auto const &record : e) + { + for (auto const &component : record.second) + { + REQUIRE(component.second.getExtent() == openPMD::Extent{5}); + } + } + } +} +} // namespace components_without_extent diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 5442247015..aee09364d6 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -768,6 +768,11 @@ TEST_CASE("issue_1744_unique_ptrs_at_close_time", "[serial]") #endif } +TEST_CASE("components_without_extent", "[serial]") +{ + components_without_extent::components_without_extent(); +} + #if openPMD_HAVE_ADIOS2 TEST_CASE("close_and_reopen_test", "[serial]") { From e5feaae726853707473223d23e99df798090a5a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 7 Apr 2025 17:17:01 +0200 Subject: [PATCH 11/20] CI fixes --- include/openPMD/RecordComponent.hpp | 35 ++++++++++++++++------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/include/openPMD/RecordComponent.hpp b/include/openPMD/RecordComponent.hpp index 610315dead..58e7d201c8 100644 --- a/include/openPMD/RecordComponent.hpp +++ b/include/openPMD/RecordComponent.hpp @@ -105,21 +105,6 @@ namespace internal }; template class BaseRecordData; - - struct HomogenizeExtents - { - std::deque without_extent; - std::optional retrieved_extent; - bool verify_homogeneous_extents = true; - - explicit HomogenizeExtents(); - HomogenizeExtents(bool verify_homogeneous_extents); - - void check_extent(Attributable const &callsite, RecordComponent &); - auto merge(Attributable const &callsite, HomogenizeExtents) - -> HomogenizeExtents &; - void homogenize(Attributable const &callsite) &&; - }; } // namespace internal template @@ -555,6 +540,26 @@ OPENPMD_protected void verifyChunk(Datatype, Offset const &, Extent const &) const; }; // RecordComponent +namespace internal +{ + // Must put this after the definition of RecordComponent due to the + // deque + struct HomogenizeExtents + { + std::deque without_extent; + std::optional retrieved_extent; + bool verify_homogeneous_extents = true; + + explicit HomogenizeExtents(); + HomogenizeExtents(bool verify_homogeneous_extents); + + void check_extent(Attributable const &callsite, RecordComponent &); + auto merge(Attributable const &callsite, HomogenizeExtents) + -> HomogenizeExtents &; + void homogenize(Attributable const &callsite) &&; + }; +} // namespace internal + } // namespace openPMD #include "openPMD/UndefDatatypeMacros.hpp" From 4fbd00e5ff3cb1f5fe6d20817ea958b26b8f55ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 7 Apr 2025 18:10:20 +0200 Subject: [PATCH 12/20] Extend test --- .../components_without_extent.cpp | 214 +++++++++++++++++- 1 file changed, 212 insertions(+), 2 deletions(-) diff --git a/test/Files_SerialIO/components_without_extent.cpp b/test/Files_SerialIO/components_without_extent.cpp index 50a876c334..552bacf73e 100644 --- a/test/Files_SerialIO/components_without_extent.cpp +++ b/test/Files_SerialIO/components_without_extent.cpp @@ -9,9 +9,10 @@ namespace components_without_extent { -auto components_without_extent() -> void +constexpr char const *filepath = "../samples/components_without_extent.bp5"; + +void particle_offset_without_extent() { - auto filepath = "../samples/components_without_extent.bp5"; // write { openPMD::Series write(filepath, openPMD::Access::CREATE); @@ -45,4 +46,213 @@ auto components_without_extent() -> void } } } + +void particles_without_any_extent() +{ + // write + { + openPMD::Series write(filepath, openPMD::Access::CREATE); + auto it0 = write.writeIterations()[0]; + auto e = it0.particles["e"]; + for (auto comp_id : {"x", "y", "z"}) + { + auto position_comp = e["position"][comp_id]; + position_comp.resetDataset({openPMD::Datatype::INT, {}}); + position_comp.makeConstant(0); + + auto offset_comp = e["positionOffset"][comp_id]; + offset_comp.resetDataset({openPMD::Datatype::INT, {}}); + offset_comp.makeConstant(0); + } + write.close(); + } + + // read + { + openPMD::Series read(filepath, openPMD::Access::READ_RANDOM_ACCESS); + REQUIRE(!read.snapshots()[0].particles.contains("e")); + } + + { + openPMD::Series read( + filepath, + openPMD::Access::READ_RANDOM_ACCESS, + R"({"verify_homogeneous_extents": false})"); + REQUIRE(read.snapshots()[0].particles.contains("e")); + auto e = read.snapshots()[0].particles["e"]; + for (auto const &record : e) + { + for (auto const &component : record.second) + { + REQUIRE(component.second.getExtent() == openPMD::Extent{}); + } + } + } +} + +void particles_without_inconsistent_extent() +{ + // write + { + openPMD::Series write(filepath, openPMD::Access::CREATE); + auto it0 = write.writeIterations()[0]; + auto e = it0.particles["e"]; + for (auto comp_id : {"x", "y", "z"}) + { + auto position_comp = e["position"][comp_id]; + position_comp.resetDataset({openPMD::Datatype::INT, {5}}); + position_comp.makeConstant(0); + + auto offset_comp = e["positionOffset"][comp_id]; + offset_comp.resetDataset({openPMD::Datatype::INT, {10}}); + offset_comp.makeConstant(0); + } + write.close(); + } + + // read + { + openPMD::Series read(filepath, openPMD::Access::READ_RANDOM_ACCESS); + REQUIRE(!read.snapshots()[0].particles.contains("e")); + } + + { + openPMD::Series read( + filepath, + openPMD::Access::READ_RANDOM_ACCESS, + R"({"verify_homogeneous_extents": false})"); + REQUIRE(read.snapshots()[0].particles.contains("e")); + auto e = read.snapshots()[0].particles["e"]; + for (auto const &component : e["position"]) + { + REQUIRE(component.second.getExtent() == openPMD::Extent{5}); + } + for (auto const &component : e["positionOffset"]) + { + REQUIRE(component.second.getExtent() == openPMD::Extent{10}); + } + } +} + +void meshes_with_incomplete_extent() +{ + // write + { + openPMD::Series write(filepath, openPMD::Access::CREATE); + auto it0 = write.writeIterations()[0]; + auto E = it0.meshes["E"]; + for (auto comp_id : {"x"}) + { + auto comp = E[comp_id]; + comp.resetDataset({openPMD::Datatype::FLOAT, {5}}); + std::unique_ptr data{new float[5]}; + std::iota(data.get(), data.get() + 5, 0); + comp.storeChunk(std::move(data), {0}, {5}); + } + for (auto comp_id : {"y", "z"}) + { + auto comp = E[comp_id]; + comp.resetDataset({openPMD::Datatype::INT, {}}); + comp.makeConstant(0); + } + write.close(); + } + + // read + { + openPMD::Series read(filepath, openPMD::Access::READ_RANDOM_ACCESS); + auto E = read.snapshots()[0].meshes["E"]; + for (auto const &component : E) + { + REQUIRE(component.second.getExtent() == openPMD::Extent{5}); + } + } +} + +void meshes_with_inconsistent_extent() +{ + // write + { + openPMD::Series write(filepath, openPMD::Access::CREATE); + auto it0 = write.writeIterations()[0]; + auto E = it0.meshes["E"]; + size_t i = 1; + for (auto comp_id : {"x", "y", "z"}) + { + size_t extent = i++ * 5; + auto comp = E[comp_id]; + comp.resetDataset({openPMD::Datatype::FLOAT, {extent}}); + std::unique_ptr data{new float[extent]}; + std::iota(data.get(), data.get() + extent, 0); + comp.storeChunk(std::move(data), {0}, {extent}); + } + write.close(); + } + + // read + { + openPMD::Series read(filepath, openPMD::Access::READ_RANDOM_ACCESS); + REQUIRE(!read.snapshots()[0].meshes.contains("E")); + } + + // read + { + openPMD::Series read( + filepath, + openPMD::Access::READ_RANDOM_ACCESS, + R"({"verify_homogeneous_extents": false})"); + auto E = read.snapshots()[0].meshes["E"]; + size_t i = 1; + for (auto const &component : E) + { + REQUIRE(component.second.getExtent() == openPMD::Extent{5 * i++}); + } + } +} + +void meshes_without_any_extent() +{ + // write + { + openPMD::Series write(filepath, openPMD::Access::CREATE); + auto it0 = write.writeIterations()[0]; + auto E = it0.meshes["E"]; + for (auto comp_id : {"x", "y", "z"}) + { + auto comp = E[comp_id]; + comp.resetDataset({openPMD::Datatype::FLOAT, {}}); + comp.makeConstant(0); + } + write.close(); + } + + // read + { + openPMD::Series read(filepath, openPMD::Access::READ_RANDOM_ACCESS); + REQUIRE(!read.snapshots()[0].meshes.contains("E")); + } + + // read + { + openPMD::Series read( + filepath, + openPMD::Access::READ_RANDOM_ACCESS, + R"({"verify_homogeneous_extents": false})"); + auto E = read.snapshots()[0].meshes["E"]; + for (auto const &component : E) + { + REQUIRE(component.second.getExtent() == openPMD::Extent{}); + } + } +} + +auto components_without_extent() -> void +{ + particle_offset_without_extent(); + particles_without_any_extent(); + particles_without_inconsistent_extent(); + meshes_with_incomplete_extent(); + meshes_with_inconsistent_extent(); + meshes_without_any_extent(); +} } // namespace components_without_extent From 5f81fcf659aec0255252d7e665553276dd06f371 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 9 Apr 2025 14:33:16 +0200 Subject: [PATCH 13/20] Use JSON backend for this test --- test/Files_SerialIO/components_without_extent.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Files_SerialIO/components_without_extent.cpp b/test/Files_SerialIO/components_without_extent.cpp index 552bacf73e..5997d8c5fd 100644 --- a/test/Files_SerialIO/components_without_extent.cpp +++ b/test/Files_SerialIO/components_without_extent.cpp @@ -9,7 +9,7 @@ namespace components_without_extent { -constexpr char const *filepath = "../samples/components_without_extent.bp5"; +constexpr char const *filepath = "../samples/components_without_extent.json"; void particle_offset_without_extent() { From f547a477b7176d5ebae270913b906914e7811e6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 9 Apr 2025 15:21:01 +0200 Subject: [PATCH 14/20] Fix unfinished_iteration_test --- test/SerialIOTest.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index aee09364d6..042b609220 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -7074,8 +7074,11 @@ void unfinished_iteration_test( auto tryReading = [&config, file, encoding]( Access access, std::string const &additionalConfig = "{}") { + auto merged_config = json::merge( + json::merge(config, additionalConfig), + R"({"verify_homogeneous_extents": false})"); { - Series read(file, access, json::merge(config, additionalConfig)); + Series read(file, access, merged_config); std::vector iterations; std::cout << "\n\n\nGoing to list iterations in " << file @@ -7126,7 +7129,7 @@ void unfinished_iteration_test( if (encoding == IterationEncoding::fileBased && access == Access::READ_ONLY) { - Series read(file, access, json::merge(config, additionalConfig)); + Series read(file, access, merged_config); if (additionalConfig == "{}") { // Eager parsing, defective iteration has already been removed From ac9a161c6a2b264f46ceb238ada7471fd0231f2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 9 Apr 2025 16:57:05 +0200 Subject: [PATCH 15/20] Bugfix: skip skipped compoments --- src/Mesh.cpp | 2 ++ src/ParticleSpecies.cpp | 2 ++ src/Record.cpp | 30 ++++++++++++++++++------------ 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/Mesh.cpp b/src/Mesh.cpp index 02dc4de42b..6b38e7ef4f 100644 --- a/src/Mesh.cpp +++ b/src/Mesh.cpp @@ -632,6 +632,7 @@ void Mesh::read() << "' and will skip it due to read error:\n" << err.what() << std::endl; map.forget(component); + continue; } homogenizeExtents.check_extent(*this, rc); } @@ -660,6 +661,7 @@ void Mesh::read() << "' and will skip it due to read error:\n" << err.what() << std::endl; map.forget(component); + continue; } homogenizeExtents.check_extent(*this, rc); } diff --git a/src/ParticleSpecies.cpp b/src/ParticleSpecies.cpp index b1a725b0f7..0476cb419e 100644 --- a/src/ParticleSpecies.cpp +++ b/src/ParticleSpecies.cpp @@ -98,6 +98,7 @@ void ParticleSpecies::read() << err.what() << std::endl; map.forget(record_name); + continue; } homogenizeExtents.merge(*this, std::move(recordExtents)); } @@ -143,6 +144,7 @@ void ParticleSpecies::read() map.forget(record_name); //(*this)[record_name].erase(RecordComponent::SCALAR); // this->erase(record_name); + continue; } homogenizeExtents.merge(*this, std::move(recordExtents)); } diff --git a/src/Record.cpp b/src/Record.cpp index c101d44f2e..25d896e901 100644 --- a/src/Record.cpp +++ b/src/Record.cpp @@ -117,18 +117,22 @@ auto Record::read() -> internal::HomogenizeExtents }; if (scalar()) { - /* using operator[] will incorrectly update parent */ - try - { - T_RecordComponent::read(/* require_unit_si = */ true); - } - catch (error::ReadError const &err) - { - std::cerr << "Cannot read scalar record component and will skip it " - "due to read error:\n" - << err.what() << std::endl; - } - check_extent(*this); + [&]() { + /* using operator[] will incorrectly update parent */ + try + { + T_RecordComponent::read(/* require_unit_si = */ true); + } + catch (error::ReadError const &err) + { + std::cerr + << "Cannot read scalar record component and will skip it " + "due to read error:\n" + << err.what() << std::endl; + return; // from lambda + } + check_extent(*this); + }(); } else { @@ -153,6 +157,7 @@ auto Record::read() -> internal::HomogenizeExtents << "' and will skip it due to read error:\n" << err.what() << std::endl; this->container().erase(component); + continue; } check_extent(rc); } @@ -181,6 +186,7 @@ auto Record::read() -> internal::HomogenizeExtents << "' and will skip it due to read error:\n" << err.what() << std::endl; this->container().erase(component); + continue; } check_extent(rc); } From f5ac5328c5b82eac445a6672738d68057bcc5769 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 9 Apr 2025 18:52:03 +0200 Subject: [PATCH 16/20] Use UNDEFINED_EXTENT for this instead of empty extent --- include/openPMD/Dataset.hpp | 21 +++++++++++++--- include/openPMD/IO/IOTask.hpp | 1 + include/openPMD/RecordComponent.hpp | 2 ++ src/Dataset.cpp | 9 +++++++ src/IO/ADIOS/ADIOS2IOHandler.cpp | 12 ++++++++++ src/IO/HDF5/HDF5IOHandler.cpp | 10 ++++++++ src/RecordComponent.cpp | 13 +++++----- .../components_without_extent.cpp | 24 +++++++++++++------ 8 files changed, 75 insertions(+), 17 deletions(-) diff --git a/include/openPMD/Dataset.hpp b/include/openPMD/Dataset.hpp index 80513683f9..507b68d350 100644 --- a/include/openPMD/Dataset.hpp +++ b/include/openPMD/Dataset.hpp @@ -54,9 +54,21 @@ class Dataset */ JOINED_DIMENSION = std::numeric_limits::max(), /** - * Some backends (i.e. JSON and TOML in template mode) support the - * creation of dataset with undefined datatype and extent. - * The extent should be given as {UNDEFINED_EXTENT} for that. + * In some use cases, the extent needs not be specified. + * For these, specify Extent{UNDEFINED_EXTENT}. + * Use cases: + * + * 1. Some backends (i.e. JSON and TOML in template mode) support the + * creation of dataset with undefined datatype and extent. + * The extent should be given as {UNDEFINED_EXTENT} for that. + * 2. With openPMD 2.0, the shape of constant components may be omitted + * in writing if it is defined somewhere else as part + * of the same Mesh / Species. + * (https://github.com/openPMD/openPMD-standard/pull/289) + * When reading such datasets, the openPMD-api will try to fill in + * the missing extents, so the extent for constistently-defined + * datasets should ideally not be reported by the read-side API + * as undefined. */ UNDEFINED_EXTENT = std::numeric_limits::max() - 1 }; @@ -87,5 +99,8 @@ class Dataset std::optional joinedDimension() const; static std::optional joinedDimension(Extent const &); + + bool undefinedExtent() const; + static bool undefinedExtent(Extent const &); }; } // namespace openPMD diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index 4c82cee174..08ee29bfff 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -22,6 +22,7 @@ #include "openPMD/ChunkInfo.hpp" #include "openPMD/Dataset.hpp" +#include "openPMD/Error.hpp" #include "openPMD/IterationEncoding.hpp" #include "openPMD/Streaming.hpp" #include "openPMD/auxiliary/Export.hpp" diff --git a/include/openPMD/RecordComponent.hpp b/include/openPMD/RecordComponent.hpp index 58e7d201c8..d06b4213f4 100644 --- a/include/openPMD/RecordComponent.hpp +++ b/include/openPMD/RecordComponent.hpp @@ -158,6 +158,8 @@ class RecordComponent : public BaseRecordComponent * * Shrinking any dimension's extent. * * Changing the number of dimensions. * + * The dataset extent may be empty to indicate undefined extents. + * * Backend support for resizing datasets: * * JSON: Supported * * ADIOS2: Supported as of ADIOS2 2.7.0 diff --git a/src/Dataset.cpp b/src/Dataset.cpp index a56c566805..f0c39cfc9d 100644 --- a/src/Dataset.cpp +++ b/src/Dataset.cpp @@ -95,4 +95,13 @@ std::optional Dataset::joinedDimension(Extent const &extent) } return res; } + +bool Dataset::undefinedExtent() const +{ + return undefinedExtent(extent); +} +bool Dataset::undefinedExtent(Extent const &e) +{ + return e.size() == 1 && e.at(0) == Dataset::UNDEFINED_EXTENT; +} } // namespace openPMD diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 5d3ce17f36..b5583da9c7 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -833,6 +833,12 @@ void ADIOS2IOHandlerImpl::createDataset( "only is not possible."); } + if (Dataset::undefinedExtent(parameters.extent)) + { + throw error::OperationUnsupportedInBackend( + "ADIOS2", "No support for Datasets with undefined extent."); + } + if (!writable->written) { /* Sanitize name */ @@ -993,6 +999,12 @@ void ADIOS2IOHandlerImpl::extendDataset( VERIFY_ALWAYS( access::write(m_handler->m_backendAccess), "[ADIOS2] Cannot extend datasets in read-only mode."); + if (Dataset::undefinedExtent(parameters.extent)) + { + throw error::OperationUnsupportedInBackend( + "ADIOS2", "No support for Datasets with undefined extent."); + } + setAndGetFilePosition(writable); auto file = refreshFileFromParent(writable, /* preferParentFile = */ false); std::string name = nameOfVariable(writable); diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp index f75fe7a527..a63de927c0 100644 --- a/src/IO/HDF5/HDF5IOHandler.cpp +++ b/src/IO/HDF5/HDF5IOHandler.cpp @@ -834,6 +834,11 @@ void HDF5IOHandlerImpl::createDataset( error::throwOperationUnsupportedInBackend( "HDF5", "Joined Arrays currently only supported in ADIOS2"); } + else if (Dataset::undefinedExtent(parameters.extent)) + { + throw error::OperationUnsupportedInBackend( + "HDF5", "No support for Datasets with undefined extent."); + } if (!writable->written) { @@ -1114,6 +1119,11 @@ void HDF5IOHandlerImpl::extendDataset( error::throwOperationUnsupportedInBackend( "HDF5", "Joined Arrays currently only supported in ADIOS2"); } + else if (Dataset::undefinedExtent(parameters.extent)) + { + throw error::OperationUnsupportedInBackend( + "HDF5", "No support for Datasets with undefined extent."); + } File file = requireFile("extendDataset", writable, /* checkParent = */ true); diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index b49e93b873..c61fe82c83 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -77,7 +77,7 @@ or alternatively the JSON option {"verify_homogeneous_extents": false}. Attributable const &callsite, RecordComponent &rc) { auto extent = rc.getExtent(); - if (extent.empty()) + if (Dataset::undefinedExtent(extent)) { without_extent.emplace_back(rc); } @@ -226,9 +226,8 @@ RecordComponent &RecordComponent::resetDataset(Dataset d) rc.m_hasBeenExtended = true; } - // @todo check this while flushing - // if (d.extent.empty()) - // throw std::runtime_error("Dataset extent must be at least 1D."); + if (d.extent.empty()) + throw std::runtime_error("Dataset extent must be at least 1D."); if (d.empty()) { if (d.extent.empty()) @@ -291,7 +290,7 @@ Extent RecordComponent::getExtent() const } else { - return {}; + return {Dataset::UNDEFINED_EXTENT}; } } @@ -422,7 +421,7 @@ void RecordComponent::flush( } auto constant_component_write_shape = [&]() { auto extent = getExtent(); - return !extent.empty() && + return !Dataset::undefinedExtent(extent) && std::none_of(extent.begin(), extent.end(), [](auto val) { return val == Dataset::JOINED_DIMENSION; }); @@ -561,7 +560,7 @@ void RecordComponent::readBase(bool require_unit_si) if (!containsAttribute("shape")) { setWritten(false, Attributable::EnqueueAsynchronously::No); - resetDataset(Dataset(dtype, {})); + resetDataset(Dataset(dtype, {Dataset::UNDEFINED_EXTENT})); setWritten(true, Attributable::EnqueueAsynchronously::No); return; diff --git a/test/Files_SerialIO/components_without_extent.cpp b/test/Files_SerialIO/components_without_extent.cpp index 5997d8c5fd..0fcc3f9e3a 100644 --- a/test/Files_SerialIO/components_without_extent.cpp +++ b/test/Files_SerialIO/components_without_extent.cpp @@ -27,7 +27,8 @@ void particle_offset_without_extent() position_comp.storeChunk(std::move(data), {0}, {5}); auto offset_comp = e["positionOffset"][comp_id]; - offset_comp.resetDataset({openPMD::Datatype::INT, {}}); + offset_comp.resetDataset( + {openPMD::Datatype::INT, {openPMD::Dataset::UNDEFINED_EXTENT}}); offset_comp.makeConstant(0); } write.close(); @@ -57,11 +58,13 @@ void particles_without_any_extent() for (auto comp_id : {"x", "y", "z"}) { auto position_comp = e["position"][comp_id]; - position_comp.resetDataset({openPMD::Datatype::INT, {}}); + position_comp.resetDataset( + {openPMD::Datatype::INT, {openPMD::Dataset::UNDEFINED_EXTENT}}); position_comp.makeConstant(0); auto offset_comp = e["positionOffset"][comp_id]; - offset_comp.resetDataset({openPMD::Datatype::INT, {}}); + offset_comp.resetDataset( + {openPMD::Datatype::INT, {openPMD::Dataset::UNDEFINED_EXTENT}}); offset_comp.makeConstant(0); } write.close(); @@ -84,7 +87,9 @@ void particles_without_any_extent() { for (auto const &component : record.second) { - REQUIRE(component.second.getExtent() == openPMD::Extent{}); + REQUIRE( + component.second.getExtent() == + openPMD::Extent{openPMD::Dataset::UNDEFINED_EXTENT}); } } } @@ -152,7 +157,8 @@ void meshes_with_incomplete_extent() for (auto comp_id : {"y", "z"}) { auto comp = E[comp_id]; - comp.resetDataset({openPMD::Datatype::INT, {}}); + comp.resetDataset( + {openPMD::Datatype::INT, {openPMD::Dataset::UNDEFINED_EXTENT}}); comp.makeConstant(0); } write.close(); @@ -220,7 +226,9 @@ void meshes_without_any_extent() for (auto comp_id : {"x", "y", "z"}) { auto comp = E[comp_id]; - comp.resetDataset({openPMD::Datatype::FLOAT, {}}); + comp.resetDataset( + {openPMD::Datatype::FLOAT, + {openPMD::Dataset::UNDEFINED_EXTENT}}); comp.makeConstant(0); } write.close(); @@ -241,7 +249,9 @@ void meshes_without_any_extent() auto E = read.snapshots()[0].meshes["E"]; for (auto const &component : E) { - REQUIRE(component.second.getExtent() == openPMD::Extent{}); + REQUIRE( + component.second.getExtent() == + openPMD::Extent{openPMD::Dataset::UNDEFINED_EXTENT}); } } } From dc9e3e6845c85589b6c70702e93584c69859d029 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 11 Apr 2025 19:33:03 +0200 Subject: [PATCH 17/20] Document verify_homogeneous_extents option --- docs/source/details/backendconfig.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/source/details/backendconfig.rst b/docs/source/details/backendconfig.rst index 9d5e1dcf2c..af3d562196 100644 --- a/docs/source/details/backendconfig.rst +++ b/docs/source/details/backendconfig.rst @@ -94,6 +94,11 @@ Using the Streaming API (i.e. ``SeriesInterface::readIteration()``) will do this Parsing eagerly might be very expensive for a Series with many iterations, but will avoid bugs by forgotten calls to ``Iteration::open()``. In complex environments, calling ``Iteration::open()`` on an already open environment does no harm (and does not incur additional runtime cost for additional ``open()`` calls). +As of openPMD-api 0.17.0, the parser verifies that all records within a mesh or within a particle species have consistent shapes / extents. +This is used for filling in the shape for constant components that do not define it. +In order to skip this check in the error case, the key ``{"verify_homogeneous_extents": false}`` may be set (alternatively ``export OPENPMD_VERIFY_HOMOGENEOUS_EXTENTS=0`` will do the same). +This will help read datasets with inconsistent metadata definitions. + The key ``resizable`` can be passed to ``Dataset`` options. It if set to ``{"resizable": true}``, this declares that it shall be allowed to increased the ``Extent`` of a ``Dataset`` via ``resetDataset()`` at a later time, i.e., after it has been first declared (and potentially written). For HDF5, resizable Datasets come with a performance penalty. From 93864451f17bd7a090efcfda99397575b3560025 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 15 Jul 2025 16:22:28 +0200 Subject: [PATCH 18/20] Fix typo --- include/openPMD/Dataset.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/openPMD/Dataset.hpp b/include/openPMD/Dataset.hpp index 507b68d350..e45605903d 100644 --- a/include/openPMD/Dataset.hpp +++ b/include/openPMD/Dataset.hpp @@ -59,7 +59,7 @@ class Dataset * Use cases: * * 1. Some backends (i.e. JSON and TOML in template mode) support the - * creation of dataset with undefined datatype and extent. + * creation of datasets with undefined datatype and extent. * The extent should be given as {UNDEFINED_EXTENT} for that. * 2. With openPMD 2.0, the shape of constant components may be omitted * in writing if it is defined somewhere else as part From 8b3d50d2e11221ff4d8b89583d0458ddecab0fa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 22 Jul 2025 18:00:47 +0200 Subject: [PATCH 19/20] Adapt test to stricter parsing --- .../read_variablebased_randomaccess.cpp | 78 ++++++++++--------- 1 file changed, 43 insertions(+), 35 deletions(-) diff --git a/test/Files_ParallelIO/read_variablebased_randomaccess.cpp b/test/Files_ParallelIO/read_variablebased_randomaccess.cpp index 577f19ae61..7cce50d2df 100644 --- a/test/Files_ParallelIO/read_variablebased_randomaccess.cpp +++ b/test/Files_ParallelIO/read_variablebased_randomaccess.cpp @@ -6,6 +6,7 @@ #include #include +#include #if openPMD_HAVE_ADIOS2 && openPMD_HAVE_MPI #include @@ -120,8 +121,6 @@ static void create_file_in_serial(bool use_group_table) "__openPMD_groups/data", step, "", "/", true); IO.DefineAttribute( "__openPMD_groups/data/meshes", step, "", "/", true); - IO.DefineAttribute( - "__openPMD_groups/data/meshes/theta", step, "", "/", true); } std::vector data{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; @@ -129,15 +128,6 @@ static void create_file_in_serial(bool use_group_table) if (step % 2 == 1) { engine.Put(variable2, data.data()); - if (use_group_table) - { - IO.DefineAttribute( - "__openPMD_groups/data/meshes/e_chargeDensity", - step, - "", - "/", - true); - } } engine.EndStep(); @@ -147,13 +137,16 @@ static void create_file_in_serial(bool use_group_table) } } -auto read_file_in_parallel(bool use_group_table) -> void +auto read_file_in_parallel( + std::optional const &dont_verify_homogeneous_extents) -> void { openPMD::Series read( "../samples/read_variablebased_randomaccess.bp", openPMD::Access::READ_ONLY, MPI_COMM_WORLD, - "adios2.engine.type = \"bp5\""); + json::merge( + "adios2.engine.type = \"bp5\"", + dont_verify_homogeneous_extents.value_or("{}"))); for (auto &[index, iteration] : read.snapshots()) { auto data = iteration.meshes["theta"].loadChunk({0}, {10}); @@ -163,42 +156,56 @@ auto read_file_in_parallel(bool use_group_table) -> void REQUIRE(data.get()[i] == int(i)); } // clang-format off - /* - * Step 0: - * uint64_t /data/snapshot attr = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9} - * Step 1: - * int32_t /data/meshes/e_chargeDensity {10} - * uint64_t /data/snapshot attr = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19} - * Step 2: - * uint64_t /data/snapshot attr = {20, 21, 22, 23, 24, 25, 26, 27, 28, 29} - * Step 3: - * int32_t /data/meshes/e_chargeDensity {10} - * uint64_t /data/snapshot attr = {30, 31, 32, 33, 34, 35, 36, 37, 38, 39} - * Step 4: - * uint64_t /data/snapshot attr = {40, 41, 42, 43, 44, 45, 46, 47, 48, 49} - */ + /* + * Step 0: + * uint64_t /data/snapshot attr = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9} + * Step 1: + * int32_t /data/meshes/e_chargeDensity {10} + * uint64_t /data/snapshot attr = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19} + * Step 2: + * uint64_t /data/snapshot attr = {20, 21, 22, 23, 24, 25, 26, 27, 28, 29} + * Step 3: + * int32_t /data/meshes/e_chargeDensity {10} + * uint64_t /data/snapshot attr = {30, 31, 32, 33, 34, 35, 36, 37, 38, 39} + * Step 4: + * uint64_t /data/snapshot attr = {40, 41, 42, 43, 44, 45, 46, 47, 48, 49} + */ // clang-format on size_t adios_step = index / 10; // 10 iterations per step bool step_has_charge_density = adios_step % 2 == 1; - if (use_group_table) + // Without a group table, the groups need to be recovered from + // attributes and variables found in the ADIOS2 file. But since the + // e_chargeDensity mesh exists only in a subselection of steps, its + // attributes will leak into the other steps, making the API see just an + // empty mesh. The behavior now depends on how strictly we are parsing: + // + // 1. If verify_homogeneous_extent == true (default): The reader will + // notice that no extent is defined anywhere, the mesh will be erased + // with a warning. + // 2. If verify_homogeneous_extent == false: An empty mesh will be + // returned. + if (!dont_verify_homogeneous_extents.has_value()) { REQUIRE( iteration.meshes.contains("e_chargeDensity") == step_has_charge_density); + if (step_has_charge_density) + { + REQUIRE(iteration.meshes["e_chargeDensity"].scalar()); + } } else { - // Without a group table, the groups need to be recovered from - // attributes and variables found in the ADIOS2 file. But since the - // e_chargeDensity mesh exists only in a subselection of steps, its - // attributes will leak into the other steps, making the API think - // that there is data where there is none. REQUIRE(iteration.meshes.contains("e_chargeDensity")); // Only when the variable is also found, the reading routines will // correctly determine that this is a scalar mesh. REQUIRE( iteration.meshes["e_chargeDensity"].scalar() == step_has_charge_density); + if (!step_has_charge_density) + { + REQUIRE(iteration.meshes["e_chargeDensity"].size() == 0); + } } if (step_has_charge_density) { @@ -222,13 +229,14 @@ auto read_variablebased_randomaccess() -> void create_file_in_serial(true); } MPI_Barrier(MPI_COMM_WORLD); - read_file_in_parallel(true); + // read_file_in_parallel(std::nullopt); if (rank == 0) { create_file_in_serial(false); } MPI_Barrier(MPI_COMM_WORLD); - read_file_in_parallel(false); + read_file_in_parallel(std::nullopt); + read_file_in_parallel(R"({"verify_homogeneous_extents": false})"); } } // namespace read_variablebased_randomaccess #else From 6257ac726d2fabc6c56dd935b15323e32147ec19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 14 Aug 2025 17:16:53 +0200 Subject: [PATCH 20/20] Relax JSON schema --- share/openPMD/json_schema/record_component.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/share/openPMD/json_schema/record_component.toml b/share/openPMD/json_schema/record_component.toml index 9a4741a9f2..09e6bca54e 100644 --- a/share/openPMD/json_schema/record_component.toml +++ b/share/openPMD/json_schema/record_component.toml @@ -44,7 +44,10 @@ title = "Either array or constant" [allOf.if] required = ["attributes"] -properties.attributes.required = ["shape", "value"] +# shape optional as of https://github.com/openPMD/openPMD-api/pull/1661 +# JSON schema not powerful enough to check that the shape must be defined +# at least once (just like we do not check that the shape is consistent). +properties.attributes.required = ["value"] [allOf.then] title = "Constant dataset"