Merge pull request #575 from sebproell/input-file-in-sections

sebproell · web-flow · commit 8c4306f7ea00 · 2025-03-28T12:49:58.000+01:00
Use InputFile::match_section
diff --git a/src/core/fem/src/condition/4C_fem_condition_definition.cpp b/src/core/fem/src/condition/4C_fem_condition_definition.cpp
@@ -64,30 +64,27 @@ void Core::Conditions::ConditionDefinition::read(Core::IO::InputFile& input,
       all_of(specs_),
   });
 
-  for (const auto& fragment : input.in_section(section_name()))
+  Core::IO::InputParameterContainer container;
+  try
   {
-    std::optional<Core::IO::InputParameterContainer> data;
-    try
-    {
-      data = fragment.match(condition_spec);
-    }
-    catch (const Core::Exception& e)
-    {
-      FOUR_C_THROW("Failed to match condition specification in section '{}'. The error was:\n{}.",
-          section_name().c_str(), e.what());
-    }
-    if (!data)
-    {
-      FOUR_C_THROW(
-          "Failed to match condition specification in section '{}'.", sectionname_.c_str());
-    }
+    input.match_section(section_name(), container);
+  }
+  catch (const Core::Exception& e)
+  {
+    FOUR_C_THROW("Failed to match condition specification in section '{}'. The error was:\n{}.",
+        section_name(), e.what());
+  }
 
+
+  for (const auto& condition_data :
+      container.get_or<std::vector<Core::IO::InputParameterContainer>>(section_name(), {}))
+  {
     // Read a one-based condition number but convert it to zero-based for internal use.
-    const int dobjid = data->get<int>("E") - 1;
+    const int dobjid = condition_data.get<int>("E") - 1;
 
     std::shared_ptr<Core::Conditions::Condition> condition =
         std::make_shared<Core::Conditions::Condition>(dobjid, condtype_, buildgeometry_, gtype_);
-    condition->parameters() = *std::move(data);
+    condition->parameters() = condition_data;
 
     //------------------------------- put condition in map of conditions
     cmap.insert(std::pair<int, std::shared_ptr<Core::Conditions::Condition>>(dobjid, condition));
diff --git a/src/core/io/src/4C_io_input_file.cpp b/src/core/io/src/4C_io_input_file.cpp
@@ -309,18 +309,32 @@ namespace Core::IO
                std::all_of(section_name.begin() + 5, section_name.end(),
                    [](const char c) { return std::isdigit(c); });
       }
+
+      bool is_legacy_section(const std::string& section_name) const
+      {
+        return std::ranges::any_of(
+            legacy_section_names_, [&](const auto& name) { return name == section_name; });
+      }
+
+      InputFile::FragmentIteratorRange in_section(const std::string& section_name) const
+      {
+        static const std::vector<InputFile::Fragment> empty;
+
+        if (!content_by_section_.contains(section_name))
+        {
+          return std::views::all(empty);
+        }
+
+        // Take a const reference to the section content to match the return type.
+        const auto& lines = content_by_section_.at(section_name).fragments;
+        return std::views::all(lines);
+      }
     };
 
   }  // namespace Internal
 
   namespace
   {
-    /**
-     * Sections that contain at least this number of entries are considered huge and are only
-     * available on rank 0.
-     */
-    constexpr std::size_t huge_section_threshold = 10'000;
-
     //! The different ways we want to handle sections in the input file.
     enum class SectionType
     {
@@ -720,31 +734,28 @@ namespace Core::IO
     // Communicate the dat content
     if (Core::Communication::my_mpi_rank(pimpl_->comm_) == 0)
     {
-      // Temporarily move the sections that are not huge into a separate map.
-      std::unordered_map<std::string, Internal::SectionContent> non_huge_sections;
+      // Temporarily move the sections that we want to broadcast into a separate map.
+      std::unordered_map<std::string, Internal::SectionContent> non_legacy_sections;
 
       for (auto&& [section_name, content] : pimpl_->content_by_section_)
       {
-        if (std::holds_alternative<Internal::SectionContent::DatContent>(content.content))
+        if (std::holds_alternative<Internal::SectionContent::DatContent>(content.content) &&
+            !pimpl_->is_legacy_section(section_name))
         {
-          if (content.as_dat().lines.size() < huge_section_threshold)
-          {
-            non_huge_sections[section_name] = std::move(content);
-          }
+          non_legacy_sections[section_name] = std::move(content);
         }
       }
 
-      Core::Communication::broadcast(non_huge_sections, 0, pimpl_->comm_);
+      Core::Communication::broadcast(non_legacy_sections, 0, pimpl_->comm_);
 
-      // Move the non-huge sections back into the main map.
-      for (auto&& [section_name, content] : non_huge_sections)
+      for (auto&& [section_name, content] : non_legacy_sections)
       {
         pimpl_->content_by_section_[section_name] = std::move(content);
       }
     }
     else
     {
-      // Other ranks receive the non-huge sections.
+      // Other ranks receive the non-legacy sections.
       Core::Communication::broadcast(pimpl_->content_by_section_, 0, pimpl_->comm_);
     }
 
@@ -754,7 +765,7 @@ namespace Core::IO
       {
         ryml::Tree tree_with_small_sections = init_yaml_tree_with_exceptions();
         tree_with_small_sections.rootref() |= ryml::MAP;
-        // Go through the tree and drop the huge sections from the tree.
+        // Go through the tree and drop the legacy sections from the tree.
         for (auto file_node : pimpl_->yaml_tree_.rootref())
         {
           auto new_file_node = tree_with_small_sections.rootref().append_child();
@@ -763,8 +774,7 @@ namespace Core::IO
 
           for (auto section_node : file_node.children())
           {
-            if (section_node.is_map() ||
-                (section_node.is_seq() && section_node.num_children() < huge_section_threshold))
+            if (!pimpl_->is_legacy_section(to_string(section_node.key())))
             {
               // Copy the node to the new tree.
               auto new_section_node = new_file_node.append_child();
@@ -773,10 +783,8 @@ namespace Core::IO
           }
         }
 
-        std::stringstream ss;
-        ss << tree_with_small_sections;
-        std::string yaml_str = ss.str();
-        Core::Communication::broadcast(yaml_str, /*root*/ 0, pimpl_->comm_);
+        auto serialized_tree = ryml::emitrs_yaml<std::string>(tree_with_small_sections);
+        Core::Communication::broadcast(serialized_tree, /*root*/ 0, pimpl_->comm_);
       }
       else
       {
@@ -836,48 +844,15 @@ namespace Core::IO
     }
   }
 
-  InputFile::FragmentIteratorRange InputFile::in_section(const std::string& section_name) const
-  {
-    static const std::vector<Fragment> empty;
-
-    // Early return in case the section does not exist at all.
-    const bool known_somewhere = has_section(section_name);
-    if (!known_somewhere)
-    {
-      return std::views::all(empty);
-    }
-
-    const bool locally_known = pimpl_->content_by_section_.contains(section_name);
-    const bool known_everywhere = Core::Communication::all_reduce<bool>(
-        locally_known, [](const bool& r, const bool& in) { return r && in; }, pimpl_->comm_);
-
-    if (known_everywhere)
-    {
-      // Take a const reference to the section content to match the return type.
-      const auto& lines = pimpl_->content_by_section_.at(section_name).fragments;
-      return std::views::all(lines);
-    }
-    else
-    // Distribute the content of the section to all ranks.
-    {
-      FOUR_C_ASSERT((!locally_known && (Core::Communication::my_mpi_rank(pimpl_->comm_) > 0)) ||
-                        (locally_known && (Core::Communication::my_mpi_rank(pimpl_->comm_) == 0)),
-          "Implementation error: section should be known on rank 0 and unknown on others.");
-
-      auto& content = pimpl_->content_by_section_[section_name];
-      Core::Communication::broadcast(content, 0, pimpl_->comm_);
-      content.set_up_fragments();
-
-      const auto& lines = pimpl_->content_by_section_.at(section_name).fragments;
-      return std::views::all(lines);
-    }
-  }
-
-
 
   InputFile::FragmentIteratorRange InputFile::in_section_rank_0_only(
       const std::string& section_name) const
   {
+    FOUR_C_ASSERT_ALWAYS(pimpl_->is_legacy_section(section_name),
+        "You tried to process section '{}' on rank 0 only, but this feature is meant for special "
+        "legacy sections. Please use match_section() instead.",
+        section_name);
+
     if (Core::Communication::my_mpi_rank(pimpl_->comm_) == 0 &&
         pimpl_->content_by_section_.contains(section_name))
     {
@@ -952,7 +927,7 @@ namespace Core::IO
       if (list_spec)
       {
         std::vector<InputParameterContainer> list_entries;
-        for (const auto& line : in_section(section_name))
+        for (const auto& line : pimpl_->in_section(section_name))
         {
           Core::IO::ValueParser parser{line.get_as_dat_style_string(),
               {.user_scope_message =
@@ -977,7 +952,7 @@ namespace Core::IO
         // Create a group in the dat file format by starting with the section name.
         std::stringstream flattened_dat;
         flattened_dat << std::quoted(section_name) << " ";
-        for (const auto& line : in_section(section_name))
+        for (const auto& line : pimpl_->in_section(section_name))
         {
           std::string line_str(line.get_as_dat_style_string());
           // Split the line into key-value according to the dat file format. Quote keys and values
@@ -1084,7 +1059,7 @@ namespace Core::IO
         auto section = tree.rootref().append_child();
         section << ryml::key(section_name);
         section |= ryml::SEQ;
-        for (const auto& line : in_section(section_name))
+        for (const auto& line : pimpl_->in_section(section_name))
         {
           section.append_child() = ryml::csubstr(
               line.get_as_dat_style_string().data(), line.get_as_dat_style_string().size());
diff --git a/src/core/io/src/4C_io_input_file.hpp b/src/core/io/src/4C_io_input_file.hpp
@@ -40,19 +40,16 @@ namespace Core::IO
    * This class encapsulates input files independent of their format.
    *
    * Objects of this class read the content of a file and grant access to it. The input is not
-   * yet interpreted in any way. Input files contain different sections. A section either contains
-   * key-value pairs or a list of arbitrary lines of text. The content is accessed via the
-   * in_section() function which return InputFile::Fragment objects. These hide what exactly is
-   * contained and in which format. The content can be matched against an expected InputSpec to
-   * extract meaningful data.
+   * yet interpreted in any way. Input files contain different sections. The content can be matched
+   * against an expected InputSpec via match_section() to extract meaningful data.
    *
    * Sections may appear in an arbitrary order and each section name must be unique. An exception is
    * the special section named "INCLUDES" which can contain a list of other files that should be
    * read in addition to the current file.
    *
    * Three file formats are supported: the custom .dat file format and the standard .yaml (or .yml)
    * and .json formats. The format of a file is detected based on its ending. If the ending is not
-   * one of the above mentioned, the file is assumed to be in the .dat format.
+   * one of the above-mentioned, the file is assumed to be in the .dat format.
    * Included files do not have to use the same format as the including file.
    *
    * The following example shows the structure of a .dat file:
@@ -86,12 +83,13 @@ namespace Core::IO
    * SECTION2:
    *   - A line with content that is not a key-value pair.
    *   - It can contain anything.
+   *   - We call these unstructured sections "legacy sections".
    * @endcode
    *
    *
-   * @note The file is only read on rank 0 to save memory. Sections that are huge are only
-   * distributed to other ranks if they are accessed through line_in_section(). If you only
-   * want to read a section on rank 0, use in_section_rank_0_only().
+   * @note The file is only read on rank 0 to save memory. All sections are broadcast to all other
+   * ranks, except for the legacy sections (see the constructor). Legacy sections need to be
+   * consumed on rank 0 with the help of in_section_rank_0_only().
    */
   class InputFile
   {
@@ -213,31 +211,10 @@ namespace Core::IO
      */
     [[nodiscard]] std::filesystem::path file_for_section(const std::string& section_name) const;
 
-    /**
-     * Get a range of Fragments inside a section. A Fragment always contains meaningful content,
-     * i.e., something other than whitespace or comments. The usual way to interact with the
-     * Fragments is as follows:
-     *
-     * @code
-     *   for (const auto& input_fragment : input.in_section("section_name"))
-     *   {
-     *      auto parameters = fragment.match(spec);
-     *   }
-     * @endcode
-     *
-     * @return A range of Fragments which encapsulate the content of the section. Use the
-     * Fragment::match() function to extract the content.
-     *
-     * @note This is a collective call that needs to be called on all MPI ranks in the
-     * communicator associated with this object. Depending on the section size, the content
-     * might need to be distributed from rank 0 to all other ranks. This happens automatically.
-     */
-    FragmentIteratorRange in_section(const std::string& section_name) const;
 
     /**
-     * This function is similar to in_section(), but it only returns the lines on rank 0 and
-     * returns an empty range on all other ranks. This is useful for sections that might be huge
-     * and are not processed on all ranks.
+     * Returns the lines in a section on rank 0 and returns an empty range on all other ranks. This
+     * only works for legacy sections.
      */
     FragmentIteratorRange in_section_rank_0_only(const std::string& section_name) const;
 
@@ -247,7 +224,7 @@ namespace Core::IO
      * that you do not need to pass an InputSpec to this function, since the InputFile object
      * already knows about the expected format of the section. Nevertheless, this function only
      * makes sense for sections with a known InputSpec. Legacy string sections cannot use this
-     * function and must be processed with in_section() or in_section_rank_0_only().
+     * function and must be processed with in_section_rank_0_only().
      */
     void match_section(const std::string& section_name, InputParameterContainer& container) const;
 
diff --git a/src/core/io/src/4C_io_input_parameter_container.hpp b/src/core/io/src/4C_io_input_parameter_container.hpp
@@ -21,6 +21,7 @@
 #include <map>
 #include <optional>
 #include <ostream>
+#include <ranges>
 #include <string>
 #include <typeindex>
 #include <vector>
@@ -50,6 +51,8 @@ namespace Core::IO
      */
     using List = std::vector<InputParameterContainer>;
 
+    using GroupStorage = std::map<std::string, InputParameterContainer>;
+
     /**
      * \brief Add @data to the container at the given key @name.
      *
@@ -68,6 +71,11 @@ namespace Core::IO
      */
     [[nodiscard]] const InputParameterContainer& group(const std::string& name) const;
 
+    /**
+     * Get a range view of all groups in the container.
+     */
+    [[nodiscard]] auto groups() const;
+
     /**
      * Check if a group with the name @p name exists.
      */
@@ -266,6 +274,11 @@ namespace Core::IO::Internal::InputParameterContainerImplementation
 }  // namespace Core::IO::Internal::InputParameterContainerImplementation
 
 
+inline auto Core::IO::InputParameterContainer::groups() const
+{
+  return std::ranges::subrange(groups_);
+}
+
 
 template <typename T>
 const T& Core::IO::InputParameterContainer::get(const std::string& name) const
diff --git a/src/core/io/tests/4C_io_input_file_test.cpp b/src/core/io/tests/4C_io_input_file_test.cpp
diff --git a/src/global_data/4C_global_data_read.cpp b/src/global_data/4C_global_data_read.cpp
