Merge pull request PowerGridModel#1276 from PowerGridModel/feature/owning-dataset-in-cpp-api

TonyXiang8787 · web-flow · commit 2095f643b47c · 2026-01-26T14:17:04.000Z
Move owning dataset buffer creation logic into cpp api
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -1,11 +1,6 @@
 {
-  "cmake.buildDirectory": "${workspaceFolder}/cpp_build/${buildType}",
   "C_Cpp.autoAddFileAssociations": false,
   "cmake.configureOnOpen": true,
-  "cmake.generator": "Ninja",
-  "cmake.configureSettings": {
-    "POWER_GRID_MODEL_SANITIZER": 1
-  },
   "testMate.cpp.test.executables": "${command:cmake.buildDirectory}/bin/power_grid_model_*_tests*",
   "testMate.cpp.debug.configTemplate": {
     "type": "cppvsdbg",
@@ -21,6 +16,7 @@
   },
   "doxdocgen.generic.filteredKeywords": ["PGM_API"],
   "doxdocgen.generic.includeTypeAtReturn": false,
+  "python.defaultInterpreterPath": "${workspaceFolder}/.venv",
   "python.testing.pytestArgs": ["tests", "--no-cov"],
   "python.testing.unittestEnabled": false,
   "python.testing.pytestEnabled": true,
@@ -31,6 +27,9 @@
   },
   "C_Cpp.formatting": "clangFormat",
   "C_Cpp.clang_format_fallbackStyle": "LLVM",
+  "[python]": {
+    "editor.defaultFormatter": "charliermarsh.ruff"
+  },
   "[markdown]": {
     "editor.defaultFormatter": "DavidAnson.vscode-markdownlint",
     "editor.rulers": [120]
diff --git a/power_grid_model_c/power_grid_model_cpp/include/power_grid_model_cpp/buffer.hpp b/power_grid_model_c/power_grid_model_cpp/include/power_grid_model_cpp/buffer.hpp
@@ -17,6 +17,8 @@ class Buffer {
     Buffer(MetaComponent const* component, Idx size)
         : component_{component}, size_{size}, buffer_{handle_.call_with(PGM_create_buffer, component, size)} {};
 
+    Buffer() : component_{nullptr}, size_{0}, buffer_{nullptr} {};
+
     RawDataConstPtr get() const { return buffer_.get(); }
     RawDataPtr get() { return buffer_.get(); }
 
diff --git a/power_grid_model_c/power_grid_model_cpp/include/power_grid_model_cpp/dataset.hpp b/power_grid_model_c/power_grid_model_cpp/include/power_grid_model_cpp/dataset.hpp
@@ -9,9 +9,14 @@
 #include "basics.hpp"
 #include "buffer.hpp"
 #include "handle.hpp"
+#include "utils.hpp"
 
 #include "power_grid_model_c/dataset.h"
 
+#include <map>
+#include <set>
+#include <variant>
+
 namespace power_grid_model_cpp {
 class ComponentTypeNotFound : public PowerGridError {
   public:
@@ -210,14 +215,139 @@ class DatasetConst {
     DatasetInfo info_;
 };
 
+class AttributeBuffer {
+  private:
+    using VariantType = std::variant<std::monostate, std::vector<ID>, std::vector<IntS>, std::vector<double>,
+                                     std::vector<std::array<double, 3>>>;
+
+    struct BufferCreator {
+        Idx size;
+        template <class T> VariantType operator()() const { return std::vector<T>(size, nan_value<T>()); }
+    };
+
+    struct PtrGetter {
+        // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members)
+        AttributeBuffer& buffer;
+        template <class T> RawDataPtr operator()() const { return std::get<std::vector<T>>(buffer.buffer_).data(); }
+    };
+
+  public:
+    AttributeBuffer() = default;
+
+    explicit AttributeBuffer(MetaAttribute const* attribute, Idx size)
+        : attribute_{attribute},
+          buffer_{pgm_type_func_selector(MetaData::attribute_ctype(attribute), BufferCreator{size})} {}
+
+    RawDataPtr get() { return pgm_type_func_selector(MetaData::attribute_ctype(attribute_), PtrGetter{*this}); }
+
+  private:
+    MetaAttribute const* attribute_{nullptr};
+    VariantType buffer_;
+};
+
 struct OwningMemory {
     std::vector<Buffer> buffers;
     std::vector<std::vector<Idx>> indptrs;
+    std::vector<std::vector<AttributeBuffer>> attribute_buffers;
 };
 
 struct OwningDataset {
     DatasetMutable dataset;
     OwningMemory storage{};
+
+    OwningDataset(DatasetWritable& writable_dataset, bool enable_columnar_buffers = false)
+        : dataset{writable_dataset.get_info().name(), writable_dataset.get_info().is_batch(),
+                  writable_dataset.get_info().batch_size()},
+          storage{} {
+        auto const& info = writable_dataset.get_info();
+        Idx const batch_size = info.batch_size();
+        auto const& dataset_name = info.name();
+
+        for (Idx component_idx{}; component_idx < info.n_components(); ++component_idx) {
+            auto const& component_name = info.component_name(component_idx);
+            auto const* const component_meta = MetaData::get_component_by_name(dataset_name, component_name);
+            Idx const component_size = info.component_total_elements(component_idx);
+            Idx const elements_per_scenario = info.component_elements_per_scenario(component_idx);
+
+            auto& current_indptr = storage.indptrs.emplace_back(elements_per_scenario < 0 ? batch_size + 1 : 0);
+            if (!current_indptr.empty()) {
+                current_indptr.at(0) = 0;
+                current_indptr.at(batch_size) = component_size;
+            }
+            Idx* const indptr = current_indptr.empty() ? nullptr : current_indptr.data();
+            if (info.has_attribute_indications(component_idx) && enable_columnar_buffers) {
+                auto& current_buffer = storage.buffers.emplace_back();
+                writable_dataset.set_buffer(component_name, indptr, current_buffer);
+                dataset.add_buffer(component_name, elements_per_scenario, component_size, indptr, current_buffer);
+                auto const& attribute_indications = info.attribute_indications(component_idx);
+                auto& current_attribute_buffers = storage.attribute_buffers.emplace_back();
+                for (auto const& attribute_name : attribute_indications) {
+                    auto const* const attribute_meta =
+                        MetaData::get_attribute_by_name(dataset_name, component_name, attribute_name);
+                    current_attribute_buffers.emplace_back(attribute_meta, component_size);
+                    writable_dataset.set_attribute_buffer(component_name, attribute_name,
+                                                          current_attribute_buffers.back().get());
+                    dataset.add_attribute_buffer(component_name, attribute_name,
+                                                 current_attribute_buffers.back().get());
+                }
+            } else {
+                auto& current_buffer = storage.buffers.emplace_back(component_meta, component_size);
+                storage.attribute_buffers.emplace_back(); // empty attribute buffers
+                writable_dataset.set_buffer(component_name, indptr, current_buffer);
+                dataset.add_buffer(component_name, elements_per_scenario, component_size, indptr, current_buffer);
+            }
+        }
+    }
+
+    OwningDataset(
+        OwningDataset const& ref_dataset, std::string const& dataset_name, bool is_batch = false, Idx batch_size = 1,
+        std::map<MetaComponent const*, std::set<MetaAttribute const*>> const& output_component_attribute_filters = {})
+        : dataset{dataset_name, is_batch, batch_size}, storage{} {
+        DatasetInfo const& ref_info = ref_dataset.dataset.get_info();
+        bool const enable_filters = !output_component_attribute_filters.empty();
+
+        for (Idx component_idx{}; component_idx != ref_info.n_components(); ++component_idx) {
+            auto const& component_name = ref_info.component_name(component_idx);
+            auto const& component_meta = MetaData::get_component_by_name(dataset_name, component_name);
+            // skip components not in the filter
+            if (enable_filters &&
+                output_component_attribute_filters.find(component_meta) == output_component_attribute_filters.end()) {
+                continue;
+            }
+
+            // get size info from reference dataset
+            Idx const component_elements_per_scenario = ref_info.component_elements_per_scenario(component_idx);
+            if (component_elements_per_scenario < 0) {
+                throw PowerGridError{"Cannot create result dataset for component with variable size per scenario"};
+            }
+            Idx const component_size = component_elements_per_scenario * batch_size;
+            storage.indptrs.emplace_back();
+
+            auto const component_filter_it = output_component_attribute_filters.find(component_meta);
+            std::set<MetaAttribute const*> const& attribute_filter =
+                component_filter_it != output_component_attribute_filters.end() ? component_filter_it->second
+                                                                                : std::set<MetaAttribute const*>{};
+            if (attribute_filter.empty()) {
+                // create full row buffer
+                auto& component_buffer = storage.buffers.emplace_back(component_meta, component_size);
+                storage.attribute_buffers.emplace_back(); // empty attribute buffers
+                dataset.add_buffer(component_name, component_elements_per_scenario, component_size, nullptr,
+                                   component_buffer);
+            } else {
+                // push nullptr as row buffer, and start attribute buffers
+                auto& component_buffer = storage.buffers.emplace_back();
+                storage.attribute_buffers.emplace_back();
+                dataset.add_buffer(component_name, component_elements_per_scenario, component_size, nullptr,
+                                   component_buffer);
+                for (auto const* const attribute_meta : attribute_filter) {
+                    auto const attribute_name = MetaData::attribute_name(attribute_meta);
+                    auto& attribute_buffer =
+                        storage.attribute_buffers.back().emplace_back(attribute_meta, component_size);
+                    dataset.add_attribute_buffer(component_name, attribute_name, attribute_buffer.get());
+                }
+            }
+        }
+    }
 };
 } // namespace power_grid_model_cpp
 
diff --git a/power_grid_model_c/power_grid_model_cpp/include/power_grid_model_cpp/serialization.hpp b/power_grid_model_c/power_grid_model_cpp/include/power_grid_model_cpp/serialization.hpp
@@ -14,6 +14,8 @@
 #include "power_grid_model_c/serialization.h"
 
 #include <cstring>
+#include <filesystem>
+#include <fstream>
 
 namespace power_grid_model_cpp {
 class Deserializer {
@@ -83,42 +85,52 @@ class Serializer {
     }
 
     std::string get_to_zero_terminated_string(Idx use_compact_list, Idx indent) {
-        return std::string{
-            handle_.call_with(PGM_serializer_get_to_zero_terminated_string, get(), use_compact_list, indent)};
+        return std::string{get_to_zero_terminated_c_string(use_compact_list, indent)};
+    }
+
+    char const* get_to_zero_terminated_c_string(Idx use_compact_list, Idx indent) {
+        return handle_.call_with(PGM_serializer_get_to_zero_terminated_string, get(), use_compact_list, indent);
     }
 
   private:
     power_grid_model_cpp::Handle handle_{};
     detail::UniquePtr<RawSerializer, &PGM_destroy_serializer> serializer_;
 };
 
-inline OwningDataset create_owning_dataset(DatasetWritable& writable_dataset) {
-    auto const& info = writable_dataset.get_info();
-    bool const is_batch = info.is_batch();
-    Idx const batch_size = info.batch_size();
-    auto const& dataset_name = info.name();
-    DatasetMutable dataset_mutable{dataset_name, is_batch, batch_size};
-    OwningMemory storage{};
-
-    for (Idx component_idx{}; component_idx < info.n_components(); ++component_idx) {
-        auto const& component_name = info.component_name(component_idx);
-        auto const& component_meta = MetaData::get_component_by_name(dataset_name, component_name);
-        Idx const component_size = info.component_total_elements(component_idx);
-        Idx const elements_per_scenario = info.component_elements_per_scenario(component_idx);
-
-        auto& current_indptr = storage.indptrs.emplace_back(elements_per_scenario < 0 ? batch_size + 1 : 0);
-        if (!current_indptr.empty()) {
-            current_indptr.at(0) = 0;
-            current_indptr.at(batch_size) = component_size;
+inline OwningDataset load_dataset(std::filesystem::path const& path, PGM_SerializationFormat serialization_format,
+                                  bool enable_columnar_buffers = false) {
+    auto read_file = [](std::filesystem::path const& read_file_path) {
+        std::ifstream f{read_file_path, std::ios::binary | std::ios::ate};
+        if (!f) {
+            throw std::runtime_error("Failed to open file: " + read_file_path.string());
         }
-        Idx* const indptr = current_indptr.empty() ? nullptr : current_indptr.data();
-        auto& current_buffer = storage.buffers.emplace_back(component_meta, component_size);
-        writable_dataset.set_buffer(component_name, indptr, current_buffer);
-        dataset_mutable.add_buffer(component_name, elements_per_scenario, component_size, indptr, current_buffer);
-    }
-    return OwningDataset{// NOLINT(modernize-use-designated-initializers)
-                         std::move(dataset_mutable), std::move(storage)};
+        auto const file_size = f.tellg();
+        f.seekg(0, std::ios::beg);
+        std::vector<char> buffer(static_cast<size_t>(file_size));
+        f.read(buffer.data(), file_size);
+        return buffer;
+    };
+
+    Deserializer deserializer{read_file(path), serialization_format};
+    auto& writable_dataset = deserializer.get_dataset();
+    OwningDataset dataset{writable_dataset, enable_columnar_buffers};
+    deserializer.parse_to_buffer();
+    return dataset;
 }
+
+inline void save_dataset(std::filesystem::path const& path, DatasetConst const& dataset,
+                         PGM_SerializationFormat serialization_format, Idx use_compact_list, Idx indent = 2) {
+    Serializer serializer{dataset, serialization_format};
+    std::string_view const serialized_data = serialization_format == PGM_msgpack
+                                                 ? serializer.get_to_binary_buffer(use_compact_list)
+                                                 : serializer.get_to_zero_terminated_c_string(use_compact_list, indent);
+    if (std::ofstream f{path, std::ios::binary}; f.is_open()) {
+        f << serialized_data;
+    } else {
+        throw std::runtime_error{"Failed to open file for writing: " + path.string()};
+    };
+}
+
 } // namespace power_grid_model_cpp
 
 #endif // POWER_GRID_MODEL_CPP_SERIALIZATION_HPP
diff --git a/pyproject.toml b/pyproject.toml
@@ -46,7 +46,13 @@ lint = [
     "clang-format ~= 18.0", # C++ formatter
 ]
 
-dev = [{ include-group = "test" }, { include-group = "lint" }]
+example = ["ipykernel"]
+
+dev = [
+    { include-group = "test" },
+    { include-group = "lint" },
+    { include-group = "example" },
+]
 
 code-generation = ["dataclasses_json", "jinja2"]
 
diff --git a/tests/cpp_validation_tests/test_validation.cpp b/tests/cpp_validation_tests/test_validation.cpp
@@ -41,43 +41,6 @@ auto read_json(std::filesystem::path const& path) {
     return j;
 }
 
-OwningDataset create_result_dataset(OwningDataset const& input, std::string const& dataset_name, bool is_batch = false,
-                                    Idx batch_size = 1) {
-    DatasetInfo const& input_info = input.dataset.get_info();
-
-    OwningDataset result{.dataset = DatasetMutable{dataset_name, is_batch, batch_size}, .storage{}};
-
-    for (Idx component_idx{}; component_idx != input_info.n_components(); ++component_idx) {
-        auto const& component_name = input_info.component_name(component_idx);
-        auto const& component_meta = MetaData::get_component_by_name(dataset_name, component_name);
-        Idx const component_elements_per_scenario = input_info.component_elements_per_scenario(component_idx);
-        Idx const component_size = input_info.component_total_elements(component_idx);
-
-        auto& current_indptr = result.storage.indptrs.emplace_back(
-            input_info.component_elements_per_scenario(component_idx) < 0 ? batch_size + 1 : 0);
-        Idx const* const indptr = current_indptr.empty() ? nullptr : current_indptr.data();
-        auto& current_buffer = result.storage.buffers.emplace_back(component_meta, component_size);
-        result.dataset.add_buffer(component_name, component_elements_per_scenario, component_size, indptr,
-                                  current_buffer);
-    }
-    return result;
-}
-
-OwningDataset load_dataset(std::filesystem::path const& path) {
-    auto read_file = [](std::filesystem::path const& read_file_path) {
-        std::ifstream const f{read_file_path};
-        std::ostringstream buffer;
-        buffer << f.rdbuf();
-        return buffer.str();
-    };
-
-    Deserializer deserializer{read_file(path), PGM_json};
-    auto& writable_dataset = deserializer.get_dataset();
-    auto dataset = create_owning_dataset(writable_dataset);
-    deserializer.parse_to_buffer();
-    return dataset;
-}
-
 template <typename T> std::string get_as_string(T const& attribute_value) {
     std::stringstream sstr;
     sstr << std::setprecision(16);
@@ -571,17 +534,17 @@ struct ValidationCase {
 ValidationCase create_validation_case(CaseParam const& param, std::string const& output_type) {
     // input
     ValidationCase validation_case{.param = param,
-                                   .input = load_dataset(param.case_dir / "input.json"),
+                                   .input = load_dataset(param.case_dir / "input.json", PGM_json, true),
                                    .output = std::nullopt,
                                    .update_batch = std::nullopt,
                                    .output_batch = std::nullopt};
 
     // output and update
     if (!param.is_batch) {
-        validation_case.output = load_dataset(param.case_dir / (output_type + ".json"));
+        validation_case.output = load_dataset(param.case_dir / (output_type + ".json"), PGM_json);
     } else {
-        validation_case.update_batch = load_dataset(param.case_dir / "update_batch.json");
-        validation_case.output_batch = load_dataset(param.case_dir / (output_type + "_batch.json"));
+        validation_case.update_batch = load_dataset(param.case_dir / "update_batch.json", PGM_json);
+        validation_case.output_batch = load_dataset(param.case_dir / (output_type + "_batch.json"), PGM_json);
     }
     return validation_case;
 }
@@ -643,7 +606,7 @@ void validate_single_case(CaseParam const& param) {
     execute_test(param, [&param](Subcase& subcase) {
         auto const output_prefix = get_output_type(param.calculation_type, param.sym);
         auto const validation_case = create_validation_case(param, output_prefix);
-        auto const result = create_result_dataset(validation_case.output.value(), output_prefix);
+        OwningDataset const result{validation_case.output.value(), output_prefix};
 
         // create and run model
         auto const& options = get_options(param);
@@ -661,8 +624,7 @@ void validate_batch_case(CaseParam const& param) {
         auto const validation_case = create_validation_case(param, output_prefix);
         auto const& info = validation_case.update_batch.value().dataset.get_info();
         Idx const batch_size = info.batch_size();
-        auto const batch_result =
-            create_result_dataset(validation_case.output_batch.value(), output_prefix, true, batch_size);
+        OwningDataset const batch_result{validation_case.output_batch.value(), output_prefix, true, batch_size};
 
         // create model
         Model model{50.0, validation_case.input.dataset};
diff --git a/tests/data/power_flow/1os2msr/README.md b/tests/data/power_flow/1os2msr/README.md
@@ -0,0 +1,9 @@
+<!--
+SPDX-FileCopyrightText: Contributors to the Power Grid Model project <powergridmodel@lfenergy.org>
+
+SPDX-License-Identifier: MPL-2.0
+-->
+
+# Simple test case: 1 primary substation (OS), 2 secondary substations (MSR)
+
+The input data has some attribut indications to test also columnar deserializer.
diff --git a/tests/data/power_flow/1os2msr/input.json b/tests/data/power_flow/1os2msr/input.json
diff --git a/tests/native_api_tests/load_dataset.hpp b/tests/native_api_tests/load_dataset.hpp
diff --git a/uv.lock b/uv.lock
