create res tables only for elements that exist in the network (#2813)

* initial impl

* fix a lot a res_ entry access errors

* fix non existant res errors

* fix consistency checks

* fix runpp3ph

* fix missing res errors

* fix keys in util

* fix util.py

* fix for ucte test

* update power system test case jsons

* update networks, fix utils

* fix rundcpp test

* fix adapt result tables in replace

* add gen to pf_3ph relevant elements

* fix get result from gen 3ph

* fixed elements_to_deserialize breaking custom key compatibility.

* removed empty res_ tables from new networks when using metadata feature

* fixed useages of res_ tables while not present
removed _empty_res_ tables from net and created own object to get them
fixed sc tests ignoring columns order

---------

Co-authored-by: KS_HTK <2981026+KS-HTK@users.noreply.github.com>

Author: heckstrahler

pandapower/auxiliary.py

@@ -5,7 +5,7 @@
 
 import numbers
 import warnings
-from collections.abc import MutableMapping, Iterable, Collection
+from collections.abc import Iterable, Collection
 from importlib.metadata import PackageNotFoundError
 from importlib.metadata import version as version_str
 from typing import (

pandapower/convert_format.py

@@ -44,6 +44,12 @@ def convert_format(net, elements_to_deserialize=None, drop_invalid_geodata=False
         if not bool(cols.issubset(net.load.columns)):
             for col in cols:
                 add_column_to_df(net, "load", col)
+
+        # drop empty res_ tables and _empty_res_ tables
+        for key in list(net.keys()):  # conversion to list required because of dict modification while iteration
+            if key.startswith("_empty_res_") or (key.startswith("res_") and len(net[key]) == 0):
+                del net[key]
+
     if net_format_version < Version("3.1.0"):
         _convert_q_capability_characteristic(net)
     if Version("3.0.0") <= net_format_version < Version("3.1.3"):
@@ -155,12 +161,12 @@ def _restore_index_names(net):
     """Restores dataframes index names stored as dictionary. With newer pp to_json() this
     information is stored to the dataframe its self.
     """
-    if "index_names" in net.keys():
+    if "index_names" in net:
         if not isinstance(net["index_names"], dict):
             raise ValueError("To restore the index names of the dataframes, a dict including this "
                              f"information is expected, not {type(net['index_names'])}")
         for key, index_name in net["index_names"].items():
-            if key in net.keys():
+            if key in net:
                 net[key].index.name = index_name
         del net["index_names"]
 
@@ -208,19 +214,19 @@ def _convert_trafo_controller_parameter_names(net):
         controller = net.controller.at[ctrl_idx, "object"]
         if issubclass(type(controller), TrafoController):
 
-            if "tid" in controller.__dict__.keys():
+            if "tid" in controller.__dict__:
                 controller.__dict__["element_index"] = controller.__dict__.pop("tid")
-            elif "transformer_index" in controller.__dict__.keys():
+            elif "transformer_index" in controller.__dict__:
                 controller.__dict__["element_index"] = controller.__dict__.pop("transformer_index")
 
-            if "trafotable" in controller.__dict__.keys():
+            if "trafotable" in controller.__dict__:
                 controller.__dict__["element"] = controller.__dict__.pop("trafotable")
-                if "trafotype" in controller.__dict__.keys():
+                if "trafotype" in controller.__dict__:
                     del controller.__dict__["trafotype"]
-            elif "trafotype" in controller.__dict__.keys():
+            elif "trafotype" in controller.__dict__:
                 controller.__dict__["element"] = controller.__dict__.pop("trafotype")
 
-            if "controlled_bus" in controller.__dict__.keys():
+            if "controlled_bus" in controller.__dict__:
                 controller.__dict__["trafobus"] = controller.__dict__.pop("controlled_bus")
 
 
@@ -289,8 +295,8 @@ def _add_nominal_power(net):
 def _add_missing_tables(net):
     net_new = pandapowerNet(name='missing_tables_net')
     net_new.name = ""  # name is set to avoid warnings, then unset here to avoid adding it to any network
-    for key in net_new.keys():
-        if key.startswith("_empty_res") or key not in net.keys():
+    for key in net_new:
+        if key.startswith("_empty_res") or key not in net:
             net[key] = net_new[key]
 
 
@@ -374,7 +380,7 @@ def _rename_columns(net, elements_to_deserialize):
         if "controller" in net:
             net["controller"] = net["controller"].rename(columns={"controller": "object"})
 
-    if _check_elements_to_deserialize('res_line_3ph', elements_to_deserialize):
+    if 'res_line_3ph' in net and _check_elements_to_deserialize('res_line_3ph', elements_to_deserialize):
         if "p_a_l_mw" in net.res_line_3ph:
             net['res_line_3ph'] = net['res_line_3ph'].rename(columns={
                 'p_a_l_mw': 'pl_a_mw',
@@ -385,7 +391,7 @@ def _rename_columns(net, elements_to_deserialize):
                 'q_c_l_mvar': 'ql_c_mvar',
             })
 
-    if _check_elements_to_deserialize('res_trafo_3ph', elements_to_deserialize):
+    if 'res_trafo_3ph' in net and _check_elements_to_deserialize('res_trafo_3ph', elements_to_deserialize):
         if "p_a_l_mw" in net.res_trafo_3ph:
             net['res_trafo_3ph'] = net['res_trafo_3ph'].rename(columns={
                 'p_a_l_mw': 'pl_a_mw',
@@ -517,16 +523,16 @@ def _add_missing_columns(net, elements_to_deserialize):
         net.switch['in_ka'] = np.nan
 
     # Update the switch table with 'in_ka'
-    if _check_elements_to_deserialize('res_switch', elements_to_deserialize) and \
-            'p_from_mw' not in net.res_switch:
-        net.res_switch['p_from_mw'] = np.nan
-        net.res_switch['q_from_mvar'] = np.nan
-        net.res_switch['p_to_mw'] = np.nan
-        net.res_switch['q_to_mvar'] = np.nan
+    if ('res_switch' in net and _check_elements_to_deserialize('res_switch', elements_to_deserialize) and
+            'p_from_mw' not in net.res_switch):
+            net.res_switch['p_from_mw'] = np.nan
+            net.res_switch['q_from_mvar'] = np.nan
+            net.res_switch['p_to_mw'] = np.nan
+            net.res_switch['q_to_mvar'] = np.nan
 
     # Update the switch table with 'in_ka'
-    if _check_elements_to_deserialize('res_switch_est', elements_to_deserialize) and \
-            'p_from_mw' not in net.res_switch_est:
+    if ('res_switch_est' in net and _check_elements_to_deserialize('res_switch_est', elements_to_deserialize) and
+            'p_from_mw' not in net.res_switch_est):
         net.res_switch_est['p_from_mw'] = np.nan
         net.res_switch_est['q_from_mvar'] = np.nan
         net.res_switch_est['p_to_mw'] = np.nan
@@ -560,8 +566,8 @@ def _add_missing_columns(net, elements_to_deserialize):
             "slack_weight" not in net.xward:
         net.xward['slack_weight'] = 0.0
 
-    if _check_elements_to_deserialize('res_line_3ph', elements_to_deserialize) and \
-        "p_c_from_mw" not in net.res_line_3ph:
+    if ('res_line_3ph' in net and _check_elements_to_deserialize('res_line_3ph', elements_to_deserialize) and
+            "p_c_from_mw" not in net.res_line_3ph):
             net.res_line_3ph['p_c_from_mw'] = np.nan
             net.res_line_3ph['loading_a_percent'] = np.nan
             net.res_line_3ph['loading_b_percent'] = np.nan
@@ -570,8 +576,8 @@ def _add_missing_columns(net, elements_to_deserialize):
 
 def _update_trafo_type_parameter_names(net):
     for element in ('trafo', 'trafo3w'):
-        for type in net.std_types[element].keys():
-            keys = {col: _update_column(col) for col in net.std_types[element][type].keys() if
+        for type in net.std_types[element]:
+            keys = {col: _update_column(col) for col in net.std_types[element][type] if
                     col.startswith("tp") or col.startswith("vsc")}
             for old_key, new_key in keys.items():
                 net.std_types[element][type][new_key] = net.std_types[element][type].pop(old_key)
@@ -614,7 +620,7 @@ def _set_data_type_of_columns(net):
 
 def _convert_to_mw(net):
     replace = [("kw", "mw"), ("kvar", "mvar"), ("kva", "mva")]
-    for element in net.keys():
+    for element in net:
         if isinstance(net[element], pd.DataFrame):
             for old, new in replace:
                 diff = {column: column.replace(old, new) for column in net[element].columns if
@@ -682,7 +688,7 @@ def _convert_objects(net, elements_to_deserialize):
     """
     _check_elements_to_deserialize('controller', elements_to_deserialize)
     if _check_elements_to_deserialize('controller', elements_to_deserialize) and \
-            "controller" in net.keys():
+            "controller" in net:
         for obj in net["controller"].object.values:
             _update_object_attributes(obj)
 

pandapower/converter/cim/cim2pp/build_pp_net.py

@@ -25,7 +25,6 @@
 
 
 class CimConverter:
-
     def __init__(self, cim_parser: cim_classes.CimParser, converter_classes: Dict,
                  cim_version: str | None = None, **kwargs):
         self.logger = logging.getLogger(self.__class__.__name__)

pandapower/converter/cim/cim_tools.py

@@ -1,10 +1,11 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2016-2026 by University of Kassel and Fraunhofer Institute for Energy Economics
 # and Energy System Technology (IEE), Kassel. All rights reserved.
+
 import logging
 import os
 import json
 from typing import Dict, List
+
 import numpy as np
 import pandas as pd
 
@@ -33,21 +34,25 @@ def get_pp_net_special_columns_dict() -> Dict[str, str]:
 
 def extend_pp_net_cim(net: pandapowerNet, override: bool = True) -> pandapowerNet:
     """
-    Extend pandapower element DataFrames with special columns for the CIM converter, e.g. a column for the RDF ID.
-    :param net: The pandapower net to extend.
-    :param override: If True, all existing special CIM columns will be overwritten (content will be erased). If False,
-    only missing columns will be created. Optional, default: True
-    :return: The extended pandapower network.
+    Extend pandapower network with special element for the CIM converter.
+
+    ..note::
+        The CIM converter creates a pandapower network with metadata key "cim".
+        See :func:`pandapower.network.pandapowerNet.__init__`
+
+    Parameters:
+        net: The pandapower net to extend.
+        override: If True, net.CGMES will be overwritten (content will be erased). If False,
+            only missing element will be created.
+
+    Returns:
+        Reference to the input pandapower network (input will be modified).
     """
     # some special items
-    if override:
+    if 'CGMES' not in net or override:
         net['CGMES'] = {}
+    if 'BaseVoltage' not in net['CGMES'] or override:
         net['CGMES']['BaseVoltage'] = pd.DataFrame(None, columns=['rdfId', 'nominalVoltage'])
-    else:
-        if 'CGMES' not in net:
-            net['CGMES'] = {}
-        if 'BaseVoltage' not in net['CGMES']:
-            net['CGMES']['BaseVoltage'] = pd.DataFrame(None, columns=['rdfId', 'nominalVoltage'])
 
     return net