Merge branch 'develop' into 'master'

v2.3.7

See merge request iek-3/shared-code/fine!347

Author: kknos

.gitlab-ci.yml

@@ -2,7 +2,7 @@ image: mambaorg/micromamba
 
 stages:
   - test
-  - build
+  - deploy
 
 variables:
   DOCKER_HOST: tcp://docker:2375
@@ -38,6 +38,8 @@ variables:
       when: never
     - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
       when: never
+    - if: $CI_COMMIT_TAG
+      when: never
   retry: 1
 
 
@@ -65,6 +67,8 @@ variables:
         - requirements.yml
         - requirements_dev.yml
       when: never
+    - if: $CI_COMMIT_TAG
+      when: never
     - when: on_success
 
 
@@ -100,14 +104,16 @@ variables:
       when: never
     - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
       when: never
+    - if: $CI_COMMIT_TAG
+      when: never
     - changes:
         - pyproject.toml
         - requirements.yml
         - requirements_dev.yml
       when: on_success
 
 .build_template:
-  stage: build
+  stage: deploy
   image: docker@sha256:c8bb6fa5388b56304dd770c4bc0478de81ce18540173b1a589178c0d31bfce90
   services:
     - docker:dind@sha256:c8bb6fa5388b56304dd770c4bc0478de81ce18540173b1a589178c0d31bfce90
@@ -121,6 +127,8 @@ test-pypi:
     - python -m pip install .[develop]
     - python -m pytest -n auto test/
   rules:
+    - if: $CI_COMMIT_TAG
+      when: never
     - if: '$CI_COMMIT_BRANCH == "master"'
     - if: '$CI_COMMIT_BRANCH == "develop"'
     - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master"'
@@ -203,27 +211,12 @@ test-codestyle:
 
 # Deployment
 
-build-master-latest:
-  extends: .build_template
-  script:
-    # Login to the DockerHub repo using a specialized access token.
-    # Then, build the docker image with the tested code and tag it
-    # with the current version, as well as latest.
-    # Afterwards, push to DockerHub.
-    - docker login -u fzjiek3 -p $DOCKER_AT
-    - docker build -t "fzjiek3/fine:latest" .
-    - docker push fzjiek3/fine:latest
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "schedule"
-      when: never
-    - if: $CI_COMMIT_BRANCH == "master"
-
 build-tag:
   extends: .build_template
   script:
     - docker login -u fzjiek3 -p $DOCKER_AT
-    - docker build -t fzjiek3/fine:${CI_COMMIT_TAG} .
-    - docker push fzjiek3/fine:${CI_COMMIT_TAG}
+    - docker build -t fzjiek3/fine:${CI_COMMIT_TAG} -t fzjiek3/fine:latest .
+    - docker push fzjiek3/fine --all-tags
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule"
       when: never
@@ -242,14 +235,21 @@ build-dev:
     - if: $CI_COMMIT_BRANCH == "develop"
 
 pypi-upload:
-  stage: build
+  stage: deploy
+  image: python:3.12
+  before_script:
+    - python3 -m pip install --upgrade build
+    - python3 -m pip install --upgrade twine
+  variables:
+    TWINE_USERNAME: $PYPI_USERNAME
+    TWINE_PASSWORD: $PYPI_PASSWORD
   script:
-    - micromamba install -c conda-forge -n base -y python=3.10
-    - python -m pip install --upgrade build
-    - python -m pip install --upgrade twine
-    - python -m build
-    - python -m twine upload -u __token__ -p $PYPI_TOKEN  dist/*
+    # Test if the version defined in `pyproject.toml` is the same as the tag
+    - PYPROJECT_VERSION=$(grep -m 1 version pyproject.toml | tr -s ' ' | tr -d '"' | tr -d "'" | cut -d' ' -f3)
+    - test PYPROJECT_VERSION = v${CI_COMMIT_TAG}    
+    # Build and push to pypi
+    - python3 -m build
+    - python3 -m twine upload dist/*
   rules:
     - if: $CI_COMMIT_BRANCH == "master" && $CI_COMMIT_TAG
-  when: manual
 

docs/source/conf.py

@@ -80,7 +80,7 @@
 # The short X.Y version.
 version = "2.3"
 # The full version, including alpha/beta/rc tags.
-release = "2.3.6"
+release = "2.3.7"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

fine/IOManagement/utilsIO.py

@@ -223,7 +223,7 @@ def generateIterationDicts(component_dict, investmentPeriods):
     return df_iteration_dict, series_iteration_dict, constants_iteration_dict
 
 
-def addDFVariablesToXarray(xr_ds, component_dict, df_iteration_dict):
+def addDFVariablesToXarray(xr_ds, component_dict, df_iteration_dict, locations):
     """Adds all variables whose data is contained in a pd.DataFrame to xarray dataset.
     These variables are normally regional time series (dimensions - space, time)
 
@@ -238,10 +238,33 @@ def addDFVariablesToXarray(xr_ds, component_dict, df_iteration_dict):
         values - list of tuple of component class and component name
     :type df_iteration_dict: dict
 
+    :param locations: esM locations
+    :type locations: list
+
     :return: xr_ds
     """
+    # Treat transmission data separately
+    df_iteration_dict_orig = df_iteration_dict.copy()
+    df_iteration_dict_transm = {}
+    df_iteration_dict = {}
+    for variable_description, description_tuple_list in df_iteration_dict_orig.items():
+        for description_tuple in description_tuple_list:
+            # check if data is transmission and time dependent
+            if "Transmission" in description_tuple[0]:
+                # add "2dim" to variable_description
+                if variable_description not in df_iteration_dict_transm.keys():
+                    df_iteration_dict_transm[variable_description] = []
+                df_iteration_dict_transm[variable_description].append(description_tuple)
 
-    for variable_description, description_tuple_list in df_iteration_dict.items():
+            else:
+                if variable_description not in df_iteration_dict.keys():
+                    df_iteration_dict[variable_description] = []
+                df_iteration_dict[variable_description].append(description_tuple)
+
+    for (
+        variable_description,
+        description_tuple_list,
+    ) in df_iteration_dict_transm.items():
         df_dict = {}
 
         for description_tuple in description_tuple_list:
@@ -260,22 +283,48 @@ def addDFVariablesToXarray(xr_ds, component_dict, df_iteration_dict):
                 data = component_dict[classname][component][variable_description]
 
             multi_index_dataframe = data.stack()
-            if "Period" in multi_index_dataframe.index.names:
-                multi_index_dataframe.index.set_names("time", level=1, inplace=True)
-                multi_index_dataframe.index.set_names("space", level=2, inplace=True)
+            if set(locations) == set(
+                component_dict[classname][component][
+                    variable_description
+                ].index.to_list()
+            ):
+                multi_index_dataframe.index.set_names("space", level=0, inplace=True)
+                multi_index_dataframe.index.set_names("space_2", level=1, inplace=True)
             else:
+                # split X_X into multiindex
                 multi_index_dataframe.index.set_names("time", level=0, inplace=True)
                 multi_index_dataframe.index.set_names("space", level=1, inplace=True)
+                # use regex to split via location names
+                import re
+
+                pattern = re.compile("(" + "|".join(locations) + ")")
+                space_index = multi_index_dataframe.index.get_level_values(
+                    "space"
+                ).str.findall(pattern)
+                time_index = multi_index_dataframe.index.get_level_values("time")
+                # reconstruct multiindex
+                multi_index_dataframe.index = pd.MultiIndex.from_tuples(
+                    [
+                        (time_index[i], space_index[i][0], space_index[i][1])
+                        for i in range(len(space_index))
+                    ],
+                    names=["time", "space", "space_2"],
+                )
 
             df_dict[df_description] = multi_index_dataframe
 
         df_variable = pd.concat(df_dict)
         df_variable.index.set_names("component", level=0, inplace=True)
 
         ds_component = xr.Dataset()
-        ds_component[f"ts_{variable_description}"] = (
-            df_variable.sort_index().to_xarray()
-        )
+        if "time" in df_variable.index.names:
+            ds_component[f"ts_{variable_description}"] = (
+                df_variable.sort_index().to_xarray()
+            )
+        else:
+            ds_component[f"2d_{variable_description}"] = (
+                df_variable.sort_index().to_xarray()
+            )
 
         for comp in df_variable.index.get_level_values(0).unique():
             this_class = comp.split("; ")[0]
@@ -294,6 +343,61 @@ def addDFVariablesToXarray(xr_ds, component_dict, df_iteration_dict):
             except Exception:
                 pass
 
+    for variable_description, description_tuple_list in df_iteration_dict.items():
+        df_dict = {}
+
+        for description_tuple in description_tuple_list:
+            classname, component = description_tuple
+
+            df_description = f"{classname}; {component}"
+
+            # If a . is present in variable name, then the data would be
+            # another level further in the component_dict
+            if "." in variable_description:
+                [var_name, subvar_name] = variable_description.split(".")
+                if subvar_name.isdigit():
+                    subvar_name = int(subvar_name)
+                data = component_dict[classname][component][var_name][subvar_name]
+            else:
+                data = component_dict[classname][component][variable_description]
+
+            multi_index_dataframe = data.stack()
+            if "Period" in multi_index_dataframe.index.names:
+                multi_index_dataframe.index.set_names("time", level=1, inplace=True)
+                multi_index_dataframe.index.set_names("space", level=2, inplace=True)
+            else:
+                multi_index_dataframe.index.set_names("time", level=0, inplace=True)
+                multi_index_dataframe.index.set_names("space", level=1, inplace=True)
+
+            df_dict[df_description] = multi_index_dataframe
+
+        # check if there is data
+        if len(df_dict) > 0:
+            df_variable = pd.concat(df_dict)
+            df_variable.index.set_names("component", level=0, inplace=True)
+
+            ds_component = xr.Dataset()
+            ds_component[f"ts_{variable_description}"] = (
+                df_variable.sort_index().to_xarray()
+            )
+
+            for comp in df_variable.index.get_level_values(0).unique():
+                this_class = comp.split("; ")[0]
+                this_comp = comp.split("; ")[1]
+
+                this_ds_component = (
+                    ds_component.sel(component=comp)
+                    .squeeze()
+                    .reset_coords(names=["component"], drop=True)
+                )
+
+                try:
+                    xr_ds[this_class][this_comp] = xr.merge(
+                        [xr_ds[this_class][this_comp], this_ds_component]
+                    )
+                except Exception:
+                    pass
+
     return xr_ds
 
 
@@ -624,10 +728,27 @@ def addTimeSeriesVariableToDict(
         df = comp_var_xr.to_series()
     elif drop_component:
         df = comp_var_xr.drop("component").to_dataframe().unstack(level=1)
+    elif "space_2" in comp_var_xr.dims:
+        df = comp_var_xr.to_dataframe().squeeze()
+        # merge space and space_2 levels
+        space_index = df.index.get_level_values("space")
+        space_2_index = df.index.get_level_values("space_2")
+        new_space_index = [
+            f"{space_index[i]}_{space_2_index[i]}" for i in range(len(space_index))
+        ]
+        df.index = pd.MultiIndex.from_tuples(
+            [
+                (df.index.get_level_values("time")[i], new_space_index[i])
+                for i in range(len(new_space_index))
+            ],
+            names=["time", "space"],
+        )
+        df = df.unstack()
+        df = df.dropna(axis=1, how="all")
     else:
         df = comp_var_xr.to_dataframe().unstack(level=1)
 
-    if isinstance(df, pd.DataFrame):
+    if isinstance(df, pd.DataFrame) and "space_2" not in comp_var_xr.dims:
         if len(df.columns) > 1:
             df.columns = df.columns.droplevel(0)
 

fine/IOManagement/xarrayIO.py

@@ -46,7 +46,7 @@ def convertOptimizationInputToDatasets(esM, useProcessedValues=False):
         }
 
     # STEP 4. Add all df variables to xr_ds
-    xr_dss = utilsIO.addDFVariablesToXarray(xr_dss, component_dict, df_iteration_dict)
+    xr_dss = utilsIO.addDFVariablesToXarray(xr_dss, component_dict, df_iteration_dict, list(esM.locations))
 
     # STEP 5. Add all series variables to xr_ds
     locations = sorted(esm_dict["locations"])

fine/energySystemModel.py

@@ -2,6 +2,7 @@
 import os
 import time
 import warnings
+import importlib.util
 
 import gurobi_logtools as glt
 import pandas as pd
@@ -2031,7 +2032,11 @@ def optimize(
         ################################################################################################################
 
         # Set which solver should solve the specified optimization problem
-        optimizer = opt.SolverFactory(solver)
+        if solver == "gurobi" and importlib.util.find_spec('gurobipy'):
+            # Use the direct gurobi solver that uses the Python API.
+            optimizer = opt.SolverFactory(solver, solver_io="python")
+        else:
+            optimizer = opt.SolverFactory(solver)
 
         # Set, if specified, the time limit
         if self.solverSpecs["timeLimit"] is not None and solver == "gurobi":

fine/transmission.py

@@ -409,15 +409,15 @@ def __init__(
         # operationRateMax
         self.operationRateMax = operationRateMax
         self.fullOperationRateMax = utils.checkAndSetInvestmentPeriodTimeSeries(
-            esM, name, operationRateMax, self.locationalEligibility
+            esM, name, operationRateMax, self.locationalEligibility, "2dim"
         )
         self.aggregatedOperationRateMax = dict.fromkeys(esM.investmentPeriods)
         self.processedOperationRateMax = dict.fromkeys(esM.investmentPeriods)
-
+        
         # operationRateFix
         self.operationRateFix = operationRateFix
         self.fullOperationRateFix = utils.checkAndSetInvestmentPeriodTimeSeries(
-            esM, name, operationRateFix, self.locationalEligibility
+            esM, name, operationRateFix, self.locationalEligibility, "2dim"
         )
         self.aggregatedOperationRateFix = dict.fromkeys(esM.investmentPeriods)
         self.processedOperationRateFix = dict.fromkeys(esM.investmentPeriods)