Merge branch 'interpolation' into outputs

tsmbland · web-flow · commit 7c8669b16bcd · 2025-02-05T10:10:41.000Z
diff --git a/docs/inputs/commodities_io.rst b/docs/inputs/commodities_io.rst
@@ -79,5 +79,3 @@ The input data has to be provided for the base year. Additional years within the
 framework of the overall simulation can be defined. In this case, MUSE would interpolate
 the values between the provided periods and assume a constant value afterwards. The additional
 years at which input data for input/output commodities, are defined needs to equal for :ref:`inputs-technodata` and :ref:`inputs-technodata-ts`.
-
-Interpolation is activated only if the feature *interpolation_mode = 'Active'* is defined in the TOML file.
diff --git a/docs/inputs/technodata.rst b/docs/inputs/technodata.rst
@@ -166,5 +166,3 @@ The input data has to be provided for the base year. Additional years within the
 framework of the overall simulation can be defined. In this case, MUSE would interpolate
 the values between the provided periods and assume a constant value afterwards. The additional
 years at which input data for techno-data are defined need to equal for :ref:`inputs-iocomms` and :ref:`inputs-technodata-ts`.
-
-Interpolation is activated only if the feature *interpolation_mode = 'Active'* is defined in the TOML file.
diff --git a/docs/inputs/technodata_timeslices.rst b/docs/inputs/technodata_timeslices.rst
@@ -44,5 +44,3 @@ The input data has to be provided for the base year. Additional years within the
 framework of the overall simulation can be defined. In this case, MUSE would interpolate
 the values between the provided periods and assume a constant value afterwards. The additional
 years at which input data for techno-data timeslices, need to equal those for :ref:`inputs-iocomms` and :ref:`inputs-technodata`.
-
-Interpolation is activated only if the feature *interpolation_mode = 'Active'* is defined in the TOML file.
diff --git a/docs/inputs/toml.rst b/docs/inputs/toml.rst
@@ -26,7 +26,7 @@ a whole.
    time_framework = [2020, 2025, 2030, 2035, 2040, 2045, 2050]
    regions = ["USA"]
    interest_rate = 0.1
-   interpolation_mode = 'Active'
+   interpolation_mode = 'linear'
    log_level = 'info'
 
    equilibrium_variable = 'demand'
@@ -293,8 +293,9 @@ A sector accepts these attributes:
    Defaults to "last".
 
 *interpolation*
-   Interpolation method user when filling in missing values. Available interpolation
-   methods depend on the underlying `scipy method's kind attribute`_.
+   Interpolation method used to fill missing years in the *technodata* (defaults to "linear").
+   Available interpolation methods depend on the underlying `scipy method's kind attribute`_.
+   Years outside the data range will always be back/forward filled with the closest available data.
 
    .. _scipy method's kind attribute: https://docs.scipy.org/doc/scipy/reference/generated/scipy.interpolate.interp1d.html
 
diff --git a/docs/tutorial-code/add-agent/1-single-objective/settings.toml b/docs/tutorial-code/add-agent/1-single-objective/settings.toml
@@ -3,7 +3,7 @@
 time_framework = [2020, 2025, 2030, 2035, 2040, 2045, 2050]
 regions = ["R1"]
 interest_rate = 0.1
-interpolation_mode = 'Active'
+interpolation_mode = 'linear'
 log_level = 'info'
 excluded_commodities = ["wind"]
 # Convergence parameters
diff --git a/docs/tutorial-code/add-agent/2-multiple-objective/settings.toml b/docs/tutorial-code/add-agent/2-multiple-objective/settings.toml
@@ -3,7 +3,7 @@
 time_framework = [2020, 2025, 2030, 2035, 2040, 2045, 2050]
 regions = ["R1"]
 interest_rate = 0.1
-interpolation_mode = 'Active'
+interpolation_mode = 'linear'
 log_level = 'info'
 excluded_commodities = ["wind"]
 # Convergence parameters
diff --git a/docs/tutorial-code/add-correlation-demand/1-correlation/settings.toml b/docs/tutorial-code/add-correlation-demand/1-correlation/settings.toml
@@ -3,7 +3,7 @@
 time_framework = [2020, 2025, 2030, 2035, 2040, 2045, 2050]
 regions = ["R1"]
 interest_rate = 0.1
-interpolation_mode = 'Active'
+interpolation_mode = 'linear'
 log_level = 'info'
 excluded_commodities = ["wind"]
 # Convergence parameters
diff --git a/docs/tutorial-code/add-new-technology/1-introduction/settings.toml b/docs/tutorial-code/add-new-technology/1-introduction/settings.toml
@@ -3,7 +3,7 @@
 time_framework = [2020, 2025, 2030, 2035, 2040, 2045, 2050]
 regions = ["R1"]
 interest_rate = 0.1
-interpolation_mode = 'Active'
+interpolation_mode = 'linear'
 log_level = 'info'
 excluded_commodities = ["wind", "solar"]
 # Convergence parameters
diff --git a/docs/tutorial-code/add-new-technology/2-scenario/settings.toml b/docs/tutorial-code/add-new-technology/2-scenario/settings.toml
@@ -3,7 +3,7 @@
 time_framework = [2020, 2025, 2030, 2035, 2040, 2045, 2050]
 regions = ["R1"]
 interest_rate = 0.1
-interpolation_mode = 'Active'
+interpolation_mode = 'linear'
 log_level = 'info'
 excluded_commodities = ["wind", "solar"]
 # Convergence parameters
diff --git a/docs/tutorial-code/add-region/1-new-region/settings.toml b/docs/tutorial-code/add-region/1-new-region/settings.toml
@@ -3,7 +3,7 @@
 time_framework = [2020, 2025, 2030, 2035, 2040, 2045, 2050]
 regions = ["R1", "R2"]
 interest_rate = 0.1
-interpolation_mode = 'Active'
+interpolation_mode = 'linear'
 log_level = 'info'
 excluded_commodities = ["wind"]
 # Convergence parameters
diff --git a/docs/tutorial-code/add-service-demand/1-exogenous-demand/settings.toml b/docs/tutorial-code/add-service-demand/1-exogenous-demand/settings.toml
@@ -3,7 +3,7 @@
 time_framework = [2020, 2025, 2030, 2035, 2040, 2045, 2050]
 regions = ["R1"]
 interest_rate = 0.1
-interpolation_mode = 'Active'
+interpolation_mode = 'linear'
 log_level = 'info'
 excluded_commodities = ["wind"]
 # Convergence parameters
diff --git a/docs/tutorial-code/carbon-budget/1-carbon-budget/settings.toml b/docs/tutorial-code/carbon-budget/1-carbon-budget/settings.toml
@@ -3,7 +3,7 @@
 time_framework = [2020, 2025, 2030, 2035]
 regions = ["R1"]
 interest_rate = 0.1
-interpolation_mode = 'Active'
+interpolation_mode = 'linear'
 log_level = 'info'
 excluded_commodities = ["wind"]
 # Convergence parameters
diff --git a/docs/tutorial-code/min-max-timeslice-constraints/1-min-constraint/settings.toml b/docs/tutorial-code/min-max-timeslice-constraints/1-min-constraint/settings.toml
@@ -3,7 +3,7 @@
 time_framework = [2020, 2025, 2030, 2035, 2040, 2045, 2050]
 regions = ["R1"]
 interest_rate = 0.1
-interpolation_mode = 'Active'
+interpolation_mode = 'linear'
 log_level = 'info'
 excluded_commodities = ["wind"]
 # Convergence parameters
diff --git a/docs/tutorial-code/min-max-timeslice-constraints/2-max-constraint/settings.toml b/docs/tutorial-code/min-max-timeslice-constraints/2-max-constraint/settings.toml
@@ -3,7 +3,7 @@
 time_framework = [2020, 2025, 2030, 2035, 2040, 2045, 2050]
 regions = ["R1"]
 interest_rate = 0.1
-interpolation_mode = 'Active'
+interpolation_mode = 'linear'
 log_level = 'info'
 excluded_commodities = ["wind"]
 # Convergence parameters
diff --git a/docs/tutorial-code/modify-timing-data/1-modify-timeslices/settings.toml b/docs/tutorial-code/modify-timing-data/1-modify-timeslices/settings.toml
@@ -3,7 +3,7 @@
 time_framework = [2020, 2025, 2030, 2035, 2040, 2045, 2050]
 regions = ["R1"]
 interest_rate = 0.1
-interpolation_mode = 'Active'
+interpolation_mode = 'linear'
 log_level = 'info'
 excluded_commodities = ["wind"]
 # Convergence parameters
diff --git a/docs/tutorial-code/modify-timing-data/2-modify-time-framework/settings.toml b/docs/tutorial-code/modify-timing-data/2-modify-time-framework/settings.toml
@@ -3,7 +3,7 @@
 time_framework = [2020, 2022, 2024, 2026, 2028, 2030, 2032, 2034, 2036, 2038, 2040]
 regions = ["R1"]
 interest_rate = 0.1
-interpolation_mode = 'Active'
+interpolation_mode = 'linear'
 log_level = 'info'
 excluded_commodities = ["wind"]
 # Convergence parameters
diff --git a/docs/tutorial-code/new-decision-metric/settings.toml b/docs/tutorial-code/new-decision-metric/settings.toml
@@ -3,7 +3,7 @@ plugins = "{path}/new_decision.py"
 time_framework = [2020, 2025, 2030, 2035, 2040, 2045, 2050]
 regions = ["R1"]
 interest_rate = 0.1
-interpolation_mode = 'Active'
+interpolation_mode = 'linear'
 log_level = 'info'
 excluded_commodities = ["wind", "solar"]
 # Convergence parameters
diff --git a/docs/user-guide/modify-timing-data.ipynb b/docs/user-guide/modify-timing-data.ipynb
@@ -149,21 +149,6 @@
     "```"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "We also have to modity the `forecast` value for each sector. For example, for the power sector the `sectors.power.subsectors.all` section should now look as follows:\n",
-    "\n",
-    "```toml\n",
-    "[sectors.power.subsectors.all]\n",
-    "agents = '{path}/technodata/Agents.csv'\n",
-    "existing_capacity = '{path}/technodata/power/ExistingCapacity.csv'\n",
-    "lpsolver = \"scipy\"\n",
-    "forecast = 2\n",
-    "```"
-   ]
-  },
   {
    "cell_type": "markdown",
    "metadata": {},
diff --git a/src/muse/agents/agent.py b/src/muse/agents/agent.py
@@ -297,7 +297,7 @@ def next(
         """
         from logging import getLogger
 
-        from muse.utilities import reduce_assets
+        from muse.utilities import interpolate_capacity, reduce_assets
 
         # Check inputs
         assert len(market.year) == 2
@@ -340,9 +340,10 @@ def next(
         search = search.sel(asset=condtechs)
 
         # Calculate capacity in current and investment year
-        capacity = reduce_assets(
-            self.assets.capacity, coords=("technology", "region")
-        ).interp(year=[current_year, investment_year])
+        capacity = interpolate_capacity(
+            reduce_assets(self.assets.capacity, coords=("technology", "region")),
+            year=[current_year, investment_year],
+        )
 
         # Calculate constraints
         constraints = self.constraints(
diff --git a/src/muse/data/default_settings.toml b/src/muse/data/default_settings.toml
@@ -3,7 +3,7 @@
 time_framework = 'REQUIRED'
 regions = 'REQUIRED'
 interest_rate = 0.1
-interpolation_mode = 'Active'
+interpolation_mode = 'linear'
 log_level = 'info'
 # Convergence parameters
 equilibrium = true
diff --git a/src/muse/data/example/default/settings.toml b/src/muse/data/example/default/settings.toml
@@ -3,7 +3,7 @@
 time_framework = [2020, 2025, 2030, 2035, 2040, 2045, 2050]
 regions = ["R1"]
 interest_rate = 0.1
-interpolation_mode = 'Active'
+interpolation_mode = 'linear'
 log_level = 'info'
 excluded_commodities = ["wind"]
 # Convergence parameters
diff --git a/src/muse/data/example/default_retro/settings.toml b/src/muse/data/example/default_retro/settings.toml
@@ -3,7 +3,7 @@
 time_framework = [2020, 2025, 2030, 2035, 2040, 2045, 2050]
 regions = ["R1"]
 interest_rate = 0.1
-interpolation_mode = 'Active'
+interpolation_mode = 'linear'
 log_level = 'info'
 excluded_commodities = ["wind"]
 # Convergence parameters
diff --git a/src/muse/data/example/default_timeslice/settings.toml b/src/muse/data/example/default_timeslice/settings.toml
@@ -3,7 +3,7 @@
 time_framework = [2020, 2025, 2030, 2035, 2040, 2045, 2050]
 regions = ["R1"]
 interest_rate = 0.1
-interpolation_mode = 'Active'
+interpolation_mode = 'linear'
 log_level = 'info'
 excluded_commodities = ["wind"]
 # Convergence parameters
diff --git a/src/muse/data/example/minimum_service/settings.toml b/src/muse/data/example/minimum_service/settings.toml
@@ -3,7 +3,7 @@
 time_framework = [2010, 2015, 2020, 2025, 2030, 2035, 2040, 2045, 2050]
 regions = ["R1"]
 interest_rate = 0.1
-interpolation_mode = 'Active'
+interpolation_mode = 'linear'
 log_level = 'info'
 excluded_commodities = ["fuel1", "fuel2", "fuel3"]
 # Convergence parameters
diff --git a/src/muse/data/example/trade/settings.toml b/src/muse/data/example/trade/settings.toml
@@ -3,7 +3,7 @@
 time_framework = [2020, 2025, 2030, 2035]
 regions = ["R1", "R2"]
 interest_rate = 0.1
-interpolation_mode = 'Active'
+interpolation_mode = 'linear'
 log_level = 'info'
 excluded_commodities = ["wind"]
 # Convergence parameters
diff --git a/src/muse/demand_share.py b/src/muse/demand_share.py
@@ -246,21 +246,27 @@ def new_and_retro(
 
     from muse.commodities import is_enduse
     from muse.quantities import maximum_production
-    from muse.utilities import agent_concatenation, broadcast_over_assets, reduce_assets
+    from muse.utilities import (
+        agent_concatenation,
+        broadcast_over_assets,
+        interpolate_capacity,
+        reduce_assets,
+    )
 
     current_year, investment_year = map(int, market.year.values)
 
     def decommissioning(capacity, technologies):
         return decommissioning_demand(
             technologies=technologies,
-            capacity=capacity.interp(
-                year=[current_year, investment_year], kwargs={"fill_value": 0.0}
+            capacity=interpolate_capacity(
+                capacity, year=[current_year, investment_year]
             ),
             timeslice_level=timeslice_level,
         )
 
-    capacity = reduce_assets([u.assets.capacity for u in agents]).interp(
-        year=[current_year, investment_year], kwargs={"fill_value": 0.0}
+    capacity = interpolate_capacity(
+        reduce_assets([u.assets.capacity for u in agents]),
+        year=[current_year, investment_year],
     )
 
     # Select technodata for assets
@@ -288,8 +294,8 @@ def decommissioning(capacity, technologies):
     id_to_share: MutableMapping[Hashable, xr.DataArray] = {}
     for region in demands.region.values:
         retro_capacity: MutableMapping[Hashable, xr.DataArray] = {
-            agent.uuid: agent.assets.capacity.interp(
-                year=[current_year, investment_year]
+            agent.uuid: interpolate_capacity(
+                agent.assets.capacity, year=[current_year, investment_year]
             )
             for agent in agents
             if agent.category == "retrofit" and agent.region == region
@@ -382,15 +388,20 @@ def standard_demand(
 
     from muse.commodities import is_enduse
     from muse.quantities import maximum_production
-    from muse.utilities import agent_concatenation, broadcast_over_assets, reduce_assets
+    from muse.utilities import (
+        agent_concatenation,
+        broadcast_over_assets,
+        interpolate_capacity,
+        reduce_assets,
+    )
 
     current_year, investment_year = map(int, market.year.values)
 
     def decommissioning(capacity, technologies):
         return decommissioning_demand(
             technologies=technologies,
-            capacity=capacity.interp(
-                year=[current_year, investment_year], kwargs={"fill_value": 0.0}
+            capacity=interpolate_capacity(
+                capacity, year=[current_year, investment_year]
             ),
             timeslice_level=timeslice_level,
         )
@@ -401,8 +412,9 @@ def decommissioning(capacity, technologies):
             raise RetrofitAgentInStandardDemandShare()
 
     # Calculate existing capacity
-    capacity = reduce_assets([agent.assets.capacity for agent in agents]).interp(
-        year=[current_year, investment_year], kwargs={"fill_value": 0.0}
+    capacity = interpolate_capacity(
+        reduce_assets([agent.assets.capacity for agent in agents]),
+        year=[current_year, investment_year],
     )
 
     # Select technodata for assets
@@ -425,8 +437,8 @@ def decommissioning(capacity, technologies):
     for region in demands.region.values:
         # Calculate current capacity
         current_capacity: MutableMapping[Hashable, xr.DataArray] = {
-            agent.uuid: agent.assets.capacity.interp(
-                year=[current_year, investment_year]
+            agent.uuid: interpolate_capacity(
+                agent.assets.capacity, year=[current_year, investment_year]
             )
             for agent in agents
             if agent.region == region
@@ -481,16 +493,21 @@ def unmet_forecasted_demand(
 ) -> xr.DataArray:
     """Forecast demand that cannot be serviced by non-decommissioned current assets."""
     from muse.commodities import is_enduse
-    from muse.utilities import broadcast_over_assets, reduce_assets
+    from muse.utilities import (
+        broadcast_over_assets,
+        interpolate_capacity,
+        reduce_assets,
+    )
 
     current_year, investment_year = map(int, market.year.values)
 
     comm_usage = technologies.comm_usage.sel(commodity=market.commodity)
     smarket: xr.Dataset = market.where(is_enduse(comm_usage), 0)
 
     # Calculate existing capacity
-    capacity = reduce_assets([agent.assets.capacity for agent in agents]).interp(
-        year=[current_year, investment_year], kwargs={"fill_value": 0.0}
+    capacity = interpolate_capacity(
+        reduce_assets([agent.assets.capacity for agent in agents]),
+        year=[current_year, investment_year],
     )
 
     # Select data for future years
diff --git a/src/muse/examples.py b/src/muse/examples.py
@@ -204,15 +204,19 @@ def mca_market(model: str = "default") -> xr.Dataset:
         path = copy_model(model, tmpdir)
         settings = read_settings(path / "settings.toml")
 
-        market = read_initial_market(
-            settings.global_input_files.projections,
-            base_year_export=getattr(
-                settings.global_input_files, "base_year_export", None
-            ),
-            base_year_import=getattr(
-                settings.global_input_files, "base_year_import", None
-            ),
-        ).sel(region=settings.regions, year=settings.time_framework)
+        market = (
+            read_initial_market(
+                settings.global_input_files.projections,
+                base_year_export=getattr(
+                    settings.global_input_files, "base_year_export", None
+                ),
+                base_year_import=getattr(
+                    settings.global_input_files, "base_year_import", None
+                ),
+            )
+            .sel(region=settings.regions)
+            .interp(year=settings.time_framework, method=settings.interpolation_mode)
+        )
         market["supply"] = drop_timeslice(zeros_like(market.exports))
         market["consumption"] = drop_timeslice(zeros_like(market.exports))
 
diff --git a/src/muse/filters.py b/src/muse/filters.py
diff --git a/src/muse/investments.py b/src/muse/investments.py
diff --git a/src/muse/mca.py b/src/muse/mca.py
diff --git a/src/muse/readers/toml.py b/src/muse/readers/toml.py
diff --git a/src/muse/sectors/sector.py b/src/muse/sectors/sector.py
diff --git a/src/muse/sectors/subsector.py b/src/muse/sectors/subsector.py
diff --git a/src/muse/utilities.py b/src/muse/utilities.py
diff --git a/tests/conftest.py b/tests/conftest.py
diff --git a/tests/test_constraints.py b/tests/test_constraints.py
diff --git a/tests/test_demand_share.py b/tests/test_demand_share.py
diff --git a/tests/test_utilities.py b/tests/test_utilities.py
