Switch model to use only public version of database (#81)

* new names for the scenarios

* delete public config

* not used in pypsa-de

* discontinue fallback_reference_scenario

* hard-code 2020 values from internal db

* remove unused code

* simplify readme

* whitespace

* remove unused code

* convert everything to MWh

* convert to Mwh instead of Gj

* fix pandas warning

* rename prefix

* fix empty plot

* fix overlooked conversion from Joule

* fix mobility demand

* fix rounding error in exporter

* increased tolerance for comparison

* switch to Mix value from steel production from Forecast 2020

* more correction of hardcoded steel values

* bump version

* add changes

---------

Co-authored-by: JulianGeis <JulianGeis@gmx.net>

Author: Micha

.github/workflows/validate.yaml

@@ -54,9 +54,9 @@ jobs:
         # <scenario>/<plot_name>.png
         plots: >
           "
-          KN2045_Bal_v4/ariadne/primary_energy.png
-          KN2045_Bal_v4/ariadne/secondary_energy.png
-          KN2045_Bal_v4/ariadne/final_energy.png
+          KN2045_Mix/ariadne/primary_energy.png
+          KN2045_Mix/ariadne/secondary_energy.png
+          KN2045_Mix/ariadne/final_energy.png
           ariadne_comparison/Trade-Secondary-Energy-Hydrogen-Volume.png
           ariadne_comparison/Trade-Secondary-Energy-Liquids-Hydrogen-Volume.png
           ariadne_comparison/Capacity-Electricity-Solar.png

CHANGELOG.md

@@ -1,4 +1,5 @@
 # Changelog
+- 0.3: workflow is all public now, no longer requires credentials to internal data
 - Allowing myopic optimization until 2050
 - CHP plants located in onshore regions without district heating are represented as EOP plants
 - updating technology cost data to v0.11.0

CITATION.cff

@@ -2,7 +2,7 @@ cff-version: 1.2.0
 message: "pypsa-de"
 title: "PyPSA-DE - Hochaufgelöstes, sektorengekoppeltes Modell des deutschen Energiesystems"
 repository: https://github.com/PyPSA/pypsa-de
-version: 0.2.0
+version: 0.3.0
 doi: "10.5281/zenodo.15096969"
 date-released: "Mar 27, 2025"
 license: MIT

README.md

@@ -24,42 +24,11 @@ Where `{os}` should be replaced with your operating system, e.g. for linux the c
 conda env create -f envs/linux-pinned.yaml
 ```
 
-## Connecting to the Ariadne-Database
-
-### For external users: Use config.public.yaml
-
-The default workflow configured for this repository assumes access to the internal Ariadne2 database. The database will soon be publicly available. Until then, users that do not have the required login details can run the analysis based on the data published during the [first phase of the Ariadne project](https://data.ece.iiasa.ac.at/ariadne/).
-
-This is possible by providing an additional config to the snakemake workflow. For every `snakemake COMMAND` specified in the instructions below, public users should use:
-
-```
-snakemake COMMAND --configfile=config/config.public.yaml
-```
-
-The additional config file specifies the required database, model, and scenario names for Ariadne1. If public users wish to edit the default scenario specifications, they can do so by changing `scenarios.public.yaml` to `scenarios.manual.yaml`. More details on using scenarios are given below.
-
-### For internal users: Provide login details
-
-The snakemake rule `retrieve_ariadne_database` logs into the interal Ariadne IIASA Database via the [`pyam`](https://pyam-iamc.readthedocs.io/en/stable/tutorials/iiasa.html) package. The credentials for logging into this database have to be stored locally on your machine with `ixmp4`. To do this activate the project environment and run
-
-```
-ixmp4 login <username>
-```
-
-You will be prompted to enter your `<password>`.
-
-Caveat: These credentials are stored on your machine in plain text.
-
-To switch between internal and public use, the command `ixmp4 logout` may be necessary.
-
 ## Run the analysis
 
 Before running any analysis with scenarios, the rule `build_scenarios` must be executed. This will create the file `config/scenarios.automated.yaml` which includes input data and CO2 targets from the IIASA Ariadne database as well as the specifications from the manual scenario file. [This file is specified in the  config.de.yaml via they key `run:scenarios:manual_file` and located at `config/scenarios.manual.yaml` by default].
 
     snakemake build_scenarios -f
-or in case of using the public database
-
-    snakemake build_scenarios --configfile=config/config.public.yaml -f
 
 Note that the hierarchy of scenario files is the following: `scenarios.automated.yaml` > (any `explicitly specified --configfiles`) > `config.de.yaml `> `config.default.yaml `Changes in the file `scenarios.manual.yaml `are only taken into account if the rule `build_scenarios` is executed.
 
@@ -88,7 +57,6 @@ PyPSA-DE is a softfork of PyPSA-EUR. As such, large parts of the functionality a
 - Default resolution of 16 regions in Germany and 13 region for neighboring countries
 - 10 pre-defined scenarios (1 Current Policies, 3 Net-Zero Scenarios (Balanced, Focus H2, Focus Electricity), 2 Demand Variations based on the Balanced Scenario, 4 Demand Variations Based on the Current Policies Scenario)
 - Specific cost assumption for Germany:
-
   - Gas, Oil, Coal prices
   - electrolysis and heat-pump costs
   - Infrastructure costs [according to the Netzentwicklungsplan](https://github.com/PyPSA/pypsa-ariadne/pull/193) 2021 and 2023

Snakefile

@@ -453,9 +453,6 @@ rule modify_existing_heating:
     params:
         iiasa_reference_scenario=config_provider("iiasa_database", "reference_scenario"),
         leitmodelle=config_provider("iiasa_database", "leitmodelle"),
-        fallback_reference_scenario=config_provider(
-            "iiasa_database", "fallback_reference_scenario"
-        ),
     input:
         ariadne=resources("ariadne_database.csv"),
         existing_heating="data/existing_infrastructure/existing_heating_raw.csv",
@@ -641,9 +638,7 @@ rule export_ariadne_variables:
 rule plot_ariadne_variables:
     params:
         iiasa_scenario=config_provider("iiasa_database", "reference_scenario"),
-        fallback_reference_scenario=config_provider(
-            "iiasa_database", "fallback_reference_scenario"
-        ),
+        reference_scenario=config_provider("iiasa_database", "reference_scenario"),
     input:
         exported_variables_full=RESULTS + "ariadne/exported_variables_full.xlsx",
         ariadne_database=resources("ariadne_database.csv"),

config/config.de.yaml

@@ -4,18 +4,14 @@
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#run
 run:
-  prefix: 20250414_merge_master_april
+  prefix: 20250425_public_db
   name:
-  # - CurrentPolicies
-  - KN2045_Bal_v4
-  # - KN2045_Elec_v4
-  # - KN2045_H2_v4
-  # - KN2045plus_EasyRide
-  # - KN2045plus_LowDemand
-  # - KN2045minus_WorstCase
-  # - KN2045minus_SupplyFocus
-  # - KN2045_Bal_LowDemand
-  # - KN2045_Bal_HighDemand
+  # - ExPol
+  - KN2045_Mix
+  # - KN2045_Elek
+  # - KN2045_H2
+  # - KN2045_NFniedrig
+  # - KN2045_NFhoch
   scenarios:
     enable: true
     manual_file: config/scenarios.manual.yaml
@@ -32,22 +28,20 @@ run:
   disable_progressbar: true
 
 iiasa_database:
-  db_name: ariadne2_intern
+  db_name: ariadne2
   leitmodelle:
     general: REMIND-EU v1.1
     buildings: REMod v1.0
     transport: Aladin v1
     industry: FORECAST v1.0
   scenarios:
-  - CurrentPolicies
-  - KN2045_Elec_v4
-  - KN2045_H2_v4
-  - KN2045_Bal_v4
-  - KN2045plus_EasyRide
-  - KN2045plus_LowDemand
-  - KN2045minus_WorstCase
-  - KN2045minus_SupplyFocus
-  reference_scenario: KN2045_Bal_v4
+  - ExPol
+  - KN2045_Elek
+  - KN2045_H2
+  - KN2045_Mix
+  - KN2045_NFniedrig
+  - KN2045_NFhoch
+  reference_scenario: KN2045_Mix
   region: Deutschland
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#foresight
@@ -298,7 +292,6 @@ sector:
     rural: true
   co2_spatial: true
   biomass_spatial: true
-  #TBD what to include in config
   #relax so no infeasibility in 2050 with no land transport demand
   min_part_load_fischer_tropsch: 0.
   regional_methanol_demand: true  #set to true if regional CO2 constraints needed