Maintenance: improve iiasa_database handling, update_config_from_wildcards, raise errors (#102)

* * rename iiasa_database section to pypsa-de

* require retrieve config to be active

* relocate ariadne_databse from resources to data

* remove unused params from some rules

* remove unused config

* changelog and scenario name

* make sure every logger.error is actually raised

* setting scenario snakemake.config everywhere

* minor simplifications

* logger.info instead of print

* fix some warnings from plotting scripts

* eliminate some globals

---------

Co-authored-by: Lukas Trippe <[email protected]>

Author: Michael Lindner

.github/workflows/test.yaml

@@ -69,7 +69,7 @@ jobs:
         path: |
           data
           cutouts
-          resources/ariadne_database.csv
+          data/ariadne_database.csv
         key: data-cutouts-${{ env.WEEK }}
 
     - uses: conda-incubator/setup-miniconda@v3

CHANGELOG.md

@@ -1,4 +1,5 @@
 # Changelog
+- Simplified IIASA database download
 - Bugfix: Enforce stricter H2 derivative import limit to avoid that exports of one type of derivative compensate for imports of another
 - Added an option to source mobility demand from UBA MWMS (Projektionsbericht 2025) for the years 2025-2035
 - Renamed functions and script for exogenous mobility demand

README.md

@@ -77,7 +77,7 @@ PyPSA-DE is a softfork of PyPSA-EUR. As such, large parts of the functionality a
 
 ## New Config Options
 
-- `iiasa_database` - interaction with IIASA database. Specify a database, and `leitmodelle` for demand and co2 emissions data in specific sectors
+- `pypsa-de` - interaction with IIASA database. Specify a database, and `leitmodelle` for demand and co2 emissions data in specific sectors
 - `wasserstoff_kernnetz` - configure which parts of the Wasserstoff Kernnetz should be included in the model
 - `new_decentral_fossil_boiler_ban` - specify in which country and which years to ban fossil boilers
 - `coal_generation_ban` - specify in which country and which years to ban electricity generation from coal

Snakefile

@@ -302,33 +302,6 @@ rule clean:
         print("Data downloaded to data/ has not been cleaned.")
 
 
-rule retrieve_egon_data:
-    output:
-        spatial="data/egon/demandregio_spatial_2018.json",
-        mapping="data/egon/mapping_technologies.json",
-    shell:
-        """
-        mkdir -p data/egon
-        curl -o {output.spatial} "https://api.opendata.ffe.de/demandregio/demandregio_spatial?id_spatial=5&year=2018"
-        curl -o {output.mapping} "https://api.opendata.ffe.de/demandregio/demandregio_spatial_description?id_spatial=5"
-        """
-
-
-rule retrieve_ariadne_database:
-    params:
-        db_name=config["iiasa_database"]["db_name"],
-        leitmodelle=config["iiasa_database"]["leitmodelle"],
-        scenarios=config["iiasa_database"]["scenarios"],
-    output:
-        data="resources/ariadne_database.csv",
-    log:
-        "logs/retrieve_ariadne_database.log",
-    resources:
-        mem_mb=1000,
-    script:
-        "scripts/pypsa-de/retrieve_ariadne_database.py"
-
-
 rule modify_cost_data:
     params:
         file_path="ariadne-data/costs/",
@@ -354,23 +327,81 @@ rule modify_cost_data:
         "scripts/pypsa-de/modify_cost_data.py"
 
 
+if config["enable"]["retrieve"]:
+
+    rule retrieve_ariadne_database:
+        output:
+            data="data/ariadne_database.csv",
+        log:
+            "logs/retrieve_ariadne_database.log",
+        resources:
+            mem_mb=1000,
+        script:
+            "scripts/pypsa-de/retrieve_ariadne_database.py"
+
+
+if config["enable"]["retrieve"]:
+
+    rule retrieve_ariadne_template:
+        input:
+            storage(
+                "https://github.com/iiasa/ariadne-intern-workflow/raw/main/attachments/2025-01-27_template_Ariadne.xlsx",
+                keep_local=True,
+            ),
+        output:
+            "data/template_ariadne_database.xlsx",
+        run:
+            move(input[0], output[0])
+
+
+if config["enable"]["retrieve"]:
+
+    rule retrieve_mastr:
+        input:
+            storage(
+                "https://zenodo.org/records/8225106/files/bnetza_open_mastr_2023-08-08_B.zip",
+                keep_local=True,
+            ),
+        params:
+            "data/mastr",
+        output:
+            "data/mastr/bnetza_open_mastr_2023-08-08_B_biomass.csv",
+            "data/mastr/bnetza_open_mastr_2023-08-08_B_combustion.csv",
+        run:
+            unpack_archive(input[0], params[0])
+
+
+if config["enable"]["retrieve"]:
+
+    rule retrieve_egon_data:
+        output:
+            spatial="data/egon/demandregio_spatial_2018.json",
+            mapping="data/egon/mapping_technologies.json",
+        shell:
+            """
+            mkdir -p data/egon
+            curl -o {output.spatial} "https://api.opendata.ffe.de/demandregio/demandregio_spatial?id_spatial=5&year=2018"
+            curl -o {output.mapping} "https://api.opendata.ffe.de/demandregio/demandregio_spatial_description?id_spatial=5"
+            """
+
+
 if config["enable"]["retrieve"] and config["enable"].get("retrieve_cost_data", True):
 
     ruleorder: modify_cost_data > retrieve_cost_data
 
 
 rule build_exogenous_mobility_data:
     params:
-        reference_scenario=config_provider("iiasa_database", "reference_scenario"),
+        reference_scenario=config_provider("pypsa-de", "reference_scenario"),
         planning_horizons=config_provider("scenario", "planning_horizons"),
-        leitmodelle=config_provider("iiasa_database", "leitmodelle"),
-        ageb_for_mobility=config_provider("iiasa_database", "ageb_for_mobility"),
-        uba_for_mobility=config_provider("iiasa_database", "uba_for_mobility"),
+        leitmodelle=config_provider("pypsa-de", "leitmodelle"),
+        ageb_for_mobility=config_provider("pypsa-de", "ageb_for_mobility"),
+        uba_for_mobility=config_provider("pypsa-de", "uba_for_mobility"),
         shipping_oil_share=config_provider("sector", "shipping_oil_share"),
         aviation_demand_factor=config_provider("sector", "aviation_demand_factor"),
         energy_totals_year=config_provider("energy", "energy_totals_year"),
     input:
-        ariadne="resources/ariadne_database.csv",
+        ariadne="data/ariadne_database.csv",
         energy_totals=resources("energy_totals.csv"),
     output:
         mobility_data=resources(
@@ -599,11 +630,8 @@ ruleorder: modify_industry_demand > build_industrial_production_per_country_tomo
 
 
 rule modify_existing_heating:
-    params:
-        iiasa_reference_scenario=config_provider("iiasa_database", "reference_scenario"),
-        leitmodelle=config_provider("iiasa_database", "leitmodelle"),
     input:
-        ariadne="resources/ariadne_database.csv",
+        ariadne="data/ariadne_database.csv",
         existing_heating="data/existing_infrastructure/existing_heating_raw.csv",
     output:
         existing_heating=resources("existing_heating.csv"),
@@ -615,21 +643,6 @@ rule modify_existing_heating:
         "scripts/pypsa-de/modify_existing_heating.py"
 
 
-rule retrieve_mastr:
-    input:
-        storage(
-            "https://zenodo.org/records/8225106/files/bnetza_open_mastr_2023-08-08_B.zip",
-            keep_local=True,
-        ),
-    params:
-        "data/mastr",
-    output:
-        "data/mastr/bnetza_open_mastr_2023-08-08_B_biomass.csv",
-        "data/mastr/bnetza_open_mastr_2023-08-08_B_combustion.csv",
-    run:
-        unpack_archive(input[0], params[0])
-
-
 rule build_existing_chp_de:
     params:
         district_heating_subnodes=config_provider(
@@ -658,9 +671,9 @@ rule build_existing_chp_de:
 
 rule modify_industry_demand:
     params:
-        reference_scenario=config_provider("iiasa_database", "reference_scenario"),
+        reference_scenario=config_provider("pypsa-de", "reference_scenario"),
     input:
-        ariadne="resources/ariadne_database.csv",
+        ariadne="data/ariadne_database.csv",
         industrial_production_per_country_tomorrow=resources(
             "industrial_production_per_country_tomorrow_{planning_horizons}.csv"
         ),
@@ -722,18 +735,6 @@ rule cluster_wasserstoff_kernnetz:
         "scripts/pypsa-de/cluster_wasserstoff_kernnetz.py"
 
 
-rule download_ariadne_template:
-    input:
-        storage(
-            "https://github.com/iiasa/ariadne-intern-workflow/raw/main/attachments/2025-01-27_template_Ariadne.xlsx",
-            keep_local=True,
-        ),
-    output:
-        "data/template_ariadne_database.xlsx",
-    run:
-        move(input[0], output[0])
-
-
 rule export_ariadne_variables:
     params:
         planning_horizons=config_provider("scenario", "planning_horizons"),
@@ -794,11 +795,10 @@ rule export_ariadne_variables:
 
 rule plot_ariadne_variables:
     params:
-        iiasa_scenario=config_provider("iiasa_database", "reference_scenario"),
-        reference_scenario=config_provider("iiasa_database", "reference_scenario"),
+        reference_scenario=config_provider("pypsa-de", "reference_scenario"),
     input:
         exported_variables_full=RESULTS + "ariadne/exported_variables_full.xlsx",
-        ariadne_database="resources/ariadne_database.csv",
+        ariadne_database="data/ariadne_database.csv",
     output:
         primary_energy=RESULTS + "ariadne/primary_energy.png",
         primary_energy_detailed=RESULTS + "ariadne/primary_energy_detailed.png",
@@ -863,9 +863,9 @@ rule ariadne_all:
 rule build_scenarios:
     params:
         scenarios=config["run"]["name"],
-        leitmodelle=config["iiasa_database"]["leitmodelle"],
+        leitmodelle=config["pypsa-de"]["leitmodelle"],
     input:
-        ariadne_database="resources/ariadne_database.csv",
+        ariadne_database="data/ariadne_database.csv",
         scenario_yaml=config["run"]["scenarios"]["manual_file"],
     output:
         scenario_yaml=config["run"]["scenarios"]["file"],

config/config.de.yaml

@@ -4,7 +4,7 @@
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#run
 run:
-  prefix: 20250901_h2_deriv_fix
+  prefix: 20250811_improve_database
   name:
   # - ExPol
   - KN2045_Mix
@@ -27,20 +27,12 @@ run:
     - modify_existing_heating
   disable_progressbar: true
 
-iiasa_database:
-  db_name: ariadne2
-  leitmodelle:
+pypsa-de:
+  leitmodelle: # Model data downloaded from public IIASA database 'ariadne2'
     general: REMIND-EU v1.1
     buildings: REMod v1.0
     transport: Aladin v1
     industry: FORECAST v1.0
-  scenarios:
-  - ExPol
-  - KN2045_Elek
-  - KN2045_H2
-  - KN2045_Mix
-  - KN2045_NFniedrig
-  - KN2045_NFhoch
   reference_scenario: KN2045_Mix
   region: Deutschland
   ageb_for_mobility: true # In 2020 use AGEB data for final energy demand and KBA for vehicles

config/scenarios.manual.yaml

@@ -8,7 +8,7 @@ ExPol:
 # Kernnetz is not assumed to be built
 # Not forcing renewables after 2030
 
-  iiasa_database:
+  pypsa-de:
     reference_scenario: ExPol
 
   co2_budget_DE_source: UBA
@@ -80,7 +80,7 @@ KN2045_Elek:
 # Ausbau der erneuerbaren Stromerzeugung erreicht politisch gesetzte Ziele
 # Importe erneuerbar erzeugter Energien auf mittlerem Niveau
 
-  iiasa_database:
+  pypsa-de:
     reference_scenario: KN2045_Elek
 
   solving:
@@ -135,7 +135,7 @@ KN2045_H2:
 # Direkte Elektrifizierung spielt dennoch wesentliche Rolle bei der Dekarbonisierung der Endenergie
 # Ausbau der erneuerbaren Stromerzeugung erreicht politisch gesetzte Ziele
 
-  iiasa_database:
+  pypsa-de:
     reference_scenario: KN2045_H2
 
   solving:
@@ -204,7 +204,7 @@ KN2045_NFniedrig:
 # Importe erneuerbar erzeugter Energien auf mittlerem Niveau
 # dient als Referenzszenario in der Familie der Ariadne-Szenarien
 
-  iiasa_database:
+  pypsa-de:
     reference_scenario: KN2045_NFniedrig
 
   sector:
@@ -224,7 +224,7 @@ KN2045_NFhoch:
 # Importe erneuerbar erzeugter Energien auf mittlerem Niveau
 # dient als Referenzszenario in der Familie der Ariadne-Szenarien
 
-  iiasa_database:
+  pypsa-de:
     reference_scenario: KN2045_NFhoch
 
   sector:

config/scenarios.sysgf.yaml

@@ -5,7 +5,7 @@
 
 Baseline:
   foresight: myopic
-  iiasa_database:
+  pypsa-de:
     reference_scenario: KN2045_Mix
   co2_budget_DE_source: UBA
 

scripts/add_existing_baseyear.py

@@ -444,6 +444,7 @@ def add_power_capacities_installed_before_baseyear(
                         bus0 = spatial.biogas.df.loc[new_capacity.index, "nodes"].values
                     else:
                         logger.error(f"Generator {generator} not recognized.")
+                        raise ValueError(f"Generator {generator} not recognized.")
 
                     # We assume the electrical efficiency of a CHP for the biomass and biogas power plants
                     # The EOP from technology data seems to be somewhat too efficient

scripts/build_industry_sector_ratios.py

@@ -1645,6 +1645,9 @@ def other_industrial_sectors():
         logger.error(
             "Fractions for steam processing do not sum to 1! Please check config[industry][steam_biomass_fraction], config[industry][steam_hydrogen_fraction] and config[industry][steam_electricity_fraction]!"
         )
+        raise ValueError(
+            "Fractions for steam processing do not sum to 1! Please check config[industry][steam_biomass_fraction], config[industry][steam_hydrogen_fraction] and config[industry][steam_electricity_fraction]!"
+        )
 
     logger.info(
         f"Steam processing fractions set to {params['steam_biomass_fraction']} biomass, {params['steam_hydrogen_fraction']} hydrogen and {params['steam_electricity_fraction']} electricity."

scripts/pypsa-de/additional_functionality.py

@@ -384,6 +384,9 @@ def add_national_co2_budgets(n, snakemake, national_co2_budgets, investment_year
             logger.error(
                 f"CO2 budget for countries other than `DE` is not yet supported. Found country {ct}. Please check the config file."
             )
+            raise NotImplementedError(
+                f"CO2 budget for countries other than `DE` is not yet supported. Found country {ct}. Please check the config file."
+            )
 
         limit = co2_total_totals[ct] * national_co2_budgets[ct][investment_year]
         logger.info(