Merge branch 'main' into fix-power-limit

Author: Michael Lindner

.github/workflows/codeql.yaml

@@ -59,7 +59,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v3
+      uses: github/codeql-action/init@v4
       with:
         languages: ${{ matrix.language }}
         build-mode: ${{ matrix.build-mode }}
@@ -87,6 +87,6 @@ jobs:
         exit 1
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v3
+      uses: github/codeql-action/analyze@v4
       with:
         category: "/language:${{matrix.language}}"

.github/workflows/test.yaml

@@ -69,7 +69,6 @@ jobs:
         path: |
           data
           cutouts
-          resources/ariadne_database.csv
         key: data-cutouts-${{ env.WEEK }}
 
     - uses: conda-incubator/setup-miniconda@v3

.pre-commit-config.yaml

@@ -5,7 +5,7 @@ ci:
 
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v5.0.0
+  rev: v6.0.0
   hooks:
   - id: check-merge-conflict
   - id: check-added-large-files
@@ -14,7 +14,7 @@ repos:
 # Run ruff to lint and format
 - repo: https://github.com/astral-sh/ruff-pre-commit
   # Ruff version.
-  rev: v0.12.2
+  rev: v0.13.3
   hooks:
     # Run the linter.
   - id: ruff
@@ -43,11 +43,11 @@ repos:
   rev: v2.15.0
   hooks:
   - id: pretty-format-yaml
-    exclude: (pinned\.yaml|\.lock\.yaml)$
+    exclude: (\.lock\.yaml)$
     args: [--autofix, --indent, "2", --preserve-quotes]
 
   # Format Snakemake rule / workflow files
 - repo: https://github.com/snakemake/snakefmt
-  rev: v0.11.0
+  rev: v0.11.2
   hooks:
   - id: snakefmt

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 - Bugfix: Enforce stricter power import limit to avoid that import from one country compensate from exports to another
+- Added the IIASA database to the repository and disabled re-downloading it by default.
+- Simplified IIASA database download, rename `iiasa_database` config section to `pypsa-de`
+- Updated technology-data to v0.13.4
 - Bugfix: Enforce stricter H2 derivative import limit to avoid that exports of one type of derivative compensate for imports of another
 - Added an option to source mobility demand from UBA MWMS (Projektionsbericht 2025) for the years 2025-2035
 - Renamed functions and script for exogenous mobility demand

README.md

@@ -77,7 +77,7 @@ PyPSA-DE is a softfork of PyPSA-EUR. As such, large parts of the functionality a
 
 ## New Config Options
 
-- `iiasa_database` - interaction with IIASA database. Specify a database, and `leitmodelle` for demand and co2 emissions data in specific sectors
+- `pypsa-de` - interaction with IIASA database. Specify a database, and `leitmodelle` for demand and co2 emissions data in specific sectors
 - `wasserstoff_kernnetz` - configure which parts of the Wasserstoff Kernnetz should be included in the model
 - `new_decentral_fossil_boiler_ban` - specify in which country and which years to ban fossil boilers
 - `coal_generation_ban` - specify in which country and which years to ban electricity generation from coal
@@ -87,6 +87,15 @@ PyPSA-DE is a softfork of PyPSA-EUR. As such, large parts of the functionality a
 - `co2_budget_DE_source` specify the carbon trajectory for Germany: Following the projections of the Umweltbundestamt (`UBA`) or targeting net zero with the Klimaschutzgesetz(`KSG`)
 - `costs:NEP` and `costs:transmission` - specify which year of the Netzentwicklungsplan should be used as basis for the transmission line costs (`2021,2023`) and if new HVDC links should be built with `overhead` or `underground` cables
 
+## Data sources
+
+`ariadne-data/ariadne-database.csv`
+
+* **Source:** Kopernikus Projekt Ariadne
+* **Link:** [Szenarien-Explorer](https://ariadne2.apps.ece.iiasa.ac.at/explorer)
+* **License:** CC-BY 4.0
+* **Description:** Results from the [Ariadne Report: Die Energiewende kosteneffizient gestalten](https://ariadneprojekt.de/publikation/report-szenarien-zur-klimaneutralitat-2045/)
+
 ## License
 
 The code in this repo is MIT licensed, see `./LICENSE.md`.

Snakefile

@@ -302,33 +302,6 @@ rule clean:
         print("Data downloaded to data/ has not been cleaned.")
 
 
-rule retrieve_egon_data:
-    output:
-        spatial="data/egon/demandregio_spatial_2018.json",
-        mapping="data/egon/mapping_technologies.json",
-    shell:
-        """
-        mkdir -p data/egon
-        curl -o {output.spatial} "https://api.opendata.ffe.de/demandregio/demandregio_spatial?id_spatial=5&year=2018"
-        curl -o {output.mapping} "https://api.opendata.ffe.de/demandregio/demandregio_spatial_description?id_spatial=5"
-        """
-
-
-rule retrieve_ariadne_database:
-    params:
-        db_name=config["iiasa_database"]["db_name"],
-        leitmodelle=config["iiasa_database"]["leitmodelle"],
-        scenarios=config["iiasa_database"]["scenarios"],
-    output:
-        data="resources/ariadne_database.csv",
-    log:
-        "logs/retrieve_ariadne_database.log",
-    resources:
-        mem_mb=1000,
-    script:
-        "scripts/pypsa-de/retrieve_ariadne_database.py"
-
-
 rule modify_cost_data:
     params:
         file_path="ariadne-data/costs/",
@@ -354,23 +327,87 @@ rule modify_cost_data:
         "scripts/pypsa-de/modify_cost_data.py"
 
 
+if config["pypsa-de"]["retrieve_ariadne_database"]:
+
+    rule retrieve_ariadne_database:
+        output:
+            data="data/ariadne_database.csv",
+        log:
+            "logs/retrieve_ariadne_database.log",
+        resources:
+            mem_mb=1000,
+        script:
+            "scripts/pypsa-de/retrieve_ariadne_database.py"
+
+def get_ariadne_database(w):
+    if config["pypsa-de"]["retrieve_ariadne_database"]:
+        return "data/ariadne_database.csv"
+    else:
+        return "ariadne-data/ariadne_database.csv"
+
+
+if config["enable"]["retrieve"]:
+
+    rule retrieve_ariadne_template:
+        input:
+            storage(
+                "https://github.com/iiasa/ariadne-intern-workflow/raw/main/attachments/2025-01-27_template_Ariadne.xlsx",
+                keep_local=True,
+            ),
+        output:
+            "data/template_ariadne_database.xlsx",
+        run:
+            move(input[0], output[0])
+
+
+if config["enable"]["retrieve"]:
+
+    rule retrieve_mastr:
+        input:
+            storage(
+                "https://zenodo.org/records/8225106/files/bnetza_open_mastr_2023-08-08_B.zip",
+                keep_local=True,
+            ),
+        params:
+            "data/mastr",
+        output:
+            "data/mastr/bnetza_open_mastr_2023-08-08_B_biomass.csv",
+            "data/mastr/bnetza_open_mastr_2023-08-08_B_combustion.csv",
+        run:
+            unpack_archive(input[0], params[0])
+
+
+if config["enable"]["retrieve"]:
+
+    rule retrieve_egon_data:
+        output:
+            spatial="data/egon/demandregio_spatial_2018.json",
+            mapping="data/egon/mapping_technologies.json",
+        shell:
+            """
+            mkdir -p data/egon
+            curl -o {output.spatial} "https://api.opendata.ffe.de/demandregio/demandregio_spatial?id_spatial=5&year=2018"
+            curl -o {output.mapping} "https://api.opendata.ffe.de/demandregio/demandregio_spatial_description?id_spatial=5"
+            """
+
+
 if config["enable"]["retrieve"] and config["enable"].get("retrieve_cost_data", True):
 
     ruleorder: modify_cost_data > retrieve_cost_data
 
 
 rule build_exogenous_mobility_data:
     params:
-        reference_scenario=config_provider("iiasa_database", "reference_scenario"),
+        reference_scenario=config_provider("pypsa-de", "reference_scenario"),
         planning_horizons=config_provider("scenario", "planning_horizons"),
-        leitmodelle=config_provider("iiasa_database", "leitmodelle"),
-        ageb_for_mobility=config_provider("iiasa_database", "ageb_for_mobility"),
-        uba_for_mobility=config_provider("iiasa_database", "uba_for_mobility"),
+        leitmodelle=config_provider("pypsa-de", "leitmodelle"),
+        ageb_for_mobility=config_provider("pypsa-de", "ageb_for_mobility"),
+        uba_for_mobility=config_provider("pypsa-de", "uba_for_mobility"),
         shipping_oil_share=config_provider("sector", "shipping_oil_share"),
         aviation_demand_factor=config_provider("sector", "aviation_demand_factor"),
         energy_totals_year=config_provider("energy", "energy_totals_year"),
     input:
-        ariadne="resources/ariadne_database.csv",
+        ariadne=get_ariadne_database,
         energy_totals=resources("energy_totals.csv"),
     output:
         mobility_data=resources(
@@ -599,11 +636,8 @@ ruleorder: modify_industry_demand > build_industrial_production_per_country_tomo
 
 
 rule modify_existing_heating:
-    params:
-        iiasa_reference_scenario=config_provider("iiasa_database", "reference_scenario"),
-        leitmodelle=config_provider("iiasa_database", "leitmodelle"),
     input:
-        ariadne="resources/ariadne_database.csv",
+        ariadne=get_ariadne_database,
         existing_heating="data/existing_infrastructure/existing_heating_raw.csv",
     output:
         existing_heating=resources("existing_heating.csv"),
@@ -615,21 +649,6 @@ rule modify_existing_heating:
         "scripts/pypsa-de/modify_existing_heating.py"
 
 
-rule retrieve_mastr:
-    input:
-        storage(
-            "https://zenodo.org/records/8225106/files/bnetza_open_mastr_2023-08-08_B.zip",
-            keep_local=True,
-        ),
-    params:
-        "data/mastr",
-    output:
-        "data/mastr/bnetza_open_mastr_2023-08-08_B_biomass.csv",
-        "data/mastr/bnetza_open_mastr_2023-08-08_B_combustion.csv",
-    run:
-        unpack_archive(input[0], params[0])
-
-
 rule build_existing_chp_de:
     params:
         district_heating_subnodes=config_provider(
@@ -658,9 +677,9 @@ rule build_existing_chp_de:
 
 rule modify_industry_demand:
     params:
-        reference_scenario=config_provider("iiasa_database", "reference_scenario"),
+        reference_scenario=config_provider("pypsa-de", "reference_scenario"),
     input:
-        ariadne="resources/ariadne_database.csv",
+        ariadne=get_ariadne_database,
         industrial_production_per_country_tomorrow=resources(
             "industrial_production_per_country_tomorrow_{planning_horizons}.csv"
         ),
@@ -722,18 +741,6 @@ rule cluster_wasserstoff_kernnetz:
         "scripts/pypsa-de/cluster_wasserstoff_kernnetz.py"
 
 
-rule download_ariadne_template:
-    input:
-        storage(
-            "https://github.com/iiasa/ariadne-intern-workflow/raw/main/attachments/2025-01-27_template_Ariadne.xlsx",
-            keep_local=True,
-        ),
-    output:
-        "data/template_ariadne_database.xlsx",
-    run:
-        move(input[0], output[0])
-
-
 rule export_ariadne_variables:
     params:
         planning_horizons=config_provider("scenario", "planning_horizons"),
@@ -794,11 +801,10 @@ rule export_ariadne_variables:
 
 rule plot_ariadne_variables:
     params:
-        iiasa_scenario=config_provider("iiasa_database", "reference_scenario"),
-        reference_scenario=config_provider("iiasa_database", "reference_scenario"),
+        reference_scenario=config_provider("pypsa-de", "reference_scenario"),
     input:
         exported_variables_full=RESULTS + "ariadne/exported_variables_full.xlsx",
-        ariadne_database="resources/ariadne_database.csv",
+        ariadne_database=get_ariadne_database,
     output:
         primary_energy=RESULTS + "ariadne/primary_energy.png",
         primary_energy_detailed=RESULTS + "ariadne/primary_energy_detailed.png",
@@ -863,9 +869,9 @@ rule ariadne_all:
 rule build_scenarios:
     params:
         scenarios=config["run"]["name"],
-        leitmodelle=config["iiasa_database"]["leitmodelle"],
+        leitmodelle=config["pypsa-de"]["leitmodelle"],
     input:
-        ariadne_database="resources/ariadne_database.csv",
+        ariadne_database=get_ariadne_database,
         scenario_yaml=config["run"]["scenarios"]["manual_file"],
     output:
         scenario_yaml=config["run"]["scenarios"]["file"],