Fix and add pre-commit checks (#66)

* add config

* first run

* Fix and add pre-commit

Author: lkstrp

.github/workflows/test.yaml

@@ -60,7 +60,7 @@ jobs:
     - name: Setup cache keys
       run: |
         echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts
-  
+
     - uses: actions/cache@v4
       with:
         path: |
@@ -86,7 +86,7 @@ jobs:
       run: |
         conda env update -n pypsa-de -f ${{ env.env_file }}
         echo "Run conda list" && conda list
-    
+
     - name: Setup ixmp4 access
       run: |
         mkdir -p ~/.local/share/ixmp4/
@@ -158,7 +158,7 @@ jobs:
         else
             echo "pinned=false" >> $GITHUB_ENV
         fi
-    
+
     - name: Setup cache keys
       if: env.pinned == 'false'
       run: |
@@ -218,4 +218,4 @@ jobs:
           logs
           .snakemake/log
           results
-        retention-days: 3
+        retention-days: 3

.pre-commit-config.yaml

@@ -1,9 +1,53 @@
 ci:
   autoupdate_schedule: monthly
 
+exclude: "^LICENSES"
+
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
   rev: v5.0.0
   hooks:
+  - id: check-merge-conflict
   - id: check-added-large-files
-    args: ['--maxkb=2000']
+    args: ["--maxkb=2000"]
+
+# Run ruff to lint and format
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  # Ruff version.
+  rev: v0.11.2
+  hooks:
+    # Run the linter.
+  - id: ruff
+    args: [--fix]
+    # Run the formatter.
+  - id: ruff-format
+
+  # Find common spelling mistakes in comments and docstrings
+- repo: https://github.com/codespell-project/codespell
+  rev: v2.4.1
+  hooks:
+  - id: codespell
+    args: ['--ignore-regex="(\b[A-Z]+\b)"', '--ignore-words-list=fom,appartment,bage,ore,setis,tabacco,berfore,vor,pris,GuD,Dezember,Juni,Juli,Produktion,WorstCase'] # Ignore capital case words, e.g. country codes
+    types_or: [python, rst, markdown]
+    files: ^(scripts|doc)/
+
+# Remove output from Jupyter notebooks
+- repo: https://github.com/aflc/pre-commit-jupyter
+  rev: v1.2.1
+  hooks:
+  - id: jupyter-notebook-cleanup
+    args: ['--remove-kernel-metadata']
+
+  # YAML formatting
+- repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
+  rev: v2.14.0
+  hooks:
+  - id: pretty-format-yaml
+    exclude: pinned\.yaml$
+    args: [--autofix, --indent, "2", --preserve-quotes]
+
+  # Format Snakemake rule / workflow files
+- repo: https://github.com/snakemake/snakefmt
+  rev: v0.11.0
+  hooks:
+  - id: snakefmt

Snakefile

@@ -332,7 +332,9 @@ rule modify_prenetwork:
         scale_capacity=config_provider("scale_capacity"),
     input:
         costs_modifications="ariadne-data/costs_{planning_horizons}-modifications.csv",
-        network=resources("networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_brownfield.nc"),
+        network=resources(
+            "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_brownfield.nc"
+        ),
         wkn=lambda w: (
             resources("wasserstoff_kernnetz_base_s_{clusters}.csv")
             if config_provider("wasserstoff_kernnetz", "enable")(w)

config/test/config.dach.yaml

@@ -54,4 +54,4 @@ electricity:
 solving:
   solver:
     name: highs
-    options: highs-simplex
+    options: highs-simplex

scripts/_helpers.py

@@ -47,7 +47,7 @@ def get_scenarios(run):
         fn = Path(scenario_config["file"])
         if fn.exists():
             scenarios = yaml.safe_load(fn.read_text())
-            if scenarios == None:
+            if scenarios is None:
                 print(
                     "WARNING! Scenario management enabled but scenarios file appears to be empty."
                 )

scripts/add_brownfield.py

@@ -12,15 +12,13 @@
 import pypsa
 import xarray as xr
 from add_existing_baseyear import add_build_year_to_new_assets
-from pypsa.clustering.spatial import normed_or_uniform
 
 from scripts._helpers import (
     configure_logging,
     get_snapshots,
     set_scenario_config,
     update_config_from_wildcards,
 )
-from add_existing_baseyear import add_build_year_to_new_assets
 
 logger = logging.getLogger(__name__)
 idx = pd.IndexSlice

scripts/add_existing_baseyear.py

@@ -7,8 +7,6 @@
 """
 
 import logging
-import os
-import sys
 from types import SimpleNamespace
 
 import country_converter as coco
@@ -25,9 +23,8 @@
     update_config_from_wildcards,
 )
 from scripts.add_electricity import sanitize_carriers
-from scripts.definitions.heat_sector import HeatSector
+from scripts.build_powerplants import add_custom_powerplants
 from scripts.definitions.heat_system import HeatSystem
-from scripts.definitions.heat_system_type import HeatSystemType
 from scripts.prepare_sector_network import (
     cluster_heat_buses,
     define_spatial,
@@ -39,8 +36,6 @@
 idx = pd.IndexSlice
 spatial = SimpleNamespace()
 
-from scripts.build_powerplants import add_custom_powerplants
-
 
 def add_build_year_to_new_assets(n: pypsa.Network, baseyear: int) -> None:
     """
@@ -500,8 +495,8 @@ def add_chp_plants(n, grouping_years, costs, baseyear):
 
     # drop assets which are already phased out / decommissioned
     # drop hydro, waste and oil fueltypes for CHP
-    limit = np.max(grouping_years)
-    drop_fueltypes = ["Hydro", "Other", "Waste", "nicht biogener Abfall"]
+    limit = np.max(grouping_years)  # noqa
+    drop_fueltypes = ["Hydro", "Other", "Waste", "nicht biogener Abfall"]  # noqa
     chp = ppl.query(
         "Set == 'CHP' and (DateOut >= @baseyear or DateOut != DateOut) and (DateIn <= @limit or DateIn != DateIn) and Fueltype not in @drop_fueltypes"
     ).copy()
@@ -516,7 +511,9 @@ def add_chp_plants(n, grouping_years, costs, baseyear):
     chp["lifetime"] = (chp.DateOut - chp["grouping_year"] + 1).fillna(
         snakemake.params.costs["fill_values"]["lifetime"]
     )
-    chp = chp.loc[chp.grouping_year + chp.lifetime > baseyear] # in add_brownfield this is build_year + lifetime <= baseyear
+    chp = chp.loc[
+        chp.grouping_year + chp.lifetime > baseyear
+    ]  # in add_brownfield this is build_year + lifetime <= baseyear
 
     # check if the CHPs were read in from MaStR for Germany
     if "Capacity_thermal" in chp.columns:
@@ -1054,7 +1051,6 @@ def add_heating_capacities_installed_before_baseyear(
             )
 
             # delete links with capacities below threshold
-            threshold = snakemake.params.existing_capacities["threshold_capacity"]
             n.remove(
                 "Link",
                 [

scripts/build_daily_heat_demand.py

@@ -20,9 +20,10 @@
 import geopandas as gpd
 import numpy as np
 import xarray as xr
-from scripts._helpers import configure_logging, get_snapshots, set_scenario_config
 from dask.distributed import Client, LocalCluster
 
+from scripts._helpers import configure_logging, get_snapshots, set_scenario_config
+
 logger = logging.getLogger(__name__)
 
 

scripts/build_hac_features.py

@@ -9,10 +9,11 @@
 
 import atlite
 import geopandas as gpd
-from scripts._helpers import configure_logging, get_snapshots, set_scenario_config
 from atlite.aggregate import aggregate_matrix
 from dask.distributed import Client
 
+from scripts._helpers import configure_logging, get_snapshots, set_scenario_config
+
 logger = logging.getLogger(__name__)
 
 if __name__ == "__main__":

scripts/build_hourly_heat_demand.py

@@ -14,10 +14,12 @@
 
 import pandas as pd
 import xarray as xr
+
 from scripts._helpers import (
     configure_logging,
     generate_periodic_profiles,
     get_snapshots,
+    mock_snakemake,
     set_scenario_config,
 )