Merge branch 'improve-transport-sector' of github.com:PyPSA/pypsa-de into improve-transport-sector

Author: lindnemi

.github/workflows/push-images.yaml

@@ -9,7 +9,7 @@ on:
 
 env:
   IMAGE_NAME: pypsa/eur-dev-env
-  BASE_ENV: envs/linux-pinned.yaml
+  BASE_ENV: envs/linux-64.lock.yaml
 
 jobs:
   push-image:
@@ -32,7 +32,7 @@ jobs:
         username: ${{ github.repository_owner }}
         password: ${{ secrets.REGISTRY_TOKEN }}
 
-    - name: 'Build new image' # only build if the pinned envs file was modified
+    - name: 'Build new image' # only build if the locked envs file was modified
       if: env.hash_last_changed == github.sha || github.event_name == 'workflow_dispatch'
       run: |
         docker build . --file docker/dev-env/Dockerfile --tag ghcr.io/${{ env.IMAGE_NAME }}:${{ github.sha }}

.github/workflows/test.yaml

@@ -40,21 +40,25 @@ jobs:
     - name: Setup env file path (ubuntu)
       if: matrix.os == 'ubuntu'
       run: |
-        echo "env_file=envs/linux-pinned.yaml" >> $GITHUB_ENV
+        echo "env_file=envs/linux-64.lock.yaml" >> $GITHUB_ENV
 
     - name: Setup env file path (macos and windows)
       if: matrix.os != 'ubuntu'
       run: |
-        echo "env_file=envs/${{ matrix.os }}-pinned.yaml" >> $GITHUB_ENV
+        if [[ "${{ matrix.os }}" == "macos" ]]; then
+          echo "env_file=envs/osx-arm64.lock.yaml" >> $GITHUB_ENV
+        else
+          echo "env_file=envs/win-64.lock.yaml" >> $GITHUB_ENV
+        fi
 
     - name: Use base env file if it was changed
       run: |
         git fetch origin master
         if git diff --name-only origin/master | grep '${{ env.BASE_ENV }}'; then
-          echo "Base env ${{ env.BASE_ENV }} changed. Using it instead of pinned envs."
+          echo "Base env ${{ env.BASE_ENV }} changed. Using it instead of locked envs."
           echo "env_file=${{ env.BASE_ENV }}" >> $GITHUB_ENV
         else
-          echo "Base env ${{ env.BASE_ENV }} not changed. Using pinned envs."
+          echo "Base env ${{ env.BASE_ENV }} not changed. Using locked envs."
         fi
 
     - name: Setup cache keys
@@ -66,6 +70,7 @@ jobs:
         path: |
           data
           cutouts
+          resources/ariadne_database.csv
         key: data-cutouts-${{ env.WEEK }}
 
     - uses: conda-incubator/setup-miniconda@v3
@@ -87,19 +92,6 @@ jobs:
         conda env update -n pypsa-de -f ${{ env.env_file }}
         echo "Run conda list" && conda list
 
-    - name: Setup ixmp4 access
-      run: |
-        mkdir -p ~/.local/share/ixmp4/
-        cat > ~/.local/share/ixmp4/credentials.toml << 'EOL'
-        ${{ secrets.IXMP4_ACCESS }}
-        EOL
-      shell: bash
-
-    # - name: Run pylint check on scripts
-    #   # check for undefined variables to reuse functions across scripts
-    #   run: |
-    #     pylint --disable=all --enable=E0601,E0606 --output-format=parseable scripts/add_* scripts/prepare_* scripts/solve_* 
-
     - name: Run snakemake test workflows
       run: |
         make test
@@ -118,104 +110,3 @@ jobs:
           .snakemake/log
           results
         retention-days: 3
-
-  run-tests-on-dev-deps:
-    name: Inhouse
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        inhouse:
-        - pypsa
-        - atlite
-        - powerplantmatching
-        - linopy
-
-    defaults:
-      run:
-        shell: bash -l {0}
-
-    steps:
-    - uses: actions/checkout@v4
-
-    - name: Use base env file if it was changed
-      run: |
-        git fetch origin master
-        if git diff --name-only origin/master | grep '${{ env.BASE_ENV }}'; then
-          echo "Base env ${{ env.BASE_ENV }} changed. Using it instead of pinned envs."
-          echo "env_file=${{ env.BASE_ENV }}" >> $GITHUB_ENV
-        else
-          echo "Base env ${{ env.BASE_ENV }} not changed. Using pinned envs."
-          echo "env_file=envs/linux-pinned.yaml" >> $GITHUB_ENV
-        fi
-
-    # Only run checks if package is not pinned
-    - name: Check if inhouse package is pinned
-      run: |
-        grep_line=$(grep -- '- ${{ matrix.inhouse }}' ${{ env.BASE_ENV }})
-        if [[ $grep_line == *"<"* || $grep_line == *"=="* ]]; then
-            echo "pinned=true" >> $GITHUB_ENV
-        else
-            echo "pinned=false" >> $GITHUB_ENV
-        fi
-
-    - name: Setup cache keys
-      if: env.pinned == 'false'
-      run: |
-        echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts
-
-    - uses: actions/cache@v4
-      if: env.pinned == 'false'
-      with:
-        path: |
-          data
-          cutouts
-        key: data-cutouts-${{ env.WEEK }}
-
-    - uses: conda-incubator/setup-miniconda@v3
-      if: env.pinned == 'false'
-      with:
-        activate-environment: pypsa-de
-
-    - name: Cache Conda env
-      if: env.pinned == 'false'
-      uses: actions/cache@v4
-      with:
-        path: ${{ env.CONDA }}/envs
-        key: conda-${{ runner.os }}-${{ runner.arch }}-${{  matrix.inhouse  }}-${{ hashFiles(format('{0}', env.env_file)) }}
-      id: cache-env
-
-    - name: Update environment
-      if: env.pinned == 'false' && steps.cache-env.outputs.cache-hit != 'true'
-      run: |
-        conda env update -n pypsa-de -f ${{ env.env_file }}
-        echo "Run conda list" && conda list
-
-    - name: Setup ixmp4 access
-      run: |
-        mkdir -p ~/.local/share/ixmp4/
-        cat > ~/.local/share/ixmp4/credentials.toml << 'EOL'
-        ${{ secrets.IXMP4_ACCESS }}
-        EOL
-      shell: bash
-
-    - name: Install inhouse packages from master
-      if: env.pinned == 'false'
-      run: |
-        python -m pip install git+https://github.com/PyPSA/${{ matrix.inhouse }}.git@master
-
-    - name: Run snakemake test workflows
-      if: env.pinned == 'false'
-      run: |
-        make test
-
-    - name: Upload artifacts
-      if: env.pinned == 'false' && always()
-      uses: actions/upload-artifact@v4
-      with:
-        name: results-inhouse-${{ matrix.inhouse }}
-        path: |
-          logs
-          .snakemake/log
-          results
-        retention-days: 3

.github/workflows/update-pinned-env.yaml

@@ -1,5 +1,4 @@
-name: Update pinned envs
-
+name: Update locked envs
 on:
   push:
     paths:
@@ -12,22 +11,13 @@ env:
   BASE_ENV: envs/environment.yaml
 
 jobs:
-  update-pinned-environment:
+  update-locked-environment:
     if: ${{ github.ref == 'refs/heads/main' }}
     name: Update pinned envs
-    runs-on: ${{ matrix.os }}-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu, macos, windows]
-        include:
-        - os: ubuntu
-          suffix: "linux"
-        - os: macos
-          suffix: "macos"
-        - os: windows
-          suffix: "windows"
-
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -l {0}
     steps:
     - uses: actions/checkout@v4
 
@@ -39,53 +29,72 @@ jobs:
         channel-priority: strict
         environment-file: ${{ env.BASE_ENV }}
 
-    - name: Update pinned environment per OS
+    - name: Install conda-lock
       run: |
-        conda env export --name ${{ github.event.repository.name }} --no-builds -c conda-forge -c bioconda --override-channels > envs/${{ matrix.suffix }}-pinned.yaml
+        conda install -c conda-forge conda-lock
 
-    # - name: Add SPDX header
-    #   if: ${{ matrix.suffix != 'windows' }}
-    #   run: |
-    #     SPDX_HEADER="# SPDX-FileCopyrightText: Contributors to PyPSA-Eur <https://github.com/pypsa/pypsa-eur>\n# SPDX-License-Identifier: CC0-1.0\n"
-    #     echo -e "$SPDX_HEADER" | cat - envs/${{ matrix.suffix }}-pinned.yaml > temp && mv temp envs/${{ matrix.suffix }}-pinned.yaml
+    - name: Generate lockfiles for all platforms
+      run: |
+        conda-lock -f ${{ env.BASE_ENV }} \
+        -p linux-64 -p osx-64 -p win-64 -p osx-arm64 \
+        -k env --filename-template "envs/{platform}.lock"
 
-    # - name: Add SPDX header (windows)
-    #   if: ${{ matrix.suffix == 'windows' }}
-    #   run: |
-    #     $SPDX_HEADER = "# SPDX-FileCopyrightText: Contributors to PyPSA-Eur <https://github.com/pypsa/pypsa-eur>`r`n# SPDX-License-Identifier: CC0-1.0`r`n`r`n"
-    #     $CurrentContent = Get-Content "envs/${{ matrix.suffix }}-pinned.yaml" -Raw
-    #     $NewContent = $SPDX_HEADER + $CurrentContent
-    #     $NewContent | Set-Content "envs/${{ matrix.suffix }}-pinned.yaml"
+        # Rename to .yaml extension
+        for file in envs/*.lock.yml; do
+          mv "$file" "${file%.yml}.yaml"
+        done
+    - name: Add SPDX headers to lock files
+      run: |
+        SPDX_HEADER="# SPDX-FileCopyrightText: Contributors to PyPSA-Eur <https://github.com/pypsa/pypsa-eur>\n# SPDX-License-Identifier: CC0-1.0\n"
+
+        # Add header to all generated lock files
+        for file in envs/*.lock.yaml; do
+          echo "Adding header to $file"
+          echo -e "$SPDX_HEADER" | cat - "$file" > temp && mv temp "$file"
+        done
 
-    - name: Cache environment files
+    - name: Insert environment name in lock files
+      run: |
+        for file in envs/*.lock.yaml; do
+          if [ -f "$file" ]; then
+            echo "Processing $file"
+            if ! grep -q "name: pypsa-de" "$file"; then
+              # Insert name: pypsa-de before channels section
+              sed -i '7a name: pypsa-de' "$file"
+            else
+              echo "name: pypsa-de already exists in $file"
+            fi
+          fi
+        done
+
+
+    - name: Upload artifacts
       uses: actions/upload-artifact@v4
       with:
-        name: ${{ matrix.suffix }}-pinned
-        path: envs/${{ matrix.suffix }}-pinned.yaml
+        name: lockfiles
+        path: envs/*.lock.yaml
 
   create-pull-request:
-    needs: update-pinned-environment
+    needs: update-locked-environment
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
     - name: Download all artifacts
       uses: actions/download-artifact@v4
-
-    - name: Prepare files for commit
-      run: |
-        mkdir -p envs
-        mv linux-pinned/* envs/linux-pinned.yaml
-        mv macos-pinned/* envs/macos-pinned.yaml
-        mv windows-pinned/* envs/windows-pinned.yaml
+      with:
+        name: lockfiles
+        path: envs/
 
     - name: Create Pull Request
       uses: peter-evans/create-pull-request@v7
       with:
         token: ${{ secrets.GITHUB_TOKEN }}
-        branch: update-pinned-environment
-        title: "[github-actions.ci] Update pinned envs"
+        branch: update-locked-environment
+        title: "[github-actions.ci] Update locked envs"
         body: |
-          Automatically generated PR to update pinned environment files for Windows, macOS, and Linux.
+          Automatically generated PR to update locked environment files for Windows, macOS, and Linux.
+
+          These files were generated using conda-lock for improved dependency resolution and reproducibility.
 
           **Note: Do not merge without manual test execution. Either update the branch to trigger tests, or use `workflow_dispatch` to run tests manually. Unlike standard PRs, tests will not run automatically.**
-        commit-message: "Update pinned environment files for all platforms"
+        commit-message: "Update locked environment files for all platforms"

.gitignore

@@ -87,3 +87,8 @@ cutouts
 local
 # private dev folder
 dev/*
+
+# ai coding tools
+CLAUDE.md
+.claude/
+.github/copilot-instructions.md

.pre-commit-config.yaml

@@ -1,8 +1,8 @@
+exclude: "^LICENSES"
+
 ci:
   autoupdate_schedule: quarterly
 
-exclude: "^LICENSES"
-
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
   rev: v5.0.0
@@ -14,7 +14,7 @@ repos:
 # Run ruff to lint and format
 - repo: https://github.com/astral-sh/ruff-pre-commit
   # Ruff version.
-  rev: v0.11.8
+  rev: v0.12.2
   hooks:
     # Run the linter.
   - id: ruff
@@ -27,7 +27,7 @@ repos:
   rev: v2.4.1
   hooks:
   - id: codespell
-    args: ['--ignore-regex="(\b[A-Z]+\b)"', '--ignore-words-list=fom,appartment,bage,ore,setis,tabacco,berfore,vor,pris,GuD,Dezember,Juni,Juli,Produktion,WorstCase'] # Ignore capital case words, e.g. country codes
+    args: ['--ignore-regex="(\b[A-Z]+\b)"', '--ignore-words-list=fom,appartment,bage,ore,setis,tabacco,berfore,vor,pris,bund,GuD,Dezember,Juni,Juli,Produktion,WorstCase'] # Ignore capital case words, e.g. country codes
     types_or: [python, rst, markdown]
     files: ^(scripts|doc)/
 
@@ -40,10 +40,10 @@ repos:
 
   # YAML formatting
 - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
-  rev: v2.14.0
+  rev: v2.15.0
   hooks:
   - id: pretty-format-yaml
-    exclude: pinned\.yaml$
+    exclude: (pinned\.yaml|\.lock\.yaml)$
     args: [--autofix, --indent, "2", --preserve-quotes]
 
   # Format Snakemake rule / workflow files