Merge remote-tracking branch 'upstream/master' into merge-upstream-26-01-05

Author: lindnemi

.dockerignore

@@ -0,0 +1,6 @@
+# SPDX-FileCopyrightText: Contributors to PyPSA-Eur <https://github.com/pypsa/pypsa-eur>
+#
+# SPDX-License-Identifier: CC0-1.0
+
+*
+!pixi.*

.gitattributes

@@ -4,3 +4,5 @@
 
 *.h5 filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
+# SCM syntax highlighting & preventing 3-way merges
+pixi.lock merge=binary linguist-language=YAML linguist-generated=true

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -58,7 +58,7 @@ body:
   attributes:
     label: Installed Versions
     description: >
-      Please share information on your environment. Paste the output below.  For conda ``conda env export`` and for pip ``pip freeze``.
+      Please share information on your environment. Paste the output below.  For conda ``conda env export`` and for pixi, the content of ``pixi.lock``.
     value: >
       <details>
 

.github/workflows/release.yaml

@@ -0,0 +1,93 @@
+name: Release
+
+on:
+  push:
+    tags:
+    - v*.*.*
+
+env:
+  PREPARATION_COMMIT: '[github-actions.ci] prepare release ${{ github.ref_name }}'
+  BASE_ENV: envs/environment.yaml
+
+jobs:
+  check-preparation:
+    name: Check if release is prepared
+    runs-on: ubuntu-latest
+    outputs:
+      prepared: ${{ steps.validate.outputs.prepared }}
+    steps:
+    - uses: actions/checkout@v6
+
+    - name: Validate commit message
+      id: validate
+      run: |
+        # Check if last commit is the expected commit message
+        COMMIT_MESSAGE=$(git log -1 --pretty=%B)
+        echo "Expected: '${{ env.PREPARATION_COMMIT }}'"
+        echo "Received: '$COMMIT_MESSAGE'"
+
+        prepared="false"
+        if [[ "$COMMIT_MESSAGE" == "${{ env.PREPARATION_COMMIT }}" ]]; then
+          prepared="true"
+        fi
+
+        echo "prepared=$prepared" >> $GITHUB_OUTPUT
+
+  prepare-release:
+    name: Prepare release
+    needs: [check-preparation]
+    if: ${{ needs.check-preparation.outputs.prepared == 'false' }}
+    runs-on: ubuntu-latest
+    steps:
+    - name: Generate token for PyPSA Bot
+      id: generate-token
+      uses: actions/create-github-app-token@v2
+      with:
+        app-id: ${{ vars.PYPSA_BOT_ID }}
+        private-key: ${{ secrets.PYPSA_BOT_PRIVATE_KEY }}
+
+    - uses: actions/checkout@v6
+      with:
+        fetch-depth: 0
+
+    - name: Find the branch for commit/ tag
+      run: |
+        branch=$(git branch -r --contains ${{ github.sha }} | grep -v 'HEAD' | head -n 1 | sed 's|origin/||' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
+        echo "Branch found: $branch"
+        echo "BRANCH_NAME=$branch" >> $GITHUB_ENV
+
+    - uses: actions/checkout@v6
+      with:
+        fetch-depth: 0
+        ref: ${{ env.BRANCH_NAME }}
+        token: ${{ steps.generate-token.outputs.token }}
+
+    - name: Setup Pixi
+      uses: prefix-dev/setup-pixi@v0.9.3
+      with:
+        pixi-version: v0.59.0
+        cache: true
+        # Do not cache in branches
+        cache-write: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
+
+    # Start of preparation script
+
+    - name: Update DAGs in documentation
+      run: |
+        pixi run update-dags
+
+    # End of preparation script
+
+    - name: Remove previous tag
+      run: |
+        git tag -d ${{ github.ref_name }}
+        git push origin --delete ${{ github.ref_name }}
+
+    - name: Commit changes
+      uses: stefanzweifel/git-auto-commit-action@v7
+      with:
+        branch: ${{ env.BRANCH_NAME }}
+        commit_message: '${{ env.PREPARATION_COMMIT }}'
+        tagging_message: '${{ github.ref_name }}' # add tag again
+        push_options: '${{ github.ref_name }}'
+        add_options: '-u' # Never add untracked files

.github/workflows/test.yaml

@@ -16,9 +16,6 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
-env:
-  BASE_ENV: envs/environment.yaml
-
 jobs:
   run-tests:
     name: OS
@@ -36,67 +33,75 @@ jobs:
     steps:
     - uses: actions/checkout@v6
 
-    - name: Setup env file path (ubuntu)
-      if: matrix.os == 'ubuntu'
-      run: |
-        echo "env_file=envs/linux-64.lock.yaml" >> $GITHUB_ENV
-
-    - name: Setup env file path (macos and windows)
-      if: matrix.os != 'ubuntu'
-      run: |
-        if [[ "${{ matrix.os }}" == "macos" ]]; then
-          echo "env_file=envs/osx-arm64.lock.yaml" >> $GITHUB_ENV
-        else
-          echo "env_file=envs/win-64.lock.yaml" >> $GITHUB_ENV
-        fi
-
-    - name: Use base env file if it was changed
+    - uses: dorny/paths-filter@v3
+      id: filter
+      with:
+        filters: |
+          src:
+            - 'scripts/**'
+            - 'rules/**'
+            - 'data/**'
+            - 'Snakefile'
+            - 'config/**'
+            - 'test/**'
+            - 'pixi.toml'
+            - 'pixi.lock'
+            - '.github/workflows/test.yaml'
+
+    - name: Free up disk space
       run: |
-        git fetch origin ${{ github.event.repository.default_branch }}
-        if git diff --name-only origin/${{ github.event.repository.default_branch }} | grep '${{ env.BASE_ENV }}'; then
-          echo "Base env ${{ env.BASE_ENV }} changed. Using it instead of locked envs."
-          echo "env_file=${{ env.BASE_ENV }}" >> $GITHUB_ENV
-        else
-          echo "Base env ${{ env.BASE_ENV }} not changed. Using locked envs."
-        fi
+        echo "Initial disk space"
+        df -h
+        echo "Free up disk space"
+        sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+        sudo docker image prune --all --force
+        sudo docker builder prune -a --force
+        echo "Final disk space"
+        df -h
+    - name: Skip - no source changes
+      if: steps.filter.outputs.src != 'true' && github.event_name != 'schedule'
+      run: echo "Skipping tests because no source code changes detected"
+
+    - name: Setup Pixi
+      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
+      uses: prefix-dev/setup-pixi@v0.9.3
+      with:
+        pixi-version: v0.59.0
+        cache: true
+        # Do not cache in branches
+        cache-write: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
 
     - name: Setup cache keys
+      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
       run: |
         echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts
 
     - uses: actions/cache@v5
+      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
       with:
         path: |
           data
           cutouts
         key: data-cutouts-${{ env.WEEK }}
 
-    - uses: conda-incubator/setup-miniconda@v3
-      with:
-        miniforge-version: latest
-        activate-environment: pypsa-de
-        channel-priority: strict
-
-    - name: Cache Conda env
-      uses: actions/cache@v5
-      with:
-        path: ${{ env.CONDA }}/envs
-        key: conda-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles(format('{0}', env.env_file)) }}
-      id: cache-env
-
-    - name: Update environment
-      if: steps.cache-env.outputs.cache-hit != 'true'
+    - name: Run pylint check on scripts
+      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
+      # check for undefined variables to reuse functions across scripts
       run: |
-        conda env update -n pypsa-de -f ${{ env.env_file }}
-        echo "Run conda list" && conda list
+        pixi run pylint --disable=all --enable=E0601,E0606 --output-format=parseable scripts/add_* scripts/prepare_* scripts/solve_*
 
     - name: Run snakemake test workflows
+      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
+      env:
+        SNAKEMAKE_STORAGE_CACHED_HTTP_CACHE: ""
+        SNAKEMAKE_STORAGE_CACHED_HTTP_SKIP_REMOTE_CHECKS: "1"
       run: |
-        make test
+        pixi run integration-tests
 
     - name: Run unit tests
+      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
       run: |
-        make unit-test
+        pixi run unit-tests
 
     - name: Upload artifacts
       if: always()
@@ -108,3 +113,7 @@ jobs:
           .snakemake/log
           results
         retention-days: 3
+
+    - name: Show remaining disk space
+      if: always()
+      run: df -h

.github/workflows/update-lockfile.yaml

@@ -0,0 +1,68 @@
+name: Update locked envs
+on:
+  push:
+    paths:
+    - pixi.toml
+  schedule:
+  - cron: "0 8 1,16 * *" # Bi-weekly
+  workflow_dispatch:
+
+jobs:
+  update-locked-environment:
+    if: ${{ github.ref == 'refs/heads/master' }}
+    name: Update lockfiles
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -l {0}
+    steps:
+    - uses: actions/checkout@v6
+
+    - name: Setup Pixi
+      uses: prefix-dev/setup-pixi@v0.9.3
+      with:
+        pixi-version: v0.59.0
+
+    - name: Update lockfile
+      run: |
+        rm pixi.lock
+        pixi install --all
+
+    - name: Create new conda lock files
+      run: |
+        pixi workspace export conda-explicit-spec -e default envs
+        for f in envs/*_conda_spec*; do mv "$f" "${f/_conda_spec/.pin}"; done
+
+    - name: Create new conda environment file
+      run: |
+        pixi workspace export conda-environment -e default envs/environment.yaml -n pypsa-eur
+
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v6
+      with:
+        name: lockfiles
+        path: |
+          pixi.lock
+          envs/
+
+  create-pull-request:
+    needs: update-locked-environment
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v6
+    - name: Download all artifacts
+      uses: actions/download-artifact@v7
+      with:
+        name: lockfiles
+
+    - name: Create Pull Request
+      uses: peter-evans/create-pull-request@v8
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+        branch: update-locked-environment
+        title: "[github-actions.ci] Update locked envs"
+        body: |
+          Automatically generated PR to update the pixi lockfile.
+
+          **Note: Do not merge without manual test execution. Either update the branch to trigger tests, or use `workflow_dispatch` to run tests manually. Unlike standard PRs, tests will not run automatically.**
+        commit-message: "Update locked environment files for all platforms"