Merge remote-tracking branch 'upstream' into merge-master-march

Author: lindnemi

.git-blame-ignore-revs

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2022 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: Contributors to PyPSA-Eur <https://github.com/pypsa/pypsa-eur>
 #
 # SPDX-License-Identifier: CC0-1.0
 
@@ -8,6 +8,9 @@
 
 # Custom commits
 9865a970893d9e515786f33c629b14f71645bf1e
+6a974d0bba5f6d45957f0196b77fd49dbbbbf777
+151dc73b40dfbe78ba08d5698b39b1cf2e5a5ff5
+
 # DE model specific
 ca1f43c23f1c0936cb9f9e1062854fdf0060c409
 

.gitattributes

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: Contributors to PyPSA-Eur <https://github.com/pypsa/pypsa-eur>
 #
 # SPDX-License-Identifier: CC0-1.0
 

.github/workflows/push-images.yaml

@@ -0,0 +1,52 @@
+name: Push images
+
+on:
+  push:
+
+    branches:
+    - master
+  workflow_dispatch:
+
+env:
+  IMAGE_NAME: pypsa/eur-dev-env
+  BASE_ENV: envs/linux-pinned.yaml
+
+jobs:
+  push-image:
+    name: dev-env
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: 'Get relevant env'
+      run: |
+        hash_last_changed=$(git log -1 --pretty=format:%H -- ${{ env.BASE_ENV }})
+        echo "hash_last_changed=$hash_last_changed" >> $GITHUB_ENV
+
+    - name: 'Login to GitHub Container Registry'
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ secrets.REGISTRY_TOKEN }}
+
+    - name: 'Build new image' # only build if the pinned envs file was modified
+      if: env.hash_last_changed == github.sha || github.event_name == 'workflow_dispatch'
+      run: |
+        docker build . --file docker/dev-env/Dockerfile --tag ghcr.io/${{ env.IMAGE_NAME }}:${{ github.sha }}
+        docker push ghcr.io/${{ env.IMAGE_NAME }}:${{ github.sha }}
+
+        # Add latest tag if on main branch
+        if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+          docker tag ghcr.io/${{ env.IMAGE_NAME }}:${{ github.sha }} ghcr.io/${{ env.IMAGE_NAME }}:latest
+          docker push ghcr.io/${{ env.IMAGE_NAME }}:latest
+        fi
+
+    - name: 'Add SHA tag to existing image' # when rebuild is not needed
+      if: env.hash_last_changed != github.sha
+      run: |-
+        docker pull ghcr.io/${{ env.IMAGE_NAME }}:${{ env.hash_last_changed }}
+        docker tag ghcr.io/${{ env.IMAGE_NAME }}:${{ env.hash_last_changed }} ghcr.io/${{ env.IMAGE_NAME }}:${{ github.sha }}
+        docker push ghcr.io/${{ env.IMAGE_NAME }}:${{ github.sha }}

.github/workflows/test.yaml

@@ -1,7 +1,3 @@
-# SPDX-FileCopyrightText: : 2021-2024 The PyPSA-Eur Authors
-#
-# SPDX-License-Identifier: CC0-1.0
-
 name: Test workflows
 
 on:
@@ -12,28 +8,27 @@ on:
     branches:
     - main
   schedule:
-  - cron: "0 5 * * *"
+  - cron: "0 5 * * 1-6"
+  - cron: "0 5 * * 0"
+  workflow_dispatch:
 
 # Cancel any in-progress runs when a new run is triggered
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 env:
-  ENV_FILE: envs/environment.yaml
-  ENV_FIXED_FILE: envs/environment.fixed.yaml
+  BASE_ENV: envs/environment.yaml
 
 jobs:
   run-tests:
     name: OS
+    runs-on: ${{ matrix.os }}-latest
     strategy:
       fail-fast: false
       matrix:
-        os:
-        - macos
-        - windows
-        - ubuntu
-    runs-on: ${{ matrix.os }}-latest
+        # Run windows only on scheduled runs on Sundays, otherwise ignore
+        os: ${{ github.event.schedule == '0 5 * * 0' && fromJson('["ubuntu", "macos", "windows"]') || fromJson('["ubuntu", "macos"]') }}
 
     defaults:
       run:
@@ -42,73 +37,83 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - name: Setup secrets & cache dates
+    - name: Setup env file path (ubuntu)
+      if: matrix.os == 'ubuntu'
+      run: |
+        echo "env_file=envs/linux-pinned.yaml" >> $GITHUB_ENV
+
+    - name: Setup env file path (macos and windows)
+      if: matrix.os != 'ubuntu'
+      run: |
+        echo "env_file=envs/${{ matrix.os }}-pinned.yaml" >> $GITHUB_ENV
+
+    - name: Use base env file if it was changed
       run: |
-        echo -ne "url: ${CDSAPI_URL}\nkey: ${CDSAPI_TOKEN}\n" > ~/.cdsapirc
-        echo "week=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts
-        echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_ENV # env
+        git fetch origin master
+        if git diff --name-only origin/master | grep '${{ env.BASE_ENV }}'; then
+          echo "Base env ${{ env.BASE_ENV }} changed. Using it instead of pinned envs."
+          echo "env_file=${{ env.BASE_ENV }}" >> $GITHUB_ENV
+        else
+          echo "Base env ${{ env.BASE_ENV }} not changed. Using pinned envs."
+        fi
+
+    - name: Setup cache keys
+      run: |
+        echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts
 
     - uses: actions/cache@v4
       with:
         path: |
           data
           cutouts
-        key: data-cutouts-${{ env.week }}
+        key: data-cutouts-${{ env.WEEK }}
 
     - uses: conda-incubator/setup-miniconda@v3
       with:
+        miniforge-version: latest
         activate-environment: pypsa-eur
+        channel-priority: strict
 
     - name: Cache Conda env
       uses: actions/cache@v4
       with:
         path: ${{ env.CONDA }}/envs
-        key: conda-${{ runner.os }}-${{ runner.arch }}-${{ env.today }}-${{ hashFiles(format('{0}', env.ENV_FILE)) }}
+        key: conda-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles(format('{0}', env.env_file)) }}
       id: cache-env
 
     - name: Update environment
       if: steps.cache-env.outputs.cache-hit != 'true'
-      run: conda env update -n pypsa-eur -f ${{ env.ENV_FILE }}
-
-    - name: Log env diff to environment.fixed.yaml
-      if: ${{ matrix.os == 'ubuntu' }}
       run: |
-        # Get fixed environment of current env
-        conda env export --name ${{ github.event.repository.name }} --no-builds | sed 's/^name: ${{ github.event.repository.name }}$/name: ${{ github.event.repository.name }}-fixed/' > current-env.yaml
-
-        # Add SPDX header
-        SPDX_HEADER="# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors\n# SPDX-License-Identifier: CC0-1.0\n"
-        echo -e "$SPDX_HEADER" | cat - current-env.yaml > temp && mv temp current-env.yaml
-
-        # Format with pre-commit (it differs from the conda output)
-        pip install pre-commit
-        pre-commit run --files current-env.yaml || true
+        conda env update -n pypsa-eur -f ${{ env.env_file }}
+        echo "Run conda list" && conda list
 
-        # Get diff
-        diff ${{ env.ENV_FIXED_FILE }} current-env.yaml > diff.txt || true
-
-        # Format
-        { echo -e "**Environment diff**\n\`\`\` diff\n"; cat diff.txt; echo -e "\n\`\`\`"; } > temp && mv temp diff.txt
-
-        # Write to summary
-        cat diff.txt >> $GITHUB_STEP_SUMMARY
-      shell: bash
+    - name: Run pylint check on scripts
+      # check for undefined variables to reuse functions across scripts
+      run: |
+        pylint --disable=all --enable=E0601 --output-format=parseable scripts/add_* scripts/prepare_* scripts/solve_* 
 
     - name: Run snakemake test workflows
       run: |
         make test
 
+    - name: Run unit tests
+      run: |
+        make unit-test
+
     - name: Upload artifacts
+      if: always()
       uses: actions/upload-artifact@v4
       with:
-        name: logs-${{ matrix.inhouse }}
+        name: results-${{ matrix.os }}
         path: |
           logs
           .snakemake/log
+          results
         retention-days: 3
 
   run-tests-on-dev-deps:
     name: Inhouse
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
@@ -117,7 +122,6 @@ jobs:
         - atlite
         - powerplantmatching
         - linopy
-    runs-on: ubuntu-latest
 
     defaults:
       run:
@@ -126,30 +130,39 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
+    - name: Use base env file if it was changed
+      run: |
+        git fetch origin master
+        if git diff --name-only origin/master | grep '${{ env.BASE_ENV }}'; then
+          echo "Base env ${{ env.BASE_ENV }} changed. Using it instead of pinned envs."
+          echo "env_file=${{ env.BASE_ENV }}" >> $GITHUB_ENV
+        else
+          echo "Base env ${{ env.BASE_ENV }} not changed. Using pinned envs."
+          echo "env_file=envs/linux-pinned.yaml" >> $GITHUB_ENV
+        fi
+
     # Only run checks if package is not pinned
     - name: Check if inhouse package is pinned
       run: |
-        grep_line=$(grep -- '- ${{ matrix.inhouse }}' ${{ env.ENV_FILE }})
+        grep_line=$(grep -- '- ${{ matrix.inhouse }}' ${{ env.BASE_ENV }})
         if [[ $grep_line == *"<"* || $grep_line == *"=="* ]]; then
             echo "pinned=true" >> $GITHUB_ENV
         else
             echo "pinned=false" >> $GITHUB_ENV
         fi
 
-    - name: Setup secrets & cache dates
+    - name: Setup cache keys
       if: env.pinned == 'false'
       run: |
-        echo -ne "url: ${CDSAPI_URL}\nkey: ${CDSAPI_TOKEN}\n" > ~/.cdsapirc
-        echo "week=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts
-        echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_ENV # env
+        echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts
 
     - uses: actions/cache@v4
       if: env.pinned == 'false'
       with:
         path: |
           data
           cutouts
-        key: data-cutouts-${{ env.week }}
+        key: data-cutouts-${{ env.WEEK }}
 
     - uses: conda-incubator/setup-miniconda@v3
       if: env.pinned == 'false'
@@ -161,12 +174,14 @@ jobs:
       uses: actions/cache@v4
       with:
         path: ${{ env.CONDA }}/envs
-        key: conda-${{ runner.os }}-${{ runner.arch }}-${{  matrix.inhouse  }}-${{ env.today }}-${{ hashFiles(format('{0}', env.ENV_FILE)) }}
+        key: conda-${{ runner.os }}-${{ runner.arch }}-${{  matrix.inhouse  }}-${{ hashFiles(format('{0}', env.env_file)) }}
       id: cache-env
 
     - name: Update environment
       if: env.pinned == 'false' && steps.cache-env.outputs.cache-hit != 'true'
-      run: conda env update -n pypsa-eur -f ${{ env.ENV_FILE }}
+      run: |
+        conda env update -n pypsa-eur -f ${{ env.env_file }}
+        echo "Run conda list" && conda list
 
     - name: Install inhouse packages from master
       if: env.pinned == 'false'
@@ -179,11 +194,12 @@ jobs:
         make test
 
     - name: Upload artifacts
-      if: env.pinned == 'false'
+      if: env.pinned == 'false' && always()
       uses: actions/upload-artifact@v4
       with:
-        name: logs-${{ matrix.inhouse }}
+        name: results-inhouse-${{ matrix.inhouse }}
         path: |
           logs
           .snakemake/log
+          results
         retention-days: 3

.github/workflows/update-pinned-env.yaml

@@ -34,8 +34,9 @@ jobs:
     - name: Setup conda
       uses: conda-incubator/setup-miniconda@v3
       with:
+        miniforge-version: latest
         activate-environment: ${{ github.event.repository.name }}
-        channels: bioconda,conda-forge,defaults
+        channel-priority: strict
         environment-file: ${{ env.BASE_ENV }}
 
     - name: Update pinned environment per OS
@@ -83,5 +84,8 @@ jobs:
         token: ${{ secrets.GITHUB_TOKEN }}
         branch: update-pinned-environment
         title: "[github-actions.ci] Update pinned envs"
-        body: "Automatically generated PR to update pinned environment files for Windows, macOS, and Linux."
+        body: |
+          Automatically generated PR to update pinned environment files for Windows, macOS, and Linux.
+
+          **Note: Do not merge without manual test execution. Either update the branch to trigger tests, or use `workflow_dispatch` to run tests manually. Unlike standard PRs, tests will not run automatically.**
         commit-message: "Update pinned environment files for all platforms"

.github/workflows/validate.yaml

@@ -4,7 +4,6 @@ on:
   pull_request:
     branches:
     - master
-    - main
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -15,8 +14,12 @@ jobs:
     name: Run validation
     if: github.event.pull_request.head.repo.full_name == github.repository
     runs-on: self-hosted
+    outputs:
+      validation_failed: ${{ steps.check_validation.outputs.failed }}
     steps:
     - uses: PyPSA/pypsa-validator@v0.2.5
+      id: validation
+      continue-on-error: true
       with:
         step: run-self-hosted-validation
         env_file: envs/environment.yaml
@@ -25,11 +28,21 @@ jobs:
         main_command: "ariadne_all"
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    - name: Check validation status
+      id: check_validation
+      run: |
+        if [[ "${{ steps.validation.outcome }}" == "failure" ]]; then
+          echo "failed=true" >> $GITHUB_OUTPUT
+        else
+          echo "failed=false" >> $GITHUB_OUTPUT
+        fi
 
   create-report:
     name: Create report
-    if: github.event.pull_request.head.repo.full_name == github.repository
     needs: run-validation
+    if: |
+      github.event.pull_request.head.repo.full_name == github.repository && 
+      needs.run-validation.outputs.validation_failed != 'true'
     runs-on: ubuntu-latest
     steps:
     - uses: PyPSA/pypsa-validator@v0.2.5