Merge pull request #87 from alliander-opensource/release/1.4

Release version 1.4

Author: TonyXiang8787

.clang-format

@@ -15,4 +15,5 @@ AllowShortLoopsOnASingleLine: false
 BreakBeforeBraces: Custom
 BraceWrapping:
   BeforeCatch: true
-  BeforeElse: true
+  BeforeElse: true
+

.github/workflows/black-and-clang-format.yml renamed to .github/workflows/check-code-quality.yml

@@ -4,12 +4,12 @@
 
 
 # GitHub Action that uses
-#     Black to reformat the Python code in an incoming pull request.
+#     isort, black, mypy and pylint to reformat the Python code in an incoming pull request.
 #     clang-format to reformat the C++ code in an incoming pull request.
 # If all code in the pull request is compliant with Black and clang-format then this Action
 # does nothing. Otherwise, it will print the files which need to be reformatted and raise an error.
 
-name: Format Code
+name: Check Code Quality
 
 on:
   # run pipeline on push event of main or release branch
@@ -21,8 +21,8 @@ on:
   pull_request:
 
 jobs:
-  code-format-check:
-
+  check-code-quality:
+    if: (github.event_name == 'push') || (!startsWith(github.head_ref, 'release'))
     runs-on: ubuntu-latest
 
     steps:
@@ -33,23 +33,41 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: 3.8
-        
-      - name: Install Black and clang-format
+
+      - name: Upgrade pip
+        run: pip install --upgrade pip
+
+      - name: Install and run isort
+        run: |
+          pip install isort
+          isort .
+
+      - name: Install and run black
         run: |
           pip install black
-          sudo apt-get update && sudo apt-get install -y clang-format
+          black .
 
-      - name: Run black
-        run: black .
+      - name: Install and run mypy
+        run: |
+          pip install mypy
+          mypy .
+
+      - name: Install and run pylint
+        run: |
+          pip install pylint .
+          pylint power_grid_model
+          git restore README.md
 
-      - name: Run clang-format
-        run: find . -regex '.*\.\(cpp\|hpp\|cc\|cxx\)' -exec clang-format -style=file -i {} \;
+      - name: Install and run clang-format
+        run: |
+          sudo apt-get update && sudo apt-get install -y clang-format
+          find . -regex '.*\.\(cpp\|hpp\|cc\|cxx\)' -exec clang-format -style=file -i {} \;
 
       - name: If needed raise error
         run: |
 
           if [[ `git status --porcelain --untracked-files=no` ]]; then
-            echo "Formatting not correct! See blow the files which need to be reformatted!"
+            echo "Formatting not correct! See below the files which need to be reformatted!"
             git status --porcelain --untracked-files=no
             exit 1
           fi

.github/workflows/main.yml

@@ -22,29 +22,34 @@ on:
 jobs:
 
   build-cpp-test-linux:
-    runs-on: ubuntu-latest
-    container: ghcr.io/alliander-opensource/power-grid-model-build-env:latest
+    if: (github.event_name == 'push')  || (github.event_name == 'workflow_dispatch') || (!startsWith(github.head_ref, 'release'))
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
         build-option: [ Debug, Release ]
-        sparse-solver: [ EIGEN, MKL, MKL_RUNTIME ]
+        cpp-compiler: [g++-11, clang++-14]
+    env:
+      CMAKE_PREFIX_PATH: /home/linuxbrew/.linuxbrew
+      CXX: ${{ matrix.cpp-compiler }}
 
     steps:
       - uses: actions/checkout@v3
-
+      - name: Install packages
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y ninja-build
+      - name: Install C++ dependencies
+        run: |
+          brew install boost eigen nlohmann-json doctest
       - name: Run build script
-        run: ./build.sh  ${{ matrix.build-option }} ${{ matrix.sparse-solver }}
-
+        run: ./build.sh  ${{ matrix.build-option }}
 
   build-cpp-test-windows:
+    if: (github.event_name == 'push')  || (github.event_name == 'workflow_dispatch') || (!startsWith(github.head_ref, 'release'))
     runs-on: windows-latest
     strategy:
       matrix:
         build-option: [ Debug, Release ]
-        sparse-solver: [ EIGEN, MKL, MKL_RUNTIME ]
-    env:
-      MKL_INCLUDE: C:\conda_envs\cpp_pkgs\Library\include
-      MKL_LIB: C:\conda_envs\cpp_pkgs\Library\lib
 
     steps:
       - uses: actions/checkout@v3
@@ -54,61 +59,48 @@ jobs:
           & "$env:CONDA\condabin\conda" init
 
       - name: Install conda environment
-        # TODO removed pinned v2 of catch2, see https://github.com/alliander-opensource/power-grid-model/issues/73
         run: |
-          conda create --yes -p C:\conda_envs\cpp_pkgs -c conda-forge boost-cpp eigen nlohmann_json mkl mkl-devel mkl-include catch2==2.13.9
+          conda create --yes -p C:\conda_envs\cpp_pkgs -c conda-forge boost-cpp eigen nlohmann_json doctest
 
       - name: Build and test
         run: |
           $vsPath = &(Join-Path ${env:ProgramFiles(x86)} '\Microsoft Visual Studio\Installer\vswhere.exe') -property installationpath
           Import-Module (Join-Path $vsPath 'Common7\Tools\Microsoft.VisualStudio.DevShell.dll')
           Enter-VsDevShell -VsInstallPath $vsPath -SkipAutomaticLocation -DevCmdArguments '-arch=x64 -host_arch=x64'
-          $env:Path += ";C:\conda_envs\cpp_pkgs\Library\bin"
-          mkdir cpp_build_${{ matrix.build-option }}_${{ matrix.sparse-solver }}
-          cd cpp_build_${{ matrix.build-option }}_${{ matrix.sparse-solver }}
+          mkdir cpp_build_${{ matrix.build-option }}
+          cd cpp_build_${{ matrix.build-option }}
           # generate cmake cache
           cmake .. `
             -G "Ninja" `
             -DCMAKE_BUILD_TYPE=${{ matrix.build-option }} `
-            -DPOWER_GRID_MODEL_SPARSE_SOLVER=${{ matrix.sparse-solver }} `
             -DCMAKE_PREFIX_PATH=C:\conda_envs\cpp_pkgs\Library `
             -DPOWER_GRID_MODEL_BUILD_BENCHMARK=1
           # build
-          cmake --build . --verbose
+          cmake --build . --verbose -j 1
           # test
           .\tests\cpp_unit_tests\power_grid_model_unit_tests.exe
 
-
   build-cpp-test-macos:
+    if: (github.event_name == 'push')  || (github.event_name == 'workflow_dispatch') || (!startsWith(github.head_ref, 'release'))
     runs-on: macos-latest
     strategy:
       matrix:
         build-option: [ Debug, Release ]
-        sparse-solver: [ EIGEN, MKL, MKL_RUNTIME ]
     env:
       CC: clang
       CXX: clang++
-      MKL_INCLUDE: /usr/local/include
-      MKL_LIB: /usr/local/lib
       CMAKE_PREFIX_PATH: /usr/local
 
     steps:
       - uses: actions/checkout@v3
 
       - name: Install cpp dependencies
-        # TODO removed pinned v2 of catch2, see https://github.com/alliander-opensource/power-grid-model/issues/73
         run: |
-          brew install ninja boost eigen nlohmann-json 
-          curl https://raw.githubusercontent.com/Homebrew/homebrew-core/5e5abb11bf49787d01164c4066119365262c21ed/Formula/catch2.rb > $(find $(brew --repository) -name catch2.rb) && brew reinstall catch2
-          sudo pip3 install mkl mkl-devel mkl-include
+          brew install ninja boost eigen nlohmann-json doctest
 
       - name: Build and test
         run: |
-          # environment
-          export LD_LIBRARY_PATH=${MKL_LIB}:${LD_LIBRARY_PATH}
-          # bash
-          ./build.sh  ${{ matrix.build-option }} ${{ matrix.sparse-solver }}
-
+          ./build.sh  ${{ matrix.build-option }} 
 
   build-and-test-python:
     strategy:
@@ -119,61 +111,39 @@ jobs:
             os: ubuntu-latest
             cc: gcc
             cxx: g++
-            boost:
-            eigen:
             cibw_build: "cp*-manylinux_x86_64 cp*-manylinux_aarch64"
-            test_cmd: >
-              pytest {package}/tests &&
-              LD_LIBRARY_PATH= pytest {package}/tests
             archs: "x86_64 aarch64"
           - platform: macos
             os: macos-latest
             cc: clang
             cxx: clang++
-            boost: /usr/local/include
-            eigen: /usr/local/include/eigen3
             cibw_build: cp*-macosx_*
             archs: "x86_64 arm64"
-            test_cmd: >
-              pytest {package}/tests &&
-              LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH} pytest {package}/tests
           - platform: windows
             os: windows-latest
             cc:
             cxx:
-            boost: C:\conda_envs\cpp_pkgs\Library\include
-            eigen: C:\conda_envs\cpp_pkgs\Library\include\eigen3
             cibw_build: cp*-win_amd64
             archs: AMD64
-            test_cmd: pytest {package}/tests
 
     runs-on: ${{ matrix.os }}
     needs: [build-cpp-test-linux, build-cpp-test-windows, build-cpp-test-macos]
 
     env:
       CC: ${{ matrix.cc }}
       CXX: ${{ matrix.cxx }}
-      BOOST_INCLUDE: ${{ matrix.boost }}
-      EIGEN_INCLUDE: ${{ matrix.eigen }}
       CIBW_BUILD_VERBOSITY: 1
       CIBW_BUILD_FRONTEND: "build"
       CIBW_BUILD: ${{ matrix.cibw_build }}
       CIBW_ARCHS: ${{ matrix.archs }}
-      CIBW_TEST_REQUIRES: pytest pytest-cov
-      CIBW_TEST_COMMAND: ${{ matrix.test_cmd }}
+      CIBW_TEST_EXTRAS: "dev"
+      CIBW_TEST_COMMAND: pytest {package}/tests
       # Skip trying to test arm64 builds on Intel Macs
       CIBW_TEST_SKIP: "*-macosx_arm64 *-macosx_universal2:arm64"
-      CIBW_MANYLINUX_X86_64_IMAGE: ghcr.io/alliander-opensource/power-grid-model-build-env
-      CIBW_MANYLINUX_AARCH64_IMAGE: ghcr.io/alliander-opensource/power-grid-model-build-env-arm64
-      CIBW_ENVIRONMENT_PASS_LINUX: GITHUB_SHA GITHUB_REF GITHUB_RUN_NUMBER
-      CIBW_BEFORE_ALL_MACOS: >
-        which clang++ &&
-        clang++ --version &&
-        brew install boost eigen &&
-        sudo pip3 install mkl mkl-devel mkl-include
+      CIBW_ENVIRONMENT_PASS_LINUX: GITHUB_SHA GITHUB_REF GITHUB_RUN_NUMBER CC CXX
       MACOSX_DEPLOYMENT_TARGET: 10.15
-      CIBW_BEFORE_ALL_WINDOWS : >
-        conda create --yes -p C:\conda_envs\cpp_pkgs -c conda-forge boost-cpp eigen mkl
+      CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
+      CIBW_MANYLINUX_AARCH64_IMAGE: manylinux2014
 
     steps:
       - uses: actions/checkout@v3
@@ -182,11 +152,6 @@ jobs:
         if: matrix.platform == 'linux'
         uses: docker/setup-qemu-action@v2
 
-      - name: Activate conda for windows
-        if: matrix.platform == 'windows'
-        run: |
-          & "$env:CONDA\condabin\conda" init
-
       - name: Build wheels
         uses: pypa/[email protected]
 
@@ -196,7 +161,6 @@ jobs:
           name: wheelhouse
           path: ./wheelhouse/*.whl
 
-
   publish-wheels:
     needs: build-and-test-python
     runs-on: ubuntu-latest
@@ -247,3 +211,4 @@ jobs:
           tag_name: ${{ steps.tag.outputs.tag }}
           prerelease: ${{ contains(steps.tag.outputs.tag, 'rc') }}
           generate_release_notes: true
+          target_commitish: ${{ github.sha }}

.github/workflows/reuse-compliance.yml

@@ -16,6 +16,7 @@ on:
 
 jobs:
   reuse-compliance-check:
+    if: (github.event_name == 'push') || (!startsWith(github.head_ref, 'release'))
     runs-on: ubuntu-latest
     steps:
     - name: checkout

.github/workflows/sonar.yml

@@ -9,28 +9,45 @@ on:
   push:
     branches:
       - main
+      - 'release/**'
+
   # run pipeline on pull request
   pull_request:
 
 
 jobs:
   sonar-cloud:
+    if: (github.event_name == 'push') || (!startsWith(github.head_ref, 'release'))
     name: SonarCloud
-    runs-on: ubuntu-latest
-    container: ghcr.io/alliander-opensource/power-grid-model-build-env-sonar:latest
+    runs-on: ubuntu-22.04
     env:
       SONAR_SCANNER_VERSION: 4.4.0.2170
       SONAR_SERVER_URL: "https://sonarcloud.io"
       BUILD_WRAPPER_OUT_DIR: build_wrapper_output_directory # Directory where build-wrapper output will be placed
+      CMAKE_PREFIX_PATH: /home/linuxbrew/.linuxbrew
+      LLVM_COV: llvm-cov-14
+      CC: clang-14
+      CXX: clang++-14
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
+      - name: Install packages
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y ninja-build lcov gcovr
+      - name: Install C++ dependencies
+        run: |
+          brew install boost eigen nlohmann-json doctest
       - name: Set up JDK 11
         uses: actions/setup-java@v3
         with:
           java-version: 11
           distribution: 'microsoft'
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
       - name: Download and set up sonar-scanner
         env:
           SONAR_SCANNER_DOWNLOAD_URL: https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-${{ env.SONAR_SCANNER_VERSION }}-linux.zip
@@ -49,16 +66,15 @@ jobs:
 
       - name: Python test and coverage
         run: |
-          /opt/venv/bin/pip install -e .
-          /opt/venv/bin/pip install -r dev-requirements.txt
-          /opt/venv/bin/pytest
+          pip install -e .[dev]
+          pytest
           # remove cython generated cpp file
           rm -rf src/power_grid_model/*.cpp
 
       - name: Run build-wrapper for C++
         run: |
           mkdir cpp_build
-          cmake -S . -B cpp_build -GNinja -DCMAKE_BUILD_TYPE=Debug -DPOWER_GRID_MODEL_SPARSE_SOLVER=EIGEN -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH} 
+          cmake -S . -B cpp_build -GNinja -DCMAKE_BUILD_TYPE=Debug -DPOWER_GRID_MODEL_COVERAGE=1
           VERBOSE=1 build-wrapper-linux-x86-64 --out-dir ${{ env.BUILD_WRAPPER_OUT_DIR }} cmake --build cpp_build/
 
       - name: C++ test and coverage
@@ -71,7 +87,8 @@ jobs:
           sed -i -r "s/\s*branchesToCover\s*=\s*\"[0-9]+\"\s+coveredBranches\s*=\s*\"[0-9]+\"//g" cpp_coverage.xml
 
       - name: Run sonar-scanner
-        if: github.event_name == 'push'  # only run sonar server in push event
+        # only run sonar server in push event or pull request event from own repo
+        if: github.event_name == 'push' || ${{ github.event.pull_request.head.repo.owner.login }} == 'alliander-opensource'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}