Merge pull request #106 from alliander-opensource/feature/mypy

Add mypy to pre-commit (and introduce data_types.py)

Author: TonyXiang8787

.github/workflows/black-and-clang-format.yml renamed to .github/workflows/check-code-quality.yml

@@ -4,12 +4,12 @@
 
 
 # GitHub Action that uses
-#     Black to reformat the Python code in an incoming pull request.
+#     isort, black, mypy and pylint to reformat the Python code in an incoming pull request.
 #     clang-format to reformat the C++ code in an incoming pull request.
 # If all code in the pull request is compliant with Black and clang-format then this Action
 # does nothing. Otherwise, it will print the files which need to be reformatted and raise an error.
 
-name: Format Code
+name: Check Code Quality
 
 on:
   # run pipeline on push event of main or release branch
@@ -21,7 +21,7 @@ on:
   pull_request:
 
 jobs:
-  code-format-check:
+  check-code-quality:
     if: (github.event_name == 'push') || (!startsWith(github.head_ref, 'release'))
     runs-on: ubuntu-latest
 
@@ -33,23 +33,41 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: 3.8
-        
-      - name: Install Black and clang-format
+
+      - name: Upgrade pip
+        run: pip install --upgrade pip
+
+      - name: Install and run isort
+        run: |
+          pip install isort
+          isort .
+
+      - name: Install and run black
         run: |
           pip install black
-          sudo apt-get update && sudo apt-get install -y clang-format
+          black .
 
-      - name: Run black
-        run: black .
+      - name: Install and run mypy
+        run: |
+          pip install mypy
+          mypy .
+
+      - name: Install and run pylint
+        run: |
+          pip install pylint .
+          pylint power_grid_model
+          git restore README.md
 
-      - name: Run clang-format
-        run: find . -regex '.*\.\(cpp\|hpp\|cc\|cxx\)' -exec clang-format -style=file -i {} \;
+      - name: Install and run clang-format
+        run: |
+          sudo apt-get update && sudo apt-get install -y clang-format
+          find . -regex '.*\.\(cpp\|hpp\|cc\|cxx\)' -exec clang-format -style=file -i {} \;
 
       - name: If needed raise error
         run: |
 
           if [[ `git status --porcelain --untracked-files=no` ]]; then
-            echo "Formatting not correct! See blow the files which need to be reformatted!"
+            echo "Formatting not correct! See below the files which need to be reformatted!"
             git status --porcelain --untracked-files=no
             exit 1
           fi

.pre-commit-config.yaml

@@ -10,11 +10,16 @@ repos:
   - repo: https://github.com/pycqa/isort
     rev: 5.10.1
     hooks:
-     - id: isort
+      - id: isort
   - repo: https://github.com/psf/black
     rev: 22.6.0
     hooks:
       - id: black
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.971
+    hooks:
+      - id: mypy
+        files: ^(src|tests|scripts)/.+\.py$
   - repo: local
     hooks:
       - id: pylint

CONTRIBUTING.md

@@ -64,9 +64,10 @@ corrections) to your code (style) before each commit. It is up to the developer
 use this tool or not. The goal is to make sure that each commit will pass the quality checks in the github actions
 workflow. Currently, these hooks are defined in [`.pre-commit-config.yaml`](.pre-commit-config.yaml):
 * **reuse**: check if all licence headers and files are in place
-* **isort**: sort import statements
-* **black**: check and correct code style
-* **pylint**: check code style
+* **isort**: group and sort import statements 
+* **black**: check and correct code style in a very strict manner
+* **mypy**: checks type hinting and data types in general (static type checker) 
+* **pylint**: check code style and comments
 * **pytest**: run all unit tests
 
 You can manually run pre-commit whenever you like:

README.md

@@ -6,7 +6,7 @@ SPDX-License-Identifier: MPL-2.0
 [![PyPI version](https://badge.fury.io/py/power-grid-model.svg)](https://badge.fury.io/py/power-grid-model)
 [![License: MIT](https://img.shields.io/badge/License-MPL2.0-informational.svg)](https://github.com/alliander-opensource/power-grid-model/blob/main/LICENSE)
 [![Build and Test C++ and Python](https://github.com/alliander-opensource/power-grid-model/actions/workflows/main.yml/badge.svg)](https://github.com/alliander-opensource/power-grid-model/actions/workflows/main.yml)
-[![Format Code](https://github.com/alliander-opensource/power-grid-model/actions/workflows/black-and-clang-format.yml/badge.svg)](https://github.com/alliander-opensource/power-grid-model/actions/workflows/black-and-clang-format.yml)
+[![Check Code Quality](https://github.com/alliander-opensource/power-grid-model/actions/workflows/check-code-quality.yml/badge.svg)](https://github.com/alliander-opensource/power-grid-model/actions/workflows/check-code-quality.yml)
 [![REUSE Compliance Check](https://github.com/alliander-opensource/power-grid-model/actions/workflows/reuse-compliance.yml/badge.svg)](https://github.com/alliander-opensource/power-grid-model/actions/workflows/reuse-compliance.yml)
 
 [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=alliander-opensource_power-grid-model&metric=alert_status)](https://sonarcloud.io/summary/new_code?id=alliander-opensource_power-grid-model)

docs/python-api-reference.md

@@ -17,7 +17,7 @@ The Python API consists of the following components:
   This is a pure Python module.
 * `power_grid_model.PowerGridModel`: the main class.
   This is inside a C++ extension module.
-* `power_grid_model.manual_testing`: containing the functions for load and save test dataset.
+* `power_grid_model.utils`: containing the functions for load and save test dataset.
   See [Make Test Dataset](../examples/Make%20Test%20Dataset.ipynb) for examples of how to make test datasets.
 * `power_grid_model.validation`: optional validation and assertion functions.
   See [Validation Examples](../examples/Validation%20Examples.ipynb) for more information on how to validate input 

examples/Make Test Dataset.ipynb

@@ -342,9 +342,9 @@
    "source": [
     "# Helper Functions to Import and Export\n",
     "\n",
-    "In the module `power_grid_model.manual_testing` we have some helper functions to import a json file to a `power-grid-model` compatible dataset, or the other way around. \n",
+    "In the module `power_grid_model.utils` we have some helper functions to import a json file to a `power-grid-model` compatible dataset, or the other way around. \n",
     "\n",
-    "Please refer to the [source code](../src/power_grid_model/manual_testing.py) for detailed function signature.\n",
+    "Please refer to the [source code](../src/power_grid_model/utils.py) for detailed function signature.\n",
     "\n",
     "In this notebook we export the example network from [Power Flow](./Power%20Flow%20Example.ipynb) to json. "
    ]
@@ -422,7 +422,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from power_grid_model.manual_testing import export_json_data\n",
+    "from power_grid_model.utils import export_json_data\n",
     "import tempfile\n",
     "from pathlib import Path\n",
     "\n",
@@ -560,7 +560,7 @@
    "source": [
     "# round trip and run power flow\n",
     "\n",
-    "from power_grid_model.manual_testing import import_json_data\n",
+    "from power_grid_model.utils import import_json_data\n",
     "\n",
     "imported_data = import_json_data(temp_path / \"input.json\", \"input\")\n",
     "\n",

pyproject.toml

@@ -46,6 +46,7 @@ dynamic = ["version"]
 [project.optional-dependencies]
 dev = [
     "pre-commit",
+    "pylint",
     "pytest",
     "pytest-cov",
 ]
@@ -69,8 +70,15 @@ line-length = 120
 target-version = ['py38']
 
 [tool.isort]
-src_paths = ["src", "tests/unit", "scripts"]
 profile = "black"
+line_length = 120
 
 [tool.pylint]
 max-line-length = 120
+
+[tool.mypy]
+follow_imports = "silent"
+ignore_missing_imports = true
+show_column_numbers = true
+non_interactive = true
+install_types = true

scripts/validate_batch_data.py

@@ -3,22 +3,16 @@
 # SPDX-License-Identifier: MPL-2.0
 
 from pathlib import Path
-from typing import cast
 
 from power_grid_model import CalculationType
-from power_grid_model.manual_testing import import_json_data
-from power_grid_model.validation import (
-    InputData,
-    UpdateData,
-    errors_to_string,
-    validate_batch_data,
-)
+from power_grid_model.utils import import_input_data, import_update_data
+from power_grid_model.validation import errors_to_string, validate_batch_data
 
 input_file = Path("../tests/data/power_flow/dummy-test-batch/input.json")
 update_file = Path("../tests/data/power_flow/dummy-test-batch/update_batch.json")
 
-input_data = cast(InputData, import_json_data(json_file=input_file, data_type="input"))
-update_data = cast(UpdateData, import_json_data(json_file=update_file, data_type="update"))
+input_data = import_input_data(json_file=input_file)
+update_data = import_update_data(json_file=update_file)
 
 update_errors = validate_batch_data(
     input_data=input_data, update_data=update_data, calculation_type=CalculationType.power_flow, symmetric=True

scripts/validate_input_data.py

@@ -3,15 +3,14 @@
 # SPDX-License-Identifier: MPL-2.0
 
 from pathlib import Path
-from typing import cast
 
 from power_grid_model import CalculationType
-from power_grid_model.manual_testing import import_json_data
-from power_grid_model.validation import InputData, errors_to_string, validate_input_data
+from power_grid_model.utils import import_input_data
+from power_grid_model.validation import errors_to_string, validate_input_data
 
 input_file = Path("../tests/data/state_estimation/dummy-test-sym/input.json")
 
-input_data = cast(InputData, import_json_data(json_file=input_file, data_type="input"))
+input_data = import_input_data(json_file=input_file)
 
 input_errors = validate_input_data(
     input_data=input_data, calculation_type=CalculationType.state_estimation, symmetric=True

setup.py

@@ -9,6 +9,7 @@
 from itertools import chain
 from pathlib import Path
 from sysconfig import get_paths
+from typing import List
 
 # noinspection PyPackageRequirements
 import Cython.Compiler.Main as CythonCompiler
@@ -88,10 +89,10 @@ def generate_build_ext(pkg_dir: Path, pkg_name: str):
         str(pkg_dir / "include"),  # The include-folder of the repo self
     ]
     # compiler and link flag
-    cflags = []
-    lflags = []
-    library_dirs = []
-    libraries = []
+    cflags: List[str] = []
+    lflags: List[str] = []
+    library_dirs: List[str] = []
+    libraries: List[str] = []
     # macro
     define_macros = [
         ("EIGEN_MPL2_ONLY", "1"),  # only MPL-2 part of eigen3