From d63707f71d399682dbff8851d03c796c6a8c1359 Mon Sep 17 00:00:00 2001 From: Vincent Koppen Date: Fri, 17 Jan 2025 14:05:39 +0100 Subject: [PATCH 01/11] feat(action): add dco Signed-off-by: Vincent Koppen --- .github/dco.yml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .github/dco.yml diff --git a/.github/dco.yml b/.github/dco.yml new file mode 100644 index 0000000..ab7845f --- /dev/null +++ b/.github/dco.yml @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Contributors to the Power Grid Model project +# +# SPDX-License-Identifier: MPL-2.0 + +allowRemediationCommits: + individual: true + thirdParty: true + From cc0573431a448b1661f1f83e4acc3ebb72a26da0 Mon Sep 17 00:00:00 2001 From: Vincent Koppen Date: Fri, 17 Jan 2025 14:05:57 +0100 Subject: [PATCH 02/11] feat(action): add dependabot Signed-off-by: Vincent Koppen --- .github/dependabot.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..a885093 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: Contributors to the Power Grid Model project +# +# SPDX-License-Identifier: MPL-2.0 + + +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" From a60b448017bebb9a403417cada1a3f0597391ce0 Mon Sep 17 00:00:00 2001 From: Vincent Koppen Date: Fri, 17 Jan 2025 14:07:28 +0100 Subject: [PATCH 03/11] feat(action): add build test and sonarcloud Signed-off-by: Vincent Koppen --- .github/workflows/build-test-and-sonar.yml | 173 +++++++++++++++++++++ VERSION | 2 +- pyproject.toml | 4 + set_pypi_version.py | 70 +++++++++ 4 files changed, 248 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/build-test-and-sonar.yml create mode 100644 set_pypi_version.py diff --git a/.github/workflows/build-test-and-sonar.yml b/.github/workflows/build-test-and-sonar.yml new file mode 100644 index 0000000..2be0e27 --- /dev/null +++ b/.github/workflows/build-test-and-sonar.yml @@ -0,0 +1,173 @@ +# SPDX-FileCopyrightText: Contributors to the Power Grid Model project +# +# SPDX-License-Identifier: MPL-2.0 + + +name: Build, Test, Sonar and Publish + +on: + push: + branches: + - main + - feature/github_actions + # run pipeline on pull request + pull_request: + # run pipeline on merge queue + merge_group: + # run pipeline from another workflow + workflow_call: + inputs: + create_release: + type: boolean + description: Create a (pre-)release when CI passes + default: false + required: false + # run this workflow manually from the Actions tab + workflow_dispatch: + inputs: + create_release: + type: boolean + description: Create a (pre-)release when CI passes + default: false + required: true + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-main + cancel-in-progress: true + +jobs: + + build-python: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.version }} + steps: + + - name: Checkout source code + uses: actions/checkout@v4 + + - name: Setup Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Build + run: | + pip install requests build + python set_pypi_version.py + python -m build --outdir wheelhouse . + + - name: Save version + id: version + run: echo "version=$(cat PYPI_VERSION)" >> $GITHUB_OUTPUT + + - name: Store built wheel file + uses: actions/upload-artifact@v4 + with: + name: power-grid-model-ds + path: wheelhouse/ + + sonar-cloud: + if: false #skip until open source + permissions: + contents: write + runs-on: ubuntu-latest + steps: + + - name: Checkout source code + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis + + - name: Setup Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install in develop mode + run: | + pip install -e .[dev] + + - name: Test and Coverage + run: | + coverage run -m pytest + coverage xml + coverage report --fail-under=80 + + - name: SonarCloud Scan + if: ${{ (github.event_name == 'push') || (github.event.pull_request.head.repo.owner.login == 'PowerGridModel') }} + uses: SonarSource/sonarqube-scan-action@v4 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + + tests: + needs: build-python + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python: ["3.10", "3.11", "3.12", "3.13"] + fail-fast: false + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout source code + uses: actions/checkout@v4 + + - name: Setup Python ${{ matrix.python }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + + - name: Load built wheel file + uses: actions/download-artifact@v4 + with: + name: power-grid-model-ds + path: wheelhouse/ + + - name: Install built wheel file + run: pip install power-grid-model-ds[dev]==${{ needs.build-python.outputs.version }} --find-links=wheelhouse + + - name: Unit test and coverage + run: pytest --verbose + + publish: + needs: + - build-python + - tests + - sonar-cloud + permissions: + contents: write + env: + TWINE_USERNAME: ${{ secrets.PYPI_USER }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASS }} + runs-on: ubuntu-latest + steps: + - name: Setup Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Load built wheel file + uses: actions/download-artifact@v4 + with: + name: power-grid-model-ds + path: wheelhouse/ + + # - name: Upload wheels + # if: (github.event_name == 'push') || ((github.event_name == 'workflow_dispatch') && (github.event.inputs.create_release == 'true')) + # run: | + # pip install twine + # echo "Publish to PyPI..." + # twine upload --verbose wheelhouse/* + + # - name: Release + # if: (github.event_name == 'push') || ((github.event_name == 'workflow_dispatch') && (github.event.inputs.create_release == 'true')) + # uses: softprops/action-gh-release@v2 + # with: + # files: | + # ./wheelhouse/* + # tag_name: v${{ needs.build-python.outputs.version }} + # prerelease: ${{github.ref != 'refs/heads/main'}} + # generate_release_notes: true + # target_commitish: ${{ github.sha }} diff --git a/VERSION b/VERSION index 8a9ecc2..171538e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.1 \ No newline at end of file +0.0 \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index d27ec8f..975f351 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -104,3 +104,7 @@ unfixable = [] [tool.mypy] disable_error_code = ["assignment", "import-untyped"] + +[tool.coverage.run] +relative_files = true +branch = true diff --git a/set_pypi_version.py b/set_pypi_version.py new file mode 100644 index 0000000..61eb728 --- /dev/null +++ b/set_pypi_version.py @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: Contributors to the Power Grid Model project +# +# SPDX-License-Identifier: MPL-2.0 + +# script to set version dynamically +# read VERSION and PyPI, set PYPI_VERSION + + +import os +from pathlib import Path + +import requests + + +def set_version(pkg_dir: Path): + with open(pkg_dir / "VERSION") as f: + version = f.read().strip().strip("\n") + major, minor = (int(x) for x in version.split(".")) + latest_major, latest_minor, latest_patch = get_pypi_latest() + # get version + version = get_new_version(major, minor, latest_major, latest_minor, latest_patch) + # mutate version in GitHub Actions + if ("GITHUB_SHA" in os.environ) and ("GITHUB_REF" in os.environ) and ("GITHUB_RUN_NUMBER" in os.environ): + sha = os.environ["GITHUB_SHA"] + ref = os.environ["GITHUB_REF"] + build_number = os.environ["GITHUB_RUN_NUMBER"] + # short hash number in numeric + short_hash = f"{int(f'0x{sha[0:6]}', base=16):08}" + + if "main" in ref: + # main branch + # major.minor.patch + # do nothing + pass + else: + # feature branch + # major.minor.patch a 1 build_number short_hash + version += f"a1{build_number}{short_hash}" + with open(pkg_dir / "PYPI_VERSION", "w") as f: + f.write(version) + + +def get_pypi_latest(): + response = requests.get("https://pypi.org/pypi/power-grid-model-ds/json") + if response.status_code == 404: + return 0, 0, 0 + data = response.json() + version = str(data["info"]["version"]) + return (int(x) for x in version.split(".")) + + +def get_new_version(major, minor, latest_major, latest_minor, latest_patch): + if (major > latest_major) or ((major == latest_major) and minor > latest_minor): + # brand-new version with patch zero + return f"{major}.{minor}.0" + + if major == latest_major and minor == latest_minor: + # current version, increment path + return f"{major}.{minor}.{latest_patch + 1}" + + # does not allow building older version + raise ValueError( + "Invalid version number!\n" + f"latest version: {latest_major}.{latest_minor}.{latest_patch}\n" + f"to be built version: {major}.{minor}\n" + ) + + +if __name__ == "__main__": + set_version(Path(__file__).parent) From a04ff49695bfd50a579cdb7327a50e3d09caa449 Mon Sep 17 00:00:00 2001 From: Vincent Koppen Date: Fri, 17 Jan 2025 14:11:34 +0100 Subject: [PATCH 04/11] fix: correct path creation in test Signed-off-by: Vincent Koppen --- tests/unit/model/grids/test_grid_base.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/unit/model/grids/test_grid_base.py b/tests/unit/model/grids/test_grid_base.py index 08dfe26..8f9160a 100644 --- a/tests/unit/model/grids/test_grid_base.py +++ b/tests/unit/model/grids/test_grid_base.py @@ -5,7 +5,6 @@ """Grid tests""" import dataclasses -from pathlib import Path import numpy as np import pytest @@ -294,8 +293,8 @@ def test_from_txt_file_with_branch_ids(tmp_path): np.testing.assert_array_equal([95, 91, 92, 93, 94, 96, 97, 98], grid.branches.id) -def test_from_txt_file_conflicting_ids(): - txt_file = Path("/tmp/tmp_grid") +def test_from_txt_file_conflicting_ids(tmp_path): + txt_file = tmp_path / "tmp_grid" txt_file.write_text("S1 2\n1 3", encoding="utf-8") with pytest.raises(ValueError): From 0ef327a2ad815c7066d4c6f210cf2ac2cc89c76f Mon Sep 17 00:00:00 2001 From: Vincent Koppen Date: Fri, 17 Jan 2025 14:14:10 +0100 Subject: [PATCH 05/11] feat(action): check-blocking-label Signed-off-by: Vincent Koppen --- .github/workflows/check-blocking-labels.yml | 46 +++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/check-blocking-labels.yml diff --git a/.github/workflows/check-blocking-labels.yml b/.github/workflows/check-blocking-labels.yml new file mode 100644 index 0000000..09f5c65 --- /dev/null +++ b/.github/workflows/check-blocking-labels.yml @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: Contributors to the Power Grid Model project +# +# SPDX-License-Identifier: MPL-2.0 + + +name: Check Blocking Labels + +on: + # run pipeline on pull request + pull_request: + types: + - opened + - synchronize + - labeled + - unlabeled + # run pipeline on merge queue + merge_group: + # run this workflow manually from the Actions tab + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-blocking-labels + cancel-in-progress: true + +jobs: + check-blocking-labels: + runs-on: ubuntu-latest + steps: + + - name: do-not-merge + if: contains(github.event.pull_request.labels.*.name, 'do-not-merge') + run: | + echo "This pull request should not be merged (do-not-merge)" + exit 1 + + - name: merge-target-first + if: contains(github.event.pull_request.labels.*.name, 'merge-target-first') + run: | + echo "The target branch of this PR should be merged first (merge-target-first)" + exit 2 + + - name: needs-unit-tests + if: contains(github.event.pull_request.labels.*.name, 'needs-unit-tests') + run: | + echo "This pull request needs (more) unit tests before it may be merged (needs-unit-tests)" + exit 3 From 619f43b55cd908683172191e1042bcf0a6932f41 Mon Sep 17 00:00:00 2001 From: Vincent Koppen Date: Fri, 17 Jan 2025 14:17:32 +0100 Subject: [PATCH 06/11] feat(action): add check-code-quality Signed-off-by: Vincent Koppen --- .github/workflows/check-code-quality.yml | 44 ++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .github/workflows/check-code-quality.yml diff --git a/.github/workflows/check-code-quality.yml b/.github/workflows/check-code-quality.yml new file mode 100644 index 0000000..e28fe1f --- /dev/null +++ b/.github/workflows/check-code-quality.yml @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Contributors to the Power Grid Model project +# +# SPDX-License-Identifier: MPL-2.0 + + +name: Check Code Quality + +on: + push: + branches: + - main + - feature/github_actions + # run pipeline on pull request + pull_request: + # run pipeline on merge queue + merge_group: + # run pipeline from another workflow + workflow_call: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-code-quality + cancel-in-progress: true + +jobs: + check-code-quality: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.11 + + - name: Upgrade pip + run: pip install --upgrade pip + + - name: Install dependencies + run: pip install -e .[dev] + + - name: Run pre-commit on all files + run: pre-commit run --all-files \ No newline at end of file From 49c22a12211165da60a50173185b2b893b5e3d05 Mon Sep 17 00:00:00 2001 From: Vincent Koppen Date: Fri, 17 Jan 2025 14:19:40 +0100 Subject: [PATCH 07/11] feat(action): add dco-merge-group Signed-off-by: Vincent Koppen --- .github/workflows/dco-merge-group.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/dco-merge-group.yml diff --git a/.github/workflows/dco-merge-group.yml b/.github/workflows/dco-merge-group.yml new file mode 100644 index 0000000..6b4ab94 --- /dev/null +++ b/.github/workflows/dco-merge-group.yml @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Contributors to the Power Grid Model project +# +# SPDX-License-Identifier: MPL-2.0 + +name: DCO for merge groups +# Workaround because DCO plugin does not run on merge group. See https://github.com/dcoapp/app/issues/199 + +# Controls when the workflow will run +on: + # run pipeline on merge queue because DCO plugin does not + merge_group: + # Any other signals are handled by the actual DCO plugin + +jobs: + dco-merge-group: + name: DCO + runs-on: ubuntu-latest + if: ${{ github.actor != 'dependabot[bot]' }} + steps: + - name: "Workaround for DCO on merge groups" + run: | + echo "Workaround: signal DCO for merge queues because DCO plugin does not run for merge queues. See https://github.com/dcoapp/app/issues/199" From feaaed393662091f43cb6fe6170ed7cd94333f34 Mon Sep 17 00:00:00 2001 From: Vincent Koppen Date: Fri, 17 Jan 2025 14:20:31 +0100 Subject: [PATCH 08/11] feat(action): add reuse-compliance Signed-off-by: Vincent Koppen --- .github/workflows/reuse-compliance.yml | 32 ++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/workflows/reuse-compliance.yml diff --git a/.github/workflows/reuse-compliance.yml b/.github/workflows/reuse-compliance.yml new file mode 100644 index 0000000..dbed588 --- /dev/null +++ b/.github/workflows/reuse-compliance.yml @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Contributors to the Power Grid Model project +# +# SPDX-License-Identifier: MPL-2.0 + + +name: REUSE Compliance Check + +on: + push: + branches: + - main + # run pipeline on pull request + pull_request: + # run pipeline on merge queue + merge_group: + # run pipeline from another workflow + workflow_call: + # run this workflow manually from the Actions tab + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-reuse-compliance + cancel-in-progress: true + +jobs: + reuse-compliance-check: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v4 + - name: REUSE Compliance Check + uses: fsfe/reuse-action@v5 From 7a87705c10a2a4e2451f97ae18ac6bdff8c40429 Mon Sep 17 00:00:00 2001 From: Vincent Koppen Date: Fri, 17 Jan 2025 14:21:04 +0100 Subject: [PATCH 09/11] feat(action): add nightly Signed-off-by: Vincent Koppen --- .github/workflows/nightly.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/nightly.yml diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 0000000..8f2bfcf --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Contributors to the Power Grid Model project +# +# SPDX-License-Identifier: MPL-2.0 + +name: Nightly build + +# Controls when the workflow will run +on: + workflow_dispatch: + schedule: + - cron: "0 2 * * *" # Based on UTC time + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-nightly + cancel-in-progress: true + +jobs: + main: + uses: "./.github/workflows/build-test-and-sonar.yml" + permissions: + contents: write + with: + create_release: false + + check-code-quality: + uses: "./.github/workflows/check-code-quality.yml" + + reuse-compliance: + uses: "./.github/workflows/reuse-compliance.yml" From 1c7a775c4d9af53f9c67610cc3065485d682efa2 Mon Sep 17 00:00:00 2001 From: Vincent Koppen Date: Fri, 17 Jan 2025 14:24:06 +0100 Subject: [PATCH 10/11] feat(action): enable sonarcloud Signed-off-by: Vincent Koppen --- .github/workflows/build-test-and-sonar.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-test-and-sonar.yml b/.github/workflows/build-test-and-sonar.yml index 2be0e27..06af212 100644 --- a/.github/workflows/build-test-and-sonar.yml +++ b/.github/workflows/build-test-and-sonar.yml @@ -68,7 +68,6 @@ jobs: path: wheelhouse/ sonar-cloud: - if: false #skip until open source permissions: contents: write runs-on: ubuntu-latest From fdf0644171d3748b92cece2199d1da930bba45be Mon Sep 17 00:00:00 2001 From: Vincent Koppen Date: Fri, 17 Jan 2025 14:37:43 +0100 Subject: [PATCH 11/11] fix: add sonarcloud config Signed-off-by: Vincent Koppen --- .github/workflows/build-test-and-sonar.yml | 1 - .github/workflows/check-code-quality.yml | 1 - sonar-project.properties | 13 +++++++++++++ 3 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 sonar-project.properties diff --git a/.github/workflows/build-test-and-sonar.yml b/.github/workflows/build-test-and-sonar.yml index 06af212..d936325 100644 --- a/.github/workflows/build-test-and-sonar.yml +++ b/.github/workflows/build-test-and-sonar.yml @@ -9,7 +9,6 @@ on: push: branches: - main - - feature/github_actions # run pipeline on pull request pull_request: # run pipeline on merge queue diff --git a/.github/workflows/check-code-quality.yml b/.github/workflows/check-code-quality.yml index e28fe1f..38c8f20 100644 --- a/.github/workflows/check-code-quality.yml +++ b/.github/workflows/check-code-quality.yml @@ -9,7 +9,6 @@ on: push: branches: - main - - feature/github_actions # run pipeline on pull request pull_request: # run pipeline on merge queue diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 0000000..c320719 --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: Contributors to the Power Grid Model project +# +# SPDX-License-Identifier: MPL-2.0 + +sonar.projectKey=PowerGridModel_power-grid-model-ds +sonar.organization=powergridmodel + + +sonar.sources=src +sonar.tests=tests +sonar.sourceEncoding=UTF-8 +sonar.python.version=3 +sonar.python.coverage.reportPaths=coverage.xml