Pin python dependencies with Poetry (#60)

Ported from canonical/mysql-k8s-operator#273

Author: carlcsaposs-canonical

.github/renovate.json5

@@ -0,0 +1,16 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": ["github>canonical/data-platform//renovate_presets/charm.json5"],
+  "reviewers": ["team:data-platform-mysql"],
+  "packageRules": [
+    // Later rules override earlier rules
+
+    // MySQL 8.X does not follow semantic versioning (e.g. 8.0.1 -> 8.0.2 can include a breaking change)
+    // Therefore, use a separate Renovate group so that it has a separate PR
+    {
+      "matchManagers": ["poetry"],
+      "matchPackageNames": ["mysql-connector-python"],
+      "groupName": "MySQL Connector/Python"
+    }
+  ]
+}

.github/workflows/ci.yaml

@@ -21,22 +21,24 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-      - name: Install tox
-        # TODO: Consider replacing with custom image on self-hosted runner OR pinning version
-        run: python3 -m pip install tox
+      - name: Install tox & poetry
+        run: |
+          pipx install tox
+          pipx install poetry
       - name: Run linters
         run: tox run -e lint
 
   unit-test:
-    name: Unit tests
+    name: Unit test charm
     runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-      - name: Install tox
-        # TODO: Consider replacing with custom image on self-hosted runner OR pinning version
-        run: python3 -m pip install tox
+      - name: Install tox & poetry
+        run: |
+          pipx install tox
+          pipx install poetry
       - name: Run tests
         run: tox run -e unit
       - name: Upload Coverage to Codecov
@@ -61,57 +63,86 @@ jobs:
 
   build:
     name: Build charms
-    uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v2
+    uses: canonical/data-platform-workflows/.github/workflows/[email protected]
+    with:
+      charmcraft-snap-revision: 1349  # version 2.3.0
     permissions:
       actions: write  # Needed to manage GitHub Actions cache
 
-  integration-test:
+  gh-hosted-collect-integration-tests:
+    name: (GH hosted) Collect integration test groups
+    needs:
+      - lint
+      - unit-test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install tox & poetry
+        run: |
+          pipx install tox
+          pipx install poetry
+      - name: Select test stability level
+        id: select-test-stability
+        run: |
+          if [[ "${{ github.event_name }}" == "schedule" ]]
+          then
+            echo Running unstable and stable tests
+            echo "mark_expression=" >> "$GITHUB_OUTPUT"
+          else
+            echo Skipping unstable tests
+            echo "mark_expression=not unstable" >> "$GITHUB_OUTPUT"
+          fi
+      - name: Collect test groups
+        id: collect-groups
+        run: tox run -e integration -- tests/integration -m '${{ steps.select-test-stability.outputs.mark_expression }}' --collect-groups
+    outputs:
+      groups: ${{ steps.collect-groups.outputs.groups }}
+
+  gh-hosted-integration-test:
     strategy:
       fail-fast: false
       matrix:
-        tox-environments:
-          # TODO: uncomment when the following bug is resolved:
-          # https://bugs.launchpad.net/charm-keystone/+bug/1990243
-          # - integration-shared-db
-          - integration-database
+        groups: ${{ fromJSON(needs.gh-hosted-collect-integration-tests.outputs.groups) }}
         ubuntu-versions:
           # Update whenever charmcraft.yaml is changed
           - series: focal
             bases-index: 0
           - series: jammy
             bases-index: 1
-    name: ${{ matrix.tox-environments }} | ${{ matrix.ubuntu-versions.series }}
+    name: (GH hosted) ${{ matrix.groups.job_name }} | ${{ matrix.ubuntu-versions.series }}
     needs:
       - lint
       - unit-test
       - build
+      - gh-hosted-collect-integration-tests
     runs-on: ubuntu-latest
     timeout-minutes: 120
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+      - name: Install tox & poetry
+        run: |
+          pipx install tox
+          pipx install poetry
       - name: Setup operator environment
-        # TODO: Replace with custom image on self-hosted runner
         uses: charmed-kubernetes/actions-operator@main
         with:
           provider: lxd
-          bootstrap-options: "--agent-version 2.9.42"
       - name: Download packed charm(s)
         uses: actions/download-artifact@v3
         with:
           name: ${{ needs.build.outputs.artifact-name }}
-      - name: Select tests
-        id: select-tests
+      - name: Select test stability level
+        id: select-test-stability
         run: |
-          if [ "${{ github.event_name }}" == "schedule" ]
+          if [[ "${{ github.event_name }}" == "schedule" ]]
           then
             echo Running unstable and stable tests
-            echo "mark_expression=" >> $GITHUB_OUTPUT
+            echo "mark_expression=" >> "$GITHUB_OUTPUT"
           else
             echo Skipping unstable tests
-            echo "mark_expression=not unstable" >> $GITHUB_OUTPUT
+            echo "mark_expression=not unstable" >> "$GITHUB_OUTPUT"
           fi
       - name: Run integration tests
-        run: tox run -e ${{ matrix.tox-environments }} -- -m '${{ steps.select-tests.outputs.mark_expression }}' --mysql-router-charm-series=${{ matrix.ubuntu-versions.series }} --mysql-router-charm-bases-index=${{ matrix.ubuntu-versions.bases-index }}
-        env:
-          CI_PACKED_CHARMS: ${{ needs.build.outputs.charms }}
+        run: tox run -e integration -- "${{ matrix.groups.path_to_test_file }}" --group="${{ matrix.groups.group_number }}" -m '${{ steps.select-test-stability.outputs.mark_expression }}' --mysql-router-charm-series=${{ matrix.ubuntu-versions.series }} --mysql-router-charm-bases-index=${{ matrix.ubuntu-versions.bases-index }}

.github/workflows/release.yaml

@@ -17,14 +17,14 @@ jobs:
 
   build:
     name: Build charm
-    uses: canonical/data-platform-workflows/.github/workflows/build_charm_without_cache.yaml@v2
+    uses: canonical/data-platform-workflows/.github/workflows/build_charm_without_cache.yaml@v5.0.1
 
   release:
     name: Release charm
     needs:
       - ci-tests
       - build
-    uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v2
+    uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v5.0.1
     with:
       channel: dpe/edge
       artifact-name: ${{ needs.build.outputs.artifact-name }}

.github/workflows/sync_issue_to_jira.yaml

@@ -9,7 +9,7 @@ on:
 jobs:
   sync:
     name: Sync GitHub issue to Jira
-    uses: canonical/data-platform-workflows/.github/workflows/sync_issue_to_jira.yaml@v2
+    uses: canonical/data-platform-workflows/.github/workflows/sync_issue_to_jira.yaml@v5.0.1
     with:
       jira-base-url: https://warthogs.atlassian.net
       jira-project-key: DPE

.gitignore

@@ -1,3 +1,6 @@
+/requirements.txt
+/requirements-last-build.txt
+
 venv/
 build/
 *.charm

CONTRIBUTING.md

@@ -56,11 +56,18 @@ git clone https://github.com/canonical/mysql-router-operator.git
 cd mysql-router-operator
 ```
 
-You can create an environment for development with `tox`:
+Install `tox` and `poetry`
+```shell
+python3 -m pip install --user pipx
+python3 -m pipx ensurepath
+pipx install tox
+pipx install poetry
+```
+
+You can create an environment for development:
 
 ```shell
-tox devenv -e integration
-source venv/bin/activate
+poetry install
 ```
 
 ### Testing
@@ -78,7 +85,7 @@ tox                      # runs 'lint' and 'unit' environments
 Build the charm in this git repository using:
 
 ```shell
-charmcraft pack
+tox run -e build
 ```
 
 ### Deploy

charmcraft.yaml

@@ -20,6 +20,13 @@ bases:
         channel: "22.04"
 parts:
   charm:
+    override-pull: |
+      craftctl default
+      if [[ ! -f requirements.txt ]]
+      then
+          echo 'ERROR: Use "tox run -e build" instead of calling "charmcraft pack" directly' >&2
+          exit 1
+      fi
     charm-entrypoint: src/machine_charm.py
     charm-binary-python-packages:
       - mysql-connector-python==8.0.32