[DPE-2789] update data-platform-workflows (#263)

* Tweak workflows

* Initial pass

* Tweak conftest

* Move renovate configs

* Add wrapper

* Copy from mysql-k8s

* Linting

* Partial merge

* Saner marks

* Tweak excludes

* Bump dpw version

* Bump deps

* Tweak backup tests

* Conditional free space

* Revert "Conditional free space"

This reverts commit c4f72d2.

* Bump jira sync version

* Bump lock

* Remove schedule and call

* Bump libs and actions

Author: dragomirp

.github/renovate.json5

@@ -0,0 +1,16 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": ["github>canonical/data-platform//renovate_presets/charm.json5"],
+  "reviewers": ["dragomirp", "marceloneppel", "taurus-forever"],
+  "packageRules": [
+    // Later rules override earlier rules
+  ],
+  "regexManagers": [
+    {
+      "fileMatch": ["(^|/)([\\w-]*)charmcraft\\.ya?ml$"],
+      "matchStrings": ["- (?<depName>.*?)(?:\\[.*?\\])?==(?<currentValue>.*?) +# renovate"],
+      "datasourceTemplate": "pypi",
+      "versioningTemplate": "loose"
+    }
+  ]
+}

.github/workflows/ci.yaml

@@ -22,45 +22,40 @@ on:
 jobs:
   lint:
     name: Lint
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Install tox
-        # TODO: Consider replacing with custom image on self-hosted runner OR pinning version
-        run: python3 -m pip install tox
-      - name: Run linters
-        run: tox run -e lint
+    uses: canonical/data-platform-workflows/.github/workflows/[email protected]
 
   unit-test:
-    name: Unit tests
+    name: Unit test charm
     runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
-      - name: Install tox
-        # TODO: Consider replacing with custom image on self-hosted runner OR pinning version
-        run: python3 -m pip install tox
+        uses: actions/checkout@v4
+      - name: Install tox & poetry
+        run: |
+          pipx install tox
+          pipx install poetry
       - name: Run tests
         run: tox run -e unit
       - name: Upload Coverage to Codecov
         uses: codecov/codecov-action@v3
 
   build:
-    name: Build charms
-    uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v2
+    name: Build charm
+    uses: canonical/data-platform-workflows/.github/workflows/[email protected]
+    permissions:
+      actions: write  # Needed to manage GitHub Actions cache
 
-  integration-test:
+  gh-hosted-integration-test:
     strategy:
       fail-fast: false
-      max-parallel: 6
       matrix:
-        tox-environments:
+        tox-environment:
           - backup-integration
           - charm-integration
+          - database-relation-integration
           - db-relation-integration
+          - db-admin-relation-integration
           - ha-replication-integration
           - ha-self-healing-integration
           - ha-restore-cluster-integration
@@ -69,22 +64,21 @@ jobs:
           - tls-integration
           - upgrade-integration
           - upgrade-from-stable-integration
-        agent-versions:
-          - "2.9.45"  # renovate: latest juju 2
-          - "3.1.6"  # renovate: latest juju 3
-        free-disk-space:
-          - false
+        juju-snap-channel:  ["2.9/stable", "3.1/stable"]
+        exclude:
+          # Admin tests need landscape on juju 2
+          - tox-environment: db-admin-relation-integration
+            juju-snap-channel: "3.1/stable"
         include:
-          - tox-environments: database-relation-integration
-            agent-versions: "2.9.45"  # renovate: latest juju 2
-            free-disk-space: true
-          - tox-environments: database-relation-integration
-            agent-versions: "3.1.6"  # renovate: latest juju 3
-            free-disk-space: true
-          - tox-environments: db-admin-relation-integration
-            agent-versions: "2.9.45"  # renovate: latest juju 2
-            free-disk-space: true
-    name: ${{ matrix.tox-environments }} | ${{ matrix.agent-versions }}
+          - juju-snap-channel: "3.1/stable"
+            agent-version: "3.1.6"
+            libjuju-version:  "3.2.2"
+            exclude-mark:  "juju2"
+          - juju-snap-channel: "2.9/stable"
+            agent-version: "2.9.45"
+            libjuju-version:  "2.9.45.0"
+            exclude-mark:  "juju3"
+    name: ${{ matrix.juju-snap-channel }} - (GH hosted) ${{ matrix.tox-environment }}
     needs:
       - lint
       - unit-test
@@ -93,48 +87,49 @@ jobs:
     timeout-minutes: 120
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
-      - name: Free disk space
-        if: ${{ matrix.free-disk-space == true }}
+        uses: actions/checkout@v4
+      - name: Install tox & poetry
         run: |
-          sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
-          sudo rm -rf \
-            /usr/share/dotnet /usr/local/lib/android /opt/ghc \
-            /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup \
-            /usr/lib/jvm || true
-      - name: Set channel
+          pipx install tox
+          pipx install poetry
+      - name: Free up disk space
         run: |
-          juju_channel=$(echo "${{ matrix.agent-versions }}" | cut -c 1-3)
-          echo "channel=${juju_channel}/stable" >> "$GITHUB_ENV"
-          juju_major=$(echo "${{ matrix.agent-versions }}" | cut -c 1)
-          echo "libjuju=juju${juju_major}" >> "$GITHUB_ENV"
+          # From https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /usr/local/share/boost
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
       - name: Setup operator environment
-        # TODO: Replace with custom image on self-hosted runner
         uses: charmed-kubernetes/actions-operator@main
         with:
           provider: lxd
-          bootstrap-options: "--agent-version ${{ matrix.agent-versions }}"
-          juju-channel: "${{ env.channel }}"
+          bootstrap-options: "--agent-version ${{ matrix.agent-version }}"
+          juju-channel: ${{ matrix.juju-snap-channel }}
+      - name: Update python-libjuju version
+        if: ${{ matrix.juju-snap-channel == '2.9/stable' }}
+        run: poetry add --lock --group integration juju@'${{ matrix.libjuju-version }}'
       - name: Download packed charm(s)
         uses: actions/download-artifact@v3
         with:
           name: ${{ needs.build.outputs.artifact-name }}
-      - name: Select tests
-        id: select-tests
+      - name: Select test stability level
+        id: select-test-stability
         run: |
-          if [ "${{ github.event_name }}" == "schedule" ]
+          if [[ "${{ github.event_name }}" == "schedule" ]]
           then
             echo Running unstable and stable tests
-            echo "mark_expression=" >> $GITHUB_OUTPUT
+            echo "mark_expression=" >> "$GITHUB_OUTPUT"
           else
             echo Skipping unstable tests
-            echo "mark_expression=and not unstable" >> $GITHUB_OUTPUT
+            echo "mark_expression=and not unstable" >> "$GITHUB_OUTPUT"
           fi
       - name: Run integration tests
-        run: tox run -e ${{ matrix.tox-environments }}-${{ env.libjuju }} -- -m 'not not${{ env.libjuju }} ${{ steps.select-tests.outputs.mark_expression }}' --keep-models
+        run: tox run -e ${{ matrix.tox-environment }} -- -m 'not ${{ matrix.exclude-mark }} ${{ steps.select-test-stability.outputs.mark_expression }}' --keep-models
         env:
-          AWS_ACCESS_KEY: "${{ secrets.AWS_ACCESS_KEY }}"
-          AWS_SECRET_KEY: "${{ secrets.AWS_SECRET_KEY }}"
-          GCP_ACCESS_KEY: "${{ secrets.GCP_ACCESS_KEY }}"
-          GCP_SECRET_KEY: "${{ secrets.GCP_SECRET_KEY }}"
-          CI_PACKED_CHARMS: ${{ needs.build.outputs.charms }}
+          SECRETS_FROM_GITHUB: |
+            {
+              "AWS_ACCESS_KEY": "${{ secrets.AWS_ACCESS_KEY }}",
+              "AWS_SECRET_KEY": "${{ secrets.AWS_SECRET_KEY }}",
+              "GCP_ACCESS_KEY": "${{ secrets.GCP_ACCESS_KEY }}",
+              "GCP_SECRET_KEY": "${{ secrets.GCP_SECRET_KEY }}",
+            }

.github/workflows/lib-check.yaml

@@ -14,10 +14,6 @@ on:
       - 'LICENSE'
       - '**.md'
       - 'renovate.json'
-  schedule:
-    - cron: '53 0 * * *' # Daily at 00:53 UTC
-  # Triggered on push to branch "main" by .github/workflows/release.yaml
-  workflow_call:
 
 jobs:
   lib-check:
@@ -27,7 +23,7 @@ jobs:
     if: ${{ github.event.pull_request.head.repo.full_name == 'canonical/postgresql-operator' }}
     steps:
       - name: Checkout
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Check libs

.github/workflows/release.yaml

@@ -20,17 +20,19 @@ jobs:
     name: Tests
     uses: ./.github/workflows/ci.yaml
     secrets: inherit
+    permissions:
+      actions: write  # Needed to manage GitHub Actions cache
 
   build:
     name: Build charm
-    uses: canonical/data-platform-workflows/.github/workflows/build_charm_without_cache.yaml@v2
+    uses: canonical/data-platform-workflows/.github/workflows/build_charm_without_cache.yaml@v5.1.2
 
   release:
     name: Release charm
     needs:
       - ci-tests
       - build
-    uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v2
+    uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v5.1.2
     with:
       channel: 14/edge
       artifact-name: ${{ needs.build.outputs.artifact-name }}

.github/workflows/sync_issue_to_jira.yaml

@@ -9,7 +9,7 @@ on:
 jobs:
   sync:
     name: Sync GitHub issue to Jira
-    uses: canonical/data-platform-workflows/.github/workflows/sync_issue_to_jira.yaml@v2
+    uses: canonical/data-platform-workflows/.github/workflows/sync_issue_to_jira.yaml@v5.1.2
     with:
       jira-base-url: https://warthogs.atlassian.net
       jira-project-key: DPE

.gitignore

@@ -6,7 +6,8 @@ build/
 __pycache__/
 *.py[cod]
 coverage.xml
-patroni.tar.gz
+requirements.txt
+requirements-last-build.txt
 
 # PyCharm project folder.
 .idea/

charmcraft.yaml

@@ -11,6 +11,13 @@ bases:
         channel: "22.04"
 parts:
   charm:
+    override-pull: |
+      craftctl default
+      if [[ ! -f requirements.txt ]]
+      then
+          echo 'ERROR: Use "tox run -e build" instead of calling "charmcraft pack" directly' >&2
+          exit 1
+      fi
     build-packages:
       - libffi-dev
       - libssl-dev