[DPE-3064] reusable workflow (#338)

* Bump actions

* Adding marks

* Relative imports

* Switch to reusable workflow

* Add markers

* Skip password tests on juju

* Remove deps

* Borked import

* Bump workflow

* Missing import

Author: dragomirp

.github/workflows/ci.yaml

@@ -22,7 +22,7 @@ on:
 jobs:
   lint:
     name: Lint
-    uses: canonical/data-platform-workflows/.github/workflows/[email protected].1
+    uses: canonical/data-platform-workflows/.github/workflows/[email protected].3
 
   unit-test:
     name: Unit test charm
@@ -45,83 +45,35 @@ jobs:
 
   build:
     name: Build charm
-    uses: canonical/data-platform-workflows/.github/workflows/[email protected].1
+    uses: canonical/data-platform-workflows/.github/workflows/[email protected].3
     permissions:
       actions: write  # Needed to manage GitHub Actions cache
 
-  gh-hosted-integration-test:
+  integration-test:
     strategy:
       fail-fast: false
       matrix:
-        tox-environment:
-          - backup-integration
-          - charm-integration
-          - database-relation-integration
-          - db-relation-integration
-          - db-admin-relation-integration
-          - ha-replication-integration
-          - ha-self-healing-integration
-          - password-rotation-integration
-          - plugins-integration
-          - tls-integration
-          - upgrade-integration
-          - upgrade-from-stable-integration
-        juju-snap-channel:  ["2.9/stable", "3.1/stable"]
-        include:
-          - juju-snap-channel: "3.1/stable"
-            agent-version: "3.1.6"
-            libjuju-version:  "3.2.2"
-            exclude-mark:  "juju2"
-          - juju-snap-channel: "2.9/stable"
-            agent-version: "2.9.45"
-            libjuju-version:  "2.9.45.0"
-            exclude-mark:  "juju3"
-    name: ${{ matrix.juju-snap-channel }} - (GH hosted) ${{ matrix.tox-environment }}
+        juju:
+          - agent: 2.9.45
+            libjuju: ^2
+          - agent: 3.1.6
+    name: Integration test charm | ${{ matrix.juju.agent }}
     needs:
       - lint
       - unit-test
       - build
-    runs-on: ubuntu-latest
-    timeout-minutes: 120
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Install tox & poetry
-        run: |
-          pipx install tox
-          pipx install poetry
-      - name: Setup operator environment
-        uses: charmed-kubernetes/actions-operator@main
-        with:
-          provider: microk8s
-          channel: "1.28-strict/stable"
-          bootstrap-options: "--agent-version ${{ matrix.agent-version }}"
-          juju-channel: ${{ matrix.juju-snap-channel }}
-      - name: Update python-libjuju version
-        if: ${{ matrix.juju-snap-channel == '2.9/stable' }}
-        run: poetry add --lock --group integration juju@'${{ matrix.libjuju-version }}'
-      - name: Download packed charm(s)
-        uses: actions/download-artifact@v3
-        with:
-          name: ${{ needs.build.outputs.artifact-name }}
-      - name: Select test stability level
-        id: select-test-stability
-        run: |
-          if [[ "${{ github.event_name }}" == "schedule" ]]
-          then
-            echo Running unstable and stable tests
-            echo "mark_expression=" >> "$GITHUB_OUTPUT"
-          else
-            echo Skipping unstable tests
-            echo "mark_expression=and not unstable" >> "$GITHUB_OUTPUT"
-          fi
-      - name: Run integration tests
-        run: tox run -e ${{ matrix.tox-environment }} -- -m 'not ${{ matrix.exclude-mark }} ${{ steps.select-test-stability.outputs.mark_expression }}' --keep-models
-        env:
-          SECRETS_FROM_GITHUB: |
-            {
-              "AWS_ACCESS_KEY": "${{ secrets.AWS_ACCESS_KEY }}",
-              "AWS_SECRET_KEY": "${{ secrets.AWS_SECRET_KEY }}",
-              "GCP_ACCESS_KEY": "${{ secrets.GCP_ACCESS_KEY }}",
-              "GCP_SECRET_KEY": "${{ secrets.GCP_SECRET_KEY }}",
-            }
+    uses: canonical/data-platform-workflows/.github/workflows/[email protected]
+    with:
+      artifact-name: ${{ needs.build.outputs.artifact-name }}
+      cloud: microk8s
+      microk8s-snap-channel: 1.28-strict/stable
+      juju-agent-version: ${{ matrix.juju.agent }}
+      libjuju-version-constraint: ${{ matrix.juju.libjuju }}
+    secrets:
+      integration-test: |
+        {
+          "AWS_ACCESS_KEY": "${{ secrets.AWS_ACCESS_KEY }}",
+          "AWS_SECRET_KEY": "${{ secrets.AWS_SECRET_KEY }}",
+          "GCP_ACCESS_KEY": "${{ secrets.GCP_ACCESS_KEY }}",
+          "GCP_SECRET_KEY": "${{ secrets.GCP_SECRET_KEY }}",
+        }

.github/workflows/release.yaml

@@ -41,14 +41,14 @@ jobs:
 
   build:
     name: Build charm
-    uses: canonical/data-platform-workflows/.github/workflows/[email protected].1
+    uses: canonical/data-platform-workflows/.github/workflows/[email protected].3
 
   release:
     name: Release charm
     needs:
       - ci-tests
       - build
-    uses: canonical/data-platform-workflows/.github/workflows/[email protected].1
+    uses: canonical/data-platform-workflows/.github/workflows/[email protected].3
     with:
       channel: 14/edge
       artifact-name: ${{ needs.build.outputs.artifact-name }}

.github/workflows/sync_issue_to_jira.yaml

@@ -9,7 +9,7 @@ on:
 jobs:
   sync:
     name: Sync GitHub issue to Jira
-    uses: canonical/data-platform-workflows/.github/workflows/[email protected].1
+    uses: canonical/data-platform-workflows/.github/workflows/[email protected].3
     with:
       jira-base-url: https://warthogs.atlassian.net
       jira-project-key: DPE

poetry.lock

@@ -70,16 +70,14 @@ optional = true
 [tool.poetry.group.integration.dependencies]
 lightkube = "^0.15.0"
 pytest = "^7.4.3"
-pytest-github-secrets = {git = "https://github.com/canonical/data-platform-workflows", tag = "v6.3.1", subdirectory = "python/pytest_plugins/github_secrets"}
+pytest-github-secrets = {git = "https://github.com/canonical/data-platform-workflows", tag = "v6.3.3", subdirectory = "python/pytest_plugins/github_secrets"}
 pytest-operator = "^0.31.0"
-pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", tag = "v6.3.1", subdirectory = "python/pytest_plugins/pytest_operator_cache"}
-pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v6.3.1", subdirectory = "python/pytest_plugins/pytest_operator_groups"}
+pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", tag = "v6.3.3", subdirectory = "python/pytest_plugins/pytest_operator_cache"}
+pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v6.3.3", subdirectory = "python/pytest_plugins/pytest_operator_groups"}
 juju = "^3.2.2"
 psycopg2 = {version = "^2.9.9", extras = ["binary"]}
 boto3 = "^1.33.1"
 tenacity = "^8.2.3"
-ops = "^2.8.0"
-pytest-mock = "^3.12.0"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]
@@ -96,7 +94,7 @@ show_missing = true
 minversion = "6.0"
 log_cli_level = "INFO"
 asyncio_mode = "auto"
-markers = ["unstable"]
+markers = ["unstable", "juju2", "juju3", "juju_secrets"]
 
 # Formatting tools configuration
 [tool.black]

pyproject.toml

@@ -0,0 +1,2 @@
+# Copyright 2023 Canonical Ltd.
+# See LICENSE file for licensing details.

tests/integration/__init__.py

@@ -0,0 +1,2 @@
+# Copyright 2023 Canonical Ltd.
+# See LICENSE file for licensing details.

tests/integration/ha_tests/__init__.py

@@ -6,7 +6,8 @@
 from pytest_operator.plugin import OpsTest
 from tenacity import Retrying, stop_after_delay, wait_fixed
 
-from tests.integration.ha_tests.helpers import (
+from ..helpers import app_name
+from .helpers import (
     APPLICATION_NAME,
     change_patroni_setting,
     change_wal_settings,
@@ -17,7 +18,6 @@
     modify_pebble_restart_delay,
     remove_instance_isolation,
 )
-from tests.integration.helpers import app_name
 
 
 @pytest.fixture()

tests/integration/ha_tests/conftest.py

@@ -27,7 +27,7 @@
     wait_fixed,
 )
 
-from tests.integration.helpers import (
+from ..helpers import (
     APPLICATION_NAME,
     app_name,
     db_connect,