[DPE-6191] fix install timeout + fix broken CI (#373)

* fix install timeout

* update poetry + workflows

* remove fields related to packages

* fix linting issues

* duplicate requirements in separate parts

* migrate to use charm_version file per Carls suggestion

* give workflow in tox access to poetry export

* update charm version to pep format

* PR feedback

* remove build wrapper

Author: MiaAltieri

.github/workflows/ci.yaml

@@ -16,7 +16,7 @@ on:
 jobs:
   lint:
     name: Lint
-    uses: canonical/data-platform-workflows/.github/workflows/lint.yaml@v21.0.0
+    uses: canonical/data-platform-workflows/.github/workflows/lint.yaml@v24.0.6
 
   lib-check:
     name: Check libraries
@@ -54,7 +54,7 @@ jobs:
 
   build:
     name: Build charm | ${{ matrix.path }}
-    uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v21.0.0
+    uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v24.0.6
     strategy:
       matrix:
         path:
@@ -80,9 +80,9 @@ jobs:
       - lint
       - unit-test
       - build
-    uses: canonical/data-platform-workflows/.github/workflows/integration_test_charm.yaml@v21.0.0
+    uses: canonical/data-platform-workflows/.github/workflows/integration_test_charm.yaml@v24.0.6
     with:
-      artifact-prefix: packed-charm-cache-false  # TODO revert to "packed-charm-cache-true" when cache re-enabled
+      artifact-prefix: packed-charm-cache-false # TODO revert to "packed-charm-cache-true" when cache re-enabled
       cloud: microk8s
       microk8s-snap-channel: 1.29-strict/stable
       juju-agent-version: ${{ matrix.juju.agent }}

.github/workflows/release.yaml

@@ -35,16 +35,16 @@ jobs:
 
   build:
     name: Build charm
-    uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v21.0.0
+    uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v24.0.6
     with:
-      artifact-prefix: temp-release-packed-charm  # TODO: remove after caching re-enabled on PR
+      artifact-prefix: temp-release-packed-charm # TODO: remove after caching re-enabled on PR
 
   release:
     name: Release charm
     needs:
       - ci-tests
       - build
-    uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v21.0.0
+    uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v24.0.6
     with:
       channel: 6/edge
       artifact-prefix: ${{ needs.build.outputs.artifact-prefix }}

charmcraft.yaml

@@ -3,38 +3,45 @@
 
 type: "charm"
 bases:
-  - build-on:
-      - name: "ubuntu"
-        channel: "22.04"
-    run-on:
-      - name: "ubuntu"
-        channel: "22.04"
+  - name: ubuntu
+    channel: "22.04"
+    architectures: [amd64]
 parts:
-  charm:
-    charm-strict-dependencies: true
+  files:
+    plugin: dump
+    source: .
+    build-packages:
+      - git
     override-build: |
-      rustup default stable
+      # Workaround to add unique identifier (git hash) to charm version while specification
+      # DA053 - Charm versioning
+      # (https://docs.google.com/document/d/1Jv1jhWLl8ejK3iJn7Q3VbCIM9GIhp8926bgXpdtx-Sg/edit?pli=1)
+      # is pending review.
+      python3 -c 'import pathlib; import shutil; import subprocess; git_hash=subprocess.run(["git", "describe", "--always", "--dirty"], capture_output=True, check=True, encoding="utf-8").stdout; file = pathlib.Path("charm_version"); shutil.copy(file, pathlib.Path("charm_version.backup")); version = file.read_text().strip(); file.write_text(f"{version}+{git_hash}")'
       craftctl default
+    stage:
+      # Exclude requirements.txt file during staging
+      # Workaround for https://github.com/canonical/charmcraft/issues/1389 on charmcraft 2
+      - -requirements.txt
+    prime:
+      - charm_version
+      - workload_version
+  charm:
     build-snaps:
       - rustup
     build-packages:
       - libffi-dev
       - libssl-dev
       - pkg-config
-    override-pull: |
-      craftctl default
-      if [[ ! -f requirements.txt ]]
-      then
-          echo 'ERROR: Use "tox run -e build-dev" instead of calling "charmcraft pack" directly' >&2
-          exit 1
-      fi
-  files:
-    plugin: dump
-    source: .
-    prime:
-      - charm_internal_version
-      - charm_version
-      - workload_version
     override-build: |
       rustup default stable
+      # Convert subset of poetry.lock to requirements.txt
+      curl -sSL https://install.python-poetry.org | python3 -
+      /root/.local/bin/poetry self add poetry-plugin-export
+      /root/.local/bin/poetry export --only main,charm-libs --output requirements.txt
       craftctl default
+    stage:
+      # Exclude charm_version file during staging
+      - -charm_version
+    charm-strict-dependencies: true
+    charm-requirements: [requirements.txt]

poetry.lock

@@ -2,16 +2,11 @@
 # See LICENSE file for licensing details.
 
 [tool.poetry]
-name = "mongodb-k8s"
-version = "0.0.1-dev.0"
-description = ""
-license = "Apache-2.0"
-authors = [
-    "Mykola Marzhan <[email protected]>"
-]
-readme = "README.md"
-homepage = "https://charmhub.io/mongodb-k8s?channel=6/edge"
-repository = "https://github.com/canonical/mongodb-k8s-operator"
+package-mode = false
+requires-poetry = ">=2.0.0"
+
+
+
 
 [tool.poetry.dependencies]
 python = "^3.10.12"
@@ -20,7 +15,7 @@ pymongo = "^4.7.3"
 tenacity = "^8.2.3"
 pyyaml = "^6.0.1"
 jinja2 = "^3.1.3"
-poetry-core = "^1.9.0"
+poetry-core = "^2.0"
 data-platform-helpers = "^0.1.3"
 overrides = "^7.7.0"
 lightkube = "^0.15.3"
@@ -35,6 +30,9 @@ jsonschema = "^4.22.0"  # tls_certificates lib v3
 cosl = "^0.0.16"  # loki_push_api
 pyOpenSSL = "^24.2.1"
 
+[tool.poetry.requires-plugins]
+poetry-plugin-export = ">=1.8"
+
 [tool.poetry.group.format]
 optional = true
 
@@ -76,13 +74,12 @@ pytest = "^8.1.1"
 pytest-asyncio = "^0.21.1"
 pytest-mock = "^3.14.0"
 pytest-operator = "^0.36.0"
-pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", tag = "v21.0.0", subdirectory = "python/pytest_plugins/pytest_operator_cache"}
-pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v21.0.0", subdirectory = "python/pytest_plugins/pytest_operator_groups"}
-pytest-github-secrets = {git = "https://github.com/canonical/data-platform-workflows", tag = "v21.0.0", subdirectory = "python/pytest_plugins/github_secrets"}
-allure-pytest-collection-report = {git = "https://github.com/canonical/data-platform-workflows", tag = "v21.0.0", subdirectory = "python/pytest_plugins/allure_pytest_collection_report"}
+pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.6", subdirectory = "python/pytest_plugins/pytest_operator_cache"}
+pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.6", subdirectory = "python/pytest_plugins/pytest_operator_groups"}
+pytest-github-secrets = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.6", subdirectory = "python/pytest_plugins/github_secrets"}
+allure-pytest-collection-report = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.6", subdirectory = "python/pytest_plugins/allure_pytest_collection_report"}
 
 [build-system]
-requires = ["poetry-core>=1.9.0"]
 build-backend = "poetry.core.masonry.api"
 
 # Testing tools configuration

pyproject.toml

@@ -22,7 +22,7 @@ class Config:
     MONGOD_CONF_DIR = "/etc/mongod"
     MONGODB_LOG_FILENAME = "mongodb.log"
 
-    CHARM_INTERNAL_VERSION_FILE = "charm_internal_version"
+    CHARM_INTERNAL_VERSION_FILE = "charm_version"
     LICENSE_PATH = "/licenses/LICENSE"
     CONTAINER_NAME = "mongod"
     SERVICE_NAME = "mongod"

src/config.py

@@ -16,6 +16,7 @@
 
 from ..ha_tests import helpers as ha_helpers
 from ..helpers import (
+    DEPLOYMENT_TIMEOUT,
     check_or_scale_app,
     destroy_cluster,
     get_app_name,
@@ -112,7 +113,7 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None:
             await ops_test.model.wait_for_idle(
                 apps=[DATABASE_APP_NAME],
                 status="active",
-                timeout=2000,
+                timeout=DEPLOYMENT_TIMEOUT,
                 raise_on_error=False,
             )
 
@@ -416,7 +417,12 @@ async def test_restore_new_cluster(
     )
 
     await asyncio.gather(
-        ops_test.model.wait_for_idle(apps=[new_cluster_app_name], status="active", idle_period=20),
+        ops_test.model.wait_for_idle(
+            apps=[new_cluster_app_name],
+            status="active",
+            timeout=DEPLOYMENT_TIMEOUT,
+            idle_period=20,
+        ),
     )
 
     leader_unit = await helpers.get_leader_unit(ops_test, db_app_name=new_cluster_app_name)

tests/integration/backup_tests/test_backups.py

@@ -14,6 +14,7 @@
 from ..backup_tests import helpers as backup_helpers
 from ..ha_tests.helpers import deploy_and_scale_application, get_direct_mongo_client
 from ..helpers import (
+    DEPLOYMENT_TIMEOUT,
     METADATA,
     MONGOS_PORT,
     get_leader_id,
@@ -338,7 +339,7 @@ async def deploy_cluster_backup_test(
         apps=[S3_APP_NAME, config_server_name, shard_one_name, shard_two_name],
         idle_period=20,
         raise_on_blocked=False,
-        timeout=TIMEOUT,
+        timeout=DEPLOYMENT_TIMEOUT,
         raise_on_error=False,
     )
 

tests/integration/backup_tests/test_sharding_backups.py

@@ -31,6 +31,7 @@
 
 from ..helpers import (
     APP_NAME,
+    DEPLOYMENT_TIMEOUT,
     MONGOD_PORT,
     MONGOS_PORT,
     get_app_name,
@@ -193,7 +194,7 @@ async def deploy_and_scale_mongodb(
             status="active",
             raise_on_blocked=True,
             raise_on_error=False,
-            timeout=TIMEOUT,
+            timeout=DEPLOYMENT_TIMEOUT,
         )
 
     return mongodb_application_name
@@ -232,7 +233,7 @@ async def deploy_and_scale_application(ops_test: OpsTest) -> str:
             status="waiting",
             raise_on_blocked=True,
             raise_on_error=False,
-            timeout=TIMEOUT,
+            timeout=DEPLOYMENT_TIMEOUT,
         )
 
     return APPLICATION_DEFAULT_APP_NAME
@@ -275,7 +276,11 @@ async def get_process_pid(
 
 
 async def send_signal_to_pod_container_process(
-    ops_test: OpsTest, unit_name: str, container_name: str, process: str, signal_code: str
+    ops_test: OpsTest,
+    unit_name: str,
+    container_name: str,
+    process: str,
+    signal_code: str,
 ) -> None:
     """Send the specified signal to a pod container process.
 
@@ -330,7 +335,9 @@ async def get_replica_set_primary(
 ) -> Optional[Unit]:
     """Returns the primary unit name based no the replica set host."""
     with await get_direct_mongo_client(
-        ops_test, excluded=excluded, use_subprocess_to_get_password=use_subprocess_to_get_password
+        ops_test,
+        excluded=excluded,
+        use_subprocess_to_get_password=use_subprocess_to_get_password,
     ) as client:
         data = client.admin.command("replSetGetStatus")
     unit_name = host_to_unit(primary_host(data))
@@ -674,7 +681,10 @@ async def wait_until_unit_in_status(
 ) -> None:
     """Waits until a replica is in the provided status as reported by MongoDB or timeout occurs."""
     with await get_direct_mongo_client(
-        ops_test, online_unit.name, use_subprocess_to_get_password=True, app_name=app_name
+        ops_test,
+        online_unit.name,
+        use_subprocess_to_get_password=True,
+        app_name=app_name,
     ) as client:
         data = client.admin.command("replSetGetStatus")
 
@@ -809,7 +819,7 @@ def convert_time(time_as_str: str) -> int:
 def get_highest_unit(ops_test: OpsTest, app_name: str) -> Unit:
     """Retrieves the most recently added unit to the MongoDB application."""
     num_units = len(ops_test.model.applications[app_name].units)
-    highest_unit_name = f"mongodb-k8s/{num_units-1}"
+    highest_unit_name = f"mongodb-k8s/{num_units - 1}"
     for unit in ops_test.model.applications[app_name].units:
         if unit.name == highest_unit_name:
             return unit
@@ -831,7 +841,11 @@ async def are_all_db_processes_down(ops_test: OpsTest, process: str) -> bool:
             with attempt:
                 for unit in ops_test.model.applications[app].units:
                     _, raw_pid, _ = await ops_test.juju(
-                        "ssh", "--container", MONGODB_CONTAINER_NAME, unit.name, *pgrep_cmd
+                        "ssh",
+                        "--container",
+                        MONGODB_CONTAINER_NAME,
+                        unit.name,
+                        *pgrep_cmd,
                     )
 
                     # If something was returned, there is a running process.

tests/integration/ha_tests/helpers.py

@@ -25,7 +25,7 @@
 UNIT_IDS = [0, 1, 2]
 MONGOS_PORT = 27018
 MONGOD_PORT = 27017
-
+DEPLOYMENT_TIMEOUT = 2000
 TEST_DOCUMENTS = """[
     {
         \"uid\": 123,