[DPE-5386] Add microceph (local backup) integration test + bump snap version (#633)

* try add ceph test

* fix lint

* use another dir

* fix quotes

* fix echo command

* fix linting

* fix command

* fix boto3 client call

* fix s3 config

* reposition relate operation

* remove cleanup

* add backup operations

* remove DNS workaround for pgbackrest 2.48

* use newer version of pgbackrest

* review improvements

Author: Lucas Gameiro

src/constants.py

@@ -41,7 +41,7 @@
     (
         POSTGRESQL_SNAP_NAME,
         {
-            "revision": {"aarch64": "125", "x86_64": "124"},
+            "revision": {"aarch64": "132", "x86_64": "133"},
             "channel": "14/stable",
         },
     )

tests/integration/helpers.py

@@ -1125,7 +1125,6 @@ async def backup_operations(
     await ops_test.model.relate(database_app_name, tls_certificates_app_name)
     async with ops_test.fast_forward(fast_interval="60s"):
         await ops_test.model.wait_for_idle(apps=[database_app_name], status="active", timeout=1000)
-    await ops_test.model.relate(database_app_name, s3_integrator_app_name)
 
     # Configure and set access and secret keys.
     logger.info(f"configuring S3 integrator for {cloud}")
@@ -1135,6 +1134,8 @@ async def backup_operations(
         **credentials,
     )
     await action.wait()
+
+    await ops_test.model.relate(database_app_name, s3_integrator_app_name)
     async with ops_test.fast_forward(fast_interval="60s"):
         await ops_test.model.wait_for_idle(
             apps=[database_app_name, s3_integrator_app_name], status="active", timeout=1500

tests/integration/test_backups_ceph.py

@@ -0,0 +1,213 @@
+#!/usr/bin/env python3
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+import dataclasses
+import json
+import logging
+import os
+import socket
+import subprocess
+import time
+
+import boto3
+import botocore.exceptions
+import pytest
+from pytest_operator.plugin import OpsTest
+
+from . import architecture, markers
+from .helpers import (
+    backup_operations,
+)
+from .juju_ import juju_major_version
+
+logger = logging.getLogger(__name__)
+
+S3_INTEGRATOR_APP_NAME = "s3-integrator"
+if juju_major_version < 3:
+    tls_certificates_app_name = "tls-certificates-operator"
+    if architecture.architecture == "arm64":
+        tls_channel = "legacy/edge"
+    else:
+        tls_channel = "legacy/stable"
+    tls_config = {"generate-self-signed-certificates": "true", "ca-common-name": "Test CA"}
+else:
+    tls_certificates_app_name = "self-signed-certificates"
+    if architecture.architecture == "arm64":
+        tls_channel = "latest/edge"
+    else:
+        tls_channel = "latest/stable"
+    tls_config = {"ca-common-name": "Test CA"}
+
+backup_id, value_before_backup, value_after_backup = "", None, None
+
+
+@dataclasses.dataclass(frozen=True)
+class ConnectionInformation:
+    access_key_id: str
+    secret_access_key: str
+    bucket: str
+
+
+@pytest.fixture(scope="session")
+def microceph():
+    if not os.environ.get("CI") == "true":
+        raise Exception("Not running on CI. Skipping microceph installation")
+    logger.info("Setting up TLS certificates")
+    subprocess.run(["openssl", "genrsa", "-out", "./ca.key", "2048"], check=True)
+    subprocess.run(
+        [
+            "openssl",
+            "req",
+            "-x509",
+            "-new",
+            "-nodes",
+            "-key",
+            "./ca.key",
+            "-days",
+            "1024",
+            "-out",
+            "./ca.crt",
+            "-outform",
+            "PEM",
+            "-subj",
+            "/C=US/ST=Denial/L=Springfield/O=Dis/CN=www.example.com",
+        ],
+        check=True,
+    )
+    subprocess.run(["openssl", "genrsa", "-out", "./server.key", "2048"], check=True)
+    subprocess.run(
+        [
+            "openssl",
+            "req",
+            "-new",
+            "-key",
+            "./server.key",
+            "-out",
+            "./server.csr",
+            "-subj",
+            "/C=US/ST=Denial/L=Springfield/O=Dis/CN=www.example.com",
+        ],
+        check=True,
+    )
+    host_ip = socket.gethostbyname(socket.gethostname())
+    subprocess.run(
+        f'echo "subjectAltName = IP:{host_ip}" > ./extfile.cnf',
+        shell=True,
+        check=True,
+    )
+    subprocess.run(
+        [
+            "openssl",
+            "x509",
+            "-req",
+            "-in",
+            "./server.csr",
+            "-CA",
+            "./ca.crt",
+            "-CAkey",
+            "./ca.key",
+            "-CAcreateserial",
+            "-out",
+            "./server.crt",
+            "-days",
+            "365",
+            "-extfile",
+            "./extfile.cnf",
+        ],
+        check=True,
+    )
+
+    logger.info("Setting up microceph")
+    subprocess.run(["sudo", "snap", "install", "microceph", "--revision", "1169"], check=True)
+    subprocess.run(["sudo", "microceph", "cluster", "bootstrap"], check=True)
+    subprocess.run(["sudo", "microceph", "disk", "add", "loop,1G,3"], check=True)
+    subprocess.run(
+        'sudo microceph enable rgw --ssl-certificate="$(sudo base64 -w0 ./server.crt)" --ssl-private-key="$(sudo base64 -w0 ./server.key)"',
+        shell=True,
+        check=True,
+    )
+    output = subprocess.run(
+        [
+            "sudo",
+            "microceph.radosgw-admin",
+            "user",
+            "create",
+            "--uid",
+            "test",
+            "--display-name",
+            "test",
+        ],
+        capture_output=True,
+        check=True,
+        encoding="utf-8",
+    ).stdout
+    key = json.loads(output)["keys"][0]
+    key_id = key["access_key"]
+    secret_key = key["secret_key"]
+    logger.info("Creating microceph bucket")
+    for attempt in range(3):
+        try:
+            boto3.client(
+                "s3",
+                endpoint_url=f"https://{host_ip}",
+                aws_access_key_id=key_id,
+                aws_secret_access_key=secret_key,
+                verify="./ca.crt",
+            ).create_bucket(Bucket=_BUCKET)
+        except botocore.exceptions.EndpointConnectionError:
+            if attempt == 2:
+                raise
+            # microceph is not ready yet
+            logger.info("Unable to connect to microceph via S3. Retrying")
+            time.sleep(1)
+        else:
+            break
+    logger.info("Set up microceph")
+    return ConnectionInformation(key_id, secret_key, _BUCKET)
+
+
+_BUCKET = "testbucket"
+logger = logging.getLogger(__name__)
+
+
+@pytest.fixture(scope="session")
+def cloud_credentials(microceph: ConnectionInformation) -> dict[str, str]:
+    """Read cloud credentials."""
+    return {
+        "access-key": microceph.access_key_id,
+        "secret-key": microceph.secret_access_key,
+    }
+
+
+@pytest.fixture(scope="session")
+def cloud_configs(microceph: ConnectionInformation):
+    host_ip = socket.gethostbyname(socket.gethostname())
+    result = subprocess.run(
+        "sudo base64 -w0 ./ca.crt", shell=True, check=True, stdout=subprocess.PIPE, text=True
+    )
+    base64_output = result.stdout
+    return {
+        "endpoint": f"https://{host_ip}",
+        "bucket": microceph.bucket,
+        "path": "/pg",
+        "region": "",
+        "s3-uri-style": "path",
+        "tls-ca-chain": f"{base64_output}",
+    }
+
+
+@pytest.mark.group("ceph")
+@markers.amd64_only
+async def test_backup_ceph(ops_test: OpsTest, cloud_configs, cloud_credentials, charm) -> None:
+    """Build and deploy two units of PostgreSQL in microceph, test backup and restore actions."""
+    await backup_operations(
+        ops_test,
+        S3_INTEGRATOR_APP_NAME,
+        tls_certificates_app_name,
+        tls_config,
+        tls_channel,
+        cloud_credentials,
+        "ceph",
+        cloud_configs,
+        charm,
+    )