[DPE-8731] Add unit kill ha test (#31)

* Add unit kill test.

* Add seed check in integration kill test.

---------

Co-authored-by: Vladyslav Tarasenko <vladyslav.tarasenko@canonical.com>

Author: BON4

tests/integration/test_kill.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+# Copyright 2025 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+import logging
+from pathlib import Path
+
+import jubilant
+import yaml
+
+from integration.helpers.cassandra import OPERATOR_PASSWORD
+from integration.helpers.continuous_writes import ContinuousWrites
+from integration.helpers.juju import (
+    app_secret_extract,
+    get_hosts,
+    get_leader_unit,
+    get_non_leader_units,
+    scale_sequentially_to,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def test_deploy(juju: jubilant.Juju, cassandra_charm: Path, app_name: str) -> None:
+    juju.deploy(
+        cassandra_charm,
+        app=app_name,
+        config={"profile": "testing"},
+        num_units=3,
+    )
+    juju.wait(
+        ready=lambda status: jubilant.all_agents_idle(status) and jubilant.all_active(status),
+        delay=20,
+        timeout=2400,
+    )
+
+
+def test_seed_primary(juju: jubilant.Juju, app_name: str) -> None:
+    _, leader_status = get_leader_unit(juju, app_name)
+    for unit, _ in juju.status().apps[app_name].units.items():
+        assert f"{leader_status.public_address}:7000" in get_seeds(juju, unit)
+
+
+def test_kill_primary(
+    juju: jubilant.Juju, app_name: str, continuous_writes: ContinuousWrites
+) -> None:
+    continuous_writes.start(
+        hosts=get_hosts(juju, app_name),
+        password=app_secret_extract(juju, app_name, OPERATOR_PASSWORD),
+        replication_factor=3,
+    )
+
+    leader, _ = get_leader_unit(juju, app_name)
+    kill_unit(juju, leader)
+
+    juju.wait(
+        ready=lambda status: jubilant.all_agents_idle(status) and jubilant.all_active(status),
+        delay=20,
+        timeout=1800,
+    )
+    continuous_writes.assert_new_writes()
+
+    new_leader, _ = get_leader_unit(juju, app_name)
+    assert leader != new_leader
+
+
+def test_seed_new_primary(juju: jubilant.Juju, app_name: str) -> None:
+    _, leader_status = get_leader_unit(juju, app_name)
+    for unit, _ in juju.status().apps[app_name].units.items():
+        assert f"{leader_status.public_address}:7000" in get_seeds(juju, unit)
+
+
+def test_kill_subordinate(
+    juju: jubilant.Juju, app_name: str, continuous_writes: ContinuousWrites
+) -> None:
+    scale_sequentially_to(juju, app_name, 3)
+
+    subordinate = get_non_leader_units(juju, app_name)[0]
+    kill_unit(juju, subordinate)
+
+    juju.wait(
+        ready=lambda status: jubilant.all_agents_idle(status) and jubilant.all_active(status),
+        delay=20,
+        timeout=1200,
+    )
+    continuous_writes.stop_and_assert_writes()
+
+
+def kill_unit(juju: jubilant.Juju, unit: str) -> None:
+    juju.cli("remove-unit", "--force", "--no-wait", "--no-prompt", "--destroy-storage", unit)
+
+
+def get_seeds(juju: jubilant.Juju, unit: str) -> list[str]:
+    config_file = juju.ssh(
+        unit, "cat /var/snap/charmed-cassandra/current/etc/cassandra/cassandra.yaml"
+    )
+    config = yaml.load(config_file, yaml.Loader)
+    return config["seed_provider"][0]["parameters"][0]["seeds"].split(",")

tests/spread/test_kill.py/task.yaml

@@ -0,0 +1,5 @@
+summary: test_kill.py
+execute: |
+  tox run -e integration-kill -- --alluredir="$SPREAD_TASK/allure-results" --model-name=test-cas --keep-models
+artifacts:
+  - allure-results

tox.ini

@@ -67,7 +67,7 @@ commands =
                  {[vars]tests_path}/unit
     poetry run coverage report
 
-[testenv:integration-{authentication,charm,config,restart,multinode,scaling,tls,cos,provider,multicluster,network}]
+[testenv:integration-{authentication,charm,config,restart,multinode,scaling,tls,cos,provider,multicluster,network,kill}]
 description = Run integration tests
 pass_env =
     CI
@@ -83,6 +83,7 @@ set_env =
     provider: TESTFILE=test_provider.py
     multicluster: TESTFILE=test_multicluster.py
     network: TESTFILE=test_network.py
+    kill: TESTFILE=test_kill.py
 commands_pre =
     poetry install --only integration
 commands =