keep telemetry stack during upgrade (#1530)

Author: jcchr

platform/services/installer/app/commands/upgrade.py

@@ -52,6 +52,7 @@
 from platform_stages.steps.install_system_packages import install_system_packages
 from platform_stages.upgrade import upgrade_platform
 from platform_utils.grafana import flush_ingesters as flush_lgtm_stack
+from platform_utils.k8s import check_if_telemetry_stack_is_installed
 from platform_utils.management.data_folder import backup_data_folder, restore_data_folder
 from platform_utils.management.management import restore_platform, stop_platform
 from platform_utils.management.state import (
@@ -499,6 +500,10 @@ def upgrade(config_file: str | None = None, install_telemetry_stack: bool = Fals
     create_logs_dir()
     configure_logging()
     logger.info(f"{get_target_product_build()} {UpgradeCmdTexts.start_message}")
+
+    if not install_telemetry_stack:
+        install_telemetry_stack = check_if_telemetry_stack_is_installed()
+
     config = UpgradeConfig(interactive_mode=not bool(config_file), install_telemetry_stack=install_telemetry_stack)
     run_initial_checks(config)
     if config_file:

platform/services/installer/app/constants/platform.py

@@ -26,3 +26,5 @@
 GPU_PROVIDER_INTEL_ARC_A = "intel-arc-a"
 
 DEFAULT_HISTORY_MAX = 3
+
+TELEMETRY_STACK_COMPONENT = "observability"

platform/services/installer/app/platform_utils/k8s.py

@@ -20,6 +20,7 @@
 from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed
 
 from constants.paths import K3S_KUBECONFIG_PATH
+from constants.platform import PLATFORM_NAMESPACE, TELEMETRY_STACK_COMPONENT
 from platform_stages.steps.errors import FailedJobError, RestartDeploymentError
 from platform_utils.kube_config_handler import KubernetesConfigHandler
 
@@ -257,3 +258,25 @@ def delete_job(batch_api: BatchV1Api, namespace: str, job_name: str) -> None:
     except client.exceptions.ApiException as ex:
         logger.error(f"Failed to delete job '{job_name}': {ex}")
         return
+
+
+def check_if_telemetry_stack_is_installed() -> bool:
+    """
+    Check if telemetry stack is installed by verifying the existence of its deployment.
+    """
+    return _deployment_exists(name=TELEMETRY_STACK_COMPONENT, namespace=PLATFORM_NAMESPACE)
+
+
+def _deployment_exists(name: str, namespace: str) -> bool:
+    """
+    Check if a deployment with the given name exists in the specified namespace.
+    """
+    KubernetesConfigHandler(kube_config=K3S_KUBECONFIG_PATH)
+    api = client.AppsV1Api()
+    try:
+        api.read_namespaced_deployment(name=name, namespace=namespace)
+        return True
+    except ApiException as ex:
+        if ex.status == 404:
+            return False
+        raise

platform/services/installer/tests/unit/commands/test_upgrade_command.py

@@ -37,6 +37,10 @@ def test_upgrade_interactive(mocker):
     prompt_for_upgrade_config_mock: Mock = mocker.patch("commands.upgrade.prompt_for_upgrade_config")
     display_final_confirmation_mock: Mock = mocker.patch("commands.upgrade.display_final_confirmation")
     create_logs_dir_mock: Mock = mocker.patch("commands.upgrade.create_logs_dir")
+    check_if_telemetry_stack_is_installed_mock: Mock = mocker.patch(
+        "commands.upgrade.check_if_telemetry_stack_is_installed"
+    )
+    check_if_telemetry_stack_is_installed_mock.return_value = False
     is_kubernetes_running_on_k3s: Mock = mocker.patch("commands.upgrade.is_kubernetes_running_on_k3s")
     is_kubernetes_running_on_k3s.return_value = True
 
@@ -48,6 +52,7 @@ def test_upgrade_interactive(mocker):
     assert execute_upgrade_mock.call_count == 1
     assert display_final_confirmation_mock.call_count == 1
     assert not result.exception
+    assert check_if_telemetry_stack_is_installed_mock.call_count == 1
     create_logs_dir_mock.assert_called_once_with()
 
 
@@ -65,6 +70,9 @@ def test_upgrade_config_file(mocker, tmpdir, is_grafana_installed):
     create_logs_dir_mock: Mock = mocker.patch("commands.upgrade.create_logs_dir")
     is_kubernetes_running_on_k3s: Mock = mocker.patch("commands.upgrade.is_kubernetes_running_on_k3s")
     is_kubernetes_running_on_k3s.return_value = True
+    check_if_telemetry_stack_is_installed_mock: Mock = mocker.patch(
+        "commands.upgrade.check_if_telemetry_stack_is_installed", side_effect=[is_grafana_installed]
+    )
 
     kubeconfig_path = tmpdir.join("kubeconfig")
     kubeconfig_path.write("foo")
@@ -81,6 +89,7 @@ def test_upgrade_config_file(mocker, tmpdir, is_grafana_installed):
 
     run_initial_checks_mock.assert_called_once()
     run_upgrade_checks_mock.assert_called_once()
+    check_if_telemetry_stack_is_installed_mock.assert_called_once()
     is_grafana_installed_mock.assert_called_once_with(str(kubeconfig_path))
     create_logs_dir_mock.assert_called_once_with()
     assert execute_upgrade_mock.call_count == 1
@@ -91,12 +100,16 @@ def test_initial_check_failure(mocker):
     run_initial_checks_mock: Mock = mocker.patch(
         "commands.upgrade.run_initial_checks", side_effect=CumulativeCheckError
     )
+    check_if_telemetry_stack_is_installed_mock: Mock = mocker.patch(
+        "commands.upgrade.check_if_telemetry_stack_is_installed", side_effect=[False]
+    )
 
     runner = CliRunner()
     result = runner.invoke(upgrade)
 
     assert result.exception
     run_initial_checks_mock.assert_called_once()
+    check_if_telemetry_stack_is_installed_mock.assert_called_once()
 
 
 def test_upgrade_check_failure(mocker):
@@ -109,13 +122,18 @@ def test_upgrade_check_failure(mocker):
     is_kubernetes_running_on_k3s: Mock = mocker.patch("commands.upgrade.is_kubernetes_running_on_k3s")
     is_kubernetes_running_on_k3s.return_value = True
 
+    check_if_telemetry_stack_is_installed_mock: Mock = mocker.patch(
+        "commands.upgrade.check_if_telemetry_stack_is_installed", side_effect=[True]
+    )
+
     runner = CliRunner()
     result = runner.invoke(upgrade)
 
     run_initial_checks_mock.assert_called_once()
     run_upgrade_checks_mock.assert_called_once()
     create_logs_dir_mock.assert_called_once_with()
     prompt_for_upgrade_config_mock.assert_called_once()
+    check_if_telemetry_stack_is_installed_mock.assert_called_once()
     assert execute_upgrade_mock.call_count == 0
     assert is_grafana_installed_mock.call_count == 0
     assert result.exception
@@ -128,6 +146,9 @@ def test_upgrade_execution_failure(mocker):
     prompt_for_upgrade_config_mock: Mock = mocker.patch("commands.upgrade.prompt_for_upgrade_config")
     display_final_confirmation_mock: Mock = mocker.patch("commands.upgrade.display_final_confirmation")
     create_logs_dir_mock: Mock = mocker.patch("commands.upgrade.create_logs_dir")
+    check_if_telemetry_stack_is_installed_mock: Mock = mocker.patch(
+        "commands.upgrade.check_if_telemetry_stack_is_installed", side_effect=[True]
+    )
     is_kubernetes_running_on_k3s: Mock = mocker.patch("commands.upgrade.is_kubernetes_running_on_k3s")
     is_kubernetes_running_on_k3s.return_value = True
 
@@ -137,6 +158,7 @@ def test_upgrade_execution_failure(mocker):
     run_initial_checks_mock.assert_called_once()
     run_upgrade_checks_mock.assert_called_once()
     create_logs_dir_mock.assert_called_once_with()
+    check_if_telemetry_stack_is_installed_mock.assert_called_once()
     assert prompt_for_upgrade_config_mock.call_count == 1
     assert execute_upgrade_mock.call_count == 1
     assert display_final_confirmation_mock.call_count == 1