step_09_deploy_modelservice.py

"""Step 09 -- Deploy the model via the llm-d modelservice Helm chart."""

import hashlib
import time
from datetime import UTC
from pathlib import Path

from llmdbenchmark.executor.command import CommandExecutor
from llmdbenchmark.executor.context import ExecutionContext
from llmdbenchmark.executor.step import Phase, Step, StepResult
from llmdbenchmark.utilities.endpoint import resolve_direct_service_namespace


class DeployModelserviceStep(Step):
    """Deploy the model via the llm-d modelservice Helm chart."""

    def __init__(self):
        super().__init__(
            number=9,
            name="deploy_modelservice",
            description="Deploy model via modelservice Helm chart",
            phase=Phase.STANDUP,
            per_stack=True,
        )

    def should_skip(self, context: ExecutionContext) -> bool:
        return "modelservice" not in context.deployed_methods

    def execute(  # pylint: disable=too-many-branches,too-many-locals,too-many-statements
        self, context: ExecutionContext, stack_path: Path | None = None
    ) -> StepResult:
        if stack_path is None:
            return StepResult(
                step_number=self.number,
                step_name=self.name,
                success=False,
                message="No stack path provided for per-stack step",
                errors=["stack_path is required"],
            )

        errors = []
        cmd = context.require_cmd()

        namespace = context.require_namespace()
        stack_name = stack_path.name

        plan_config = self._load_stack_config(stack_path)
        release = self._require_config(plan_config, "release")
        model_id_label = plan_config.get("model_id_label", "")
        inference_port = self._require_config(  # noqa: F841
            plan_config, "vllmCommon", "inferencePort"
        )
        timeout = (
            context.modelservice_deploy_timeout
        )  # Generic timeout for all pods in step 9
        gateway_class = self._require_config(plan_config, "gateway", "className")
        direct_service_mode = gateway_class == "none"
        if direct_service_mode:
            namespace = resolve_direct_service_namespace(plan_config, namespace)

        if not context.dry_run:
            pc_error = self._check_priority_class(cmd, plan_config, context)
            if pc_error:
                errors.append(pc_error)
                return StepResult(
                    step_number=self.number,
                    step_name=self.name,
                    success=False,
                    message="PriorityClass validation failed",
                    errors=errors,
                    stack_name=stack_name,
                )

        if context.is_openshift and not context.non_admin:
            self._manage_sccs(cmd, context, plan_config, namespace)

        ms_values = self._find_yaml(stack_path, "13_ms-values")
        if not ms_values:
            return StepResult(
                step_number=self.number,
                step_name=self.name,
                success=True,
                message="No modelservice values found, skipping",
                stack_name=stack_name,
            )

        helm_dir = context.setup_helm_dir() / stack_name
        helmfile_work = helm_dir / "helmfile.yaml"

        if helmfile_work.exists():
            result = cmd.helmfile(
                "--namespace",
                namespace,
                "--selector",
                f"name={model_id_label}-ms",
                "apply",
                "-f",
                str(helmfile_work),
                "--skip-diff-on-install",
                "--skip-schema-validation",
            )
            if not result.success:
                errors.append(f"Failed to deploy modelservice: {result.stderr}")
        else:
            main_helmfile = self._find_yaml(stack_path, "10_helmfile-main")
            if main_helmfile:
                result = cmd.helmfile(
                    "--namespace",
                    namespace,
                    "--selector",
                    f"name={model_id_label}-ms",
                    "apply",
                    "-f",
                    str(main_helmfile),
                    "--skip-diff-on-install",
                    "--skip-schema-validation",
                )
                if not result.success:
                    errors.append(f"Failed to deploy modelservice: {result.stderr}")

        if direct_service_mode:
            direct_service_yaml = self._find_yaml(
                stack_path, "13a_modelservice-direct-service"
            )
            if not direct_service_yaml or not self._has_yaml_content(
                direct_service_yaml
            ):
                errors.append("Direct modelservice Service manifest was not rendered")
            else:
                result = cmd.kube("apply", "-f", str(direct_service_yaml))
                if not result.success:
                    errors.append(f"Failed to apply direct Service: {result.stderr}")

        httproute_yaml = self._find_yaml(stack_path, "08_httproute")
        if httproute_yaml and self._has_yaml_content(httproute_yaml):
            result = cmd.kube("apply", "-f", str(httproute_yaml))
            if not result.success:
                errors.append(f"Failed to apply HTTPRoute: {result.stderr}")
            elif plan_config.get("httpRoute", {}).get("mode") == "shared":
                # Shared-mode HTTPRoute references sibling InferencePools
                # that may still be installing (other stacks' `-router`
                # helm releases run in parallel with this one). Wait for
                # each referenced pool to exist so the route doesn't
                # linger in ResolvedRefs=False after step 09 returns.
                self._wait_for_sibling_inference_pools(
                    cmd,
                    context,
                    errors,
                    plan_config,
                    namespace,
                )

        if not errors:
            decode_cfg = plan_config.get("decode", {})  # noqa: F841
            expected_replicas = int(
                self._require_config(plan_config, "decode", "replicas")
            )
            is_multinode = plan_config.get("multinode", {}).get("enabled", False)
            if is_multinode:
                workers = int(
                    self._require_config(
                        plan_config, "decode", "parallelism", "workers"
                    )
                )
                expected_replicas = expected_replicas * workers

            # When decode.replicas == 0 there are no decode pods to wait for.
            if expected_replicas > 0:
                decode_wait = cmd.wait_for_pods(
                    label="llm-d.ai/role=decode",
                    namespace=namespace,
                    timeout=timeout,
                    poll_interval=10,
                    description="decode pods",
                )
                if not decode_wait.success:
                    errors.append(f"Decode pods not ready: {decode_wait.stderr}")
            else:
                context.logger.log_info(
                    "decode.replicas=0 -- skipping decode-pod wait "
                    "(FMA owns model server lifecycle when fma.enabled=true)"
                )
            if expected_replicas > 1 and not context.dry_run:
                pod_count_result = cmd.kube(
                    "get",
                    "pods",
                    "-l",
                    "llm-d.ai/role=decode",
                    "--namespace",
                    namespace,
                    "-o",
                    "jsonpath={.items[*].metadata.name}",
                )
                if pod_count_result.success:
                    actual_count = (
                        len(pod_count_result.stdout.strip().split())
                        if pod_count_result.stdout.strip()
                        else 0
                    )
                    if actual_count < expected_replicas:
                        context.logger.log_warning(
                            f"⚠️  Expected {expected_replicas} decode pods "
                            f"but found {actual_count}"
                        )
                    else:
                        context.logger.log_info(
                            f"✅ Decode pod count: {actual_count}/{expected_replicas}"
                        )

            prefill_enabled = self._require_config(plan_config, "prefill", "enabled")
            prefill_replicas = int(
                self._require_config(plan_config, "prefill", "replicas")
            )

            if prefill_enabled and prefill_replicas > 0:
                prefill_wait = cmd.wait_for_pods(
                    label="llm-d.ai/role=prefill",
                    namespace=namespace,
                    timeout=timeout,
                    poll_interval=10,
                    description="prefill pods",
                )
                if not prefill_wait.success:
                    errors.append(f"Prefill pods not ready: {prefill_wait.stderr}")

            # The llm-d-router chart migration renamed the EPP chart and
            # dropped the legacy `inferencepool=<release>-epp` label the
            # old GAIE chart added. The new
            # llm-d-router-{standalone,gateway}-dev charts apply only
            # `selectorLabels` + `modeLabels` to the Pod template; the
            # common `app.kubernetes.io/*` labels are on the Deployment,
            # not the Pod (see `charts/router/templates/_deployment.yaml`).
            #
            # The Pod's release-specific selector is gated on the chart's
            # `router.inferencePool.create` value
            # (`charts/router/templates/_helpers.tpl::selectorLabels`):
            #   - create=true  (default in BOTH chart variants)
            #                                  -> llm-d-router-gateway=<release>-epp
            #   - create=false (user opt-in)   -> llm-d-router-standalone=<release>-epp
            # Counter-intuitively, this is independent of which *chart*
            # (`-standalone-dev` vs `-gateway-dev`) is installed. The
            # chart variant only controls the outer wrapper (Envoy
            # sidecar, K8s Gateway resource), not the EPP Pod labels.
            #
            # Probe both candidate labels once each so we discover which
            # the chart actually applied, then wait on that one.
            if not direct_service_mode:
                release_epp = f"{model_id_label}-router-epp"
                chosen_label = f"llm-d-router-gateway={release_epp}"  # default
                for candidate_key in (
                    "llm-d-router-gateway",
                    "llm-d-router-standalone",
                ):
                    probe_label = f"{candidate_key}={release_epp}"
                    probe = cmd.kube(
                        "get",
                        "pods",
                        "-l",
                        probe_label,
                        "--namespace",
                        namespace,
                        "-o",
                        "jsonpath={.items[*].metadata.name}",
                        check=False,
                    )
                    if probe.success and probe.stdout.strip():
                        chosen_label = probe_label
                        break

                pool_wait = cmd.wait_for_pods(
                    label=chosen_label,
                    namespace=namespace,
                    timeout=timeout,
                    poll_interval=10,
                    description="inference pool",
                )
                if not pool_wait.success:
                    stderr_lower = pool_wait.stderr.lower()
                    if (
                        "no matching resources found" not in stderr_lower
                        and "no pods found" not in stderr_lower
                    ):
                        errors.append(f"Inference pool not ready: {pool_wait.stderr}")

        if not errors and not context.dry_run:
            self._collect_logs(cmd, context, namespace)

        if context.non_admin:
            context.logger.log_info("ℹ️  Non-admin: skipping PodMonitor creation")
        else:
            podmonitor_yaml = self._find_yaml(stack_path, "17_podmonitor")
            if not podmonitor_yaml:
                podmonitor_yaml = self._find_yaml(stack_path, "18_podmonitor")
            if podmonitor_yaml and self._has_yaml_content(podmonitor_yaml):
                # Check if PodMonitor CRD exists before attempting to apply
                crd_check = cmd.kube(
                    "get",
                    "crd",
                    "podmonitors.monitoring.coreos.com",
                    check=False,
                )
                if crd_check.success:
                    result = cmd.kube("apply", "-f", str(podmonitor_yaml))
                    if not result.success:
                        context.logger.log_warning(
                            f"PodMonitor apply failed (non-fatal): {result.stderr}"
                        )
                    else:
                        context.logger.log_info(
                            "PodMonitor created for Prometheus scraping"
                        )
                else:
                    context.logger.log_warning(
                        "PodMonitor CRD (monitoring.coreos.com/v1) not found on cluster -- "
                        "skipping PodMonitor creation. Install Prometheus Operator CRDs "
                        "or pass '--no-monitoring' to disable monitoring."
                    )
            else:
                context.logger.log_info(
                    "PodMonitor skipped (template not rendered for this configuration)"
                )

        if direct_service_mode:
            service_name = f"{model_id_label}-direct"
        elif gateway_class in ("kgateway", "agentgateway"):
            service_name = f"infra-{release}-inference-gateway"
        else:
            # Covers istio / gke / data-science-gateway-class / epponly.
            # For epponly there is no Gateway, so the EPP service is the
            # endpoint clients hit directly (port 80 -> Envoy sidecar 8081).
            service_name = f"{model_id_label}-router-epp"

        context.deployed_endpoints[stack_name] = service_name

        # In epponly mode there is no Gateway resource to label, and the
        # router chart's auto-generated route is to the EPP gRPC port which
        # we'd otherwise rewrite to point at the Gateway. Skip both.
        if gateway_class not in ("epponly", "none"):
            username = context.username or "unknown"
            cmd.kube(
                "label",
                f"gateway/infra-{release}-inference-gateway",
                f"stood-up-by={username}",
                "stood-up-from=llm-d-benchmark",
                "stood-up-via=modelservice",
                "--namespace",
                namespace,
                "--overwrite",
            )

            # GAIE Helm chart creates a route to the EPP gRPC port (wrong for
            # inference). We replace it with one targeting the gateway on
            # port 80. data-science-gateway-class manages its own route.
            if context.is_openshift and gateway_class != "data-science-gateway-class":
                route_name = f"{release}-inference-gateway-route"

                if gateway_class == "agentgateway":
                    route_service = f"infra-{release}-inference-gateway"
                else:  # istio
                    route_service = f"infra-{release}-inference-gateway-istio"

                cmd.kube(
                    "delete",
                    "route",
                    route_name,
                    "-n",
                    namespace,
                    "--ignore-not-found",
                    check=False,
                )

                cmd.kube(
                    "expose",
                    f"service/{route_service}",
                    f"--name={route_name}",
                    "--port=80",
                    "-n",
                    namespace,
                )
                context.logger.log_info(
                    f"OpenShift route '{route_name}' created to "
                    f"service/{route_service}:80"
                )

        # WVA controller is installed up-front by step_03 (admin prerequisites).
        # Here we only apply this stack's KEDA ScaledObject so the (already-running)
        # controller can manage THIS model's decode deployment via KEDA autoscaling.
        wva_config = plan_config.get("wva", {})
        if wva_config.get("enabled", False) and context.is_openshift:
            self._apply_wva_stack_resources(cmd, stack_path, errors)
            self._log_wva_stack_state(cmd, context, plan_config)

        # EPP+KEDA saturation autoscaling (controller-free alternative to WVA).
        # Per-stack: ServiceMonitor, EPP metrics RBAC, TriggerAuthentication,
        # ScaledObject.
        epp_keda_config = plan_config.get("eppKedaSaturation", {})
        if epp_keda_config.get("enabled", False) and context.is_openshift:
            self._apply_epp_keda_stack_resources(cmd, stack_path, errors)
            self._log_epp_keda_stack_state(cmd, context, plan_config)

        # Generic KEDA path — not gated on is_openshift
        keda_config = plan_config.get("keda", {})
        if keda_config.get("scaledObjects"):
            self._apply_keda_stack_resources(cmd, stack_path, errors)

        self._propagate_standup_parameters(cmd, context, plan_config)

        if not errors:
            resource_types = "deployment,service,pods"
            if not direct_service_mode:
                resource_types += ",gateway,httproute"
                if context.is_openshift:
                    resource_types += ",route"
            cmd.kube(
                "get",
                resource_types,
                "--namespace",
                namespace,
            )

        if errors:
            for err in errors:
                context.logger.log_error(f"    {err}")
            return StepResult(
                step_number=self.number,
                step_name=self.name,
                success=False,
                message="Modelservice deployment had errors",
                errors=errors,
                stack_name=stack_name,
            )

        return StepResult(
            step_number=self.number,
            step_name=self.name,
            success=True,
            message=f"Modelservice deployed for {stack_name}",
            stack_name=stack_name,
        )

    def _wait_for_sibling_inference_pools(
        self,
        cmd: CommandExecutor,
        context: ExecutionContext,
        errors: list[str],
        plan_config: dict,
        namespace: str,
    ) -> None:
        """Wait for each sibling stack's InferencePool CR to exist.

        Closes the race window between HTTPRoute apply and sibling gaie
        helm releases finishing install - without this, the shared route
        lingers in ResolvedRefs=False for the few seconds it takes other
        stacks' step 09 to finish their modelservice Helm release.

        Only runs on the shared-infra-owner stack (the one that rendered
        a non-empty HTTPRoute). Standalone siblings have no InferencePool
        and are skipped.
        """
        if context.dry_run:
            return

        siblings = plan_config.get("siblingStacks") or []
        if not siblings:
            return

        # Template uses {model_id_label}-router for the InferencePool CR
        # name. Compute each sibling's label the same way render_plans
        # does (via its Jinja filter).
        def _label_for(model_name: str) -> str:
            if not model_name:
                return ""
            model_id = model_name.replace("/", "-").replace(".", "-")
            hash_input = f"{namespace}/{model_id}" if namespace else model_id
            digest = hashlib.sha256(hash_input.encode("utf-8")).hexdigest()
            return f"{model_id[:8]}-{digest[:8]}-{model_id[-8:]}".lower()

        pool_names: list[str] = []
        for sibling in siblings:
            if not isinstance(sibling, dict) or sibling.get("standalone"):
                continue
            label = _label_for(sibling.get("modelName", ""))
            if label:
                pool_names.append(f"{label}-router")

        if not pool_names:
            return

        context.logger.log_info(
            f"    | Waiting for {len(pool_names)} sibling InferencePool(s) "
            f"to exist so the shared HTTPRoute resolves cleanly..."
        )

        # Poll up to 2 minutes. Each gaie Helm release typically finishes
        # within seconds of being applied; 2 minutes is generous.
        deadline = time.time() + 120
        missing = list(pool_names)
        while missing and time.time() < deadline:
            still_missing = []
            for name in missing:
                check = cmd.kube(
                    "get",
                    "inferencepool",
                    name,
                    "--namespace",
                    namespace,
                    "-o",
                    "name",
                    check=False,
                )
                if not check.success:
                    still_missing.append(name)
            missing = still_missing
            if missing:
                time.sleep(3)

        if missing:
            # Non-fatal: the route self-heals when pools eventually appear.
            # Log a warning so operators know the window is wider than expected.
            context.logger.log_warning(
                f"    | Shared HTTPRoute applied but {len(missing)} "
                f"InferencePool(s) still not found after 120s: "
                f"{', '.join(missing)}. Route will self-heal when they appear."
            )
        else:
            context.logger.log_info(
                f"    | All {len(pool_names)} referenced InferencePool(s) "
                f"present - shared HTTPRoute fully resolved"
            )

    def _check_priority_class(
        self,
        cmd: CommandExecutor,
        plan_config: dict,
        context: ExecutionContext,
    ) -> str | None:
        """Validate that the configured priorityClassName exists on the cluster."""
        vllm_common_pc = plan_config.get("vllmCommon", {}).get("priorityClassName", "")

        classes_to_check = set()
        for section in ["decode", "prefill"]:
            pc = plan_config.get(section, {}).get("priorityClassName") or vllm_common_pc
            if pc and pc.lower() != "none":
                classes_to_check.add(pc)

        if not classes_to_check:
            return None

        for priority_class in classes_to_check:
            result = cmd.kube(
                "get",
                "priorityclass",
                priority_class,
                "--ignore-not-found",
                "-o",
                "jsonpath={.metadata.name}",
                check=False,
            )
            if result.success and result.stdout.strip() == priority_class:
                context.logger.log_info(
                    f'PriorityClass "{priority_class}" found on cluster'
                )
                continue

            list_result = cmd.kube(
                "get",
                "priorityclass",
                "-o",
                "jsonpath={.items[*].metadata.name}",
                check=False,
            )
            available = (
                list_result.stdout.strip()
                if list_result.success
                else "(unable to list)"
            )
            return (
                f'PriorityClass "{priority_class}" does not exist on this '
                f"cluster. Available priority classes: {available}"
            )

        return None

    def _manage_sccs(
        self,
        cmd: CommandExecutor,
        context: ExecutionContext,
        plan_config: dict,
        namespace: str,
    ):
        """Add anyuid/privileged SCCs when ``runAsUser: 0`` or ``runAsGroup: 0``."""
        needs_elevated = False

        sections_to_check = [
            plan_config.get("standalone", {}),
            plan_config.get("vllmCommon", {}),
            plan_config.get("decode", {}),
            plan_config.get("prefill", {}),
        ]
        for role in ["Decode", "Prefill"]:
            sections_to_check.append(plan_config.get(f"vllmModelservice{role}", {}))

        for section in sections_to_check:
            # Check top-level securityContext
            sc = section.get("securityContext", {})
            if sc.get("runAsUser") == 0 or sc.get("runAsGroup") == 0:
                needs_elevated = True
                break
            # Check securityContext inside extraContainerConfig (used by
            # modelservice Helm chart for container-level security settings)
            extra_sc = section.get("extraContainerConfig", {}).get(
                "securityContext", {}
            )
            if extra_sc.get("runAsUser") == 0 or extra_sc.get("runAsGroup") == 0:
                needs_elevated = True
                break

        if not needs_elevated:
            context.logger.log_info(
                "ℹ️  No runAsUser:0 detected -- skipping SCC assignment"
            )
            return

        # The Helm chart creates a SA named after fullnameOverride (= model_id_label).
        # If serviceAccountOverride is set, the chart uses that instead.
        sa_override = plan_config.get("serviceAccountOverride", "")
        if sa_override:
            sa_name = sa_override
        else:
            sa_name = plan_config.get("model_id_label", "")

        context.logger.log_info(
            f"Assigning anyuid/privileged SCCs to SA '{sa_name}' "
            f"in namespace {namespace}"
        )
        for scc in ["anyuid", "privileged"]:
            cmd.kube(
                "adm",
                "policy",
                "add-scc-to-user",
                scc,
                "-z",
                sa_name,
                "-n",
                namespace,
            )

    def _collect_logs(
        self, cmd: CommandExecutor, context: ExecutionContext, namespace: str
    ):
        """Collect decode and prefill pod logs after deployment."""
        logs_dir = context.setup_logs_dir()
        for role in ["decode", "prefill"]:
            result = cmd.kube(
                "get",
                "pods",
                "-l",
                f"llm-d.ai/role={role}",
                "--namespace",
                namespace,
                "-o",
                "jsonpath={.items[*].metadata.name}",
            )
            if result.success and result.stdout.strip():
                pod_names = result.stdout.strip().split()
                for pod_name in pod_names:
                    log_result = cmd.kube(
                        "logs",
                        pod_name,
                        "--namespace",
                        namespace,
                        "--tail=-1",
                    )
                    if log_result.success:
                        log_file = logs_dir / f"{pod_name}.log"
                        log_file.write_text(log_result.stdout, encoding="utf-8")

    def _apply_wva_stack_resources(
        self,
        cmd: CommandExecutor,
        stack_path: Path,
        errors: list,
    ) -> None:
        """Apply this stack's KEDA ScaledObject to the WVA namespace.

        The WVA controller was already installed by step_03 once per unique
        wva.namespace. Here we only kubectl apply the per-stack ScaledObject so
        a single controller can manage multiple models. KEDA generates and
        manages the HPA automatically when the ScaledObject is applied.
        """
        for stem in ("28_wva-scaledobject",):
            yaml_path = self._find_yaml(stack_path, stem)
            if not (yaml_path and self._has_yaml_content(yaml_path)):
                continue
            result = cmd.kube("apply", "-f", str(yaml_path))
            if not result.success:
                errors.append(f"Failed to apply {stem}: {result.stderr}")

    def _log_wva_stack_state(
        self,
        cmd: CommandExecutor,
        context: ExecutionContext,
        plan_config: dict,
    ) -> None:
        """Log the current state of this stack's KEDA ScaledObject + HPA.

        Lets the standup output show what got created (ScaledObject status,
        HPA TARGETS/REPLICAS, etc.) without needing follow-up ``oc get``.
        Best-effort - failures here don't fail step_09. KEDA's generated HPA
        carries the same name as the ScaledObject.
        """
        wva_cfg = plan_config.get("wva", {}) or {}
        wva_ns = wva_cfg.get("namespace") or plan_config.get("namespace", {}).get(
            "name", ""
        )
        model_id_label = plan_config.get("model_id_label", "")
        if not (wva_ns and model_id_label):
            return

        resource_name = f"{model_id_label}-decode"

        for kind, label in (
            ("scaledobject.keda.sh", "ScaledObject"),
            ("hpa", "HorizontalPodAutoscaler"),
        ):
            result = cmd.kube(
                "get",
                kind,
                resource_name,
                "--namespace",
                wva_ns,
                check=False,
            )
            if result.success and result.stdout.strip():
                context.logger.log_info(f"📋 {label} state in ns/{wva_ns}:")
                for line in result.stdout.rstrip().splitlines():
                    context.logger.log_info(f"    {line}")
            else:
                context.logger.log_warning(
                    f"Could not query {label}/{resource_name} for state log: "
                    f"{result.stderr.strip()[:200] or '(empty)'}"
                )

    def _apply_epp_keda_stack_resources(
        self,
        cmd: CommandExecutor,
        stack_path: Path,
        errors: list,
    ) -> None:
        """Apply EPP+KEDA saturation autoscaling per-stack resources.

        EPP monitoring setup (ServiceMonitor, RBAC) is installed by step_03
        (admin prerequisites, once per namespace). Here we apply the
        per-stack ScaledObject so KEDA can query metrics and auto-generate
        the HPA. Multiple models scale independently via their own ScaledObjects.
        """
        for stem in ("30_keda-scaledobject",):
            yaml_path = self._find_yaml(stack_path, stem)
            if not (yaml_path and self._has_yaml_content(yaml_path)):
                continue
            result = cmd.kube("apply", "-f", str(yaml_path))
            if not result.success:
                errors.append(f"Failed to apply {stem}: {result.stderr}")

    def _apply_keda_stack_resources(
        self,
        cmd: CommandExecutor,
        stack_path: Path,
        errors: list,
    ) -> None:
        """Apply per-stack generic KEDA ScaledObjects (template 31).

        step_03 applied the TriggerAuthentication (bearer-secret) and the
        ScaledObjects template once for the first stack in each namespace.
        This method re-applies the ScaledObjects template for each additional
        stack (idempotent kubectl apply). Not gated on is_openshift.
        """
        yaml_path = self._find_yaml(stack_path, "27_keda-scaledobjects")
        if not (yaml_path and self._has_yaml_content(yaml_path)):
            return
        result = cmd.kube("apply", "-f", str(yaml_path), check=False)
        if not result.success:
            errors.append(
                f"Failed to apply keda ScaledObjects for {stack_path.name}: "
                f"{result.stderr}"
            )

    def _log_epp_keda_stack_state(
        self,
        cmd: CommandExecutor,
        context: ExecutionContext,
        plan_config: dict,
    ) -> None:
        """Log the current state of this stack's EPP+KEDA ScaledObject + HPA.

        Lets the standup output show what got created (ScaledObject
        status, HPA TARGETS/REPLICAS, etc.) without needing follow-up ``oc get``.
        Best-effort - failures here don't fail step_09.
        """
        epp_keda_cfg = plan_config.get("eppKedaSaturation", {}) or {}
        epp_keda_ns = epp_keda_cfg.get("namespace") or plan_config.get(
            "namespace", {}
        ).get("name", "")
        model_id_label = plan_config.get("model_id_label", "")
        fma_enabled = plan_config.get("fma", {}).get("enabled", False)
        hpa_name = f"{model_id_label}-{'fma' if fma_enabled else 'decode'}-saturation"

        for label, resource_name in (
            ("ScaledObject", hpa_name + "-saturation"),
            ("HPA", "keda-hpa-" + hpa_name + "-saturation"),
        ):
            result = cmd.kube(
                "get",
                resource_name.split("-")[0].lower(),
                resource_name,
                "-n",
                epp_keda_ns,
                "-o",
                "wide",
                check=False,
            )
            if result.success and result.stdout.strip():
                context.logger.log_info(f"📋 {label} state in ns/{epp_keda_ns}:")
                for line in result.stdout.rstrip().splitlines():
                    context.logger.log_info(f"    {line}")
            else:
                context.logger.log_warning(
                    f"Could not query {label}/{resource_name} for state log: "
                    f"{result.stderr.strip()[:200] or '(empty)'}"
                )

    def _propagate_standup_parameters(
        self, cmd: CommandExecutor, context: ExecutionContext, plan_config: dict
    ):
        """Persist deploy metadata as a ConfigMap so run-phase steps can read it."""
        from datetime import datetime

        from llmdbenchmark import __version__

        harness_ns = context.harness_namespace or context.require_namespace()
        cm_name = "llm-d-benchmark-standup-parameters"

        params = {
            "tool_name": "llm-d-benchmark",
            "tool_version": __version__,
            "deployed_by": context.username or "unknown",
            "deployed_at": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
            "cluster_name": context.cluster_name or "",
            "platform_type": context.platform_type,
            "namespace": context.namespace or "",
            "harness_namespace": harness_ns,
            "deploy_methods": ",".join(context.deployed_methods),
        }

        if plan_config:
            params["model_name"] = self._require_config(plan_config, "model", "name")
            params["model_short_name"] = self._require_config(
                plan_config, "model", "shortName"
            )
            params["model_huggingface_id"] = plan_config.get("model", {}).get(
                "huggingfaceId", ""
            )
            params["inference_port"] = str(
                self._require_config(plan_config, "vllmCommon", "inferencePort")
            )
            params["release"] = self._require_config(plan_config, "release")
            params["decode_replicas"] = str(
                self._require_config(plan_config, "decode", "replicas")
            )
            params["prefill_enabled"] = str(
                self._require_config(plan_config, "prefill", "enabled")
            ).lower()
            params["prefill_replicas"] = str(
                self._require_config(plan_config, "prefill", "replicas")
            )

            # Accelerator model + per-role parallelism, so the benchmark report
            # can identify the hardware and topology instead of assuming.
            accel = plan_config.get("decode", {}).get("acceleratorType", {}) or {}
            params["accelerator_model"] = accel.get("labelValue", "")
            for role in ("prefill", "decode"):
                par = plan_config.get(role, {}).get("parallelism", {}) or {}
                for cm_key, cfg_key in (
                    ("tensor", "tensor"),
                    ("data", "data"),
                    ("data_local", "dataLocal"),
                    ("workers", "workers"),
                ):
                    params[f"{role}_{cm_key}_parallelism"] = str(par.get(cfg_key, 1))
            # Gateway/LWS topology, so the report can list those components.
            params["gateway_class"] = plan_config.get("gateway", {}).get(
                "className", ""
            )
            params["multinode_enabled"] = str(
                plan_config.get("multinode", {}).get("enabled", False)
            ).lower()
            chart_versions = plan_config.get("chartVersions", {})
            if chart_versions:
                params["chart_version_modelservice"] = chart_versions.get(
                    "llmDModelservice", ""
                )
                params["chart_version_inference_pool"] = chart_versions.get(
                    "inferencePool", ""
                )
                params["chart_version_gaie"] = chart_versions.get("gaie", "")
                params["chart_version_llm_d_infra"] = chart_versions.get(
                    "llmDInfra", ""
                )

            # Container images used in this deployment
            images = plan_config.get("images", {})
            vllm_img = images.get("vllm", {})
            if vllm_img:
                repo = vllm_img.get("repository", "")
                tag = vllm_img.get("tag", "")
                params["image_vllm"] = f"{repo}:{tag}" if repo else ""

            decode_img = plan_config.get("decode", {}).get("image", {})
            if decode_img and decode_img.get("repository"):
                params["image_decode"] = (
                    f"{decode_img['repository']}:{decode_img.get('tag', 'latest')}"
                )

        literal_args = []
        for key, value in params.items():
            literal_args.append(f"--from-literal={key}={value}")

        create_args = (
            [
                "create",
                "configmap",
                cm_name,
                "--namespace",
                harness_ns,
            ]
            + literal_args
            + ["--dry-run=client", "-o", "yaml"]
        )

        result = cmd.kube(*create_args)
        if result.success:
            yaml_path = context.setup_yamls_dir() / "standup-parameters.yaml"
            yaml_path.write_text(result.stdout, encoding="utf-8")
            apply_result = cmd.kube("apply", "-f", str(yaml_path))
            if apply_result.success:
                context.logger.log_info(
                    f"📋 Deployment metadata to configmap/{cm_name} in ns/{harness_ns}"
                )
                context.logger.log_info(
                    f"   {cmd._kube_bin} get configmap {cm_name} "
                    f"-n {harness_ns} -o yaml"
                )