|
| 1 | +"""A Ray operator process proxy.""" |
| 2 | + |
| 3 | +# Internal implementation at Apple |
| 4 | +from __future__ import annotations |
| 5 | + |
| 6 | +from typing import Any |
| 7 | + |
| 8 | +from kubernetes import client |
| 9 | + |
| 10 | +from ..kernels.remotemanager import RemoteKernelManager |
| 11 | +from .k8s import KubernetesProcessProxy |
| 12 | + |
| 13 | + |
| 14 | +class RayOperatorProcessProxy(KubernetesProcessProxy): |
| 15 | + """Ray operator process proxy.""" |
| 16 | + |
| 17 | + object_kind = "RayCluster" |
| 18 | + |
| 19 | + def __init__(self, kernel_manager: RemoteKernelManager, proxy_config: dict): |
| 20 | + """Initialize the proxy.""" |
| 21 | + super().__init__(kernel_manager, proxy_config) |
| 22 | + self.group = "ray.io" |
| 23 | + self.version = "v1alpha1" |
| 24 | + self.plural = "rayclusters" |
| 25 | + |
| 26 | + async def launch_process( |
| 27 | + self, kernel_cmd: str, **kwargs: dict[str, Any] | None |
| 28 | + ) -> RayOperatorProcessProxy: |
| 29 | + """Launch the process for a kernel.""" |
| 30 | + self.kernel_resource_name = self._determine_kernel_pod_name(**kwargs) |
| 31 | + kwargs["env"]["KERNEL_RESOURCE_NAME"] = self.kernel_resource_name |
| 32 | + kwargs["env"]["KERNEL_CRD_GROUP"] = self.group |
| 33 | + kwargs["env"]["KERNEL_CRD_VERSION"] = self.version |
| 34 | + kwargs["env"]["KERNEL_CRD_PLURAL"] = self.plural |
| 35 | + |
| 36 | + await super().launch_process(kernel_cmd, **kwargs) |
| 37 | + return self |
| 38 | + |
| 39 | + def get_container_status(self, iteration: int | None) -> str: |
| 40 | + """Determines submitted Ray application status and returns unified pod state. |
| 41 | +
|
| 42 | + This method returns the pod status (not CRD status) to maintain compatibility |
| 43 | + with the base class lifecycle management. The RayCluster CRD state is checked |
| 44 | + first to ensure the cluster is healthy, but we return pod states that the |
| 45 | + base class understands: 'pending', 'running', 'failed', etc. |
| 46 | + """ |
| 47 | + application_state = None |
| 48 | + head_pod_status = None |
| 49 | + application_state = self._get_application_state() |
| 50 | + if application_state: |
| 51 | + self.log.debug( |
| 52 | + f">>> ray_operator.get_container_status: application_state {application_state}" |
| 53 | + ) |
| 54 | + |
| 55 | + # Check for CRD-level errors first |
| 56 | + if application_state in self.get_error_states(): |
| 57 | + error_message = ( |
| 58 | + f"CRD submission for kernel {self.kernel_id} failed with state: {application_state}" |
| 59 | + ) |
| 60 | + self.log.error(error_message) |
| 61 | + return "failed" # Return pod state, not CRD state |
| 62 | + |
| 63 | + # If CRD is not ready yet, return "pending" to indicate still launching |
| 64 | + if application_state != "ready": |
| 65 | + self.log.debug( |
| 66 | + f">>> ray_operator.get_container_status: CRD not ready yet, state={application_state}" |
| 67 | + ) |
| 68 | + return "pending" |
| 69 | + |
| 70 | + # CRD is ready, now check the actual pod status |
| 71 | + kernel_label_selector = "kernel_id=" + self.kernel_id + ",component=kernel" |
| 72 | + ret = None |
| 73 | + try: |
| 74 | + ret = client.CoreV1Api().list_namespaced_pod( |
| 75 | + namespace=self.kernel_namespace, label_selector=kernel_label_selector |
| 76 | + ) |
| 77 | + except client.rest.ApiException as e: |
| 78 | + if e.status == 404: |
| 79 | + self.log.debug("Resetting cluster connection info as cluster deleted") |
| 80 | + self._reset_connection_info() |
| 81 | + return None |
| 82 | + |
| 83 | + if ret and ret.items: |
| 84 | + pod_info = ret.items[0] |
| 85 | + self.log.debug( |
| 86 | + f"Cluster status {application_state}, pod status {pod_info.status.phase.lower()}" |
| 87 | + ) |
| 88 | + if pod_info.status: |
| 89 | + head_pod_status = pod_info.status.phase.lower() |
| 90 | + self.log.debug( |
| 91 | + f">>> ray_operator.get_container_status: pod_status {head_pod_status}" |
| 92 | + ) |
| 93 | + if head_pod_status == "running": |
| 94 | + self.log.debug( |
| 95 | + f"Pod Info name:{pod_info.metadata.name}, pod ip {pod_info.status.pod_ip}, host {self.container_name}" |
| 96 | + ) |
| 97 | + self.container_name = pod_info.metadata.name |
| 98 | + self.assigned_ip = pod_info.status.pod_ip |
| 99 | + self.assigned_host = self.container_name |
| 100 | + self.assigned_node_ip = pod_info.status.host_ip |
| 101 | + |
| 102 | + # only log if iteration is not None (otherwise poll() is too noisy) |
| 103 | + # check for running state to avoid double logging with superclass |
| 104 | + if iteration and head_pod_status != 'running': |
| 105 | + self.log.debug( |
| 106 | + f"{iteration}: Waiting from CRD status from resource manager {self.object_kind.lower()} in " |
| 107 | + f"namespace '{self.kernel_namespace}'. Name: '{self.kernel_resource_name}', " |
| 108 | + f"Status: CRD='{application_state}', Pod='{head_pod_status}', KernelID: '{self.kernel_id}'" |
| 109 | + ) |
| 110 | + |
| 111 | + # KEY FIX: Return pod status (not CRD state) so base class poll() works correctly |
| 112 | + final_status = head_pod_status if head_pod_status else "pending" |
| 113 | + self.log.debug( |
| 114 | + f">>> ray_operator.get_container_status: returning pod_status={final_status} " |
| 115 | + f"(CRD state was {application_state})" |
| 116 | + ) |
| 117 | + return final_status |
| 118 | + |
| 119 | + def delete_managed_object(self, termination_stati: list[str]) -> bool: |
| 120 | + """Deletes the object managed by this process-proxy |
| 121 | +
|
| 122 | + A return value of True indicates the object is considered deleted, |
| 123 | + otherwise a False or None value is returned. |
| 124 | +
|
| 125 | + Note: the caller is responsible for handling exceptions. |
| 126 | + """ |
| 127 | + delete_status = client.CustomObjectsApi().delete_namespaced_custom_object( |
| 128 | + self.group, |
| 129 | + self.version, |
| 130 | + self.kernel_namespace, |
| 131 | + self.plural, |
| 132 | + self.kernel_resource_name, |
| 133 | + grace_period_seconds=0, |
| 134 | + propagation_policy="Background", |
| 135 | + ) |
| 136 | + |
| 137 | + result = delete_status and delete_status.get("status", None) in termination_stati |
| 138 | + if result: |
| 139 | + self._reset_connection_info() |
| 140 | + return result |
| 141 | + |
| 142 | + def get_initial_states(self) -> set: |
| 143 | + """Return list of states indicating container is starting (includes running). |
| 144 | +
|
| 145 | + Note: We return pod states (not CRD states) to maintain compatibility |
| 146 | + with the base class poll() implementation, which checks if the status |
| 147 | + returned by get_container_status() is in this set. |
| 148 | + """ |
| 149 | + return ["pending", "running"] |
| 150 | + |
| 151 | + def get_error_states(self) -> set: |
| 152 | + """Return list of states indicating RayCluster has failed.""" |
| 153 | + # Ray doesn't typically use "failed" state, but we'll include common error states |
| 154 | + return {"failed", "error", "unhealthy"} |
| 155 | + |
| 156 | + def _get_ray_cluster_status(self) -> dict: |
| 157 | + try: |
| 158 | + return client.CustomObjectsApi().get_namespaced_custom_object( |
| 159 | + self.group, |
| 160 | + self.version, |
| 161 | + self.kernel_namespace, |
| 162 | + self.plural, |
| 163 | + self.kernel_resource_name, |
| 164 | + ) |
| 165 | + except client.rest.ApiException as e: |
| 166 | + if e.status == 404: |
| 167 | + self.log.debug("Resetting cluster connection info as cluster deleted") |
| 168 | + self._reset_connection_info() |
| 169 | + return None |
| 170 | + |
| 171 | + def _get_application_state(self): |
| 172 | + custom_resource = self._get_ray_cluster_status() |
| 173 | + |
| 174 | + if custom_resource is None: |
| 175 | + return None |
| 176 | + |
| 177 | + if 'status' not in custom_resource or 'state' not in custom_resource['status']: |
| 178 | + return None |
| 179 | + |
| 180 | + return custom_resource['status']['state'].lower() |
| 181 | + |
| 182 | + def _get_pod_status(self) -> str: |
| 183 | + """Get the current status of the kernel pod. |
| 184 | + Returns |
| 185 | + ------- |
| 186 | + str |
| 187 | + The pod status in lowercase (e.g., 'pending', 'running', 'failed', 'unknown'). |
| 188 | + """ |
| 189 | + pod_status = "unknown" |
| 190 | + kernel_label_selector = "kernel_id=" + self.kernel_id + ",component=kernel" |
| 191 | + ret = client.CoreV1Api().list_namespaced_pod( |
| 192 | + namespace=self.kernel_namespace, label_selector=kernel_label_selector |
| 193 | + ) |
| 194 | + if ret and ret.items: |
| 195 | + pod_info = ret.items[0] |
| 196 | + self.container_name = pod_info.metadata.name |
| 197 | + if pod_info.status: |
| 198 | + pod_status = pod_info.status.phase.lower() |
| 199 | + self.log.debug(f">>> k8s._get_pod_status: {pod_status}") |
| 200 | + |
| 201 | + return pod_status |
| 202 | + |
| 203 | + def _reset_connection_info(self): |
| 204 | + """Reset all connection-related attributes to their initial state. |
| 205 | + This is typically called when a cluster is deleted or connection is lost. |
| 206 | + """ |
| 207 | + |
| 208 | + self.assigned_host = None |
| 209 | + self.container_name = "" |
| 210 | + self.assigned_node_ip = None |
| 211 | + self.assigned_ip = None |
0 commit comments