Merge pull request kubernetes#126243 from SergeyKanzhelev/devicePluginFailures

Implement resource health in pod status (KEP 4680)

Author: k8s-ci-robot

api/openapi-spec/swagger.json

@@ -1525,6 +1525,24 @@
             "description": "AllocatedResources represents the compute resources allocated for this container by the node. Kubelet sets this value to Container.Resources.Requests upon successful pod admission and after successfully admitting desired pod resize.",
             "type": "object"
           },
+          "allocatedResourcesStatus": {
+            "description": "AllocatedResourcesStatus represents the status of various resources allocated for this Pod.",
+            "items": {
+              "allOf": [
+                {
+                  "$ref": "#/components/schemas/io.k8s.api.core.v1.ResourceStatus"
+                }
+              ],
+              "default": {}
+            },
+            "type": "array",
+            "x-kubernetes-list-map-keys": [
+              "name"
+            ],
+            "x-kubernetes-list-type": "map",
+            "x-kubernetes-patch-merge-key": "name",
+            "x-kubernetes-patch-strategy": "merge"
+          },
           "containerID": {
             "description": "ContainerID is the ID of the container in the format '<type>://<container_id>'. Where type is a container runtime identifier, returned from Version call of CRI API (for example \"containerd\").",
             "type": "string"
@@ -6598,6 +6616,24 @@
         "type": "object",
         "x-kubernetes-map-type": "atomic"
       },
+      "io.k8s.api.core.v1.ResourceHealth": {
+        "description": "ResourceHealth represents the health of a resource. It has the latest device health information. This is a part of KEP https://kep.k8s.io/4680 and historical health changes are planned to be added in future iterations of a KEP.",
+        "properties": {
+          "health": {
+            "description": "Health of the resource. can be one of:\n - Healthy: operates as normal\n - Unhealthy: reported unhealthy. We consider this a temporary health issue\n              since we do not have a mechanism today to distinguish\n              temporary and permanent issues.\n - Unknown: The status cannot be determined.\n            For example, Device Plugin got unregistered and hasn't been re-registered since.\n\nIn future we may want to introduce the PermanentlyUnhealthy Status.",
+            "type": "string"
+          },
+          "resourceID": {
+            "default": "",
+            "description": "ResourceID is the unique identifier of the resource. See the ResourceID type for more information.",
+            "type": "string"
+          }
+        },
+        "required": [
+          "resourceID"
+        ],
+        "type": "object"
+      },
       "io.k8s.api.core.v1.ResourceQuota": {
         "description": "ResourceQuota sets aggregate quota restrictions enforced per namespace",
         "properties": {
@@ -6777,6 +6813,35 @@
         },
         "type": "object"
       },
+      "io.k8s.api.core.v1.ResourceStatus": {
+        "properties": {
+          "name": {
+            "default": "",
+            "description": "Name of the resource. Must be unique within the pod and match one of the resources from the pod spec.",
+            "type": "string"
+          },
+          "resources": {
+            "description": "List of unique Resources health. Each element in the list contains an unique resource ID and resource health. At a minimum, ResourceID must uniquely identify the Resource allocated to the Pod on the Node for the lifetime of a Pod. See ResourceID type for it's definition.",
+            "items": {
+              "allOf": [
+                {
+                  "$ref": "#/components/schemas/io.k8s.api.core.v1.ResourceHealth"
+                }
+              ],
+              "default": {}
+            },
+            "type": "array",
+            "x-kubernetes-list-map-keys": [
+              "resourceID"
+            ],
+            "x-kubernetes-list-type": "map"
+          }
+        },
+        "required": [
+          "name"
+        ],
+        "type": "object"
+      },
       "io.k8s.api.core.v1.SELinuxOptions": {
         "description": "SELinuxOptions are the labels to be applied to the container",
         "properties": {

api/openapi-spec/v3/api__v1_openapi.json

@@ -813,6 +813,17 @@ func dropDisabledPodStatusFields(podStatus, oldPodStatus *api.PodStatus, podSpec
 		}
 	}
 
+	if !utilfeature.DefaultFeatureGate.Enabled(features.ResourceHealthStatus) {
+		setAllocatedResourcesStatusToNil := func(csl []api.ContainerStatus) {
+			for i := range csl {
+				csl[i].AllocatedResourcesStatus = nil
+			}
+		}
+		setAllocatedResourcesStatusToNil(podStatus.ContainerStatuses)
+		setAllocatedResourcesStatusToNil(podStatus.InitContainerStatuses)
+		setAllocatedResourcesStatusToNil(podStatus.EphemeralContainerStatuses)
+	}
+
 	// drop ContainerStatus.User field to empty (disable SupplementalGroupsPolicy)
 	if !utilfeature.DefaultFeatureGate.Enabled(features.SupplementalGroupsPolicy) && !supplementalGroupsPolicyInUse(oldPodSpec) {
 		dropUserField := func(csl []api.ContainerStatus) {

pkg/api/pod/util.go

@@ -2780,6 +2780,59 @@ type ContainerStatus struct {
 	// +featureGate=SupplementalGroupsPolicy
 	// +optional
 	User *ContainerUser
+	// AllocatedResourcesStatus represents the status of various resources
+	// allocated for this Pod.
+	// +featureGate=ResourceHealthStatus
+	// +optional
+	AllocatedResourcesStatus []ResourceStatus
+}
+
+type ResourceStatus struct {
+	Name ResourceName
+	// List of unique Resources health. Each element in the list contains a unique resource ID and resource health.
+	// At a minimum, ResourceID must uniquely identify the Resource
+	// allocated to the Pod on the Node for the lifetime of a Pod.
+	// See ResourceID type for it's definition.
+	Resources []ResourceHealth
+
+	// allow to extend this struct in future with the overall health fields or things like Device Plugin version
+}
+
+// ResourceID is calculated based on the source of this resource health information.
+// For DevicePlugin:
+//
+//	deviceplugin:DeviceID, where DeviceID is from the Device structure of DevicePlugin's ListAndWatchResponse type: https://github.com/kubernetes/kubernetes/blob/eda1c780543a27c078450e2f17d674471e00f494/staging/src/k8s.io/kubelet/pkg/apis/deviceplugin/v1alpha/api.proto#L61-L73
+//
+// DevicePlugin ID is usually a constant for the lifetime of a Node and typically can be used to uniquely identify the device on the node.
+// For DRA:
+//
+//	dra:<driver name>/<pool name>/<device name>: such a device can be looked up in the information published by that DRA driver to learn more about it. It is designed to be globally unique in a cluster.
+type ResourceID string
+
+type ResourceHealthStatus string
+
+const (
+	ResourceHealthStatusHealthy   ResourceHealthStatus = "Healthy"
+	ResourceHealthStatusUnhealthy ResourceHealthStatus = "Unhealthy"
+	ResourceHealthStatusUnknown   ResourceHealthStatus = "Unknown"
+)
+
+// ResourceHealth represents the health of a resource. It has the latest device health information.
+// This is a part of KEP https://kep.k8s.io/4680 and historical health changes are planned to be added in future iterations of a KEP.
+type ResourceHealth struct {
+	// ResourceID is the unique identifier of the resource. See the ResourceID type for more information.
+	ResourceID ResourceID
+	// Health of the resource.
+	// can be one of:
+	//  - Healthy: operates as normal
+	//  - Unhealthy: reported unhealthy. We consider this a temporary health issue
+	//               since we do not have a mechanism today to distinguish
+	//               temporary and permanent issues.
+	//  - Unknown: The status cannot be determined.
+	//             For example, Device Plugin got unregistered and hasn't been re-registered since.
+	//
+	// In future we may want to introduce the PermanentlyUnhealthy Status.
+	Health ResourceHealthStatus
 }
 
 // ContainerUser represents user identity information