Handle resource utilization calculation for GPUs exposed using DRA

Author: mtrqq

cluster-autoscaler/cloudprovider/cloud_provider.go

@@ -104,6 +104,8 @@ type GpuConfig struct {
 	DraDriverName        string
 }
 
+// ExposedViaDra determines whether a GPU described in the config
+// is exposed via device plugin or DRA driver
 func (gpu *GpuConfig) ExposedViaDra() bool {
 	return gpu.DraDriverName != ""
 }

cluster-autoscaler/cloudprovider/gce/dynamicresources.go

@@ -1,3 +1,19 @@
+/*
+Copyright 2025 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package gce
 
 import apiv1 "k8s.io/api/core/v1"

cluster-autoscaler/processors/customresources/gpu_processor.go

@@ -50,7 +50,7 @@ func (p *GpuCustomResourcesProcessor) FilterOutNodesWithUnreadyResources(context
 		_, hasGpuLabel := node.Labels[context.CloudProvider.GPULabel()]
 		gpuAllocatable, hasGpuAllocatable := node.Status.Allocatable[gpu.ResourceNvidiaGPU]
 		directXAllocatable, hasDirectXAllocatable := node.Status.Allocatable[gpu.ResourceDirectX]
-		if hasGpuLabel && ((!hasGpuAllocatable || gpuAllocatable.IsZero()) && (!hasDirectXAllocatable || directXAllocatable.IsZero())) && !gpuExposedViaDra(context, node) {
+		if hasGpuLabel && ((!hasGpuAllocatable || gpuAllocatable.IsZero()) && (!hasDirectXAllocatable || directXAllocatable.IsZero())) {
 			klog.V(3).Infof("Overriding status of node %v, which seems to have unready GPU",
 				node.Name)
 			nodesWithUnreadyGpu[node.Name] = kubernetes.GetUnreadyNodeCopy(node, kubernetes.ResourceUnready)
@@ -72,22 +72,22 @@ func (p *GpuCustomResourcesProcessor) FilterOutNodesWithUnreadyResources(context
 // GetNodeResourceTargets returns mapping of resource names to their targets.
 // This includes resources which are not yet ready to use and visible in kubernetes.
 func (p *GpuCustomResourcesProcessor) GetNodeResourceTargets(context *context.AutoscalingContext, node *apiv1.Node, nodeGroup cloudprovider.NodeGroup) ([]CustomResourceTarget, errors.AutoscalerError) {
-	if gpuExposedViaDra(context, node) {
-		return []CustomResourceTarget{}, nil
-	}
-
-	gpuTarget, err := p.getNodeGpuTarget(context.CloudProvider.GPULabel(), node, nodeGroup)
+	gpuTarget, err := p.GetNodeGpuTarget(context, node, nodeGroup)
 	return []CustomResourceTarget{gpuTarget}, err
 }
 
-// getNodeGpuTarget returns the gpu target of a given node. This includes gpus
+// GetNodeGpuTarget returns the gpu target of a given node. This includes gpus
 // that are not ready to use and visible in kubernetes.
-func (p *GpuCustomResourcesProcessor) getNodeGpuTarget(GPULabel string, node *apiv1.Node, nodeGroup cloudprovider.NodeGroup) (CustomResourceTarget, errors.AutoscalerError) {
-	gpuLabel, found := node.Labels[GPULabel]
+func (p *GpuCustomResourcesProcessor) GetNodeGpuTarget(context *context.AutoscalingContext, node *apiv1.Node, nodeGroup cloudprovider.NodeGroup) (CustomResourceTarget, errors.AutoscalerError) {
+	gpuLabel, found := node.Labels[context.CloudProvider.GPULabel()]
 	if !found {
 		return CustomResourceTarget{}, nil
 	}
 
+	if gpuExposedViaDra(context, node) {
+		return CustomResourceTarget{}, nil
+	}
+
 	gpuAllocatable, found := node.Status.Allocatable[gpu.ResourceNvidiaGPU]
 	if found && gpuAllocatable.Value() > 0 {
 		return CustomResourceTarget{gpuLabel, gpuAllocatable.Value()}, nil

cluster-autoscaler/simulator/utilization/info.go

@@ -48,7 +48,7 @@ type Info struct {
 // utilization is the sum of requests for it divided by allocatable. It also
 // returns the individual cpu, memory and gpu utilization.
 func Calculate(nodeInfo *framework.NodeInfo, skipDaemonSetPods, skipMirrorPods, draEnabled bool, gpuConfig *cloudprovider.GpuConfig, currentTime time.Time) (utilInfo Info, err error) {
-	if gpuConfig != nil {
+	if gpuConfig != nil && !gpuConfig.ExposedViaDra() {
 		gpuUtil, err := CalculateUtilizationOfResource(nodeInfo, gpuConfig.ExtendedResourceName, skipDaemonSetPods, skipMirrorPods, currentTime)
 		if err != nil {
 			klog.V(3).Infof("node %s has unready GPU resource: %s", nodeInfo.Node().Name, gpuConfig.ExtendedResourceName)

cluster-autoscaler/simulator/utilization/info_test.go

@@ -82,7 +82,7 @@ func TestCalculate(t *testing.T) {
 	SetNodeReadyState(node, true, time.Time{})
 	nodeInfo := framework.NewTestNodeInfo(node, pod, pod, pod2)
 
-	gpuConfig := getGpuConfigFromNode(nodeInfo.Node())
+	gpuConfig := getGpuConfigFromNode(nodeInfo.Node(), false)
 	utilInfo, err := Calculate(nodeInfo, false, false, false, gpuConfig, testTime)
 	assert.NoError(t, err)
 	assert.InEpsilon(t, 2.0/10, utilInfo.Utilization, 0.01)
@@ -91,15 +91,15 @@ func TestCalculate(t *testing.T) {
 	node2 := BuildTestNode("node2", 2000, -1)
 	nodeInfo = framework.NewTestNodeInfo(node2, pod, pod, pod2)
 
-	gpuConfig = getGpuConfigFromNode(nodeInfo.Node())
+	gpuConfig = getGpuConfigFromNode(nodeInfo.Node(), false)
 	_, err = Calculate(nodeInfo, false, false, false, gpuConfig, testTime)
 	assert.Error(t, err)
 
 	node3 := BuildTestNode("node3", 2000, 2000000)
 	SetNodeReadyState(node3, true, time.Time{})
 	nodeInfo = framework.NewTestNodeInfo(node3, pod, podWithInitContainers, podWithLargeNonRestartableInitContainers)
 
-	gpuConfig = getGpuConfigFromNode(nodeInfo.Node())
+	gpuConfig = getGpuConfigFromNode(nodeInfo.Node(), false)
 	utilInfo, err = Calculate(nodeInfo, false, false, false, gpuConfig, testTime)
 	assert.NoError(t, err)
 	assert.InEpsilon(t, 50.25, utilInfo.Utilization, 0.01)
@@ -113,21 +113,21 @@ func TestCalculate(t *testing.T) {
 	daemonSetPod4.Annotations = map[string]string{"cluster-autoscaler.kubernetes.io/daemonset-pod": "true"}
 
 	nodeInfo = framework.NewTestNodeInfo(node, pod, pod, pod2, daemonSetPod3, daemonSetPod4)
-	gpuConfig = getGpuConfigFromNode(nodeInfo.Node())
+	gpuConfig = getGpuConfigFromNode(nodeInfo.Node(), false)
 	utilInfo, err = Calculate(nodeInfo, true, false, false, gpuConfig, testTime)
 	assert.NoError(t, err)
 	assert.InEpsilon(t, 2.5/10, utilInfo.Utilization, 0.01)
 
 	nodeInfo = framework.NewTestNodeInfo(node, pod, pod2, daemonSetPod3)
-	gpuConfig = getGpuConfigFromNode(nodeInfo.Node())
+	gpuConfig = getGpuConfigFromNode(nodeInfo.Node(), false)
 	utilInfo, err = Calculate(nodeInfo, false, false, false, gpuConfig, testTime)
 	assert.NoError(t, err)
 	assert.InEpsilon(t, 2.0/10, utilInfo.Utilization, 0.01)
 
 	terminatedPod := BuildTestPod("podTerminated", 100, 200000)
 	terminatedPod.DeletionTimestamp = &metav1.Time{Time: testTime.Add(-10 * time.Minute)}
 	nodeInfo = framework.NewTestNodeInfo(node, pod, pod, pod2, terminatedPod)
-	gpuConfig = getGpuConfigFromNode(nodeInfo.Node())
+	gpuConfig = getGpuConfigFromNode(nodeInfo.Node(), false)
 	utilInfo, err = Calculate(nodeInfo, false, false, false, gpuConfig, testTime)
 	assert.NoError(t, err)
 	assert.InEpsilon(t, 2.0/10, utilInfo.Utilization, 0.01)
@@ -138,19 +138,19 @@ func TestCalculate(t *testing.T) {
 	}
 
 	nodeInfo = framework.NewTestNodeInfo(node, pod, pod, pod2, mirrorPod)
-	gpuConfig = getGpuConfigFromNode(nodeInfo.Node())
+	gpuConfig = getGpuConfigFromNode(nodeInfo.Node(), false)
 	utilInfo, err = Calculate(nodeInfo, false, true, false, gpuConfig, testTime)
 	assert.NoError(t, err)
 	assert.InEpsilon(t, 2.0/9.0, utilInfo.Utilization, 0.01)
 
 	nodeInfo = framework.NewTestNodeInfo(node, pod, pod2, mirrorPod)
-	gpuConfig = getGpuConfigFromNode(nodeInfo.Node())
+	gpuConfig = getGpuConfigFromNode(nodeInfo.Node(), false)
 	utilInfo, err = Calculate(nodeInfo, false, false, false, gpuConfig, testTime)
 	assert.NoError(t, err)
 	assert.InEpsilon(t, 2.0/10, utilInfo.Utilization, 0.01)
 
 	nodeInfo = framework.NewTestNodeInfo(node, pod, mirrorPod, daemonSetPod3)
-	gpuConfig = getGpuConfigFromNode(nodeInfo.Node())
+	gpuConfig = getGpuConfigFromNode(nodeInfo.Node(), false)
 	utilInfo, err = Calculate(nodeInfo, true, true, false, gpuConfig, testTime)
 	assert.NoError(t, err)
 	assert.InEpsilon(t, 1.0/8.0, utilInfo.Utilization, 0.01)
@@ -161,7 +161,7 @@ func TestCalculate(t *testing.T) {
 	RequestGpuForPod(gpuPod, 1)
 	TolerateGpuForPod(gpuPod)
 	nodeInfo = framework.NewTestNodeInfo(gpuNode, pod, pod, gpuPod)
-	gpuConfig = getGpuConfigFromNode(nodeInfo.Node())
+	gpuConfig = getGpuConfigFromNode(nodeInfo.Node(), false)
 	utilInfo, err = Calculate(nodeInfo, false, false, false, gpuConfig, testTime)
 	assert.NoError(t, err)
 	assert.InEpsilon(t, 1/1, utilInfo.Utilization, 0.01)
@@ -170,7 +170,7 @@ func TestCalculate(t *testing.T) {
 	gpuNode = BuildTestNode("gpu_node", 2000, 2000000)
 	AddGpuLabelToNode(gpuNode)
 	nodeInfo = framework.NewTestNodeInfo(gpuNode, pod, pod)
-	gpuConfig = getGpuConfigFromNode(nodeInfo.Node())
+	gpuConfig = getGpuConfigFromNode(nodeInfo.Node(), false)
 	utilInfo, err = Calculate(nodeInfo, false, false, false, gpuConfig, testTime)
 	assert.NoError(t, err)
 	assert.Zero(t, utilInfo.Utilization)
@@ -182,7 +182,8 @@ func TestCalculateWithDynamicResources(t *testing.T) {
 	gpuNode := BuildTestNode("gpuNode", 1000, 1000)
 	AddGpusToNode(gpuNode, 1)
 	AddGpuLabelToNode(gpuNode)
-	gpuConfig := getGpuConfigFromNode(gpuNode)
+	gpuConfig := getGpuConfigFromNode(gpuNode, false)
+	gpuConfigDra := getGpuConfigFromNode(gpuNode, true)
 	pod1 := BuildTestPod("pod1", 250, 0, WithNodeName("node"))
 	pod2 := BuildTestPod("pod2", 250, 0, WithNodeName("node"))
 	resourceSlice1 := &resourceapi.ResourceSlice{
@@ -345,6 +346,13 @@ func TestCalculateWithDynamicResources(t *testing.T) {
 			draEnabled:   true,
 			wantUtilInfo: Info{Utilization: 0, ResourceName: gpuConfig.ExtendedResourceName},
 		},
+		{
+			testName:     "DRA slices and claims present, DRA enabled, DRA GPU config passed -> DRA util returned",
+			nodeInfo:     nodeInfoGpuAndDra,
+			gpuConfig:    gpuConfigDra,
+			draEnabled:   true,
+			wantUtilInfo: Info{DynamicResourceUtil: 0.8, Utilization: 0.8, ResourceName: apiv1.ResourceName("driver.foo.com/node-pool1")},
+		},
 		{
 			testName:     "DRA slices and claims present, DRA enabled, error while calculating DRA util -> error returned",
 			nodeInfo:     nodeInfoIncompleteSlices,
@@ -365,15 +373,23 @@ func TestCalculateWithDynamicResources(t *testing.T) {
 	}
 }
 
-func getGpuConfigFromNode(node *apiv1.Node) *cloudprovider.GpuConfig {
+func getGpuConfigFromNode(node *apiv1.Node, dra bool) *cloudprovider.GpuConfig {
 	gpuLabel := "cloud.google.com/gke-accelerator"
 	gpuType, hasGpuLabel := node.Labels[gpuLabel]
 	gpuAllocatable, hasGpuAllocatable := node.Status.Allocatable[gpu.ResourceNvidiaGPU]
 	if hasGpuLabel || (hasGpuAllocatable && !gpuAllocatable.IsZero()) {
+		if !dra {
+			return &cloudprovider.GpuConfig{
+				Label:                gpuLabel,
+				Type:                 gpuType,
+				ExtendedResourceName: gpu.ResourceNvidiaGPU,
+			}
+		}
+
 		return &cloudprovider.GpuConfig{
-			Label:                gpuLabel,
-			Type:                 gpuType,
-			ExtendedResourceName: gpu.ResourceNvidiaGPU,
+			Label:         gpuLabel,
+			Type:          gpuType,
+			DraDriverName: "gpu.nvidia.com",
 		}
 	}
 	return nil