diff --git a/cluster-autoscaler/cloudprovider/clusterapi/README.md b/cluster-autoscaler/cloudprovider/clusterapi/README.md index f515c52ed621..da2a1887b10f 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/README.md +++ b/cluster-autoscaler/cloudprovider/clusterapi/README.md @@ -307,6 +307,30 @@ metadata: > Please see the [Cluster API Book chapter on Metadata propagation](https://cluster-api.sigs.k8s.io/reference/api/metadata-propagation) > for more information. + +#### Pre-defined csi driver information on nodes scaled from zero + +To provide CSI driver information for scale from zero, the optional +capacity annotation may be supplied as a comma separated list of driver name +and volume limit key/value pairs, as demonstrated in the example below: + +```yaml +apiVersion: cluster.x-k8s.io/v1alpha4 +kind: MachineDeployment +metadata: + annotations: + cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "5" + cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "0" + capacity.cluster-autoscaler.kubernetes.io/memory: "128G" + capacity.cluster-autoscaler.kubernetes.io/cpu: "16" + capacity.cluster-autoscaler.kubernetes.io/csi-driver: "ebs.csi.aws.com=25,efs.csi.aws.com=16" +``` + +> Note: The CSI driver information supplied through the capacity annotation +> specifies which CSI drivers will be installed on nodes scaled from zero, along +> with their respective volume limits. The format is `driver-name=volume-limit` +> with multiple drivers separated by commas. + #### Per-NodeGroup autoscaling options Custom autoscaling options per node group (MachineDeployment/MachinePool/MachineSet) can be specified as annoations with a common prefix: @@ -328,14 +352,14 @@ metadata: cluster.x-k8s.io/autoscaling-options-maxnodeprovisiontime: "20m0s" ``` -#### CPU Architecture awareness for single-arch clusters +#### CPU Architecture awareness for single-arch clusters -Users of single-arch non-amd64 clusters who are using scale from zero +Users of single-arch non-amd64 clusters who are using scale from zero support should also set the `CAPI_SCALE_ZERO_DEFAULT_ARCH` environment variable to set the architecture of the nodes they want to default the node group templates to. -The autoscaler will default to `amd64` if it is not set, and the node -group templates may not match the nodes' architecture, specifically when -the workload triggering the scale-up uses a node affinity predicate checking +The autoscaler will default to `amd64` if it is not set, and the node +group templates may not match the nodes' architecture, specifically when +the workload triggering the scale-up uses a node affinity predicate checking for the node's architecture. ## Specifying a Custom Resource Group diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup.go index 6c5db17339e9..0de26fb9234a 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup.go @@ -380,8 +380,12 @@ func (ng *nodegroup) TemplateNodeInfo() (*framework.NodeInfo, error) { if err != nil { return nil, err } + csiNode := ng.scalableResource.InstanceCSINode() nodeInfo := framework.NewNodeInfo(&node, resourceSlices, &framework.PodInfo{Pod: cloudprovider.BuildKubeProxy(ng.scalableResource.Name())}) + if csiNode != nil { + nodeInfo.AddCSINode(csiNode) + } return nodeInfo, nil } diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup_test.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup_test.go index b6524c850e0d..038b92492936 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup_test.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup_test.go @@ -27,6 +27,7 @@ import ( "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/wait" @@ -34,6 +35,7 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/config" gpuapis "k8s.io/autoscaler/cluster-autoscaler/utils/gpu" "k8s.io/client-go/tools/cache" + "k8s.io/utils/ptr" ) const ( @@ -1500,6 +1502,7 @@ func TestNodeGroupTemplateNodeInfo(t *testing.T) { expectedCapacity map[corev1.ResourceName]int64 expectedNodeLabels map[string]string expectedResourceSlice testResourceSlice + expectedCSINode *storagev1.CSINode } testCases := []struct { @@ -1650,6 +1653,49 @@ func TestNodeGroupTemplateNodeInfo(t *testing.T) { }, }, }, + { + name: "When the NodeGroup can scale from zero and CSI driver annotations are present, it creates CSINode with driver information", + nodeGroupAnnotations: map[string]string{ + memoryKey: "2048Mi", + cpuKey: "2", + csiDriverKey: "ebs.csi.aws.com=25,efs.csi.aws.com=16", + }, + config: testCaseConfig{ + expectedErr: nil, + nodeLabels: map[string]string{ + "kubernetes.io/os": "linux", + "kubernetes.io/arch": "amd64", + }, + expectedCapacity: map[corev1.ResourceName]int64{ + corev1.ResourceCPU: 2, + corev1.ResourceMemory: 2048 * 1024 * 1024, + corev1.ResourcePods: 110, + }, + expectedNodeLabels: map[string]string{ + "kubernetes.io/os": "linux", + "kubernetes.io/arch": "amd64", + "kubernetes.io/hostname": "random value", + }, + expectedCSINode: &storagev1.CSINode{ + Spec: storagev1.CSINodeSpec{ + Drivers: []storagev1.CSINodeDriver{ + { + Name: "ebs.csi.aws.com", + Allocatable: &storagev1.VolumeNodeResources{ + Count: ptr.To(int32(25)), + }, + }, + { + Name: "efs.csi.aws.com", + Allocatable: &storagev1.VolumeNodeResources{ + Count: ptr.To(int32(16)), + }, + }, + }, + }, + }, + }, + }, } test := func(t *testing.T, testConfig *TestConfig, config testCaseConfig) { @@ -1726,6 +1772,55 @@ func TestNodeGroupTemplateNodeInfo(t *testing.T) { } } } + + // Validate CSINode if expected + if config.expectedCSINode != nil { + if nodeInfo.CSINode == nil { + t.Errorf("Expected CSINode to be set, but got nil") + } else { + expectedDrivers := config.expectedCSINode.Spec.Drivers + gotDrivers := nodeInfo.CSINode.Spec.Drivers + if len(expectedDrivers) != len(gotDrivers) { + t.Errorf("Expected %d CSI drivers, but got %d", len(expectedDrivers), len(gotDrivers)) + } else { + for i, expectedDriver := range expectedDrivers { + if i >= len(gotDrivers) { + t.Errorf("Expected driver at index %d but got only %d drivers", i, len(gotDrivers)) + break + } + gotDriver := gotDrivers[i] + if expectedDriver.Name != gotDriver.Name { + t.Errorf("Expected CSI driver name at index %d to be %s, but got %s", i, expectedDriver.Name, gotDriver.Name) + } + if expectedDriver.Allocatable == nil { + if gotDriver.Allocatable != nil { + t.Errorf("Expected CSI driver Allocatable at index %d to be nil, but got non-nil", i) + } + } else { + if gotDriver.Allocatable == nil { + t.Errorf("Expected CSI driver Allocatable at index %d to be non-nil, but got nil", i) + } else { + if expectedDriver.Allocatable.Count == nil { + if gotDriver.Allocatable.Count != nil { + t.Errorf("Expected CSI driver Count at index %d to be nil, but got %d", i, *gotDriver.Allocatable.Count) + } + } else { + if gotDriver.Allocatable.Count == nil { + t.Errorf("Expected CSI driver Count at index %d to be %d, but got nil", i, *expectedDriver.Allocatable.Count) + } else if *expectedDriver.Allocatable.Count != *gotDriver.Allocatable.Count { + t.Errorf("Expected CSI driver Count at index %d to be %d, but got %d", i, *expectedDriver.Allocatable.Count, *gotDriver.Allocatable.Count) + } + } + } + } + } + } + } + } else { + if nodeInfo.CSINode != nil { + t.Errorf("Expected CSINode to be nil, but got non-nil with %d drivers", len(nodeInfo.CSINode.Spec.Drivers)) + } + } } for _, tc := range testCases { diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_unstructured.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_unstructured.go index 91c8819edfff..a9e3e74d370e 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_unstructured.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_unstructured.go @@ -30,6 +30,7 @@ import ( apiv1 "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1" resourceapi "k8s.io/api/resource/v1" + storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -413,6 +414,27 @@ func (r unstructuredScalableResource) InstanceDRADriver() string { return parseDRADriver(r.unstructured.GetAnnotations()) } +// InstanceCSINode parses CSI driver information from annotations and returns +// a CSINode object with the list of installed drivers and their volume limits. +// The annotation format is "driver-name=volume-limit,driver-name2=volume-limit2". +// Returns nil if the annotation is not present or empty. +func (r unstructuredScalableResource) InstanceCSINode() *storagev1.CSINode { + annotations := r.unstructured.GetAnnotations() + // annotation value of the form "driver1=limit1,driver2=limit2" + if val, found := annotations[csiDriverKey]; found && val != "" { + drivers := parseCSIDriverAnnotation(val) + if len(drivers) == 0 { + return nil + } + return &storagev1.CSINode{ + Spec: storagev1.CSINodeSpec{ + Drivers: drivers, + }, + } + } + return nil +} + func (r unstructuredScalableResource) readInfrastructureReferenceResource() (*unstructured.Unstructured, error) { // Cache w/ lazy loading of the infrastructure reference resource. r.infraMutex.RLock() @@ -533,6 +555,66 @@ func systemInfoFromInfrastructureObject(infraobj *unstructured.Unstructured) api return nsi } +// parseCSIDriverAnnotation parses a comma-separated list of CSI driver name and volume limit +// key/value pairs in the format "driver-name=volume-limit,driver-name2=volume-limit2". +// Returns a slice of CSINodeDriver objects with Name and Allocatable.Count set. +func parseCSIDriverAnnotation(annotationValue string) []storagev1.CSINodeDriver { + drivers := []storagev1.CSINodeDriver{} + if annotationValue == "" { + return drivers + } + + driverSpecs := strings.Split(annotationValue, ",") + for _, driverSpec := range driverSpecs { + driverSpec = strings.TrimSpace(driverSpec) + if driverSpec == "" { + continue + } + + // Split on "=" to get driver name and volume limit + parts := strings.SplitN(driverSpec, "=", 2) + if len(parts) != 2 { + klog.V(4).Infof("Invalid CSI driver spec format (expected driver-name=volume-limit): %s", driverSpec) + continue + } + + driverName := strings.TrimSpace(parts[0]) + volumeLimitStr := strings.TrimSpace(parts[1]) + + if driverName == "" { + klog.V(4).Infof("Empty driver name in CSI driver spec: %s", driverSpec) + continue + } + + // Parse volume limit as integer + volumeLimit, err := strconv.ParseInt(volumeLimitStr, 10, 32) + if err != nil { + klog.V(4).Infof("Invalid volume limit value (expected integer) in CSI driver spec %s: %v", driverSpec, err) + continue + } + + if volumeLimit < 0 { + klog.V(4).Infof("Volume limit must be non-negative in CSI driver spec: %s", driverSpec) + continue + } + + // Create CSINodeDriver with Name and optionally Allocatable.Count + // If volume limit is 0, Allocatable is not set + driver := storagev1.CSINodeDriver{ + Name: driverName, + } + if volumeLimit > 0 { + limit := int32(volumeLimit) + driver.Allocatable = &storagev1.VolumeNodeResources{ + Count: &limit, + } + } + drivers = append(drivers, driver) + } + + return drivers +} + // adapted from https://github.com/kubernetes/kubernetes/blob/release-1.25/pkg/util/taints/taints.go#L39 func parseTaint(st string) (apiv1.Taint, error) { var taint apiv1.Taint diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_unstructured_test.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_unstructured_test.go index 0749358faab2..9475d45f8808 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_unstructured_test.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_unstructured_test.go @@ -25,6 +25,7 @@ import ( "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" resourceapi "k8s.io/api/resource/v1" + storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -707,3 +708,194 @@ func TestInstanceSystemInfo(t *testing.T) { }) } } + +func TestParseCSIDriverAnnotation(t *testing.T) { + for _, tc := range []struct { + description string + annotationVal string + expected []storagev1.CSINodeDriver + }{ + { + description: "empty string", + annotationVal: "", + expected: []storagev1.CSINodeDriver{}, + }, + { + description: "single valid driver", + annotationVal: "ebs.csi.aws.com=25", + expected: []storagev1.CSINodeDriver{ + { + Name: "ebs.csi.aws.com", + Allocatable: &storagev1.VolumeNodeResources{ + Count: ptr.To(int32(25)), + }, + }, + }, + }, + { + description: "multiple valid drivers", + annotationVal: "ebs.csi.aws.com=25,efs.csi.aws.com=16,disk.csi.azure.com=16", + expected: []storagev1.CSINodeDriver{ + { + Name: "ebs.csi.aws.com", + Allocatable: &storagev1.VolumeNodeResources{ + Count: ptr.To(int32(25)), + }, + }, + { + Name: "efs.csi.aws.com", + Allocatable: &storagev1.VolumeNodeResources{ + Count: ptr.To(int32(16)), + }, + }, + { + Name: "disk.csi.azure.com", + Allocatable: &storagev1.VolumeNodeResources{ + Count: ptr.To(int32(16)), + }, + }, + }, + }, + { + description: "drivers with whitespace", + annotationVal: " ebs.csi.aws.com = 25 , efs.csi.aws.com = 16 ", + expected: []storagev1.CSINodeDriver{ + { + Name: "ebs.csi.aws.com", + Allocatable: &storagev1.VolumeNodeResources{ + Count: ptr.To(int32(25)), + }, + }, + { + Name: "efs.csi.aws.com", + Allocatable: &storagev1.VolumeNodeResources{ + Count: ptr.To(int32(16)), + }, + }, + }, + }, + { + description: "single driver with large volume limit", + annotationVal: "ebs.csi.aws.com=1000", + expected: []storagev1.CSINodeDriver{ + { + Name: "ebs.csi.aws.com", + Allocatable: &storagev1.VolumeNodeResources{ + Count: ptr.To(int32(1000)), + }, + }, + }, + }, + { + description: "invalid format - missing equals sign", + annotationVal: "ebs.csi.aws.com25", + expected: []storagev1.CSINodeDriver{}, + }, + { + description: "invalid format - empty driver name", + annotationVal: "=25", + expected: []storagev1.CSINodeDriver{}, + }, + { + description: "invalid format - non-integer volume limit", + annotationVal: "ebs.csi.aws.com=abc", + expected: []storagev1.CSINodeDriver{}, + }, + { + description: "invalid format - negative volume limit", + annotationVal: "ebs.csi.aws.com=-5", + expected: []storagev1.CSINodeDriver{}, + }, + { + description: "zero volume limit is valid", + annotationVal: "ebs.csi.aws.com=0", + expected: []storagev1.CSINodeDriver{ + { + Name: "ebs.csi.aws.com", + Allocatable: nil, + }, + }, + }, + { + description: "invalid format - fractional volume limit", + annotationVal: "ebs.csi.aws.com=25.5", + expected: []storagev1.CSINodeDriver{}, + }, + { + description: "mixed valid and invalid drivers - should skip invalid ones", + annotationVal: "ebs.csi.aws.com=25,invalid-format,efs.csi.aws.com=16,=10,disk.csi.azure.com=0", + expected: []storagev1.CSINodeDriver{ + { + Name: "ebs.csi.aws.com", + Allocatable: &storagev1.VolumeNodeResources{ + Count: ptr.To(int32(25)), + }, + }, + { + Name: "efs.csi.aws.com", + Allocatable: &storagev1.VolumeNodeResources{ + Count: ptr.To(int32(16)), + }, + }, + { + Name: "disk.csi.azure.com", + Allocatable: nil, + }, + }, + }, + { + description: "comma-separated with empty entries", + annotationVal: "ebs.csi.aws.com=25,,efs.csi.aws.com=16,", + expected: []storagev1.CSINodeDriver{ + { + Name: "ebs.csi.aws.com", + Allocatable: &storagev1.VolumeNodeResources{ + Count: ptr.To(int32(25)), + }, + }, + { + Name: "efs.csi.aws.com", + Allocatable: &storagev1.VolumeNodeResources{ + Count: ptr.To(int32(16)), + }, + }, + }, + }, + { + description: "volume limit at int32 max", + annotationVal: "ebs.csi.aws.com=2147483647", + expected: []storagev1.CSINodeDriver{ + { + Name: "ebs.csi.aws.com", + Allocatable: &storagev1.VolumeNodeResources{ + Count: ptr.To(int32(2147483647)), + }, + }, + }, + }, + } { + t.Run(tc.description, func(t *testing.T) { + got := parseCSIDriverAnnotation(tc.annotationVal) + assert.Equal(t, len(tc.expected), len(got), "expected %d drivers, got %d", len(tc.expected), len(got)) + + for i, expectedDriver := range tc.expected { + if i >= len(got) { + t.Fatalf("expected driver at index %d but got only %d drivers", i, len(got)) + } + gotDriver := got[i] + assert.Equal(t, expectedDriver.Name, gotDriver.Name, "driver name mismatch at index %d", i) + if expectedDriver.Allocatable == nil { + assert.Nil(t, gotDriver.Allocatable, "expected nil Allocatable at index %d", i) + } else { + assert.NotNil(t, gotDriver.Allocatable, "expected non-nil Allocatable at index %d", i) + if gotDriver.Allocatable != nil { + assert.NotNil(t, gotDriver.Allocatable.Count, "expected non-nil Count at index %d", i) + if gotDriver.Allocatable.Count != nil { + assert.Equal(t, *expectedDriver.Allocatable.Count, *gotDriver.Allocatable.Count, "volume limit mismatch at index %d", i) + } + } + } + } + }) + } +} diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_utils.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_utils.go index 50b7dc6993d6..5bc395bbe7c2 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_utils.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_utils.go @@ -41,6 +41,7 @@ const ( taintsKey = "capacity.cluster-autoscaler.kubernetes.io/taints" labelsKey = "capacity.cluster-autoscaler.kubernetes.io/labels" draDriverKey = "capacity.cluster-autoscaler.kubernetes.io/dra-driver" + csiDriverKey = "capacity.cluster-autoscaler.kubernetes.io/csi-driver" machineDeploymentRevisionAnnotation = "machinedeployment.clusters.x-k8s.io/revision" machineDeploymentNameLabel = "cluster.x-k8s.io/deployment-name" // UnknownArch is used if the Architecture is Unknown diff --git a/cluster-autoscaler/config/autoscaling_options.go b/cluster-autoscaler/config/autoscaling_options.go index 82c106953064..c17d0b03a92e 100644 --- a/cluster-autoscaler/config/autoscaling_options.go +++ b/cluster-autoscaler/config/autoscaling_options.go @@ -314,6 +314,8 @@ type AutoscalingOptions struct { ForceDeleteFailedNodes bool // DynamicResourceAllocationEnabled configures whether logic for handling DRA objects is enabled. DynamicResourceAllocationEnabled bool + // CSINodeAwareSchedulingEnabled configures whether logic for handling CSINode objects is enabled. + CSINodeAwareSchedulingEnabled bool // ClusterSnapshotParallelism is the maximum parallelism of cluster snapshot creation. ClusterSnapshotParallelism int // PredicateParallelism is the number of goroutines to use for running scheduler predicates. diff --git a/cluster-autoscaler/config/flags/flags.go b/cluster-autoscaler/config/flags/flags.go index 8445239a548f..9f0e221e3116 100644 --- a/cluster-autoscaler/config/flags/flags.go +++ b/cluster-autoscaler/config/flags/flags.go @@ -226,6 +226,7 @@ var ( forceDeleteLongUnregisteredNodes = flag.Bool("force-delete-unregistered-nodes", false, "Whether to enable force deletion of long unregistered nodes, regardless of the min size of the node group the belong to.") forceDeleteFailedNodes = flag.Bool("force-delete-failed-nodes", false, "Whether to enable force deletion of failed nodes, regardless of the min size of the node group the belong to.") enableDynamicResourceAllocation = flag.Bool("enable-dynamic-resource-allocation", false, "Whether logic for handling DRA (Dynamic Resource Allocation) objects is enabled.") + enableCSINodeAwareScheduling = flag.Bool("enable-csi-node-aware-scheduling", false, "Whether logic for handling CSINode objects is enabled.") clusterSnapshotParallelism = flag.Int("cluster-snapshot-parallelism", 16, "Maximum parallelism of cluster snapshot creation.") predicateParallelism = flag.Int("predicate-parallelism", 4, "Maximum parallelism of scheduler predicate checking.") checkCapacityProcessorInstance = flag.String("check-capacity-processor-instance", "", "Name of the processor instance. Only ProvisioningRequests that define this name in their parameters with the key \"processorInstance\" will be processed by this CA instance. It only refers to check capacity ProvisioningRequests, but if not empty, best-effort atomic ProvisioningRequests processing is disabled in this instance. Not recommended: Until CA 1.35, ProvisioningRequests with this name as prefix in their class will be also processed.") @@ -405,6 +406,7 @@ func createAutoscalingOptions() config.AutoscalingOptions { ForceDeleteLongUnregisteredNodes: *forceDeleteLongUnregisteredNodes, ForceDeleteFailedNodes: *forceDeleteFailedNodes, DynamicResourceAllocationEnabled: *enableDynamicResourceAllocation, + CSINodeAwareSchedulingEnabled: *enableCSINodeAwareScheduling, ClusterSnapshotParallelism: *clusterSnapshotParallelism, PredicateParallelism: *predicateParallelism, CheckCapacityProcessorInstance: *checkCapacityProcessorInstance, diff --git a/cluster-autoscaler/context/autoscaling_context.go b/cluster-autoscaler/context/autoscaling_context.go index 0010d43c2414..1258fa27d1bd 100644 --- a/cluster-autoscaler/context/autoscaling_context.go +++ b/cluster-autoscaler/context/autoscaling_context.go @@ -27,6 +27,7 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/expander" processor_callbacks "k8s.io/autoscaler/cluster-autoscaler/processors/callbacks" "k8s.io/autoscaler/cluster-autoscaler/simulator/clustersnapshot" + csinodeprovider "k8s.io/autoscaler/cluster-autoscaler/simulator/csi/provider" draprovider "k8s.io/autoscaler/cluster-autoscaler/simulator/dynamicresources/provider" "k8s.io/autoscaler/cluster-autoscaler/simulator/framework" kube_util "k8s.io/autoscaler/cluster-autoscaler/utils/kubernetes" @@ -65,6 +66,8 @@ type AutoscalingContext struct { ProvisioningRequestScaleUpMode bool // DraProvider is the provider for dynamic resources allocation. DraProvider *draprovider.Provider + // CsiProvider is the provider for CSI node aware scheduling. + CsiProvider *csinodeprovider.Provider } // AutoscalingKubeClients contains all Kubernetes API clients, @@ -112,6 +115,7 @@ func NewAutoscalingContext( remainingPdbTracker pdb.RemainingPdbTracker, clusterStateRegistry *clusterstate.ClusterStateRegistry, draProvider *draprovider.Provider, + csiProvider *csinodeprovider.Provider, ) *AutoscalingContext { return &AutoscalingContext{ AutoscalingOptions: options, @@ -125,6 +129,7 @@ func NewAutoscalingContext( RemainingPdbTracker: remainingPdbTracker, ClusterStateRegistry: clusterStateRegistry, DraProvider: draProvider, + CsiProvider: csiProvider, } } diff --git a/cluster-autoscaler/core/autoscaler.go b/cluster-autoscaler/core/autoscaler.go index 7c0d8f223102..dabfdb5dbe74 100644 --- a/cluster-autoscaler/core/autoscaler.go +++ b/cluster-autoscaler/core/autoscaler.go @@ -76,6 +76,7 @@ func NewAutoscaler(opts coreoptions.AutoscalerOptions, informerFactory informers opts.DeleteOptions, opts.DrainabilityRules, opts.DraProvider, + opts.CsiProvider, ), nil } @@ -91,14 +92,14 @@ func initializeDefaultOptions(opts *coreoptions.AutoscalerOptions, informerFacto opts.AutoscalingKubeClients = ca_context.NewAutoscalingKubeClients(opts.AutoscalingOptions, opts.KubeClient, opts.InformerFactory) } if opts.FrameworkHandle == nil { - fwHandle, err := framework.NewHandle(opts.InformerFactory, opts.SchedulerConfig, opts.DynamicResourceAllocationEnabled) + fwHandle, err := framework.NewHandle(opts.InformerFactory, opts.SchedulerConfig, opts.DynamicResourceAllocationEnabled, opts.CSINodeAwareSchedulingEnabled) if err != nil { return err } opts.FrameworkHandle = fwHandle } if opts.ClusterSnapshot == nil { - opts.ClusterSnapshot = predicate.NewPredicateSnapshot(store.NewBasicSnapshotStore(), opts.FrameworkHandle, opts.DynamicResourceAllocationEnabled, opts.PredicateParallelism) + opts.ClusterSnapshot = predicate.NewPredicateSnapshot(store.NewBasicSnapshotStore(), opts.FrameworkHandle, opts.DynamicResourceAllocationEnabled, opts.PredicateParallelism, opts.CSINodeAwareSchedulingEnabled) } if opts.RemainingPdbTracker == nil { opts.RemainingPdbTracker = pdb.NewBasicRemainingPdbTracker() diff --git a/cluster-autoscaler/core/options/autoscaler.go b/cluster-autoscaler/core/options/autoscaler.go index 0ee0dbb12ed8..b6f3c02623bd 100644 --- a/cluster-autoscaler/core/options/autoscaler.go +++ b/cluster-autoscaler/core/options/autoscaler.go @@ -28,6 +28,7 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/observers/loopstart" ca_processors "k8s.io/autoscaler/cluster-autoscaler/processors" "k8s.io/autoscaler/cluster-autoscaler/simulator/clustersnapshot" + csinodeprovider "k8s.io/autoscaler/cluster-autoscaler/simulator/csi/provider" "k8s.io/autoscaler/cluster-autoscaler/simulator/drainability/rules" draprovider "k8s.io/autoscaler/cluster-autoscaler/simulator/dynamicresources/provider" "k8s.io/autoscaler/cluster-autoscaler/simulator/framework" @@ -57,4 +58,5 @@ type AutoscalerOptions struct { DeleteOptions options.NodeDeleteOptions DrainabilityRules rules.Rules DraProvider *draprovider.Provider + CsiProvider *csinodeprovider.Provider } diff --git a/cluster-autoscaler/core/podlistprocessor/filter_out_expendable_test.go b/cluster-autoscaler/core/podlistprocessor/filter_out_expendable_test.go index dc3982e0ad25..27df2ca98b36 100644 --- a/cluster-autoscaler/core/podlistprocessor/filter_out_expendable_test.go +++ b/cluster-autoscaler/core/podlistprocessor/filter_out_expendable_test.go @@ -110,7 +110,7 @@ func TestFilterOutExpendable(t *testing.T) { t.Run(tc.name, func(t *testing.T) { processor := NewFilterOutExpendablePodListProcessor() snapshot := testsnapshot.NewTestSnapshotOrDie(t) - err := snapshot.SetClusterState(tc.nodes, nil, nil) + err := snapshot.SetClusterState(tc.nodes, nil, nil, nil) assert.NoError(t, err) pods, err := processor.Process(&ca_context.AutoscalingContext{ diff --git a/cluster-autoscaler/core/podlistprocessor/filter_out_schedulable_test.go b/cluster-autoscaler/core/podlistprocessor/filter_out_schedulable_test.go index 5eb7db7b64e5..3ddb5a488409 100644 --- a/cluster-autoscaler/core/podlistprocessor/filter_out_schedulable_test.go +++ b/cluster-autoscaler/core/podlistprocessor/filter_out_schedulable_test.go @@ -280,7 +280,7 @@ func BenchmarkFilterOutSchedulable(b *testing.B) { } clusterSnapshot := snapshotFactory() - if err := clusterSnapshot.SetClusterState(nodes, scheduledPods, nil); err != nil { + if err := clusterSnapshot.SetClusterState(nodes, scheduledPods, nil, nil); err != nil { assert.NoError(b, err) } diff --git a/cluster-autoscaler/core/scaledown/actuation/actuator.go b/cluster-autoscaler/core/scaledown/actuation/actuator.go index aa3b82beaf9b..d24809753abf 100644 --- a/cluster-autoscaler/core/scaledown/actuation/actuator.go +++ b/cluster-autoscaler/core/scaledown/actuation/actuator.go @@ -36,6 +36,7 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/simulator/clustersnapshot" "k8s.io/autoscaler/cluster-autoscaler/simulator/clustersnapshot/predicate" "k8s.io/autoscaler/cluster-autoscaler/simulator/clustersnapshot/store" + csisnapshot "k8s.io/autoscaler/cluster-autoscaler/simulator/csi/snapshot" "k8s.io/autoscaler/cluster-autoscaler/simulator/drainability/rules" drasnapshot "k8s.io/autoscaler/cluster-autoscaler/simulator/dynamicresources/snapshot" "k8s.io/autoscaler/cluster-autoscaler/simulator/options" @@ -397,7 +398,7 @@ func (a *Actuator) taintNode(node *apiv1.Node) error { } func (a *Actuator) createSnapshot(nodes []*apiv1.Node) (clustersnapshot.ClusterSnapshot, error) { - snapshot := predicate.NewPredicateSnapshot(store.NewBasicSnapshotStore(), a.autoscalingCtx.FrameworkHandle, a.autoscalingCtx.DynamicResourceAllocationEnabled, a.autoscalingCtx.PredicateParallelism) + snapshot := predicate.NewPredicateSnapshot(store.NewBasicSnapshotStore(), a.autoscalingCtx.FrameworkHandle, a.autoscalingCtx.DynamicResourceAllocationEnabled, a.autoscalingCtx.PredicateParallelism, a.autoscalingCtx.CSINodeAwareSchedulingEnabled) pods, err := a.autoscalingCtx.AllPodLister().List() if err != nil { return nil, err @@ -414,7 +415,15 @@ func (a *Actuator) createSnapshot(nodes []*apiv1.Node) (clustersnapshot.ClusterS } } - err = snapshot.SetClusterState(nodes, nonExpendableScheduledPods, draSnapshot) + var csiSnapshot *csisnapshot.Snapshot + if a.autoscalingCtx.CSINodeAwareSchedulingEnabled { + csiSnapshot, err = a.autoscalingCtx.CsiProvider.Snapshot() + if err != nil { + return nil, err + } + } + + err = snapshot.SetClusterState(nodes, nonExpendableScheduledPods, draSnapshot, csiSnapshot) if err != nil { return nil, err } diff --git a/cluster-autoscaler/core/scaledown/actuation/actuator_test.go b/cluster-autoscaler/core/scaledown/actuation/actuator_test.go index aa82ab4b8ff9..0c507834cce9 100644 --- a/cluster-autoscaler/core/scaledown/actuation/actuator_test.go +++ b/cluster-autoscaler/core/scaledown/actuation/actuator_test.go @@ -1228,7 +1228,7 @@ func runStartDeletionTest(t *testing.T, tc startDeletionTestCase, force bool) { t.Fatalf("Couldn't create daemonset lister") } - registry := kube_util.NewListerRegistry(nil, nil, podLister, pdbLister, dsLister, nil, nil, nil, nil) + registry := kube_util.NewListerRegistry(nil, nil, nil, podLister, pdbLister, dsLister, nil, nil, nil, nil) autoscalingCtx, err := NewScaleTestAutoscalingContext(opts, fakeClient, registry, provider, nil, nil) if err != nil { t.Fatalf("Couldn't set up autoscaling context: %v", err) @@ -1541,7 +1541,7 @@ func TestStartDeletionInBatchBasic(t *testing.T) { podLister := kube_util.NewTestPodLister([]*apiv1.Pod{}) pdbLister := kube_util.NewTestPodDisruptionBudgetLister([]*policyv1.PodDisruptionBudget{}) - registry := kube_util.NewListerRegistry(nil, nil, podLister, pdbLister, nil, nil, nil, nil, nil) + registry := kube_util.NewListerRegistry(nil, nil, nil, podLister, pdbLister, nil, nil, nil, nil, nil) autoscalingCtx, err := NewScaleTestAutoscalingContext(opts, fakeClient, registry, provider, nil, nil) if err != nil { t.Fatalf("Couldn't set up autoscaling context: %v", err) diff --git a/cluster-autoscaler/core/scaledown/actuation/drain_test.go b/cluster-autoscaler/core/scaledown/actuation/drain_test.go index 0b4e70599235..2914d8b413ac 100644 --- a/cluster-autoscaler/core/scaledown/actuation/drain_test.go +++ b/cluster-autoscaler/core/scaledown/actuation/drain_test.go @@ -139,7 +139,7 @@ func TestDaemonSetEvictionForEmptyNodes(t *testing.T) { provider := testprovider.NewTestCloudProviderBuilder().Build() provider.AddNodeGroup("ng1", 1, 10, 1) provider.AddNode("ng1", n1) - registry := kube_util.NewListerRegistry(nil, nil, nil, nil, nil, nil, nil, nil, nil) + registry := kube_util.NewListerRegistry(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) autoscalingCtx, err := NewScaleTestAutoscalingContext(options, fakeClient, registry, provider, nil, nil) assert.NoError(t, err) diff --git a/cluster-autoscaler/core/scaledown/actuation/group_deletion_scheduler_test.go b/cluster-autoscaler/core/scaledown/actuation/group_deletion_scheduler_test.go index 681641cbbb55..62414b9463a8 100644 --- a/cluster-autoscaler/core/scaledown/actuation/group_deletion_scheduler_test.go +++ b/cluster-autoscaler/core/scaledown/actuation/group_deletion_scheduler_test.go @@ -146,7 +146,7 @@ func TestScheduleDeletion(t *testing.T) { if err != nil { t.Fatalf("Couldn't create daemonset lister") } - registry := kube_util.NewListerRegistry(nil, nil, podLister, pdbLister, dsLister, nil, nil, nil, nil) + registry := kube_util.NewListerRegistry(nil, nil, nil, podLister, pdbLister, dsLister, nil, nil, nil, nil) autoscalingCtx, err := NewScaleTestAutoscalingContext(opts, fakeClient, registry, provider, nil, nil) if err != nil { t.Fatalf("Couldn't set up autoscaling context: %v", err) diff --git a/cluster-autoscaler/core/scaledown/actuation/softtaint_test.go b/cluster-autoscaler/core/scaledown/actuation/softtaint_test.go index fa8c5e7dc442..363afe91233e 100644 --- a/cluster-autoscaler/core/scaledown/actuation/softtaint_test.go +++ b/cluster-autoscaler/core/scaledown/actuation/softtaint_test.go @@ -67,7 +67,7 @@ func TestSoftTaintUpdate(t *testing.T) { MaxBulkSoftTaintCount: 1, MaxBulkSoftTaintTime: 3 * time.Second, } - registry := kube_util.NewListerRegistry(nil, nil, nil, nil, nil, nil, nil, nil, nil) + registry := kube_util.NewListerRegistry(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) actx, err := test.NewScaleTestAutoscalingContext(options, fakeClient, registry, provider, nil, nil) assert.NoError(t, err) @@ -151,7 +151,7 @@ func TestSoftTaintTimeLimit(t *testing.T) { MaxBulkSoftTaintCount: 10, MaxBulkSoftTaintTime: maxSoftTaintDuration, } - registry := kube_util.NewListerRegistry(nil, nil, nil, nil, nil, nil, nil, nil, nil) + registry := kube_util.NewListerRegistry(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) actx, err := test.NewScaleTestAutoscalingContext(options, fakeClient, registry, provider, nil, nil) assert.NoError(t, err) diff --git a/cluster-autoscaler/core/scaledown/eligibility/eligibility_test.go b/cluster-autoscaler/core/scaledown/eligibility/eligibility_test.go index 96a3a0f9435b..26e8d249228b 100644 --- a/cluster-autoscaler/core/scaledown/eligibility/eligibility_test.go +++ b/cluster-autoscaler/core/scaledown/eligibility/eligibility_test.go @@ -233,7 +233,7 @@ func TestFilterOutUnremovable(t *testing.T) { if err != nil { t.Fatalf("Could not create autoscaling context: %v", err) } - if err := autoscalingCtx.ClusterSnapshot.SetClusterState(tc.nodes, tc.pods, tc.draSnapshot); err != nil { + if err := autoscalingCtx.ClusterSnapshot.SetClusterState(tc.nodes, tc.pods, tc.draSnapshot, nil); err != nil { t.Fatalf("Could not SetClusterState: %v", err) } unremovableNodes := unremovable.NewNodes() diff --git a/cluster-autoscaler/core/scaledown/planner/controller_test.go b/cluster-autoscaler/core/scaledown/planner/controller_test.go index e2a013e6aaf2..58b27a0d312b 100644 --- a/cluster-autoscaler/core/scaledown/planner/controller_test.go +++ b/cluster-autoscaler/core/scaledown/planner/controller_test.go @@ -125,7 +125,7 @@ func TestReplicasCounter(t *testing.T) { jobLister, _ := kube_util.NewTestJobLister([]*batchv1.Job{job, unsetJob, jobWithSucceededReplicas}) rsLister, _ := kube_util.NewTestReplicaSetLister([]*appsv1.ReplicaSet{rs, unsetRs}) ssLister, _ := kube_util.NewTestStatefulSetLister([]*appsv1.StatefulSet{sS}) - listers := kube_util.NewListerRegistry(nil, nil, nil, nil, nil, rcLister, jobLister, rsLister, ssLister) + listers := kube_util.NewListerRegistry(nil, nil, nil, nil, nil, nil, rcLister, jobLister, rsLister, ssLister) testCases := []struct { name string ownerRef metav1.OwnerReference diff --git a/cluster-autoscaler/core/scaledown/planner/planner_test.go b/cluster-autoscaler/core/scaledown/planner/planner_test.go index 7e3036ae1763..5721a9e5ee31 100644 --- a/cluster-autoscaler/core/scaledown/planner/planner_test.go +++ b/cluster-autoscaler/core/scaledown/planner/planner_test.go @@ -487,7 +487,7 @@ func TestUpdateClusterState(t *testing.T) { } rsLister, err := kube_util.NewTestReplicaSetLister(tc.replicasSets) assert.NoError(t, err) - registry := kube_util.NewListerRegistry(nil, nil, nil, nil, nil, nil, nil, rsLister, nil) + registry := kube_util.NewListerRegistry(nil, nil, nil, nil, nil, nil, nil, nil, rsLister, nil) provider := testprovider.NewTestCloudProviderBuilder().Build() provider.AddNodeGroup("ng1", 0, 0, 0) for _, node := range tc.nodes { @@ -822,6 +822,7 @@ func TestNewPlannerWithExistingDeletionCandidateNodes(t *testing.T) { &fake.Clientset{}, kube_util.NewListerRegistry( allNodeLister, + nil, readyNodeLister, nil, nil, nil, nil, nil, nil, nil, ), diff --git a/cluster-autoscaler/core/scaledown/unneeded/nodes_test.go b/cluster-autoscaler/core/scaledown/unneeded/nodes_test.go index ca703f3428e0..e703814994eb 100644 --- a/cluster-autoscaler/core/scaledown/unneeded/nodes_test.go +++ b/cluster-autoscaler/core/scaledown/unneeded/nodes_test.go @@ -199,7 +199,7 @@ func TestRemovableAt(t *testing.T) { rsLister, err := kube_util.NewTestReplicaSetLister(nil) assert.NoError(t, err) - registry := kube_util.NewListerRegistry(nil, nil, nil, nil, nil, nil, nil, rsLister, nil) + registry := kube_util.NewListerRegistry(nil, nil, nil, nil, nil, nil, nil, nil, rsLister, nil) autoscalingCtx, err := NewScaleTestAutoscalingContext(config.AutoscalingOptions{ScaleDownSimulationTimeout: 5 * time.Minute}, &fake.Clientset{}, registry, provider, nil, nil) assert.NoError(t, err) @@ -282,7 +282,7 @@ func TestNodeLoadFromExistingTaints(t *testing.T) { readyNodeLister := kubernetes.NewTestNodeLister(nil) readyNodeLister.SetNodes(tc.allNodes) - listerRegistry := kube_util.NewListerRegistry(allNodeLister, readyNodeLister, + listerRegistry := kube_util.NewListerRegistry(allNodeLister, nil, readyNodeLister, nil, nil, nil, nil, nil, nil, nil) nodes.LoadFromExistingTaints(listerRegistry, currentTime, tc.nodeDeletionCandidateTTL) diff --git a/cluster-autoscaler/core/scaleup/orchestrator/async_initializer_test.go b/cluster-autoscaler/core/scaleup/orchestrator/async_initializer_test.go index f5aeb5927b5f..0f09c08cdeae 100644 --- a/cluster-autoscaler/core/scaleup/orchestrator/async_initializer_test.go +++ b/cluster-autoscaler/core/scaleup/orchestrator/async_initializer_test.go @@ -92,7 +92,7 @@ func TestNodePoolAsyncInitialization(t *testing.T) { }, }, } - listers := kube_util.NewListerRegistry(nil, nil, nil, nil, nil, nil, nil, nil, nil) + listers := kube_util.NewListerRegistry(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) upcomingNodeGroup := provider.BuildNodeGroup("upcoming-ng", 0, 100, 0, false, true, "T1", nil) options := config.AutoscalingOptions{AsyncNodeGroupsEnabled: true} context, err := NewScaleTestAutoscalingContext(options, &fake.Clientset{}, listers, provider, nil, nil) diff --git a/cluster-autoscaler/core/scaleup/orchestrator/orchestrator.go b/cluster-autoscaler/core/scaleup/orchestrator/orchestrator.go index 9826d704c8f9..63ca5087360a 100644 --- a/cluster-autoscaler/core/scaleup/orchestrator/orchestrator.go +++ b/cluster-autoscaler/core/scaleup/orchestrator/orchestrator.go @@ -107,8 +107,6 @@ func (o *ScaleUpOrchestrator) ScaleUp( if aErr != nil { return status.UpdateScaleUpError(&status.ScaleUpStatus{}, aErr.AddPrefix("could not get upcoming nodes: ")) } - klog.V(4).Infof("Upcoming %d nodes", len(upcomingNodes)) - nodeGroups := o.autoscalingCtx.CloudProvider.NodeGroups() if o.processors != nil && o.processors.NodeGroupListProcessor != nil { var err error @@ -140,6 +138,8 @@ func (o *ScaleUpOrchestrator) ScaleUp( schedulablePodGroups := map[string][]estimator.PodEquivalenceGroup{} var options []expander.Option + // This code here runs a simulation to see which pods can be scheduled on which node groups. + // TODO: Fix bug with CSI node not being added to the simulation. for _, nodeGroup := range validNodeGroups { schedulablePodGroups[nodeGroup.Id()] = o.SchedulablePodGroups(podEquivalenceGroups, nodeGroup, nodeInfos[nodeGroup.Id()]) } @@ -150,6 +150,7 @@ func (o *ScaleUpOrchestrator) ScaleUp( if len(option.Pods) == 0 || option.NodeCount == 0 { klog.V(4).Infof("No pod can fit to %s", nodeGroup.Id()) + klog.Infof("hemant no pod can fit to %s", nodeGroup.Id()) } else if allOrNothing && len(option.Pods) < len(unschedulablePods) { klog.V(4).Infof("Some pods can't fit to %s, giving up due to all-or-nothing scale-up strategy", nodeGroup.Id()) } else { diff --git a/cluster-autoscaler/core/scaleup/orchestrator/orchestrator_test.go b/cluster-autoscaler/core/scaleup/orchestrator/orchestrator_test.go index 82fe5b75def1..4522db448167 100644 --- a/cluster-autoscaler/core/scaleup/orchestrator/orchestrator_test.go +++ b/cluster-autoscaler/core/scaleup/orchestrator/orchestrator_test.go @@ -978,7 +978,7 @@ func runSimpleScaleUpTest(t *testing.T, config *ScaleUpTestConfig) *ScaleUpTestR extraPods[i] = buildTestPod(p) } podLister := kube_util.NewTestPodLister(pods) - listers := kube_util.NewListerRegistry(nil, nil, podLister, nil, nil, nil, nil, nil, nil) + listers := kube_util.NewListerRegistry(nil, nil, nil, podLister, nil, nil, nil, nil, nil, nil) // setup node groups var provider *testprovider.TestCloudProvider @@ -1044,7 +1044,7 @@ func runSimpleScaleUpTest(t *testing.T, config *ScaleUpTestConfig) *ScaleUpTestR // build orchestrator autoscalingCtx, err := NewScaleTestAutoscalingContext(options, &fake.Clientset{}, listers, provider, nil, nil) assert.NoError(t, err) - err = autoscalingCtx.ClusterSnapshot.SetClusterState(nodes, kube_util.ScheduledPods(pods), nil) + err = autoscalingCtx.ClusterSnapshot.SetClusterState(nodes, kube_util.ScheduledPods(pods), nil, nil) assert.NoError(t, err) nodeInfos, err := nodeinfosprovider.NewDefaultTemplateNodeInfoProvider(nil, false). Process(&autoscalingCtx, nodes, []*appsv1.DaemonSet{}, taints.TaintConfig{}, now) @@ -1136,7 +1136,7 @@ func TestScaleUpUnhealthy(t *testing.T) { pods := []*apiv1.Pod{p1, p2} podLister := kube_util.NewTestPodLister(pods) - listers := kube_util.NewListerRegistry(nil, nil, podLister, nil, nil, nil, nil, nil, nil) + listers := kube_util.NewListerRegistry(nil, nil, nil, podLister, nil, nil, nil, nil, nil, nil) provider := testprovider.NewTestCloudProviderBuilder().WithOnScaleUp(func(nodeGroup string, increase int) error { t.Fatalf("No expansion is expected, but increased %s by %d", nodeGroup, increase) @@ -1155,7 +1155,7 @@ func TestScaleUpUnhealthy(t *testing.T) { } autoscalingCtx, err := NewScaleTestAutoscalingContext(options, &fake.Clientset{}, listers, provider, nil, nil) assert.NoError(t, err) - err = autoscalingCtx.ClusterSnapshot.SetClusterState(nodes, pods, nil) + err = autoscalingCtx.ClusterSnapshot.SetClusterState(nodes, pods, nil, nil) assert.NoError(t, err) nodeInfos, _ := nodeinfosprovider.NewDefaultTemplateNodeInfoProvider(nil, false).Process(&autoscalingCtx, nodes, []*appsv1.DaemonSet{}, taints.TaintConfig{}, now) clusterState := clusterstate.NewClusterStateRegistry(provider, clusterstate.ClusterStateRegistryConfig{}, autoscalingCtx.LogRecorder, NewBackoff(), nodegroupconfig.NewDefaultNodeGroupConfigProcessor(config.NodeGroupAutoscalingOptions{MaxNodeProvisionTime: 15 * time.Minute}), asyncnodegroups.NewDefaultAsyncNodeGroupStateChecker()) @@ -1183,7 +1183,7 @@ func TestBinpackingLimiter(t *testing.T) { nodes := []*apiv1.Node{n1, n2} podLister := kube_util.NewTestPodLister([]*apiv1.Pod{}) - listers := kube_util.NewListerRegistry(nil, nil, podLister, nil, nil, nil, nil, nil, nil) + listers := kube_util.NewListerRegistry(nil, nil, nil, podLister, nil, nil, nil, nil, nil, nil) provider := testprovider.NewTestCloudProviderBuilder().WithOnScaleUp(func(nodeGroup string, increase int) error { return nil @@ -1198,7 +1198,7 @@ func TestBinpackingLimiter(t *testing.T) { autoscalingCtx, err := NewScaleTestAutoscalingContext(options, &fake.Clientset{}, listers, provider, nil, nil) assert.NoError(t, err) - err = autoscalingCtx.ClusterSnapshot.SetClusterState(nodes, nil, nil) + err = autoscalingCtx.ClusterSnapshot.SetClusterState(nodes, nil, nil, nil) assert.NoError(t, err) nodeInfos, err := nodeinfosprovider.NewDefaultTemplateNodeInfoProvider(nil, false). Process(&autoscalingCtx, nodes, []*appsv1.DaemonSet{}, taints.TaintConfig{}, now) @@ -1241,7 +1241,7 @@ func TestScaleUpNoHelp(t *testing.T) { pods := []*apiv1.Pod{p1} podLister := kube_util.NewTestPodLister(pods) - listers := kube_util.NewListerRegistry(nil, nil, podLister, nil, nil, nil, nil, nil, nil) + listers := kube_util.NewListerRegistry(nil, nil, nil, podLister, nil, nil, nil, nil, nil, nil) provider := testprovider.NewTestCloudProviderBuilder().WithOnScaleUp(func(nodeGroup string, increase int) error { t.Fatalf("No expansion is expected") @@ -1259,7 +1259,7 @@ func TestScaleUpNoHelp(t *testing.T) { } autoscalingCtx, err := NewScaleTestAutoscalingContext(options, &fake.Clientset{}, listers, provider, nil, nil) assert.NoError(t, err) - err = autoscalingCtx.ClusterSnapshot.SetClusterState(nodes, pods, nil) + err = autoscalingCtx.ClusterSnapshot.SetClusterState(nodes, pods, nil, nil) assert.NoError(t, err) nodeInfos, _ := nodeinfosprovider.NewDefaultTemplateNodeInfoProvider(nil, false).Process(&autoscalingCtx, nodes, []*appsv1.DaemonSet{}, taints.TaintConfig{}, now) clusterState := clusterstate.NewClusterStateRegistry(provider, clusterstate.ClusterStateRegistryConfig{}, autoscalingCtx.LogRecorder, NewBackoff(), nodegroupconfig.NewDefaultNodeGroupConfigProcessor(config.NodeGroupAutoscalingOptions{MaxNodeProvisionTime: 15 * time.Minute}), asyncnodegroups.NewDefaultAsyncNodeGroupStateChecker()) @@ -1411,10 +1411,10 @@ func TestComputeSimilarNodeGroups(t *testing.T) { nodeGroupSetProcessor.similarNodeGroups = append(nodeGroupSetProcessor.similarNodeGroups, provider.GetNodeGroup(ng)) } - listers := kube_util.NewListerRegistry(nil, nil, kube_util.NewTestPodLister(nil), nil, nil, nil, nil, nil, nil) + listers := kube_util.NewListerRegistry(nil, nil, nil, kube_util.NewTestPodLister(nil), nil, nil, nil, nil, nil, nil) autoscalingCtx, err := NewScaleTestAutoscalingContext(config.AutoscalingOptions{BalanceSimilarNodeGroups: tc.balancingEnabled, MaxNodeGroupBinpackingDuration: 1 * time.Second}, &fake.Clientset{}, listers, provider, nil, nil) assert.NoError(t, err) - err = autoscalingCtx.ClusterSnapshot.SetClusterState(nodes, nil, nil) + err = autoscalingCtx.ClusterSnapshot.SetClusterState(nodes, nil, nil, nil) assert.NoError(t, err) nodeInfos, _ := nodeinfosprovider.NewDefaultTemplateNodeInfoProvider(nil, false).Process(&autoscalingCtx, nodes, []*appsv1.DaemonSet{}, taints.TaintConfig{}, now) clusterState := clusterstate.NewClusterStateRegistry(provider, clusterstate.ClusterStateRegistryConfig{}, autoscalingCtx.LogRecorder, NewBackoff(), nodegroupconfig.NewDefaultNodeGroupConfigProcessor(config.NodeGroupAutoscalingOptions{MaxNodeProvisionTime: 15 * time.Minute}), asyncnodegroups.NewDefaultAsyncNodeGroupStateChecker()) @@ -1488,7 +1488,7 @@ func TestScaleUpBalanceGroups(t *testing.T) { } podLister := kube_util.NewTestPodLister(podList) - listers := kube_util.NewListerRegistry(nil, nil, podLister, nil, nil, nil, nil, nil, nil) + listers := kube_util.NewListerRegistry(nil, nil, nil, podLister, nil, nil, nil, nil, nil, nil) options := config.AutoscalingOptions{ EstimatorName: estimator.BinpackingEstimatorName, @@ -1499,7 +1499,7 @@ func TestScaleUpBalanceGroups(t *testing.T) { } autoscalingCtx, err := NewScaleTestAutoscalingContext(options, &fake.Clientset{}, listers, provider, nil, nil) assert.NoError(t, err) - err = autoscalingCtx.ClusterSnapshot.SetClusterState(nodes, podList, nil) + err = autoscalingCtx.ClusterSnapshot.SetClusterState(nodes, podList, nil, nil) assert.NoError(t, err) nodeInfos, _ := nodeinfosprovider.NewDefaultTemplateNodeInfoProvider(nil, false).Process(&autoscalingCtx, nodes, []*appsv1.DaemonSet{}, taints.TaintConfig{}, now) clusterState := clusterstate.NewClusterStateRegistry(provider, clusterstate.ClusterStateRegistryConfig{}, autoscalingCtx.LogRecorder, NewBackoff(), nodegroupconfig.NewDefaultNodeGroupConfigProcessor(config.NodeGroupAutoscalingOptions{MaxNodeProvisionTime: 15 * time.Minute}), asyncnodegroups.NewDefaultAsyncNodeGroupStateChecker()) @@ -1562,7 +1562,7 @@ func TestScaleUpAutoprovisionedNodeGroup(t *testing.T) { MaxNodeGroupBinpackingDuration: 1 * time.Second, } podLister := kube_util.NewTestPodLister([]*apiv1.Pod{}) - listers := kube_util.NewListerRegistry(nil, nil, podLister, nil, nil, nil, nil, nil, nil) + listers := kube_util.NewListerRegistry(nil, nil, nil, podLister, nil, nil, nil, nil, nil, nil) autoscalingCtx, err := NewScaleTestAutoscalingContext(options, fakeClient, listers, provider, nil, nil) assert.NoError(t, err) @@ -1614,7 +1614,7 @@ func TestScaleUpBalanceAutoprovisionedNodeGroups(t *testing.T) { MaxNodeGroupBinpackingDuration: 1 * time.Second, } podLister := kube_util.NewTestPodLister([]*apiv1.Pod{}) - listers := kube_util.NewListerRegistry(nil, nil, podLister, nil, nil, nil, nil, nil, nil) + listers := kube_util.NewListerRegistry(nil, nil, nil, podLister, nil, nil, nil, nil, nil, nil) autoscalingCtx, err := NewScaleTestAutoscalingContext(options, fakeClient, listers, provider, nil, nil) assert.NoError(t, err) @@ -1644,7 +1644,7 @@ func TestScaleUpBalanceAutoprovisionedNodeGroups(t *testing.T) { func TestScaleUpToMeetNodeGroupMinSize(t *testing.T) { podLister := kube_util.NewTestPodLister([]*apiv1.Pod{}) - listers := kube_util.NewListerRegistry(nil, nil, podLister, nil, nil, nil, nil, nil, nil) + listers := kube_util.NewListerRegistry(nil, nil, nil, podLister, nil, nil, nil, nil, nil, nil) provider := testprovider.NewTestCloudProviderBuilder().WithOnScaleUp(func(nodeGroup string, increase int) error { assert.Equal(t, "ng1", nodeGroup) assert.Equal(t, 1, increase) @@ -1678,7 +1678,7 @@ func TestScaleUpToMeetNodeGroupMinSize(t *testing.T) { assert.NoError(t, err) nodes := []*apiv1.Node{n1, n2} - err = autoscalingCtx.ClusterSnapshot.SetClusterState(nodes, nil, nil) + err = autoscalingCtx.ClusterSnapshot.SetClusterState(nodes, nil, nil, nil) assert.NoError(t, err) nodeInfos, _ := nodeinfosprovider.NewDefaultTemplateNodeInfoProvider(nil, false).Process(&autoscalingCtx, nodes, []*appsv1.DaemonSet{}, taints.TaintConfig{}, time.Now()) processors := processorstest.NewTestProcessors(&autoscalingCtx) @@ -1764,7 +1764,7 @@ func TestScaleupAsyncNodeGroupsEnabled(t *testing.T) { MaxNodeGroupBinpackingDuration: 1 * time.Second, } podLister := kube_util.NewTestPodLister([]*apiv1.Pod{}) - listers := kube_util.NewListerRegistry(nil, nil, podLister, nil, nil, nil, nil, nil, nil) + listers := kube_util.NewListerRegistry(nil, nil, nil, podLister, nil, nil, nil, nil, nil, nil) autoscalingCtx, err := NewScaleTestAutoscalingContext(options, fakeClient, listers, provider, nil, nil) assert.NoError(t, err) diff --git a/cluster-autoscaler/core/scaleup/resource/manager_test.go b/cluster-autoscaler/core/scaleup/resource/manager_test.go index 7ea408b564d6..371f9531a0e5 100644 --- a/cluster-autoscaler/core/scaleup/resource/manager_test.go +++ b/cluster-autoscaler/core/scaleup/resource/manager_test.go @@ -71,7 +71,7 @@ func TestDeltaForNode(t *testing.T) { ng := testCase.nodeGroupConfig group, nodes := newNodeGroup(t, cp, ng.Name, ng.Min, ng.Max, ng.Size, ng.CPU, ng.Mem) - err := autoscalingCtx.ClusterSnapshot.SetClusterState(nodes, nil, nil) + err := autoscalingCtx.ClusterSnapshot.SetClusterState(nodes, nil, nil, nil) assert.NoError(t, err) nodeInfos, _ := nodeinfosprovider.NewDefaultTemplateNodeInfoProvider(nil, false).Process(&autoscalingCtx, nodes, []*appsv1.DaemonSet{}, taints.TaintConfig{}, time.Now()) @@ -114,7 +114,7 @@ func TestResourcesLeft(t *testing.T) { ng := testCase.nodeGroupConfig _, nodes := newNodeGroup(t, cp, ng.Name, ng.Min, ng.Max, ng.Size, ng.CPU, ng.Mem) - err := autoscalingCtx.ClusterSnapshot.SetClusterState(nodes, nil, nil) + err := autoscalingCtx.ClusterSnapshot.SetClusterState(nodes, nil, nil, nil) assert.NoError(t, err) nodeInfos, _ := nodeinfosprovider.NewDefaultTemplateNodeInfoProvider(nil, false).Process(&autoscalingCtx, nodes, []*appsv1.DaemonSet{}, taints.TaintConfig{}, time.Now()) @@ -167,7 +167,7 @@ func TestApplyLimits(t *testing.T) { ng := testCase.nodeGroupConfig group, nodes := newNodeGroup(t, cp, ng.Name, ng.Min, ng.Max, ng.Size, ng.CPU, ng.Mem) - err := autoscalingCtx.ClusterSnapshot.SetClusterState(nodes, nil, nil) + err := autoscalingCtx.ClusterSnapshot.SetClusterState(nodes, nil, nil, nil) assert.NoError(t, err) nodeInfos, _ := nodeinfosprovider.NewDefaultTemplateNodeInfoProvider(nil, false).Process(&autoscalingCtx, nodes, []*appsv1.DaemonSet{}, taints.TaintConfig{}, time.Now()) @@ -234,7 +234,7 @@ func TestResourceManagerWithGpuResource(t *testing.T) { assert.NoError(t, err) nodes := []*corev1.Node{n1} - err = autoscalingCtx.ClusterSnapshot.SetClusterState(nodes, nil, nil) + err = autoscalingCtx.ClusterSnapshot.SetClusterState(nodes, nil, nil, nil) assert.NoError(t, err) nodeInfos, _ := nodeinfosprovider.NewDefaultTemplateNodeInfoProvider(nil, false).Process(&autoscalingCtx, nodes, []*appsv1.DaemonSet{}, taints.TaintConfig{}, time.Now()) @@ -271,7 +271,7 @@ func newCloudProvider(t *testing.T, cpu, mem int64) *testprovider.TestCloudProvi func newAutoscalingContext(t *testing.T, provider cloudprovider.CloudProvider) ca_context.AutoscalingContext { podLister := kube_util.NewTestPodLister([]*corev1.Pod{}) - listers := kube_util.NewListerRegistry(nil, nil, podLister, nil, nil, nil, nil, nil, nil) + listers := kube_util.NewListerRegistry(nil, nil, nil, podLister, nil, nil, nil, nil, nil, nil) autoscalingCtx, err := test.NewScaleTestAutoscalingContext(config.AutoscalingOptions{}, &fake.Clientset{}, listers, provider, nil, nil) assert.NoError(t, err) return autoscalingCtx diff --git a/cluster-autoscaler/core/static_autoscaler.go b/cluster-autoscaler/core/static_autoscaler.go index 4512aa5beedf..68fad8d55304 100644 --- a/cluster-autoscaler/core/static_autoscaler.go +++ b/cluster-autoscaler/core/static_autoscaler.go @@ -45,6 +45,8 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/processors/status" "k8s.io/autoscaler/cluster-autoscaler/simulator" "k8s.io/autoscaler/cluster-autoscaler/simulator/clustersnapshot" + csinodeprovider "k8s.io/autoscaler/cluster-autoscaler/simulator/csi/provider" + csisnapshot "k8s.io/autoscaler/cluster-autoscaler/simulator/csi/snapshot" "k8s.io/autoscaler/cluster-autoscaler/simulator/drainability/rules" draprovider "k8s.io/autoscaler/cluster-autoscaler/simulator/dynamicresources/provider" drasnapshot "k8s.io/autoscaler/cluster-autoscaler/simulator/dynamicresources/snapshot" @@ -141,7 +143,8 @@ func NewStaticAutoscaler( scaleUpOrchestrator scaleup.Orchestrator, deleteOptions options.NodeDeleteOptions, drainabilityRules rules.Rules, - draProvider *draprovider.Provider) *StaticAutoscaler { + draProvider *draprovider.Provider, + csiProvider *csinodeprovider.Provider) *StaticAutoscaler { klog.V(4).Infof("Creating new static autoscaler with opts: %v", opts) @@ -162,7 +165,8 @@ func NewStaticAutoscaler( debuggingSnapshotter, remainingPdbTracker, clusterStateRegistry, - draProvider) + draProvider, + csiProvider) taintConfig := taints.NewTaintConfig(opts) processors.ScaleDownCandidatesNotifier.Register(clusterStateRegistry) @@ -280,8 +284,17 @@ func (a *StaticAutoscaler) RunOnce(currentTime time.Time) caerrors.AutoscalerErr } } + var csiSnapshot *csisnapshot.Snapshot + if a.AutoscalingContext.CsiProvider != nil { + var err error + csiSnapshot, err = a.AutoscalingContext.CsiProvider.Snapshot() + if err != nil { + return caerrors.ToAutoscalerError(caerrors.ApiCallError, err) + } + } + // Get nodes and pods currently living on cluster - allNodes, readyNodes, typedErr := a.obtainNodeLists(draSnapshot) + allNodes, readyNodes, typedErr := a.obtainNodeLists(draSnapshot, csiSnapshot) if typedErr != nil { klog.Errorf("Failed to get node list: %v", typedErr) return typedErr @@ -340,7 +353,7 @@ func (a *StaticAutoscaler) RunOnce(currentTime time.Time) caerrors.AutoscalerErr } nonExpendableScheduledPods := core_utils.FilterOutExpendablePods(podsBySchedulability.Scheduled, a.ExpendablePodsPriorityCutoff) - if err := a.ClusterSnapshot.SetClusterState(allNodes, nonExpendableScheduledPods, draSnapshot); err != nil { + if err := a.ClusterSnapshot.SetClusterState(allNodes, nonExpendableScheduledPods, draSnapshot, csiSnapshot); err != nil { return caerrors.ToAutoscalerError(caerrors.InternalError, err).AddPrefix("failed to initialize ClusterSnapshot: ") } // Initialize Pod Disruption Budget tracking @@ -990,7 +1003,7 @@ func (a *StaticAutoscaler) ExitCleanUp() { a.clusterStateRegistry.Stop() } -func (a *StaticAutoscaler) obtainNodeLists(draSnapshot *drasnapshot.Snapshot) ([]*apiv1.Node, []*apiv1.Node, caerrors.AutoscalerError) { +func (a *StaticAutoscaler) obtainNodeLists(draSnapshot *drasnapshot.Snapshot, csiSnapshot *csisnapshot.Snapshot) ([]*apiv1.Node, []*apiv1.Node, caerrors.AutoscalerError) { allNodes, err := a.AllNodeLister().List() if err != nil { klog.Errorf("Failed to list all nodes: %v", err) @@ -1008,7 +1021,7 @@ func (a *StaticAutoscaler) obtainNodeLists(draSnapshot *drasnapshot.Snapshot) ([ // Treat those nodes as unready until GPU actually becomes available and let // our normal handling for booting up nodes deal with this. // TODO: Remove this call when we handle dynamically provisioned resources. - allNodes, readyNodes = a.processors.CustomResourcesProcessor.FilterOutNodesWithUnreadyResources(a.AutoscalingContext, allNodes, readyNodes, draSnapshot) + allNodes, readyNodes = a.processors.CustomResourcesProcessor.FilterOutNodesWithUnreadyResources(a.AutoscalingContext, allNodes, readyNodes, draSnapshot, csiSnapshot) allNodes, readyNodes = taints.FilterOutNodesWithStartupTaints(a.taintConfig, allNodes, readyNodes) return allNodes, readyNodes, nil } diff --git a/cluster-autoscaler/core/static_autoscaler_test.go b/cluster-autoscaler/core/static_autoscaler_test.go index 5425875cbe6a..a934a6d6ee1f 100644 --- a/cluster-autoscaler/core/static_autoscaler_test.go +++ b/cluster-autoscaler/core/static_autoscaler_test.go @@ -295,7 +295,7 @@ func setupAutoscaler(config *autoscalerSetupConfig) (*StaticAutoscaler, error) { // Create all necessary autoscaler dependencies, applying the mocks from config. processorCallbacks := newStaticAutoscalerProcessorCallbacks() - listerRegistry := kube_util.NewListerRegistry(config.mocks.allNodeLister, config.mocks.readyNodeLister, config.mocks.allPodLister, + listerRegistry := kube_util.NewListerRegistry(config.mocks.allNodeLister, nil, config.mocks.readyNodeLister, config.mocks.allPodLister, config.mocks.podDisruptionBudgetLister, config.mocks.daemonSetLister, nil, nil, nil, nil) autoscalingCtx, err := NewScaleTestAutoscalingContext(config.autoscalingOptions, &fake.Clientset{}, listerRegistry, provider, processorCallbacks, nil) if err != nil { @@ -402,7 +402,7 @@ func TestStaticAutoscalerRunOnce(t *testing.T) { setUpScaleDownActuator(&autoscalingCtx, options) - listerRegistry := kube_util.NewListerRegistry(allNodeLister, readyNodeLister, allPodListerMock, podDisruptionBudgetListerMock, daemonSetListerMock, + listerRegistry := kube_util.NewListerRegistry(allNodeLister, nil, readyNodeLister, allPodListerMock, podDisruptionBudgetListerMock, daemonSetListerMock, nil, nil, nil, nil) autoscalingCtx.ListerRegistry = listerRegistry @@ -662,7 +662,7 @@ func TestStaticAutoscalerRunOnceWithScaleDownDelayPerNG(t *testing.T) { setUpScaleDownActuator(&autoscalingCtx, options) - listerRegistry := kube_util.NewListerRegistry(allNodeLister, readyNodeLister, allPodListerMock, podDisruptionBudgetListerMock, daemonSetListerMock, + listerRegistry := kube_util.NewListerRegistry(allNodeLister, nil, readyNodeLister, allPodListerMock, podDisruptionBudgetListerMock, daemonSetListerMock, nil, nil, nil, nil) autoscalingCtx.ListerRegistry = listerRegistry @@ -815,7 +815,7 @@ func TestStaticAutoscalerRunOnceWithAutoprovisionedEnabled(t *testing.T) { processors.NodeGroupManager = nodeGroupManager processors.NodeGroupListProcessor = nodeGroupListProcessor - listerRegistry := kube_util.NewListerRegistry(allNodeLister, readyNodeLister, allPodListerMock, + listerRegistry := kube_util.NewListerRegistry(allNodeLister, nil, readyNodeLister, allPodListerMock, podDisruptionBudgetListerMock, daemonSetListerMock, nil, nil, nil, nil) autoscalingCtx.ListerRegistry = listerRegistry @@ -959,7 +959,7 @@ func TestStaticAutoscalerRunOnceWithALongUnregisteredNode(t *testing.T) { setUpScaleDownActuator(&autoscalingCtx, options) - listerRegistry := kube_util.NewListerRegistry(allNodeLister, readyNodeLister, allPodListerMock, + listerRegistry := kube_util.NewListerRegistry(allNodeLister, nil, readyNodeLister, allPodListerMock, podDisruptionBudgetListerMock, daemonSetListerMock, nil, nil, nil, nil) autoscalingCtx.ListerRegistry = listerRegistry @@ -1124,7 +1124,7 @@ func TestStaticAutoscalerRunOncePodsWithPriorities(t *testing.T) { setUpScaleDownActuator(&autoscalingCtx, options) - listerRegistry := kube_util.NewListerRegistry(allNodeLister, readyNodeLister, allPodListerMock, + listerRegistry := kube_util.NewListerRegistry(allNodeLister, nil, readyNodeLister, allPodListerMock, podDisruptionBudgetListerMock, daemonSetListerMock, nil, nil, nil, nil) autoscalingCtx.ListerRegistry = listerRegistry @@ -1256,7 +1256,7 @@ func TestStaticAutoscalerRunOnceWithFilteringOnBinPackingEstimator(t *testing.T) setUpScaleDownActuator(&autoscalingCtx, options) - listerRegistry := kube_util.NewListerRegistry(allNodeLister, readyNodeLister, allPodListerMock, + listerRegistry := kube_util.NewListerRegistry(allNodeLister, nil, readyNodeLister, allPodListerMock, podDisruptionBudgetListerMock, daemonSetListerMock, nil, nil, nil, nil) autoscalingCtx.ListerRegistry = listerRegistry @@ -1355,7 +1355,7 @@ func TestStaticAutoscalerRunOnceWithFilteringOnUpcomingNodesEnabledNoScaleUp(t * setUpScaleDownActuator(&autoscalingCtx, options) - listerRegistry := kube_util.NewListerRegistry(allNodeLister, readyNodeLister, allPodListerMock, + listerRegistry := kube_util.NewListerRegistry(allNodeLister, nil, readyNodeLister, allPodListerMock, podDisruptionBudgetListerMock, daemonSetListerMock, nil, nil, nil, nil) autoscalingCtx.ListerRegistry = listerRegistry @@ -1707,7 +1707,7 @@ func TestStaticAutoscalerRunOnceWithExistingDeletionCandidateNodes(t *testing.T) setUpScaleDownActuator(&autoscalingCtx, options) - listerRegistry := kube_util.NewListerRegistry(allNodeLister, readyNodeLister, allPodListerMock, podDisruptionBudgetListerMock, daemonSetListerMock, + listerRegistry := kube_util.NewListerRegistry(allNodeLister, nil, readyNodeLister, allPodListerMock, podDisruptionBudgetListerMock, daemonSetListerMock, nil, nil, nil, nil) autoscalingCtx.ListerRegistry = listerRegistry @@ -2459,7 +2459,7 @@ func TestStaticAutoscalerUpcomingScaleDownCandidates(t *testing.T) { readyNodeLister := kubernetes.NewTestNodeLister(readyNodes) daemonSetLister, err := kubernetes.NewTestDaemonSetLister(nil) assert.NoError(t, err) - listerRegistry := kube_util.NewListerRegistry(allNodeLister, readyNodeLister, + listerRegistry := kube_util.NewListerRegistry(allNodeLister, nil, readyNodeLister, kubernetes.NewTestPodLister(nil), kubernetes.NewTestPodDisruptionBudgetLister(nil), daemonSetLister, nil, nil, nil, nil) @@ -3255,7 +3255,7 @@ func buildStaticAutoscaler(t *testing.T, provider cloudprovider.CloudProvider, a daemonSetLister, err := kubernetes.NewTestDaemonSetLister(nil) assert.NoError(t, err) - listerRegistry := kube_util.NewListerRegistry(allNodeLister, readyNodeLister, + listerRegistry := kube_util.NewListerRegistry(allNodeLister, nil, readyNodeLister, kubernetes.NewTestPodLister(nil), kubernetes.NewTestPodDisruptionBudgetLister(nil), daemonSetLister, nil, nil, nil, nil) diff --git a/cluster-autoscaler/estimator/binpacking_estimator.go b/cluster-autoscaler/estimator/binpacking_estimator.go index 10ed1dfa3d63..7a610dbef351 100644 --- a/cluster-autoscaler/estimator/binpacking_estimator.go +++ b/cluster-autoscaler/estimator/binpacking_estimator.go @@ -48,6 +48,7 @@ type estimationState struct { newNodeNameIndex int lastNodeName string newNodeNames map[string]bool + // map of node name that has at least one pod scheduled on it newNodesWithPods map[string]bool } @@ -253,6 +254,11 @@ func (e *BinpackingNodeEstimator) addNewNodeToSnapshot( if err != nil { return err } + + if template.CSINode != nil { + newNodeInfo.AddCSINode(core_utils.CreateSanitizedCSINode(template.CSINode, newNodeInfo)) + } + if err := e.clusterSnapshot.AddNodeInfo(newNodeInfo); err != nil { return err } diff --git a/cluster-autoscaler/go.mod b/cluster-autoscaler/go.mod index ca50b2e46b64..a1ad05df1360 100644 --- a/cluster-autoscaler/go.mod +++ b/cluster-autoscaler/go.mod @@ -1,9 +1,9 @@ module k8s.io/autoscaler/cluster-autoscaler -go 1.24.0 +go 1.25.0 require ( - cloud.google.com/go/compute/metadata v0.6.0 + cloud.google.com/go/compute/metadata v0.7.0 github.com/Azure/azure-sdk-for-go v68.0.0+incompatible github.com/Azure/azure-sdk-for-go-extensions v0.1.6 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0 @@ -26,18 +26,18 @@ require ( github.com/jmespath/go-jmespath v0.4.0 github.com/json-iterator/go v1.1.12 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.22.0 - github.com/spf13/pflag v1.0.6 - github.com/stretchr/testify v1.10.0 + github.com/prometheus/client_golang v1.23.2 + github.com/spf13/pflag v1.0.9 + github.com/stretchr/testify v1.11.1 github.com/vburenin/ifacemaker v1.2.1 go.uber.org/mock v0.4.0 - golang.org/x/crypto v0.36.0 - golang.org/x/net v0.38.0 - golang.org/x/oauth2 v0.27.0 - golang.org/x/sys v0.31.0 + golang.org/x/crypto v0.41.0 + golang.org/x/net v0.43.0 + golang.org/x/oauth2 v0.30.0 + golang.org/x/sys v0.37.0 google.golang.org/api v0.151.0 - google.golang.org/grpc v1.72.1 - google.golang.org/protobuf v1.36.5 + google.golang.org/grpc v1.72.2 + google.golang.org/protobuf v1.36.8 gopkg.in/gcfg.v1 v1.2.3 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.34.1 @@ -55,7 +55,7 @@ require ( k8s.io/kube-scheduler v0.0.0 k8s.io/kubelet v0.34.1 k8s.io/kubernetes v1.34.1 - k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 + k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 sigs.k8s.io/cloud-provider-azure v1.29.4 sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.13 sigs.k8s.io/yaml v1.6.0 @@ -85,6 +85,7 @@ require ( github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect github.com/GoogleCloudPlatform/k8s-cloud-provider v1.25.0 // indirect github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab // indirect + github.com/Masterminds/semver/v3 v3.4.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/Microsoft/hnslib v0.1.1 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect @@ -94,12 +95,12 @@ require ( github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/container-storage-interface/spec v1.9.0 // indirect - github.com/containerd/containerd/api v1.8.0 // indirect + github.com/containerd/containerd/api v1.9.0 // indirect github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect - github.com/containerd/ttrpc v1.2.6 // indirect - github.com/containerd/typeurl/v2 v2.2.2 // indirect + github.com/containerd/ttrpc v1.2.7 // indirect + github.com/containerd/typeurl/v2 v2.2.3 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cyphar/filepath-securejoin v0.4.1 // indirect @@ -107,14 +108,13 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/distribution/reference v0.6.0 // indirect - github.com/docker/docker v27.1.1+incompatible // indirect github.com/docker/go-units v0.5.0 // indirect github.com/emicklei/go-restful/v3 v3.12.2 // indirect github.com/euank/go-kmsg-parser v2.0.0+incompatible // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect - github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect @@ -128,10 +128,10 @@ require ( github.com/golang-jwt/jwt/v5 v5.2.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/btree v1.1.3 // indirect - github.com/google/cadvisor v0.52.1 // indirect + github.com/google/cadvisor v0.53.0 // indirect github.com/google/cel-go v0.26.0 // indirect github.com/google/gnostic-models v0.7.0 // indirect - github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect + github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect @@ -157,52 +157,53 @@ require ( github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect - github.com/onsi/ginkgo/v2 v2.21.0 // indirect - github.com/onsi/gomega v1.35.1 // indirect - github.com/opencontainers/cgroups v0.0.1 // indirect + github.com/onsi/ginkgo/v2 v2.27.2 // indirect + github.com/onsi/gomega v1.38.2 // indirect + github.com/opencontainers/cgroups v0.0.3 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect - github.com/opencontainers/runtime-spec v1.2.0 // indirect + github.com/opencontainers/runtime-spec v1.2.1 // indirect github.com/opencontainers/selinux v1.11.1 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.62.0 // indirect - github.com/prometheus/procfs v0.15.1 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect github.com/sirupsen/logrus v1.9.3 // indirect - github.com/spf13/cobra v1.9.1 // indirect + github.com/spf13/cobra v1.10.0 // indirect github.com/stoewer/go-strcase v1.3.0 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/x448/float16 v0.8.4 // indirect - go.etcd.io/etcd/api/v3 v3.6.4 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.6.4 // indirect - go.etcd.io/etcd/client/v3 v3.6.4 // indirect + go.etcd.io/etcd/api/v3 v3.6.5 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.6.5 // indirect + go.etcd.io/etcd/client/v3 v3.6.5 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.44.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect - go.opentelemetry.io/otel v1.35.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect + go.opentelemetry.io/otel v1.36.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 // indirect - go.opentelemetry.io/otel/metric v1.35.0 // indirect - go.opentelemetry.io/otel/sdk v1.34.0 // indirect - go.opentelemetry.io/otel/trace v1.35.0 // indirect + go.opentelemetry.io/otel/metric v1.36.0 // indirect + go.opentelemetry.io/otel/sdk v1.36.0 // indirect + go.opentelemetry.io/otel/trace v1.36.0 // indirect go.opentelemetry.io/proto/otlp v1.5.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - go.yaml.in/yaml/v2 v2.4.2 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect - golang.org/x/mod v0.21.0 // indirect - golang.org/x/sync v0.12.0 // indirect - golang.org/x/term v0.30.0 // indirect - golang.org/x/text v0.23.0 // indirect + golang.org/x/mod v0.28.0 // indirect + golang.org/x/sync v0.17.0 // indirect + golang.org/x/term v0.36.0 // indirect + golang.org/x/text v0.29.0 // indirect golang.org/x/time v0.9.0 // indirect - golang.org/x/tools v0.26.0 // indirect + golang.org/x/tools v0.36.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb // indirect - gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect @@ -213,14 +214,14 @@ require ( k8s.io/cri-api v0.34.1 // indirect k8s.io/cri-client v0.0.0 // indirect k8s.io/csi-translation-lib v0.27.0 // indirect - k8s.io/gengo/v2 v2.0.0-20250604051438-85fd79dbfd9f // indirect + k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b // indirect k8s.io/kms v0.34.1 // indirect - k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect + k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect k8s.io/kubectl v0.28.0 // indirect k8s.io/mount-utils v0.26.0-alpha.0 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.4 // indirect - sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect ) @@ -229,29 +230,29 @@ replace github.com/aws/aws-sdk-go/service/eks => github.com/aws/aws-sdk-go/servi replace github.com/rancher/go-rancher => github.com/rancher/go-rancher v0.1.0 -replace k8s.io/api => k8s.io/api v0.34.1 +replace k8s.io/api => github.com/kubernetes/api v0.0.0-20251107002836-f1737241c064 -replace k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.34.1 +replace k8s.io/apiextensions-apiserver => github.com/kubernetes/apiextensions-apiserver v0.0.0-20251105125549-c4e434ca2251 -replace k8s.io/apimachinery => k8s.io/apimachinery v0.34.1 +replace k8s.io/apimachinery => github.com/kubernetes/apimachinery v0.35.0-alpha.3.0.20251106231852-6f8949260573 -replace k8s.io/apiserver => k8s.io/apiserver v0.34.1 +replace k8s.io/apiserver => github.com/kubernetes/apiserver v0.0.0-20251107005520-bce27587d0bd replace k8s.io/cli-runtime => k8s.io/cli-runtime v0.34.1 -replace k8s.io/client-go => k8s.io/client-go v0.34.1 +replace k8s.io/client-go => github.com/kubernetes/client-go v0.0.0-20251107003311-5c322d3acdac -replace k8s.io/cloud-provider => k8s.io/cloud-provider v0.34.1 +replace k8s.io/cloud-provider => github.com/kubernetes/cloud-provider v0.0.0-20251106211155-080e91c4b910 -replace k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.34.1 +replace k8s.io/cluster-bootstrap => github.com/kubernetes/cluster-bootstrap v0.0.0-20251101161919-bf1255e49e2f replace k8s.io/code-generator => k8s.io/code-generator v0.34.1 -replace k8s.io/component-base => k8s.io/component-base v0.34.1 +replace k8s.io/component-base => github.com/kubernetes/component-base v0.0.0-20251105043606-09c454e1f74b -replace k8s.io/component-helpers => k8s.io/component-helpers v0.34.1 +replace k8s.io/component-helpers => github.com/kubernetes/component-helpers v0.20.0-alpha.2.0.20251106124553-0e2bf40485ce -replace k8s.io/controller-manager => k8s.io/controller-manager v0.34.1 +replace k8s.io/controller-manager => github.com/kubernetes/controller-manager v0.20.0-alpha.1.0.20251106210955-362c11ff4757 replace k8s.io/cri-api => k8s.io/cri-api v0.34.1 @@ -259,11 +260,11 @@ replace k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.34.1 replace k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.34.1 -replace k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.34.1 +replace k8s.io/kube-controller-manager => github.com/kubernetes/kube-controller-manager v0.0.0-20251105011811-223bddf58da7 replace k8s.io/kube-proxy => k8s.io/kube-proxy v0.34.1 -replace k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.34.1 +replace k8s.io/kube-scheduler => github.com/kubernetes/kube-scheduler v0.0.0-20251106172524-0962b2d7c894 replace k8s.io/kubectl => k8s.io/kubectl v0.34.1 @@ -281,7 +282,7 @@ replace k8s.io/sample-controller => k8s.io/sample-controller v0.34.1 replace k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.34.1 -replace k8s.io/dynamic-resource-allocation => k8s.io/dynamic-resource-allocation v0.34.1 +replace k8s.io/dynamic-resource-allocation => github.com/kubernetes/dynamic-resource-allocation v0.26.0-beta.0.0.20251107013247-fafc9e288a2f replace k8s.io/kms => k8s.io/kms v0.34.1 @@ -292,3 +293,5 @@ replace k8s.io/autoscaler/cluster-autoscaler/apis => ./apis replace k8s.io/cri-client => k8s.io/cri-client v0.34.1 replace k8s.io/externaljwt => k8s.io/externaljwt v0.34.1 + +replace k8s.io/kubernetes => github.com/kubernetes/kubernetes v1.35.0-alpha.3.0.20251107154100-609e2e57dacd diff --git a/cluster-autoscaler/go.sum b/cluster-autoscaler/go.sum index 5d678a44f816..c47887563773 100644 --- a/cluster-autoscaler/go.sum +++ b/cluster-autoscaler/go.sum @@ -1,8 +1,8 @@ cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I= -cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg= +cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU= +cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo= github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go-extensions v0.1.6 h1:EXGvDcj54u98XfaI/Cy65Ds6vNsIJeGKYf0eNLB1y4Q= @@ -80,6 +80,8 @@ github.com/GoogleCloudPlatform/k8s-cloud-provider v1.25.0 h1:lwL1vLWmdBJ5h+StMEN github.com/GoogleCloudPlatform/k8s-cloud-provider v1.25.0/go.mod h1:UTfhBnADaj2rybPT049NScSh7Eall3u2ib43wmz3deg= github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab h1:UKkYhof1njT1/xq4SEg5z+VpTgjmNeHwPGRQl7takDI= github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5hodBMJ5+l/7J4xAyMeuM2PNuepvHlGs8yilUCA= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Microsoft/hnslib v0.1.1 h1:JsZy681SnvSOUAfCZVAxkX4LgQGp+CZZwPbLV0/pdF8= @@ -107,18 +109,18 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/container-storage-interface/spec v1.9.0 h1:zKtX4STsq31Knz3gciCYCi1SXtO2HJDecIjDVboYavY= github.com/container-storage-interface/spec v1.9.0/go.mod h1:ZfDu+3ZRyeVqxZM0Ds19MVLkN2d1XJ5MAfi1L3VjlT0= -github.com/containerd/containerd/api v1.8.0 h1:hVTNJKR8fMc/2Tiw60ZRijntNMd1U+JVMyTRdsD2bS0= -github.com/containerd/containerd/api v1.8.0/go.mod h1:dFv4lt6S20wTu/hMcP4350RL87qPWLVa/OHOwmmdnYc= +github.com/containerd/containerd/api v1.9.0 h1:HZ/licowTRazus+wt9fM6r/9BQO7S0vD5lMcWspGIg0= +github.com/containerd/containerd/api v1.9.0/go.mod h1:GhghKFmTR3hNtyznBoQ0EMWr9ju5AqHjcZPsSpTKutI= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/containerd/ttrpc v1.2.6 h1:zG+Kn5EZ6MUYCS1t2Hmt2J4tMVaLSFEJVOraDQwNPC4= -github.com/containerd/ttrpc v1.2.6/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o= -github.com/containerd/typeurl/v2 v2.2.2 h1:3jN/k2ysKuPCsln5Qv8bzR9cxal8XjkxPogJfSNO31k= -github.com/containerd/typeurl/v2 v2.2.2/go.mod h1:95ljDnPfD3bAbDJRugOiShd/DlAAsxGtUBhJxIn7SCk= +github.com/containerd/ttrpc v1.2.7 h1:qIrroQvuOL9HQ1X6KHe2ohc7p+HP/0VE6XPU7elJRqQ= +github.com/containerd/ttrpc v1.2.7/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o= +github.com/containerd/typeurl/v2 v2.2.3 h1:yNA/94zxWdvYACdYO8zofhrTVuQY73fFU1y++dYSw40= +github.com/containerd/typeurl/v2 v2.2.3/go.mod h1:95ljDnPfD3bAbDJRugOiShd/DlAAsxGtUBhJxIn7SCk= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= @@ -141,8 +143,8 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= -github.com/docker/docker v27.1.1+incompatible h1:hO/M4MtV36kzKldqnA37IWhebRA+LnqqcqDja6kVaKY= -github.com/docker/docker v27.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v28.2.2+incompatible h1:CjwRSksz8Yo4+RmQ339Dp/D2tGO5JxwYeqtMOEe0LDw= +github.com/docker/docker v28.2.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -165,9 +167,15 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= +github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= +github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= +github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= +github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= +github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= @@ -182,6 +190,8 @@ github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+Gr github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= +github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -216,8 +226,8 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= -github.com/google/cadvisor v0.52.1 h1:sC8SZ6jio9ds+P2dk51bgbeYeufxo55n0X3tmrpA9as= -github.com/google/cadvisor v0.52.1/go.mod h1:OAhPcx1nOm5YwMh/JhpUOMKyv1YKLRtS9KgzWPndHmA= +github.com/google/cadvisor v0.53.0 h1:pmveUw2VBlr/T2SBE9Fsp8gdLhKWyOBkECGbaas9mcI= +github.com/google/cadvisor v0.53.0/go.mod h1:Tz3zf/exzFfdWd1T/U/9eNst0ZR2C6CIV62LJATj5tg= github.com/google/cel-go v0.26.0 h1:DPGjXackMpJWH680oGY4lZhYjIameYmR+/6RBdDGmaI= github.com/google/cel-go v0.26.0/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= @@ -234,8 +244,8 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= -github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -275,6 +285,8 @@ github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbd github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= +github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI= @@ -290,6 +302,30 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kubernetes/api v0.0.0-20251107002836-f1737241c064 h1:e9IFudptRLSIVceJf5LabJGMRco6T9BCXVXW4/N9tO0= +github.com/kubernetes/api v0.0.0-20251107002836-f1737241c064/go.mod h1:KmeiqHqfbEx7y5cjOafexrDA1x0/NCS0EvYefr1M9eQ= +github.com/kubernetes/apiextensions-apiserver v0.0.0-20251105125549-c4e434ca2251 h1:ovLkHC5GA8MwBr6gmzEBcIw5lhCZsTHgsLnJTHkM978= +github.com/kubernetes/apiextensions-apiserver v0.0.0-20251105125549-c4e434ca2251/go.mod h1:1apvaCR5GyeVTY2mGBsucdeQrjy/JjJilBaYXVpI45k= +github.com/kubernetes/apimachinery v0.35.0-alpha.3.0.20251106231852-6f8949260573 h1:AYVUCkWlr+kgGG1BM6jIEXyo4yDCssj4bSH9gJIhY08= +github.com/kubernetes/apimachinery v0.35.0-alpha.3.0.20251106231852-6f8949260573/go.mod h1:dR9KPaf5L0t2p9jZg/wCGB4b3ma2sXZ2zdNqILs+Sak= +github.com/kubernetes/apiserver v0.0.0-20251107005520-bce27587d0bd h1:bLs/ApMfL2XfnsrNiL/TL4aw3udspMvVTYN4FlnBkNM= +github.com/kubernetes/apiserver v0.0.0-20251107005520-bce27587d0bd/go.mod h1:DDCWQZ0qVGBTYLDBtM1h3kqiRKcHQcWA+PWPYmHpC+E= +github.com/kubernetes/client-go v0.0.0-20251107003311-5c322d3acdac h1:hKZ43mwhjUbODb4BJmFzDsz/0mKkGl+rr+jvcpboXJ4= +github.com/kubernetes/client-go v0.0.0-20251107003311-5c322d3acdac/go.mod h1:QM7Zy4bRkvPZ8IwCto7QVHLGkeASaoOsSDtpTRqY55s= +github.com/kubernetes/cloud-provider v0.0.0-20251106211155-080e91c4b910 h1:aDR3t7Oz5ehbDzZjw0FnnRSMi7yszcsjumPbwNa7IqM= +github.com/kubernetes/cloud-provider v0.0.0-20251106211155-080e91c4b910/go.mod h1:MU0iK322F3Q7rFrGVcltfxEbGe8J14Xw/OeH51Vmu4s= +github.com/kubernetes/component-base v0.0.0-20251105043606-09c454e1f74b h1:XpDrn48ThMpzCtCeogxBXpRj106BIBfHl7ILaGdWH00= +github.com/kubernetes/component-base v0.0.0-20251105043606-09c454e1f74b/go.mod h1:ccVXlbwyM8koWd6+OylPWi8m3aunkoPzcQcSbVvTHrk= +github.com/kubernetes/component-helpers v0.20.0-alpha.2.0.20251106124553-0e2bf40485ce h1:+8ppu7ilcTpgeoNTy/NZ7u3/3vi22axQhRyyTPTUGig= +github.com/kubernetes/component-helpers v0.20.0-alpha.2.0.20251106124553-0e2bf40485ce/go.mod h1:hBkkxYVO7wXIh0RjzhUECKBc6LIsL1wzNIyKYGDW5q8= +github.com/kubernetes/controller-manager v0.20.0-alpha.1.0.20251106210955-362c11ff4757 h1:rl1N5YzG4g5Kr7uw759EjWOI8FRWgv7Au/FCN0IO93U= +github.com/kubernetes/controller-manager v0.20.0-alpha.1.0.20251106210955-362c11ff4757/go.mod h1:4D7ndnErA3yzIsCA+PbAcsHFrbpP/E2nM+tMAxCVHvI= +github.com/kubernetes/dynamic-resource-allocation v0.26.0-beta.0.0.20251107013247-fafc9e288a2f h1:8NSbfQigKVFKbgrR+B9JCDNrl255XbexAaSKYKEhrfA= +github.com/kubernetes/dynamic-resource-allocation v0.26.0-beta.0.0.20251107013247-fafc9e288a2f/go.mod h1:GlnATfc/m2eJH5uTytJx03P4zNyZzQKC0mF7gqj96zQ= +github.com/kubernetes/kube-scheduler v0.0.0-20251106172524-0962b2d7c894 h1:OtP3tOFmliD6yVqTYCiy7jVrvNqCdn2rsGmWfDLfInY= +github.com/kubernetes/kube-scheduler v0.0.0-20251106172524-0962b2d7c894/go.mod h1:fxTWPayBp8KDZ0KDhlQ3btV35RWSdWuIjC+54/e6za8= +github.com/kubernetes/kubernetes v1.35.0-alpha.3.0.20251107154100-609e2e57dacd h1:yS676rGTWAR/yln+slQxAiU8gHz6x5eboU5UHFIqjmg= +github.com/kubernetes/kubernetes v1.35.0-alpha.3.0.20251107154100-609e2e57dacd/go.mod h1:dn6voo4MTvJc53jWau65nBpLH/CD7jVb17FfCfXmq3g= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/libopenstorage/openstorage v1.0.0 h1:GLPam7/0mpdP8ZZtKjbfcXJBTIA/T1O6CBErVEFEyIM= @@ -298,10 +334,14 @@ github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffkt github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= +github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= +github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible h1:aKW/4cBs+yK6gpqU3K/oIwk9Q/XICqd3zOX/UFuvqmk= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -326,18 +366,18 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= -github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= -github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= -github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= -github.com/opencontainers/cgroups v0.0.1 h1:MXjMkkFpKv6kpuirUa4USFBas573sSAY082B4CiHEVA= -github.com/opencontainers/cgroups v0.0.1/go.mod h1:s8lktyhlGUqM7OSRL5P7eAW6Wb+kWPNvt4qvVfzA5vs= +github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns= +github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= +github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= +github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= +github.com/opencontainers/cgroups v0.0.3 h1:Jc9dWh/0YLGjdy6J/9Ln8NM5BfTA4W2BY0GMozy3aDU= +github.com/opencontainers/cgroups v0.0.3/go.mod h1:s8lktyhlGUqM7OSRL5P7eAW6Wb+kWPNvt4qvVfzA5vs= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= -github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk= -github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.2.1 h1:S4k4ryNgEpxW1dzyqffOmhI1BHYcjzU8lpJfSlR0xww= +github.com/opencontainers/runtime-spec v1.2.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/selinux v1.11.1 h1:nHFvthhM0qY8/m+vfhJylliSshm8G1jJ2jDMcgULaH8= github.com/opencontainers/selinux v1.11.1/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= @@ -347,26 +387,29 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= -github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= -github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= -github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= -github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= -github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= -github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/cobra v1.10.0 h1:a5/WeUlSDCvV5a45ljW2ZFtV0bTDpkfSAj3uqB6Sc+0= +github.com/spf13/cobra v1.10.0/go.mod h1:9dhySC7dnTtEiqzmqfkLj47BslqLCUPMXjG2lj/NgoE= +github.com/spf13/pflag v1.0.8/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -380,8 +423,16 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= github.com/vburenin/ifacemaker v1.2.1 h1:3Vq8B/bfBgjWTkv+jDg4dVL1KHt3k1K4lO7XRxYA2sk= @@ -394,18 +445,18 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.etcd.io/bbolt v1.4.2 h1:IrUHp260R8c+zYx/Tm8QZr04CX+qWS5PGfPdevhdm1I= -go.etcd.io/bbolt v1.4.2/go.mod h1:Is8rSHO/b4f3XigBC0lL0+4FwAQv3HXEEIgFMuKHceM= -go.etcd.io/etcd/api/v3 v3.6.4 h1:7F6N7toCKcV72QmoUKa23yYLiiljMrT4xCeBL9BmXdo= -go.etcd.io/etcd/api/v3 v3.6.4/go.mod h1:eFhhvfR8Px1P6SEuLT600v+vrhdDTdcfMzmnxVXXSbk= -go.etcd.io/etcd/client/pkg/v3 v3.6.4 h1:9HBYrjppeOfFjBjaMTRxT3R7xT0GLK8EJMVC4xg6ok0= -go.etcd.io/etcd/client/pkg/v3 v3.6.4/go.mod h1:sbdzr2cl3HzVmxNw//PH7aLGVtY4QySjQFuaCgcRFAI= -go.etcd.io/etcd/client/v3 v3.6.4 h1:YOMrCfMhRzY8NgtzUsHl8hC2EBSnuqbR3dh84Uryl7A= -go.etcd.io/etcd/client/v3 v3.6.4/go.mod h1:jaNNHCyg2FdALyKWnd7hxZXZxZANb0+KGY+YQaEMISo= -go.etcd.io/etcd/pkg/v3 v3.6.4 h1:fy8bmXIec1Q35/jRZ0KOes8vuFxbvdN0aAFqmEfJZWA= -go.etcd.io/etcd/pkg/v3 v3.6.4/go.mod h1:kKcYWP8gHuBRcteyv6MXWSN0+bVMnfgqiHueIZnKMtE= -go.etcd.io/etcd/server/v3 v3.6.4 h1:LsCA7CzjVt+8WGrdsnh6RhC0XqCsLkBly3ve5rTxMAU= -go.etcd.io/etcd/server/v3 v3.6.4/go.mod h1:aYCL/h43yiONOv0QIR82kH/2xZ7m+IWYjzRmyQfnCAg= +go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= +go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= +go.etcd.io/etcd/api/v3 v3.6.5 h1:pMMc42276sgR1j1raO/Qv3QI9Af/AuyQUW6CBAWuntA= +go.etcd.io/etcd/api/v3 v3.6.5/go.mod h1:ob0/oWA/UQQlT1BmaEkWQzI0sJ1M0Et0mMpaABxguOQ= +go.etcd.io/etcd/client/pkg/v3 v3.6.5 h1:Duz9fAzIZFhYWgRjp/FgNq2gO1jId9Yae/rLn3RrBP8= +go.etcd.io/etcd/client/pkg/v3 v3.6.5/go.mod h1:8Wx3eGRPiy0qOFMZT/hfvdos+DjEaPxdIDiCDUv/FQk= +go.etcd.io/etcd/client/v3 v3.6.5 h1:yRwZNFBx/35VKHTcLDeO7XVLbCBFbPi+XV4OC3QJf2U= +go.etcd.io/etcd/client/v3 v3.6.5/go.mod h1:ZqwG/7TAFZ0BJ0jXRPoJjKQJtbFo/9NIY8uoFFKcCyo= +go.etcd.io/etcd/pkg/v3 v3.6.5 h1:byxWB4AqIKI4SBmquZUG1WGtvMfMaorXFoCcFbVeoxM= +go.etcd.io/etcd/pkg/v3 v3.6.5/go.mod h1:uqrXrzmMIJDEy5j00bCqhVLzR5jEJIwDp5wTlLwPGOU= +go.etcd.io/etcd/server/v3 v3.6.5 h1:4RbUb1Bd4y1WkBHmuF+cZII83JNQMuNXzyjwigQ06y0= +go.etcd.io/etcd/server/v3 v3.6.5/go.mod h1:PLuhyVXz8WWRhzXDsl3A3zv/+aK9e4A9lpQkqawIaH0= go.etcd.io/raft/v3 v3.6.0 h1:5NtvbDVYpnfZWcIHgGRk9DyzkBIXOi8j+DDp1IcnUWQ= go.etcd.io/raft/v3 v3.6.0/go.mod h1:nLvLevg6+xrVtHUmVaTcTz603gQPHfh7kUAwV6YpfGo= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= @@ -416,24 +467,24 @@ go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelr go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.44.0/go.mod h1:uq8DrRaen3suIWTpdR/JNHCGpurSvMv9D5Nr5CU5TXc= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 h1:x7wzEgXfnzJcHDwStJT+mxOz4etr2EcexjqhBvmoakw= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0/go.mod h1:rg+RlpR5dKwaS95IyyZqj5Wd4E13lk/msnTS0Xl9lJM= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= go.opentelemetry.io/contrib/propagators/b3 v1.19.0 h1:ulz44cpm6V5oAeg5Aw9HyqGFMS6XM7untlMEhD7YzzA= go.opentelemetry.io/contrib/propagators/b3 v1.19.0/go.mod h1:OzCmE2IVS+asTI+odXQstRGVfXQ4bXv9nMBRK0nNyqQ= -go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= -go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= +go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 h1:OeNbIYk/2C15ckl7glBlOBp5+WlYsOElzTNmiPW/x60= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0/go.mod h1:7Bept48yIeqxP2OZ9/AqIpYS94h2or0aB4FypJTc8ZM= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 h1:tgJ0uaNS4c98WRNUEx5U3aDlrDOI5Rs+1Vifcw4DJ8U= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0/go.mod h1:U7HYyW0zt/a9x5J1Kjs+r1f/d4ZHnYFclhYY2+YbeoE= -go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= -go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= -go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= -go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= -go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= -go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= -go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= -go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= +go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= +go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= +go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= +go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis= +go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= +go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= +go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -444,8 +495,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= -go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -455,8 +506,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= +golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= @@ -468,8 +519,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= -golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.28.0 h1:gQBtGhjxykdjY9YhZpSlZIsbnaE2+PgjfLWUQTnoZ1U= +golang.org/x/mod v0.28.0/go.mod h1:yfB/L0NOf/kmEbXjzCPOx1iK1fRutOydrCMsqRhEBxI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -486,11 +537,11 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= -golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -499,8 +550,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -517,16 +568,16 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= +golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= -golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= -golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= +golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -535,8 +586,8 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -550,8 +601,12 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= -golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= +golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= +golang.org/x/tools/go/expect v0.1.0-deprecated h1:jY2C5HGYR5lqex3gEniOQL0r7Dq5+VGVgY1nudX5lXY= +golang.org/x/tools/go/expect v0.1.0-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= +golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= +golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGNv1ib0J382/DPCRS/BPnsGebyM1Gj5VSDpG8= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -565,15 +620,15 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb h1:p31xT4yrYrSM/G4Sn2+TNUkVhFCbG9y8itM2S6Th950= google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:jbe3Bkdp+Dh2IrslsFCklNhweNTBgSYanP1UXhJDhKg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb h1:TLPQVbx1GJ8VKZxz52VAxl1EBgKXXbTiU9Fc5fZeLn4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a h1:v2PbRU4K3llS09c7zodFpNePeamkAwG3mPrAery9VeE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.72.1 h1:HR03wO6eyZ7lknl75XlxABNVLLFc2PAb6mHlYh756mA= -google.golang.org/grpc v1.72.1/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= +google.golang.org/grpc v1.72.2 h1:TdbGzwb82ty4OusHWepvFWGLgIbNo1/SUynEN0ssqv8= +google.golang.org/grpc v1.72.2/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -583,15 +638,15 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= -google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/dnaeon/go-vcr.v3 v3.2.0 h1:Rltp0Vf+Aq0u4rQXgmXgtgoRDStTnFN83cWgSGSoRzM= gopkg.in/dnaeon/go-vcr.v3 v3.2.0/go.mod h1:2IMOnnlx9I6u9x+YBsM3tAMx6AlOxnJ0pWxQAzZ79Ag= -gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= -gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/gcfg.v1 v1.2.3 h1:m8OOJ4ccYHnx2f4gQwpno8nAX5OGOh7RLaaz0pj3Ogs= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= @@ -608,58 +663,34 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.34.1 h1:jC+153630BMdlFukegoEL8E/yT7aLyQkIVuwhmwDgJM= -k8s.io/api v0.34.1/go.mod h1:SB80FxFtXn5/gwzCoN6QCtPD7Vbu5w2n1S0J5gFfTYk= -k8s.io/apiextensions-apiserver v0.34.1 h1:NNPBva8FNAPt1iSVwIE0FsdrVriRXMsaWFMqJbII2CI= -k8s.io/apiextensions-apiserver v0.34.1/go.mod h1:hP9Rld3zF5Ay2Of3BeEpLAToP+l4s5UlxiHfqRaRcMc= -k8s.io/apimachinery v0.34.1 h1:dTlxFls/eikpJxmAC7MVE8oOeP1zryV7iRyIjB0gky4= -k8s.io/apimachinery v0.34.1/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= -k8s.io/apiserver v0.34.1 h1:U3JBGdgANK3dfFcyknWde1G6X1F4bg7PXuvlqt8lITA= -k8s.io/apiserver v0.34.1/go.mod h1:eOOc9nrVqlBI1AFCvVzsob0OxtPZUCPiUJL45JOTBG0= -k8s.io/client-go v0.34.1 h1:ZUPJKgXsnKwVwmKKdPfw4tB58+7/Ik3CrjOEhsiZ7mY= -k8s.io/client-go v0.34.1/go.mod h1:kA8v0FP+tk6sZA0yKLRG67LWjqufAoSHA2xVGKw9Of8= -k8s.io/cloud-provider v0.34.1 h1:FS+4C1vq9pIngd/5LR5Jha1sEbn+fo0HJitgZmUyBNc= -k8s.io/cloud-provider v0.34.1/go.mod h1:ghyQYfQIWZAXKNS+TEgEiQ8wPuhzIVt3wFO6rKqS/rQ= k8s.io/cloud-provider-aws v1.27.0 h1:PF8YrH8QcN6JoXB3Xxlaz84SBDYMPunJuCc0cPuCWXA= k8s.io/cloud-provider-aws v1.27.0/go.mod h1:9vUb5mnVnReSRDBWcBxB1b0HOeEc472iOPmrnwpN9SA= k8s.io/cloud-provider-gcp/providers v0.28.2 h1:I65pFTLNMQSj7YuW3Mg3pZIXmw0naCmF6TGAuz4/sZE= k8s.io/cloud-provider-gcp/providers v0.28.2/go.mod h1:P8dxRvvLtX7xUwVUzA/QOqv8taCzBaVsVMnjnpjmYXE= k8s.io/code-generator v0.34.1 h1:WpphT26E+j7tEgIUfFr5WfbJrktCGzB3JoJH9149xYc= k8s.io/code-generator v0.34.1/go.mod h1:DeWjekbDnJWRwpw3s0Jat87c+e0TgkxoR4ar608yqvg= -k8s.io/component-base v0.34.1 h1:v7xFgG+ONhytZNFpIz5/kecwD+sUhVE6HU7qQUiRM4A= -k8s.io/component-base v0.34.1/go.mod h1:mknCpLlTSKHzAQJJnnHVKqjxR7gBeHRv0rPXA7gdtQ0= -k8s.io/component-helpers v0.34.1 h1:gWhH3CCdwAx5P3oJqZKb4Lg5FYZTWVbdWtOI8n9U4XY= -k8s.io/component-helpers v0.34.1/go.mod h1:4VgnUH7UA/shuBur+OWoQC0xfb69sy/93ss0ybZqm3c= -k8s.io/controller-manager v0.34.1 h1:c9Cmun/zF740kmdRQWPGga+4MglT5SlrwsCXDS/KtJI= -k8s.io/controller-manager v0.34.1/go.mod h1:fGiJDhi3OSzSAB4f40ZkJLAqMQSag9RM+7m5BRhBO3Q= k8s.io/cri-api v0.34.1 h1:n2bU++FqqJq0CNjP/5pkOs0nIx7aNpb1Xa053TecQkM= k8s.io/cri-api v0.34.1/go.mod h1:4qVUjidMg7/Z9YGZpqIDygbkPWkg3mkS1PvOx/kpHTE= k8s.io/cri-client v0.34.1 h1:eq6FcEPDDL379w0WhPnItj2egsMZqOtU7nv1JaJmwP0= k8s.io/cri-client v0.34.1/go.mod h1:Dq6mKWV2ugO5tMv4xqVgcQ8vD7csP//e4KkzcFi2Pio= k8s.io/csi-translation-lib v0.34.1 h1:8+QMIWBwPGFsqWw9eAvimA2GaHXGgLLYT61I1NzDnXw= k8s.io/csi-translation-lib v0.34.1/go.mod h1:QXytPJ1KzYQaiMgVm82ANG+RGAUf276m8l9gFT+R6Xg= -k8s.io/dynamic-resource-allocation v0.34.1 h1:pd9qhOeAFkn8eOO4BthAiGHQc8pu+N6TK/2Fj+jaPwU= -k8s.io/dynamic-resource-allocation v0.34.1/go.mod h1:Zlpqyh6EKhTVoQDe5BS31/8oMXGfG6c12ydj3ChXyuw= -k8s.io/gengo/v2 v2.0.0-20250604051438-85fd79dbfd9f h1:SLb+kxmzfA87x4E4brQzB33VBbT2+x7Zq9ROIHmGn9Q= -k8s.io/gengo/v2 v2.0.0-20250604051438-85fd79dbfd9f/go.mod h1:EJykeLsmFC60UQbYJezXkEsG2FLrt0GPNkU5iK5GWxU= +k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b h1:gMplByicHV/TJBizHd9aVEsTYoJBnnUAT5MHlTkbjhQ= +k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b/go.mod h1:CgujABENc3KuTrcsdpGmrrASjtQsWCT7R99mEV4U/fM= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kms v0.34.1 h1:iCFOvewDPzWM9fMTfyIPO+4MeuZ0tcZbugxLNSHFG4w= k8s.io/kms v0.34.1/go.mod h1:s1CFkLG7w9eaTYvctOxosx88fl4spqmixnNpys0JAtM= -k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= -k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= -k8s.io/kube-scheduler v0.34.1 h1:S5td6VZwC3lCqERXclerDXhJ26zYc6JroY0s03+PqJ8= -k8s.io/kube-scheduler v0.34.1/go.mod h1:UiOkod/w+HKoGut9mz9ie4s4KcI82vmLFdq1iIgsmRs= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= k8s.io/kubectl v0.34.1 h1:1qP1oqT5Xc93K+H8J7ecpBjaz511gan89KO9Vbsh/OI= k8s.io/kubectl v0.34.1/go.mod h1:JRYlhJpGPyk3dEmJ+BuBiOB9/dAvnrALJEiY/C5qa6A= k8s.io/kubelet v0.34.1 h1:doAaTA9/Yfzbdq/u/LveZeONp96CwX9giW6b+oHn4m4= k8s.io/kubelet v0.34.1/go.mod h1:PtV3Ese8iOM19gSooFoQT9iyRisbmJdAPuDImuccbbA= -k8s.io/kubernetes v1.34.1 h1:F3p8dtpv+i8zQoebZeK5zBqM1g9x1aIdnA5vthvcuUk= -k8s.io/kubernetes v1.34.1/go.mod h1:iu+FhII+Oc/1gGWLJcer6wpyih441aNFHl7Pvm8yPto= k8s.io/mount-utils v0.34.1 h1:zMBEFav8Rxwm54S8srzy5FxAc4KQ3X4ZcjnqTCzHmZk= k8s.io/mount-utils v0.34.1/go.mod h1:MIjjYlqJ0ziYQg0MO09kc9S96GIcMkhF/ay9MncF0GA= -k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= -k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/cloud-provider-azure v1.29.4 h1:lW/mqq9fofs52/T+Crs6JNzzEhz0NjzQUtSXMseh67M= @@ -668,8 +699,8 @@ sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.13 h1:dxpo41/N6m2R//9fmqKgqYZ sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.13/go.mod h1:tN2BDTM6RDyQsae6JRvaaA14LVxDsRaLU3Ea2MRUBjg= sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.4 h1:CFMHYo6/OQpLTycJGQIze2pchNeJQ7L2TQC6fDo4JGY= sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.4/go.mod h1:PvXgFxPcfve6yBiWNIO/fqAMvGVC9W7qN6M2vIj4zmY= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= diff --git a/cluster-autoscaler/main.go b/cluster-autoscaler/main.go index 6800e0fec041..4c47cfda189b 100644 --- a/cluster-autoscaler/main.go +++ b/cluster-autoscaler/main.go @@ -68,6 +68,7 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/processors/scaledowncandidates/previouscandidates" "k8s.io/autoscaler/cluster-autoscaler/processors/status" provreqorchestrator "k8s.io/autoscaler/cluster-autoscaler/provisioningrequest/orchestrator" + csinodeprovider "k8s.io/autoscaler/cluster-autoscaler/simulator/csi/provider" "k8s.io/autoscaler/cluster-autoscaler/simulator/drainability/rules" "k8s.io/autoscaler/cluster-autoscaler/simulator/options" kube_util "k8s.io/autoscaler/cluster-autoscaler/utils/kubernetes" @@ -115,24 +116,30 @@ func buildAutoscaler(ctx context.Context, debuggingSnapshotter debuggingsnapshot } informerFactory := informers.NewSharedInformerFactoryWithOptions(kubeClient, 0, informers.WithTransform(trim)) - fwHandle, err := framework.NewHandle(informerFactory, autoscalingOptions.SchedulerConfig, autoscalingOptions.DynamicResourceAllocationEnabled) + fwHandle, err := framework.NewHandle(informerFactory, autoscalingOptions.SchedulerConfig, autoscalingOptions.DynamicResourceAllocationEnabled, autoscalingOptions.CSINodeAwareSchedulingEnabled) if err != nil { return nil, nil, err } deleteOptions := options.NewNodeDeleteOptions(autoscalingOptions) drainabilityRules := rules.Default(deleteOptions) + var csiProvider *csinodeprovider.Provider + if autoscalingOptions.CSINodeAwareSchedulingEnabled { + csiProvider = csinodeprovider.NewCSINodeProviderFromInformers(informerFactory) + } + var snapshotStore clustersnapshot.ClusterSnapshotStore = store.NewDeltaSnapshotStore(autoscalingOptions.ClusterSnapshotParallelism) opts := coreoptions.AutoscalerOptions{ AutoscalingOptions: autoscalingOptions, FrameworkHandle: fwHandle, - ClusterSnapshot: predicate.NewPredicateSnapshot(snapshotStore, fwHandle, autoscalingOptions.DynamicResourceAllocationEnabled, autoscalingOptions.PredicateParallelism), + ClusterSnapshot: predicate.NewPredicateSnapshot(snapshotStore, fwHandle, autoscalingOptions.DynamicResourceAllocationEnabled, autoscalingOptions.PredicateParallelism, autoscalingOptions.CSINodeAwareSchedulingEnabled), KubeClient: kubeClient, InformerFactory: informerFactory, DebuggingSnapshotter: debuggingSnapshotter, DeleteOptions: deleteOptions, DrainabilityRules: drainabilityRules, ScaleUpOrchestrator: orchestrator.New(), + CsiProvider: csiProvider, } opts.Processors = ca_processors.DefaultProcessors(autoscalingOptions) diff --git a/cluster-autoscaler/processors/customresources/csi_processor.go b/cluster-autoscaler/processors/customresources/csi_processor.go new file mode 100644 index 000000000000..ee3263a5dcb4 --- /dev/null +++ b/cluster-autoscaler/processors/customresources/csi_processor.go @@ -0,0 +1,103 @@ +package customresources + +import ( + apiv1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" + ca_context "k8s.io/autoscaler/cluster-autoscaler/context" + csisnapshot "k8s.io/autoscaler/cluster-autoscaler/simulator/csi/snapshot" + drasnapshot "k8s.io/autoscaler/cluster-autoscaler/simulator/dynamicresources/snapshot" + "k8s.io/autoscaler/cluster-autoscaler/utils/errors" + "k8s.io/autoscaler/cluster-autoscaler/utils/kubernetes" + "k8s.io/klog/v2" +) + +// CSICustomResourcesProcessor is a processor that filters out nodes with unready CSI resources. +type CSICustomResourcesProcessor struct { +} + +// FilterOutNodesWithUnreadyResources filters out nodes with unready CSI resources. +func (p *CSICustomResourcesProcessor) FilterOutNodesWithUnreadyResources(autoscalingCtx *ca_context.AutoscalingContext, allNodes, readyNodes []*apiv1.Node, _ *drasnapshot.Snapshot, csiSnapshot *csisnapshot.Snapshot) ([]*apiv1.Node, []*apiv1.Node) { + newAllNodes := make([]*apiv1.Node, 0) + newReadyNodes := make([]*apiv1.Node, 0) + nodesWithUnreadyCSI := make(map[string]*apiv1.Node) + if csiSnapshot == nil { + klog.Warningf("Cannot filter out nodes with unready CSI resources. The CSI snapshot is nil. Processing will be skipped.") + return allNodes, readyNodes + } + + for _, node := range readyNodes { + ng, err := autoscalingCtx.CloudProvider.NodeGroupForNode(node) + if err != nil { + newReadyNodes = append(newReadyNodes, node) + klog.Warningf("Failed to get node group for node %s, Skipping CSI readiness check and keeping node in ready list. Error: %v", node.Name, err) + continue + } + if ng == nil { + newReadyNodes = append(newReadyNodes, node) + continue + } + + nodeInfo, err := ng.TemplateNodeInfo() + if err != nil { + newReadyNodes = append(newReadyNodes, node) + klog.Warningf("Failed to get template node info for node group %s with error: %v", ng.Id(), err) + continue + } + + // if cloudprovider does not provide CSI related stuff, then we can skip the CSI readiness check + if nodeInfo.CSINode == nil { + newReadyNodes = append(newReadyNodes, node) + klog.Warningf("No CSI node found for node %s, Skipping CSI readiness check and keeping node in ready list.", node.Name) + continue + } + + csiNode, err := csiSnapshot.Get(node.Name) + if err != nil { + newReadyNodes = append(newReadyNodes, node) + klog.Warningf("Failed to get CSI node for node %s, Skipping CSI readiness check and keeping node in ready list. Error: %v", node.Name, err) + continue + } + + if areDriversInstalled(csiNode, nodeInfo.CSINode) { + newReadyNodes = append(newReadyNodes, node) + } else { + nodesWithUnreadyCSI[node.Name] = kubernetes.GetUnreadyNodeCopy(node, kubernetes.ResourceUnready) + } + } + for _, node := range allNodes { + if newNode, found := nodesWithUnreadyCSI[node.Name]; found { + newAllNodes = append(newAllNodes, newNode) + } else { + newAllNodes = append(newAllNodes, node) + } + } + return newAllNodes, newReadyNodes +} + +// GetNodeResourceTargets returns mapping of resource names to their targets. +// CSI processor doesn't track resource targets, so it returns an empty list. +func (p *CSICustomResourcesProcessor) GetNodeResourceTargets(autoscalingCtx *ca_context.AutoscalingContext, node *apiv1.Node, nodeGroup cloudprovider.NodeGroup) ([]CustomResourceTarget, errors.AutoscalerError) { + return []CustomResourceTarget{}, nil +} + +// CleanUp cleans up processor's internal structures. +func (p *CSICustomResourcesProcessor) CleanUp() { +} + +func areDriversInstalled(csiNode *storagev1.CSINode, templateCSINode *storagev1.CSINode) bool { + defaultDrivers := templateCSINode.Spec.Drivers + allDriversInstalled := true + + installedDrivers := make(map[string]bool) + for _, csiDriver := range csiNode.Spec.Drivers { + installedDrivers[csiDriver.Name] = true + } + for _, driver := range defaultDrivers { + if _, found := installedDrivers[driver.Name]; !found { + allDriversInstalled = false + break + } + } + return allDriversInstalled +} diff --git a/cluster-autoscaler/processors/customresources/csi_processor_test.go b/cluster-autoscaler/processors/customresources/csi_processor_test.go new file mode 100644 index 000000000000..5eb43b27b740 --- /dev/null +++ b/cluster-autoscaler/processors/customresources/csi_processor_test.go @@ -0,0 +1,347 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package customresources + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + apiv1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + testprovider "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/test" + ca_context "k8s.io/autoscaler/cluster-autoscaler/context" + "k8s.io/autoscaler/cluster-autoscaler/simulator/clustersnapshot/store" + "k8s.io/autoscaler/cluster-autoscaler/simulator/clustersnapshot/testsnapshot" + csisnapshot "k8s.io/autoscaler/cluster-autoscaler/simulator/csi/snapshot" + "k8s.io/autoscaler/cluster-autoscaler/simulator/framework" +) + +func TestFilterOutNodesWithUnreadyCSIResources(t *testing.T) { + testCases := map[string]struct { + nodeGroupsAllNodes map[string][]*apiv1.Node + nodeGroupsTemplatesCSINode map[string]*storagev1.CSINode + nodesCSINode map[string]*storagev1.CSINode + csiSnapshot *csisnapshot.Snapshot + expectedNodesReadiness map[string]bool + }{ + "1 CSI node group all totally ready": { + nodeGroupsAllNodes: map[string][]*apiv1.Node{ + "ng1": { + buildTestNode("node_1_CSI_Ready", true), + buildTestNode("node_2_CSI_Ready", true), + }, + }, + nodeGroupsTemplatesCSINode: map[string]*storagev1.CSINode{ + "ng1": createCSINode("ng1_template", []string{"driver1", "driver2"}), + }, + nodesCSINode: map[string]*storagev1.CSINode{ + "node_1_CSI_Ready": createCSINode("node_1_CSI_Ready", []string{"driver1", "driver2"}), + "node_2_CSI_Ready": createCSINode("node_2_CSI_Ready", []string{"driver1", "driver2"}), + }, + expectedNodesReadiness: map[string]bool{ + "node_1_CSI_Ready": true, + "node_2_CSI_Ready": true, + }, + }, + "1 CSI node group, one initially unready": { + nodeGroupsAllNodes: map[string][]*apiv1.Node{ + "ng1": { + buildTestNode("node_1_CSI_Ready", true), + buildTestNode("node_2_CSI_Ready", false), + }, + }, + nodeGroupsTemplatesCSINode: map[string]*storagev1.CSINode{ + "ng1": createCSINode("ng1_template", []string{"driver1", "driver2"}), + }, + nodesCSINode: map[string]*storagev1.CSINode{ + "node_1_CSI_Ready": createCSINode("node_1_CSI_Ready", []string{"driver1", "driver2"}), + "node_2_CSI_Ready": createCSINode("node_2_CSI_Ready", []string{"driver1", "driver2"}), + }, + expectedNodesReadiness: map[string]bool{ + "node_1_CSI_Ready": true, + "node_2_CSI_Ready": false, + }, + }, + "1 CSI node group, one initially ready with missing driver": { + nodeGroupsAllNodes: map[string][]*apiv1.Node{ + "ng1": { + buildTestNode("node_1_CSI_Ready", true), + buildTestNode("node_2_CSI_Unready", true), + }, + }, + nodeGroupsTemplatesCSINode: map[string]*storagev1.CSINode{ + "ng1": createCSINode("ng1_template", []string{"driver1", "driver2"}), + }, + nodesCSINode: map[string]*storagev1.CSINode{ + "node_1_CSI_Ready": createCSINode("node_1_CSI_Ready", []string{"driver1", "driver2"}), + "node_2_CSI_Unready": createCSINode("node_2_CSI_Unready", []string{"driver1"}), + }, + expectedNodesReadiness: map[string]bool{ + "node_1_CSI_Ready": true, + "node_2_CSI_Unready": false, + }, + }, + "1 CSI node group, one initially ready with no drivers": { + nodeGroupsAllNodes: map[string][]*apiv1.Node{ + "ng1": { + buildTestNode("node_1_CSI_Ready", true), + buildTestNode("node_2_CSI_Unready", true), + }, + }, + nodeGroupsTemplatesCSINode: map[string]*storagev1.CSINode{ + "ng1": createCSINode("ng1_template", []string{"driver1", "driver2"}), + }, + nodesCSINode: map[string]*storagev1.CSINode{ + "node_1_CSI_Ready": createCSINode("node_1_CSI_Ready", []string{"driver1", "driver2"}), + "node_2_CSI_Unready": createCSINode("node_2_CSI_Unready", []string{}), + }, + expectedNodesReadiness: map[string]bool{ + "node_1_CSI_Ready": true, + "node_2_CSI_Unready": false, + }, + }, + "1 CSI node group, one initially ready with extra drivers": { + nodeGroupsAllNodes: map[string][]*apiv1.Node{ + "ng1": { + buildTestNode("node_1_CSI_Ready", true), + buildTestNode("node_2_CSI_Ready", true), + }, + }, + nodeGroupsTemplatesCSINode: map[string]*storagev1.CSINode{ + "ng1": createCSINode("ng1_template", []string{"driver1", "driver2"}), + }, + nodesCSINode: map[string]*storagev1.CSINode{ + "node_1_CSI_Ready": createCSINode("node_1_CSI_Ready", []string{"driver1", "driver2", "driver3"}), + "node_2_CSI_Ready": createCSINode("node_2_CSI_Ready", []string{"driver1", "driver2"}), + }, + expectedNodesReadiness: map[string]bool{ + "node_1_CSI_Ready": true, + "node_2_CSI_Ready": true, + }, + }, + "2 node groups, one CSI with 1 driver unready node": { + nodeGroupsAllNodes: map[string][]*apiv1.Node{ + "ng1": { + buildTestNode("node_1_CSI_Ready", true), + buildTestNode("node_2_CSI_Ready", true), + buildTestNode("node_3_CSI_Unready", true), + }, + "ng2": { + buildTestNode("node_4_NonCSI_Ready", true), + buildTestNode("node_5_NonCSI_Unready", false), + }, + }, + nodeGroupsTemplatesCSINode: map[string]*storagev1.CSINode{ + "ng1": createCSINode("ng1_template", []string{"driver1", "driver2"}), + }, + nodesCSINode: map[string]*storagev1.CSINode{ + "node_1_CSI_Ready": createCSINode("node_1_CSI_Ready", []string{"driver1", "driver2"}), + "node_2_CSI_Ready": createCSINode("node_2_CSI_Ready", []string{"driver1", "driver2"}), + "node_3_CSI_Unready": createCSINode("node_3_CSI_Unready", []string{"driver1"}), + }, + expectedNodesReadiness: map[string]bool{ + "node_1_CSI_Ready": true, + "node_2_CSI_Ready": true, + "node_3_CSI_Unready": false, + "node_4_NonCSI_Ready": true, + "node_5_NonCSI_Unready": false, + }, + }, + "2 CSI node groups, each with 1 driver unready node": { + nodeGroupsAllNodes: map[string][]*apiv1.Node{ + "ng1": { + buildTestNode("node_1_CSI_Ready", true), + buildTestNode("node_2_CSI_Ready", true), + buildTestNode("node_3_CSI_Unready", true), + }, + "ng2": { + buildTestNode("node_4_CSI_Ready", true), + buildTestNode("node_5_CSI_Unready", true), + }, + }, + nodeGroupsTemplatesCSINode: map[string]*storagev1.CSINode{ + "ng1": createCSINode("ng1_template", []string{"driver1", "driver2"}), + "ng2": createCSINode("ng2_template", []string{"driver3", "driver4"}), + }, + nodesCSINode: map[string]*storagev1.CSINode{ + "node_1_CSI_Ready": createCSINode("node_1_CSI_Ready", []string{"driver1", "driver2"}), + "node_2_CSI_Ready": createCSINode("node_2_CSI_Ready", []string{"driver1", "driver2"}), + "node_3_CSI_Unready": createCSINode("node_3_CSI_Unready", []string{"driver1"}), + "node_4_CSI_Ready": createCSINode("node_4_CSI_Ready", []string{"driver3", "driver4"}), + "node_5_CSI_Unready": createCSINode("node_5_CSI_Unready", []string{"driver3"}), + }, + expectedNodesReadiness: map[string]bool{ + "node_1_CSI_Ready": true, + "node_2_CSI_Ready": true, + "node_3_CSI_Unready": false, + "node_4_CSI_Ready": true, + "node_5_CSI_Unready": false, + }, + }, + "nil CSI snapshot returns original nodes": { + nodeGroupsAllNodes: map[string][]*apiv1.Node{ + "ng1": { + buildTestNode("node_1_CSI_Ready", true), + buildTestNode("node_2_CSI_Ready", true), + }, + }, + nodeGroupsTemplatesCSINode: map[string]*storagev1.CSINode{ + "ng1": createCSINode("ng1_template", []string{"driver1", "driver2"}), + }, + nodesCSINode: map[string]*storagev1.CSINode{}, + csiSnapshot: nil, + expectedNodesReadiness: map[string]bool{ + "node_1_CSI_Ready": true, + "node_2_CSI_Ready": true, + }, + }, + "node missing from CSI snapshot stays ready": { + nodeGroupsAllNodes: map[string][]*apiv1.Node{ + "ng1": { + buildTestNode("node_1_CSI_Ready", true), + buildTestNode("node_2_CSI_Ready", true), + }, + }, + nodeGroupsTemplatesCSINode: map[string]*storagev1.CSINode{ + "ng1": createCSINode("ng1_template", []string{"driver1", "driver2"}), + }, + nodesCSINode: map[string]*storagev1.CSINode{ + "node_1_CSI_Ready": createCSINode("node_1_CSI_Ready", []string{"driver1", "driver2"}), + // node_2_CSI_Ready is missing from snapshot + }, + expectedNodesReadiness: map[string]bool{ + "node_1_CSI_Ready": true, + "node_2_CSI_Ready": true, // stays ready because error getting CSI node keeps it in ready list + }, + }, + "All together": { + nodeGroupsAllNodes: map[string][]*apiv1.Node{ + "ng1": { + buildTestNode("node_1", true), + buildTestNode("node_2", true), + buildTestNode("node_3", true), + }, + "ng2": { + buildTestNode("node_4", false), + buildTestNode("node_5", true), + }, + "ng3": { + buildTestNode("node_6", false), + buildTestNode("node_7", true), + }, + }, + nodeGroupsTemplatesCSINode: map[string]*storagev1.CSINode{ + "ng1": createCSINode("ng1_template", []string{"driver1", "driver2"}), + "ng2": createCSINode("ng2_template", []string{"driver3"}), + }, + nodesCSINode: map[string]*storagev1.CSINode{ + "node_1": createCSINode("node_1", []string{"driver1", "driver2"}), + "node_2": createCSINode("node_2", []string{"driver1"}), + "node_3": createCSINode("node_3", []string{"driver1", "driver2", "driver3"}), + "node_4": createCSINode("node_4", []string{"driver3"}), + "node_5": createCSINode("node_5", []string{}), + }, + expectedNodesReadiness: map[string]bool{ + "node_1": true, + "node_2": false, + "node_3": true, + "node_4": false, + "node_5": false, + "node_6": false, + "node_7": true, + }, + }, + } + + for tcName, tc := range testCases { + t.Run(tcName, func(t *testing.T) { + provider := testprovider.NewTestCloudProviderBuilder().Build() + machineTemplates := map[string]*framework.NodeInfo{} + initialAllNodes := []*apiv1.Node{} + initialReadyNodes := []*apiv1.Node{} + for ng, nodes := range tc.nodeGroupsAllNodes { + machineName := fmt.Sprintf("%s_machine_template", ng) + if csiNode, found := tc.nodeGroupsTemplatesCSINode[ng]; found { + machineTemplates[machineName] = framework.NewNodeInfo(buildTestNode(fmt.Sprintf("%s_template", ng), true), nil). + AddCSINode(csiNode) + } else { + machineTemplates[machineName] = framework.NewTestNodeInfo(buildTestNode(fmt.Sprintf("%s_template", ng), true)) + } + provider.AddAutoprovisionedNodeGroup(ng, 0, 20, len(nodes), machineName) + for _, node := range nodes { + initialAllNodes = append(initialAllNodes, node) + if getNodeReadiness(node) { + initialReadyNodes = append(initialReadyNodes, node) + } + provider.AddNode(ng, node) + } + } + provider.SetMachineTemplates(machineTemplates) + + var csiSnapshot *csisnapshot.Snapshot + if tc.csiSnapshot == nil { + if len(tc.nodesCSINode) > 0 { + csiSnapshot = csisnapshot.NewSnapshot(tc.nodesCSINode) + } else { + csiSnapshot = nil + } + } else { + csiSnapshot = tc.csiSnapshot + } + + clusterSnapshotStore := store.NewBasicSnapshotStore() + clusterSnapshotStore.SetClusterState([]*apiv1.Node{}, []*apiv1.Pod{}, nil, csiSnapshot) + clusterSnapshot, _, _ := testsnapshot.NewCustomTestSnapshotAndHandle(clusterSnapshotStore) + + autoscalingCtx := &ca_context.AutoscalingContext{CloudProvider: provider, ClusterSnapshot: clusterSnapshot} + processor := CSICustomResourcesProcessor{} + newAllNodes, newReadyNodes := processor.FilterOutNodesWithUnreadyResources(autoscalingCtx, initialAllNodes, initialReadyNodes, nil, csiSnapshot) + + readyNodes := make(map[string]bool) + for _, node := range newReadyNodes { + readyNodes[node.Name] = true + } + + assert.True(t, len(newAllNodes) == len(initialAllNodes), "Total number of nodes should not change") + for _, node := range newAllNodes { + gotReadiness := getNodeReadiness(node) + assert.Equal(t, tc.expectedNodesReadiness[node.Name], gotReadiness, "Node %s readiness mismatch", node.Name) + assert.Equal(t, gotReadiness, readyNodes[node.Name], "Node %s readiness consistency check", node.Name) + } + }) + } +} + +func createCSINode(nodeName string, driverNames []string) *storagev1.CSINode { + drivers := make([]storagev1.CSINodeDriver, 0, len(driverNames)) + for _, driverName := range driverNames { + drivers = append(drivers, storagev1.CSINodeDriver{ + Name: driverName, + NodeID: fmt.Sprintf("%s-%s", nodeName, driverName), + }) + } + return &storagev1.CSINode{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + }, + Spec: storagev1.CSINodeSpec{ + Drivers: drivers, + }, + } +} diff --git a/cluster-autoscaler/processors/customresources/custom_resources_processor.go b/cluster-autoscaler/processors/customresources/custom_resources_processor.go index 753d244741e3..15a065d7533e 100644 --- a/cluster-autoscaler/processors/customresources/custom_resources_processor.go +++ b/cluster-autoscaler/processors/customresources/custom_resources_processor.go @@ -20,6 +20,7 @@ import ( apiv1 "k8s.io/api/core/v1" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" ca_context "k8s.io/autoscaler/cluster-autoscaler/context" + csisnapshot "k8s.io/autoscaler/cluster-autoscaler/simulator/csi/snapshot" drasnapshot "k8s.io/autoscaler/cluster-autoscaler/simulator/dynamicresources/snapshot" "k8s.io/autoscaler/cluster-autoscaler/utils/errors" ) @@ -36,7 +37,7 @@ type CustomResourceTarget struct { type CustomResourcesProcessor interface { // FilterOutNodesWithUnreadyResources removes nodes that should have a custom resource, but don't have // it in allocatable from ready nodes list and updates their status to unready on all nodes list. - FilterOutNodesWithUnreadyResources(autoscalingCtx *ca_context.AutoscalingContext, allNodes, readyNodes []*apiv1.Node, draSnapshot *drasnapshot.Snapshot) ([]*apiv1.Node, []*apiv1.Node) + FilterOutNodesWithUnreadyResources(autoscalingCtx *ca_context.AutoscalingContext, allNodes, readyNodes []*apiv1.Node, draSnapshot *drasnapshot.Snapshot, csiSnapshot *csisnapshot.Snapshot) ([]*apiv1.Node, []*apiv1.Node) // GetNodeResourceTargets returns mapping of resource names to their targets. GetNodeResourceTargets(autoscalingCtx *ca_context.AutoscalingContext, node *apiv1.Node, nodeGroup cloudprovider.NodeGroup) ([]CustomResourceTarget, errors.AutoscalerError) // CleanUp cleans up processor's internal structures. diff --git a/cluster-autoscaler/processors/customresources/default_custom_processor.go b/cluster-autoscaler/processors/customresources/default_custom_processor.go index 35addfa901d0..54ff17ee1057 100644 --- a/cluster-autoscaler/processors/customresources/default_custom_processor.go +++ b/cluster-autoscaler/processors/customresources/default_custom_processor.go @@ -20,6 +20,7 @@ import ( apiv1 "k8s.io/api/core/v1" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" ca_context "k8s.io/autoscaler/cluster-autoscaler/context" + csisnapshot "k8s.io/autoscaler/cluster-autoscaler/simulator/csi/snapshot" drasnapshot "k8s.io/autoscaler/cluster-autoscaler/simulator/dynamicresources/snapshot" "k8s.io/autoscaler/cluster-autoscaler/utils/errors" ) @@ -31,20 +32,23 @@ type DefaultCustomResourcesProcessor struct { } // NewDefaultCustomResourcesProcessor returns an instance of DefaultCustomResourcesProcessor. -func NewDefaultCustomResourcesProcessor(draEnabled bool) CustomResourcesProcessor { +func NewDefaultCustomResourcesProcessor(draEnabled bool, csiEnabled bool) CustomResourcesProcessor { customProcessors := []CustomResourcesProcessor{&GpuCustomResourcesProcessor{}} if draEnabled { customProcessors = append(customProcessors, &DraCustomResourcesProcessor{}) } + if csiEnabled { + customProcessors = append(customProcessors, &CSICustomResourcesProcessor{}) + } return &DefaultCustomResourcesProcessor{customProcessors} } // FilterOutNodesWithUnreadyResources calls the corresponding method for internal custom resources processors in order. -func (p *DefaultCustomResourcesProcessor) FilterOutNodesWithUnreadyResources(autoscalingCtx *ca_context.AutoscalingContext, allNodes, readyNodes []*apiv1.Node, draSnapshot *drasnapshot.Snapshot) ([]*apiv1.Node, []*apiv1.Node) { +func (p *DefaultCustomResourcesProcessor) FilterOutNodesWithUnreadyResources(autoscalingCtx *ca_context.AutoscalingContext, allNodes, readyNodes []*apiv1.Node, draSnapshot *drasnapshot.Snapshot, csiSnapshot *csisnapshot.Snapshot) ([]*apiv1.Node, []*apiv1.Node) { newAllNodes := allNodes newReadyNodes := readyNodes for _, processor := range p.customResourcesProcessors { - newAllNodes, newReadyNodes = processor.FilterOutNodesWithUnreadyResources(autoscalingCtx, newAllNodes, newReadyNodes, draSnapshot) + newAllNodes, newReadyNodes = processor.FilterOutNodesWithUnreadyResources(autoscalingCtx, newAllNodes, newReadyNodes, draSnapshot, csiSnapshot) } return newAllNodes, newReadyNodes } diff --git a/cluster-autoscaler/processors/customresources/default_custom_processor_test.go b/cluster-autoscaler/processors/customresources/default_custom_processor_test.go index 713055c91525..841d76aac836 100644 --- a/cluster-autoscaler/processors/customresources/default_custom_processor_test.go +++ b/cluster-autoscaler/processors/customresources/default_custom_processor_test.go @@ -23,6 +23,7 @@ import ( "github.com/stretchr/testify/assert" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" + csisnapshot "k8s.io/autoscaler/cluster-autoscaler/simulator/csi/snapshot" drasnapshot "k8s.io/autoscaler/cluster-autoscaler/simulator/dynamicresources/snapshot" "k8s.io/autoscaler/cluster-autoscaler/utils/errors" @@ -112,7 +113,7 @@ func TestDefaultProcessorFilterOut(t *testing.T) { readyNodes = append(readyNodes, node) } } - resultedAllNodes, resultedReadyNodes := processor.FilterOutNodesWithUnreadyResources(nil, tc.allNodes, readyNodes, nil) + resultedAllNodes, resultedReadyNodes := processor.FilterOutNodesWithUnreadyResources(nil, tc.allNodes, readyNodes, nil, nil) assert.ElementsMatch(t, tc.allNodes, resultedAllNodes) assert.True(t, len(resultedReadyNodes) == len(tc.expectedReadyNodes)) for _, node := range resultedReadyNodes { @@ -174,7 +175,7 @@ type mockCustomResourcesProcessor struct { customResourceTargetsQuantity int64 } -func (m *mockCustomResourcesProcessor) FilterOutNodesWithUnreadyResources(_ *ca_context.AutoscalingContext, allNodes, readyNodes []*apiv1.Node, _ *drasnapshot.Snapshot) ([]*apiv1.Node, []*apiv1.Node) { +func (m *mockCustomResourcesProcessor) FilterOutNodesWithUnreadyResources(_ *ca_context.AutoscalingContext, allNodes, readyNodes []*apiv1.Node, _ *drasnapshot.Snapshot, _ *csisnapshot.Snapshot) ([]*apiv1.Node, []*apiv1.Node) { filteredReadyNodes := []*apiv1.Node{} for _, node := range readyNodes { if !strings.Contains(node.Name, m.nodeMark) { diff --git a/cluster-autoscaler/processors/customresources/dra_processor.go b/cluster-autoscaler/processors/customresources/dra_processor.go index defad5443556..7c80869d60ee 100644 --- a/cluster-autoscaler/processors/customresources/dra_processor.go +++ b/cluster-autoscaler/processors/customresources/dra_processor.go @@ -23,6 +23,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" ca_context "k8s.io/autoscaler/cluster-autoscaler/context" + csisnapshot "k8s.io/autoscaler/cluster-autoscaler/simulator/csi/snapshot" "k8s.io/autoscaler/cluster-autoscaler/simulator/dynamicresources/snapshot" "k8s.io/autoscaler/cluster-autoscaler/utils/errors" "k8s.io/autoscaler/cluster-autoscaler/utils/kubernetes" @@ -36,7 +37,7 @@ type DraCustomResourcesProcessor struct { // FilterOutNodesWithUnreadyResources removes nodes that should have DRA resource, but don't have // it in allocatable from ready nodes list and updates their status to unready on all nodes list. -func (p *DraCustomResourcesProcessor) FilterOutNodesWithUnreadyResources(autoscalingCtx *ca_context.AutoscalingContext, allNodes, readyNodes []*apiv1.Node, draSnapshot *snapshot.Snapshot) ([]*apiv1.Node, []*apiv1.Node) { +func (p *DraCustomResourcesProcessor) FilterOutNodesWithUnreadyResources(autoscalingCtx *ca_context.AutoscalingContext, allNodes, readyNodes []*apiv1.Node, draSnapshot *snapshot.Snapshot, _ *csisnapshot.Snapshot) ([]*apiv1.Node, []*apiv1.Node) { newAllNodes := make([]*apiv1.Node, 0) newReadyNodes := make([]*apiv1.Node, 0) nodesWithUnreadyDraResources := make(map[string]*apiv1.Node) diff --git a/cluster-autoscaler/processors/customresources/dra_processor_test.go b/cluster-autoscaler/processors/customresources/dra_processor_test.go index 59639791801b..1bb0c343d2b3 100644 --- a/cluster-autoscaler/processors/customresources/dra_processor_test.go +++ b/cluster-autoscaler/processors/customresources/dra_processor_test.go @@ -333,12 +333,12 @@ func TestFilterOutNodesWithUnreadyDRAResources(t *testing.T) { provider.SetMachineTemplates(machineTemplates) draSnapshot := drasnapshot.NewSnapshot(nil, tc.nodesSlices, nil, nil) clusterSnapshotStore := store.NewBasicSnapshotStore() - clusterSnapshotStore.SetClusterState([]*apiv1.Node{}, []*apiv1.Pod{}, draSnapshot) + clusterSnapshotStore.SetClusterState([]*apiv1.Node{}, []*apiv1.Pod{}, draSnapshot, nil) clusterSnapshot, _, _ := testsnapshot.NewCustomTestSnapshotAndHandle(clusterSnapshotStore) autoscalingCtx := &ca_context.AutoscalingContext{CloudProvider: provider, ClusterSnapshot: clusterSnapshot} processor := DraCustomResourcesProcessor{} - newAllNodes, newReadyNodes := processor.FilterOutNodesWithUnreadyResources(autoscalingCtx, initialAllNodes, initialReadyNodes, draSnapshot) + newAllNodes, newReadyNodes := processor.FilterOutNodesWithUnreadyResources(autoscalingCtx, initialAllNodes, initialReadyNodes, draSnapshot, nil) readyNodes := make(map[string]bool) for _, node := range newReadyNodes { diff --git a/cluster-autoscaler/processors/customresources/gpu_processor.go b/cluster-autoscaler/processors/customresources/gpu_processor.go index bcbd451bebb9..b5ab0d876a5c 100644 --- a/cluster-autoscaler/processors/customresources/gpu_processor.go +++ b/cluster-autoscaler/processors/customresources/gpu_processor.go @@ -20,6 +20,7 @@ import ( apiv1 "k8s.io/api/core/v1" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" ca_context "k8s.io/autoscaler/cluster-autoscaler/context" + csisnapshot "k8s.io/autoscaler/cluster-autoscaler/simulator/csi/snapshot" drasnapshot "k8s.io/autoscaler/cluster-autoscaler/simulator/dynamicresources/snapshot" "k8s.io/autoscaler/cluster-autoscaler/utils/errors" "k8s.io/autoscaler/cluster-autoscaler/utils/gpu" @@ -37,7 +38,7 @@ type GpuCustomResourcesProcessor struct { // it in allocatable from ready nodes list and updates their status to unready on all nodes list. // This is a hack/workaround for nodes with GPU coming up without installed drivers, resulting // in GPU missing from their allocatable and capacity. -func (p *GpuCustomResourcesProcessor) FilterOutNodesWithUnreadyResources(autoscalingCtx *ca_context.AutoscalingContext, allNodes, readyNodes []*apiv1.Node, _ *drasnapshot.Snapshot) ([]*apiv1.Node, []*apiv1.Node) { +func (p *GpuCustomResourcesProcessor) FilterOutNodesWithUnreadyResources(autoscalingCtx *ca_context.AutoscalingContext, allNodes, readyNodes []*apiv1.Node, _ *drasnapshot.Snapshot, _ *csisnapshot.Snapshot) ([]*apiv1.Node, []*apiv1.Node) { newAllNodes := make([]*apiv1.Node, 0) newReadyNodes := make([]*apiv1.Node, 0) nodesWithUnreadyGpu := make(map[string]*apiv1.Node) diff --git a/cluster-autoscaler/processors/customresources/gpu_processor_test.go b/cluster-autoscaler/processors/customresources/gpu_processor_test.go index 47a35b3db231..aeb71b5043eb 100644 --- a/cluster-autoscaler/processors/customresources/gpu_processor_test.go +++ b/cluster-autoscaler/processors/customresources/gpu_processor_test.go @@ -189,7 +189,7 @@ func TestFilterOutNodesWithUnreadyResources(t *testing.T) { processor := GpuCustomResourcesProcessor{} provider := testprovider.NewTestCloudProviderBuilder().Build() autoscalingCtx := &ca_context.AutoscalingContext{CloudProvider: provider} - newAllNodes, newReadyNodes := processor.FilterOutNodesWithUnreadyResources(autoscalingCtx, initialAllNodes, initialReadyNodes, nil) + newAllNodes, newReadyNodes := processor.FilterOutNodesWithUnreadyResources(autoscalingCtx, initialAllNodes, initialReadyNodes, nil, nil) foundInReady := make(map[string]bool) for _, node := range newReadyNodes { diff --git a/cluster-autoscaler/processors/nodeinfosprovider/mixed_nodeinfos_processor_test.go b/cluster-autoscaler/processors/nodeinfosprovider/mixed_nodeinfos_processor_test.go index 3065394dde55..8df06bda4797 100644 --- a/cluster-autoscaler/processors/nodeinfosprovider/mixed_nodeinfos_processor_test.go +++ b/cluster-autoscaler/processors/nodeinfosprovider/mixed_nodeinfos_processor_test.go @@ -81,11 +81,11 @@ func TestGetNodeInfosForGroups(t *testing.T) { provider2.AddNodeGroup("ng7", 1, 10, 1) // Nodegroup without nodes. podLister := kube_util.NewTestPodLister([]*apiv1.Pod{}) - registry := kube_util.NewListerRegistry(nil, nil, podLister, nil, nil, nil, nil, nil, nil) + registry := kube_util.NewListerRegistry(nil, nil, nil, podLister, nil, nil, nil, nil, nil, nil) nodes := []*apiv1.Node{justReady5, unready4, unready3, ready2, ready1, ready7, readyToBeDeleted6} snapshot := testsnapshot.NewTestSnapshotOrDie(t) - err := snapshot.SetClusterState(nodes, nil, nil) + err := snapshot.SetClusterState(nodes, nil, nil, nil) assert.NoError(t, err) autoscalingCtx := ca_context.AutoscalingContext{ @@ -168,11 +168,11 @@ func TestGetNodeInfosForGroupsCache(t *testing.T) { provider1.AddNode("ng4", ready6) podLister := kube_util.NewTestPodLister([]*apiv1.Pod{}) - registry := kube_util.NewListerRegistry(nil, nil, podLister, nil, nil, nil, nil, nil, nil) + registry := kube_util.NewListerRegistry(nil, nil, nil, podLister, nil, nil, nil, nil, nil, nil) nodes := []*apiv1.Node{unready4, unready3, ready2, ready1} snapshot := testsnapshot.NewTestSnapshotOrDie(t) - err := snapshot.SetClusterState(nodes, nil, nil) + err := snapshot.SetClusterState(nodes, nil, nil, nil) assert.NoError(t, err) // Fill cache @@ -259,11 +259,11 @@ func TestGetNodeInfosCacheExpired(t *testing.T) { // Cloud provider with TemplateNodeInfo not implemented. provider := testprovider.NewTestCloudProviderBuilder().Build() podLister := kube_util.NewTestPodLister([]*apiv1.Pod{}) - registry := kube_util.NewListerRegistry(nil, nil, podLister, nil, nil, nil, nil, nil, nil) + registry := kube_util.NewListerRegistry(nil, nil, nil, podLister, nil, nil, nil, nil, nil, nil) nodes := []*apiv1.Node{ready1} snapshot := testsnapshot.NewTestSnapshotOrDie(t) - err := snapshot.SetClusterState(nodes, nil, nil) + err := snapshot.SetClusterState(nodes, nil, nil, nil) assert.NoError(t, err) autoscalingCtx := ca_context.AutoscalingContext{ diff --git a/cluster-autoscaler/processors/podinjection/pod_injection_processor_test.go b/cluster-autoscaler/processors/podinjection/pod_injection_processor_test.go index ff14d377637b..4f10a6cba908 100644 --- a/cluster-autoscaler/processors/podinjection/pod_injection_processor_test.go +++ b/cluster-autoscaler/processors/podinjection/pod_injection_processor_test.go @@ -141,7 +141,7 @@ func TestTargetCountInjectionPodListProcessor(t *testing.T) { allPodsLister := fakeAllPodsLister{podsToList: append(append(tc.scheduledPods, tc.unschedulablePods...), tc.otherPods...)} autoscalingCtx := ca_context.AutoscalingContext{ AutoscalingKubeClients: ca_context.AutoscalingKubeClients{ - ListerRegistry: kubernetes.NewListerRegistry(nil, nil, &allPodsLister, nil, nil, nil, jobLister, replicaSetLister, statefulsetLister), + ListerRegistry: kubernetes.NewListerRegistry(nil, nil, nil, &allPodsLister, nil, nil, nil, jobLister, replicaSetLister, statefulsetLister), }, ClusterSnapshot: clusterSnapshot, } @@ -305,7 +305,7 @@ func TestGroupPods(t *testing.T) { autoscalingCtx := ca_context.AutoscalingContext{ AutoscalingKubeClients: ca_context.AutoscalingKubeClients{ - ListerRegistry: kubernetes.NewListerRegistry(nil, nil, nil, nil, nil, nil, jobLister, replicaSetLister, statefulsetLister), + ListerRegistry: kubernetes.NewListerRegistry(nil, nil, nil, nil, nil, nil, nil, jobLister, replicaSetLister, statefulsetLister), }, } controllers := listControllers(&autoscalingCtx) diff --git a/cluster-autoscaler/processors/processors.go b/cluster-autoscaler/processors/processors.go index 3285590d323d..35496033e31b 100644 --- a/cluster-autoscaler/processors/processors.go +++ b/cluster-autoscaler/processors/processors.go @@ -97,7 +97,7 @@ func DefaultProcessors(options config.AutoscalingOptions) *AutoscalingProcessors NodeGroupManager: nodegroups.NewDefaultNodeGroupManager(), AsyncNodeGroupStateChecker: asyncnodegroups.NewDefaultAsyncNodeGroupStateChecker(), NodeGroupConfigProcessor: nodegroupconfig.NewDefaultNodeGroupConfigProcessor(options.NodeGroupDefaults), - CustomResourcesProcessor: customresources.NewDefaultCustomResourcesProcessor(options.DynamicResourceAllocationEnabled), + CustomResourcesProcessor: customresources.NewDefaultCustomResourcesProcessor(options.DynamicResourceAllocationEnabled, options.CSINodeAwareSchedulingEnabled), ActionableClusterProcessor: actionablecluster.NewDefaultActionableClusterProcessor(), TemplateNodeInfoProvider: nodeinfosprovider.NewDefaultTemplateNodeInfoProvider(nil, false), ScaleDownCandidatesNotifier: scaledowncandidates.NewObserversList(), diff --git a/cluster-autoscaler/processors/test/common.go b/cluster-autoscaler/processors/test/common.go index 3201c2026dcc..29854e1b18de 100644 --- a/cluster-autoscaler/processors/test/common.go +++ b/cluster-autoscaler/processors/test/common.go @@ -52,7 +52,7 @@ func NewTestProcessors(autoscalingCtx *ca_context.AutoscalingContext) *processor NodeGroupManager: nodegroups.NewDefaultNodeGroupManager(), TemplateNodeInfoProvider: nodeinfosprovider.NewDefaultTemplateNodeInfoProvider(nil, false), NodeGroupConfigProcessor: nodegroupconfig.NewDefaultNodeGroupConfigProcessor(autoscalingCtx.NodeGroupDefaults), - CustomResourcesProcessor: customresources.NewDefaultCustomResourcesProcessor(true), + CustomResourcesProcessor: customresources.NewDefaultCustomResourcesProcessor(true, false), ActionableClusterProcessor: actionablecluster.NewDefaultActionableClusterProcessor(), ScaleDownCandidatesNotifier: scaledowncandidates.NewObserversList(), ScaleStateNotifier: nodegroupchange.NewNodeGroupChangeObserversList(), diff --git a/cluster-autoscaler/provisioningrequest/orchestrator/orchestrator_test.go b/cluster-autoscaler/provisioningrequest/orchestrator/orchestrator_test.go index 730cdcab5e6a..c81a9bf68249 100644 --- a/cluster-autoscaler/provisioningrequest/orchestrator/orchestrator_test.go +++ b/cluster-autoscaler/provisioningrequest/orchestrator/orchestrator_test.go @@ -480,7 +480,7 @@ func setupTest(t *testing.T, client *provreqclient.ProvisioningRequestClient, no } podLister := kube_util.NewTestPodLister(nil) - listers := kube_util.NewListerRegistry(nil, nil, podLister, nil, nil, nil, nil, nil, nil) + listers := kube_util.NewListerRegistry(nil, nil, nil, podLister, nil, nil, nil, nil, nil, nil) options := config.AutoscalingOptions{ MaxNodeGroupBinpackingDuration: 1 * time.Second, diff --git a/cluster-autoscaler/simulator/cluster_test.go b/cluster-autoscaler/simulator/cluster_test.go index 06b715625fd5..1bc1a06aa77d 100644 --- a/cluster-autoscaler/simulator/cluster_test.go +++ b/cluster-autoscaler/simulator/cluster_test.go @@ -80,7 +80,7 @@ func TestSimulateNodeRemoval(t *testing.T) { } rsLister, err := kube_util.NewTestReplicaSetLister(replicaSets) assert.NoError(t, err) - registry := kube_util.NewListerRegistry(nil, nil, nil, nil, nil, nil, nil, rsLister, nil) + registry := kube_util.NewListerRegistry(nil, nil, nil, nil, nil, nil, nil, nil, rsLister, nil) ownerRefs := GenerateOwnerReferences("rs", "ReplicaSet", "extensions/v1beta1", "") diff --git a/cluster-autoscaler/simulator/clustersnapshot/clustersnapshot.go b/cluster-autoscaler/simulator/clustersnapshot/clustersnapshot.go index d8dea34e7471..81c085042c88 100644 --- a/cluster-autoscaler/simulator/clustersnapshot/clustersnapshot.go +++ b/cluster-autoscaler/simulator/clustersnapshot/clustersnapshot.go @@ -20,6 +20,7 @@ import ( "errors" apiv1 "k8s.io/api/core/v1" + csisnapshot "k8s.io/autoscaler/cluster-autoscaler/simulator/csi/snapshot" drasnapshot "k8s.io/autoscaler/cluster-autoscaler/simulator/dynamicresources/snapshot" "k8s.io/autoscaler/cluster-autoscaler/simulator/framework" "k8s.io/klog/v2" @@ -77,7 +78,7 @@ type ClusterSnapshotStore interface { // SetClusterState resets the snapshot to an unforked state and replaces the contents of the snapshot // with the provided data. scheduledPods are correlated to their Nodes based on spec.NodeName. - SetClusterState(nodes []*apiv1.Node, scheduledPods []*apiv1.Pod, draSnapshot *drasnapshot.Snapshot) error + SetClusterState(nodes []*apiv1.Node, scheduledPods []*apiv1.Pod, draSnapshot *drasnapshot.Snapshot, csiSnapshot *csisnapshot.Snapshot) error // ForceAddPod adds the given Pod to the Node with the given nodeName inside the snapshot without checking scheduler predicates. ForceAddPod(pod *apiv1.Pod, nodeName string) error @@ -95,6 +96,9 @@ type ClusterSnapshotStore interface { // DraSnapshot returns an interface that allows accessing and modifying the DRA objects in the snapshot. DraSnapshot() *drasnapshot.Snapshot + // CsiSnapshot returns an interface that allows accessing and modifying the CSINode objects in the snapshot. + CsiSnapshot() *csisnapshot.Snapshot + // Fork creates a fork of snapshot state. All modifications can later be reverted to moment of forking via Revert(). // Use WithForkedSnapshot() helper function instead if possible. Fork() diff --git a/cluster-autoscaler/simulator/clustersnapshot/predicate/plugin_runner_test.go b/cluster-autoscaler/simulator/clustersnapshot/predicate/plugin_runner_test.go index 704f2bbbdab7..31992cafc3ce 100644 --- a/cluster-autoscaler/simulator/clustersnapshot/predicate/plugin_runner_test.go +++ b/cluster-autoscaler/simulator/clustersnapshot/predicate/plugin_runner_test.go @@ -319,9 +319,13 @@ func TestDebugInfo(t *testing.T) { _, _, predicateErr := defaultPluginRunner.RunFiltersOnNode(p1, "n1") assert.NotNil(t, predicateErr) - assert.Contains(t, predicateErr.FailingPredicateReasons(), "node(s) had untolerated taint {SomeTaint: WhyNot?}") - assert.Contains(t, predicateErr.Error(), "node(s) had untolerated taint {SomeTaint: WhyNot?}") + // The TaintToleration plugin now returns a generic error message: "node(s) had untolerated taint(s)" + // See: https://github.com/kubernetes/kubernetes/blob/master/pkg/scheduler/framework/plugins/tainttoleration/taint_toleration.go + assert.Contains(t, predicateErr.FailingPredicateReasons(), "node(s) had untolerated taint(s)") + assert.Contains(t, predicateErr.Error(), "node(s) had untolerated taint(s)") + // Verify that detailed taint information is still available in debugInfo assert.Contains(t, predicateErr.Error(), "RandomTaint") + assert.Contains(t, predicateErr.Error(), "SomeTaint") // with custom predicate checker @@ -360,11 +364,11 @@ func newTestPluginRunnerAndSnapshot(schedConfig *config.KubeSchedulerConfigurati schedConfig = defaultConfig } - fwHandle, err := framework.NewHandle(informers.NewSharedInformerFactory(clientsetfake.NewSimpleClientset(), 0), schedConfig, true) + fwHandle, err := framework.NewHandle(informers.NewSharedInformerFactory(clientsetfake.NewSimpleClientset(), 0), schedConfig, true, false) if err != nil { return nil, nil, err } - snapshot := NewPredicateSnapshot(store.NewBasicSnapshotStore(), fwHandle, true, 1) + snapshot := NewPredicateSnapshot(store.NewBasicSnapshotStore(), fwHandle, true, 1, false) return NewSchedulerPluginRunner(fwHandle, snapshot, 1), snapshot, nil } diff --git a/cluster-autoscaler/simulator/clustersnapshot/predicate/predicate_snapshot.go b/cluster-autoscaler/simulator/clustersnapshot/predicate/predicate_snapshot.go index d1a3b98b9c65..595b79a5e6cd 100644 --- a/cluster-autoscaler/simulator/clustersnapshot/predicate/predicate_snapshot.go +++ b/cluster-autoscaler/simulator/clustersnapshot/predicate/predicate_snapshot.go @@ -32,15 +32,17 @@ import ( // SchedulerBasedPredicateChecker to check scheduler predicates. type PredicateSnapshot struct { clustersnapshot.ClusterSnapshotStore - pluginRunner *SchedulerPluginRunner - draEnabled bool + pluginRunner *SchedulerPluginRunner + draEnabled bool + enableCSINodeAwareScheduling bool } // NewPredicateSnapshot builds a PredicateSnapshot. -func NewPredicateSnapshot(snapshotStore clustersnapshot.ClusterSnapshotStore, fwHandle *framework.Handle, draEnabled bool, parallelism int) *PredicateSnapshot { +func NewPredicateSnapshot(snapshotStore clustersnapshot.ClusterSnapshotStore, fwHandle *framework.Handle, draEnabled bool, parallelism int, enableCSINodeAwareScheduling bool) *PredicateSnapshot { snapshot := &PredicateSnapshot{ - ClusterSnapshotStore: snapshotStore, - draEnabled: draEnabled, + ClusterSnapshotStore: snapshotStore, + draEnabled: draEnabled, + enableCSINodeAwareScheduling: enableCSINodeAwareScheduling, } // Plugin runner really only needs a framework.SharedLister for running the plugins, but it also needs to run the provided Node-matching functions // which operate on *framework.NodeInfo. The only object that allows obtaining *framework.NodeInfos is PredicateSnapshot, so we have an ugly circular @@ -57,10 +59,22 @@ func (s *PredicateSnapshot) GetNodeInfo(nodeName string) (*framework.NodeInfo, e return nil, err } + wrappedNodeInfo := framework.WrapSchedulerNodeInfo(schedNodeInfo, nil, nil) + if s.draEnabled { - return s.ClusterSnapshotStore.DraSnapshot().WrapSchedulerNodeInfo(schedNodeInfo) + wrappedNodeInfo, err = s.ClusterSnapshotStore.DraSnapshot().AddDRAInfo(wrappedNodeInfo) + if err != nil { + return nil, err + } } - return framework.WrapSchedulerNodeInfo(schedNodeInfo, nil, nil), nil + + if s.enableCSINodeAwareScheduling { + wrappedNodeInfo, err = s.ClusterSnapshotStore.CsiSnapshot().AddCSINodeInfoToNodeInfo(wrappedNodeInfo) + if err != nil { + return nil, err + } + } + return wrappedNodeInfo, nil } // ListNodeInfos returns internal NodeInfos wrapping all schedulerframework.NodeInfos in the snapshot. @@ -71,15 +85,21 @@ func (s *PredicateSnapshot) ListNodeInfos() ([]*framework.NodeInfo, error) { } var result []*framework.NodeInfo for _, schedNodeInfo := range schedNodeInfos { + wrappedNodeInfo := framework.WrapSchedulerNodeInfo(schedNodeInfo, nil, nil) + var err error if s.draEnabled { - nodeInfo, err := s.ClusterSnapshotStore.DraSnapshot().WrapSchedulerNodeInfo(schedNodeInfo) + wrappedNodeInfo, err = s.ClusterSnapshotStore.DraSnapshot().AddDRAInfo(wrappedNodeInfo) + if err != nil { + return nil, err + } + } + if s.enableCSINodeAwareScheduling { + wrappedNodeInfo, err = s.ClusterSnapshotStore.CsiSnapshot().AddCSINodeInfoToNodeInfo(wrappedNodeInfo) if err != nil { return nil, err } - result = append(result, nodeInfo) - } else { - result = append(result, framework.WrapSchedulerNodeInfo(schedNodeInfo, nil, nil)) } + result = append(result, wrappedNodeInfo) } return result, nil } @@ -106,6 +126,14 @@ func (s *PredicateSnapshot) AddNodeInfo(nodeInfo *framework.NodeInfo) error { } } + if s.enableCSINodeAwareScheduling { + if nodeInfo.CSINode != nil { + if err := s.ClusterSnapshotStore.CsiSnapshot().AddCSINode(nodeInfo.CSINode); err != nil { + return err + } + } + } + return s.ClusterSnapshotStore.AddSchedulerNodeInfo(nodeInfo.ToScheduler()) } @@ -127,6 +155,12 @@ func (s *PredicateSnapshot) RemoveNodeInfo(nodeName string) error { s.ClusterSnapshotStore.DraSnapshot().RemovePodOwnedClaims(pod.Pod) } } + if s.enableCSINodeAwareScheduling { + // generally a node name is same as csi node name and hence we should be safe + if nodeInfo.CSINode != nil { + s.ClusterSnapshotStore.CsiSnapshot().RemoveCSINode(nodeName) + } + } return nil } diff --git a/cluster-autoscaler/simulator/clustersnapshot/predicate/predicate_snapshot_benchmark_test.go b/cluster-autoscaler/simulator/clustersnapshot/predicate/predicate_snapshot_benchmark_test.go index b384bcecc02f..3afd07f1909c 100644 --- a/cluster-autoscaler/simulator/clustersnapshot/predicate/predicate_snapshot_benchmark_test.go +++ b/cluster-autoscaler/simulator/clustersnapshot/predicate/predicate_snapshot_benchmark_test.go @@ -39,7 +39,7 @@ func BenchmarkAddNodeInfo(b *testing.B) { b.Run(fmt.Sprintf("%s: AddNodeInfo() %d", snapshotName, tc), func(b *testing.B) { for i := 0; i < b.N; i++ { b.StopTimer() - assert.NoError(b, clusterSnapshot.SetClusterState(nil, nil, nil)) + assert.NoError(b, clusterSnapshot.SetClusterState(nil, nil, nil, nil)) b.StartTimer() for _, node := range nodes { err := clusterSnapshot.AddNodeInfo(framework.NewTestNodeInfo(node)) @@ -61,7 +61,7 @@ func BenchmarkListNodeInfos(b *testing.B) { nodes := clustersnapshot.CreateTestNodes(tc) clusterSnapshot, err := snapshotFactory() assert.NoError(b, err) - err = clusterSnapshot.SetClusterState(nodes, nil, nil) + err = clusterSnapshot.SetClusterState(nodes, nil, nil, nil) if err != nil { assert.NoError(b, err) } @@ -91,14 +91,14 @@ func BenchmarkAddPods(b *testing.B) { clustersnapshot.AssignTestPodsToNodes(pods, nodes) clusterSnapshot, err := snapshotFactory() assert.NoError(b, err) - err = clusterSnapshot.SetClusterState(nodes, nil, nil) + err = clusterSnapshot.SetClusterState(nodes, nil, nil, nil) assert.NoError(b, err) b.ResetTimer() b.Run(fmt.Sprintf("%s: ForceAddPod() 30*%d", snapshotName, tc), func(b *testing.B) { for i := 0; i < b.N; i++ { b.StopTimer() - err = clusterSnapshot.SetClusterState(nodes, nil, nil) + err = clusterSnapshot.SetClusterState(nodes, nil, nil, nil) if err != nil { assert.NoError(b, err) } @@ -127,7 +127,7 @@ func BenchmarkForkAddRevert(b *testing.B) { clustersnapshot.AssignTestPodsToNodes(pods, nodes) clusterSnapshot, err := snapshotFactory() assert.NoError(b, err) - err = clusterSnapshot.SetClusterState(nodes, pods, nil) + err = clusterSnapshot.SetClusterState(nodes, pods, nil, nil) assert.NoError(b, err) tmpNode1 := BuildTestNode("tmp-1", 2000, 2000000) tmpNode2 := BuildTestNode("tmp-2", 2000, 2000000) diff --git a/cluster-autoscaler/simulator/clustersnapshot/predicate/predicate_snapshot_dra_benchmark_test.go b/cluster-autoscaler/simulator/clustersnapshot/predicate/predicate_snapshot_dra_benchmark_test.go index b9250c62d55a..f2bf949197bf 100644 --- a/cluster-autoscaler/simulator/clustersnapshot/predicate/predicate_snapshot_dra_benchmark_test.go +++ b/cluster-autoscaler/simulator/clustersnapshot/predicate/predicate_snapshot_dra_benchmark_test.go @@ -333,7 +333,7 @@ func BenchmarkScheduleRevert(b *testing.B) { draSnapshot.AddClaims(ownedClaims[nodeIndex]) } - err = snapshot.SetClusterState(nil, nil, draSnapshot) + err = snapshot.SetClusterState(nil, nil, draSnapshot, nil) if err != nil { b.Errorf("Failed to set cluster state: %v", err) } diff --git a/cluster-autoscaler/simulator/clustersnapshot/predicate/predicate_snapshot_test.go b/cluster-autoscaler/simulator/clustersnapshot/predicate/predicate_snapshot_test.go index 13c71b00e3d3..fc340879823f 100644 --- a/cluster-autoscaler/simulator/clustersnapshot/predicate/predicate_snapshot_test.go +++ b/cluster-autoscaler/simulator/clustersnapshot/predicate/predicate_snapshot_test.go @@ -49,14 +49,14 @@ var snapshots = map[string]func() (clustersnapshot.ClusterSnapshot, error){ if err != nil { return nil, err } - return NewPredicateSnapshot(store.NewBasicSnapshotStore(), fwHandle, true, 1), nil + return NewPredicateSnapshot(store.NewBasicSnapshotStore(), fwHandle, true, 1, false), nil }, "delta": func() (clustersnapshot.ClusterSnapshot, error) { fwHandle, err := framework.NewTestFrameworkHandle() if err != nil { return nil, err } - return NewPredicateSnapshot(store.NewDeltaSnapshotStore(16), fwHandle, true, 1), nil + return NewPredicateSnapshot(store.NewDeltaSnapshotStore(16), fwHandle, true, 1, false), nil }, } @@ -150,7 +150,7 @@ func startSnapshot(t *testing.T, snapshotFactory func() (clustersnapshot.Cluster } draSnapshot := drasnapshot.CloneTestSnapshot(state.draSnapshot) - err = snapshot.SetClusterState(state.nodes, pods, draSnapshot) + err = snapshot.SetClusterState(state.nodes, pods, draSnapshot, nil) assert.NoError(t, err) return snapshot } @@ -1332,7 +1332,7 @@ func TestSetClusterState(t *testing.T) { snapshot := startSnapshot(t, snapshotFactory, state) compareStates(t, state, getSnapshotState(t, snapshot)) - assert.NoError(t, snapshot.SetClusterState(nil, nil, nil)) + assert.NoError(t, snapshot.SetClusterState(nil, nil, nil, nil)) compareStates(t, snapshotState{draSnapshot: drasnapshot.NewEmptySnapshot()}, getSnapshotState(t, snapshot)) }) @@ -1343,7 +1343,7 @@ func TestSetClusterState(t *testing.T) { newNodes, newPods := clustersnapshot.CreateTestNodes(13), clustersnapshot.CreateTestPods(37) newPodsByNode := clustersnapshot.AssignTestPodsToNodes(newPods, newNodes) - assert.NoError(t, snapshot.SetClusterState(newNodes, newPods, nil)) + assert.NoError(t, snapshot.SetClusterState(newNodes, newPods, nil, nil)) compareStates(t, snapshotState{nodes: newNodes, podsByNode: newPodsByNode, draSnapshot: drasnapshot.NewEmptySnapshot()}, getSnapshotState(t, snapshot)) }) @@ -1366,7 +1366,7 @@ func TestSetClusterState(t *testing.T) { compareStates(t, snapshotState{nodes: allNodes, podsByNode: allPodsByNode, draSnapshot: drasnapshot.NewEmptySnapshot()}, getSnapshotState(t, snapshot)) - assert.NoError(t, snapshot.SetClusterState(nil, nil, nil)) + assert.NoError(t, snapshot.SetClusterState(nil, nil, nil, nil)) compareStates(t, snapshotState{draSnapshot: drasnapshot.NewEmptySnapshot()}, getSnapshotState(t, snapshot)) @@ -1777,7 +1777,7 @@ func TestPVCClearAndFork(t *testing.T) { volumeExists := snapshot.StorageInfos().IsPVCUsedByPods(schedulerframework.GetNamespacedName("default", "claim1")) assert.Equal(t, true, volumeExists) - assert.NoError(t, snapshot.SetClusterState(nil, nil, nil)) + assert.NoError(t, snapshot.SetClusterState(nil, nil, nil, nil)) volumeExists = snapshot.StorageInfos().IsPVCUsedByPods(schedulerframework.GetNamespacedName("default", "claim1")) assert.Equal(t, false, volumeExists) diff --git a/cluster-autoscaler/simulator/clustersnapshot/store/basic.go b/cluster-autoscaler/simulator/clustersnapshot/store/basic.go index 89dd90b402da..b6a8ae30dfa5 100644 --- a/cluster-autoscaler/simulator/clustersnapshot/store/basic.go +++ b/cluster-autoscaler/simulator/clustersnapshot/store/basic.go @@ -21,10 +21,11 @@ import ( apiv1 "k8s.io/api/core/v1" "k8s.io/autoscaler/cluster-autoscaler/simulator/clustersnapshot" + csisnapshot "k8s.io/autoscaler/cluster-autoscaler/simulator/csi/snapshot" drasnapshot "k8s.io/autoscaler/cluster-autoscaler/simulator/dynamicresources/snapshot" "k8s.io/klog/v2" fwk "k8s.io/kube-scheduler/framework" - schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" + intreeschedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" ) // BasicSnapshotStore is simple, reference implementation of ClusterSnapshotStore. @@ -32,6 +33,7 @@ import ( type BasicSnapshotStore struct { data []*internalBasicSnapshotData draSnapshot *drasnapshot.Snapshot + csiSnapshot *csisnapshot.Snapshot } type internalBasicSnapshotData struct { @@ -92,7 +94,7 @@ func (data *internalBasicSnapshotData) addPvcUsedByPod(pod *apiv1.Pod) { if volume.PersistentVolumeClaim == nil { continue } - k := schedulerframework.GetNamespacedName(nameSpace, volume.PersistentVolumeClaim.ClaimName) + k := intreeschedulerframework.GetNamespacedName(nameSpace, volume.PersistentVolumeClaim.ClaimName) _, found := data.pvcNamespacePodMap[k] if !found { data.pvcNamespacePodMap[k] = make(map[string]bool) @@ -111,7 +113,7 @@ func (data *internalBasicSnapshotData) removePvcUsedByPod(pod *apiv1.Pod) { if volume.PersistentVolumeClaim == nil { continue } - k := schedulerframework.GetNamespacedName(nameSpace, volume.PersistentVolumeClaim.ClaimName) + k := intreeschedulerframework.GetNamespacedName(nameSpace, volume.PersistentVolumeClaim.ClaimName) if _, found := data.pvcNamespacePodMap[k]; found { delete(data.pvcNamespacePodMap[k], pod.GetName()) if len(data.pvcNamespacePodMap[k]) == 0 { @@ -150,7 +152,7 @@ func (data *internalBasicSnapshotData) addNode(node *apiv1.Node) error { if _, found := data.nodeInfoMap[node.Name]; found { return fmt.Errorf("node %s already in snapshot", node.Name) } - nodeInfo := schedulerframework.NewNodeInfo() + nodeInfo := intreeschedulerframework.NewNodeInfo() nodeInfo.SetNode(node) data.nodeInfoMap[node.Name] = nodeInfo return nil @@ -171,7 +173,7 @@ func (data *internalBasicSnapshotData) addPod(pod *apiv1.Pod, nodeName string) e if _, found := data.nodeInfoMap[nodeName]; !found { return clustersnapshot.ErrNodeNotFound } - podInfo, _ := schedulerframework.NewPodInfo(pod) + podInfo, _ := intreeschedulerframework.NewPodInfo(pod) data.nodeInfoMap[nodeName].AddPodInfo(podInfo) data.addPvcUsedByPod(pod) return nil @@ -213,6 +215,16 @@ func (snapshot *BasicSnapshotStore) DraSnapshot() *drasnapshot.Snapshot { return snapshot.draSnapshot } +// CsiSnapshot returns the CSI snapshot. +func (snapshot *BasicSnapshotStore) CsiSnapshot() *csisnapshot.Snapshot { + return snapshot.csiSnapshot +} + +// CSINodes returns the CSI nodes snapshot. +func (snapshot *BasicSnapshotStore) CSINodes() fwk.CSINodeLister { + return snapshot.csiSnapshot.CSINodes() +} + // AddSchedulerNodeInfo adds a NodeInfo. func (snapshot *BasicSnapshotStore) AddSchedulerNodeInfo(nodeInfo fwk.NodeInfo) error { if err := snapshot.getInternalData().addNode(nodeInfo.Node()); err != nil { @@ -227,7 +239,7 @@ func (snapshot *BasicSnapshotStore) AddSchedulerNodeInfo(nodeInfo fwk.NodeInfo) } // SetClusterState sets the cluster state. -func (snapshot *BasicSnapshotStore) SetClusterState(nodes []*apiv1.Node, scheduledPods []*apiv1.Pod, draSnapshot *drasnapshot.Snapshot) error { +func (snapshot *BasicSnapshotStore) SetClusterState(nodes []*apiv1.Node, scheduledPods []*apiv1.Pod, draSnapshot *drasnapshot.Snapshot, csiSnapshot *csisnapshot.Snapshot) error { snapshot.clear() knownNodes := make(map[string]bool) @@ -251,6 +263,12 @@ func (snapshot *BasicSnapshotStore) SetClusterState(nodes []*apiv1.Node, schedul snapshot.draSnapshot = draSnapshot } + if csiSnapshot == nil { + snapshot.csiSnapshot = csisnapshot.NewEmptySnapshot() + } else { + snapshot.csiSnapshot = csiSnapshot + } + return nil } @@ -279,6 +297,7 @@ func (snapshot *BasicSnapshotStore) Fork() { forkData := snapshot.getInternalData().clone() snapshot.data = append(snapshot.data, forkData) snapshot.draSnapshot.Fork() + snapshot.csiSnapshot.Fork() } // Revert reverts snapshot state to moment of forking. @@ -288,6 +307,7 @@ func (snapshot *BasicSnapshotStore) Revert() { } snapshot.data = snapshot.data[:len(snapshot.data)-1] snapshot.draSnapshot.Revert() + snapshot.csiSnapshot.Revert() } // Commit commits changes done after forking. @@ -298,6 +318,7 @@ func (snapshot *BasicSnapshotStore) Commit() error { } snapshot.data = append(snapshot.data[:len(snapshot.data)-2], snapshot.data[len(snapshot.data)-1]) snapshot.draSnapshot.Commit() + snapshot.csiSnapshot.Commit() return nil } @@ -306,6 +327,7 @@ func (snapshot *BasicSnapshotStore) clear() { baseData := newInternalBasicSnapshotData() snapshot.data = []*internalBasicSnapshotData{baseData} snapshot.draSnapshot = drasnapshot.NewEmptySnapshot() + snapshot.csiSnapshot = csisnapshot.NewEmptySnapshot() } // implementation of SharedLister interface @@ -314,30 +336,35 @@ type basicSnapshotStoreNodeLister BasicSnapshotStore type basicSnapshotStoreStorageLister BasicSnapshotStore // NodeInfos exposes snapshot as NodeInfoLister. -func (snapshot *BasicSnapshotStore) NodeInfos() schedulerframework.NodeInfoLister { +func (snapshot *BasicSnapshotStore) NodeInfos() fwk.NodeInfoLister { return (*basicSnapshotStoreNodeLister)(snapshot) } // StorageInfos exposes snapshot as StorageInfoLister. -func (snapshot *BasicSnapshotStore) StorageInfos() schedulerframework.StorageInfoLister { +func (snapshot *BasicSnapshotStore) StorageInfos() fwk.StorageInfoLister { return (*basicSnapshotStoreStorageLister)(snapshot) } // ResourceClaims exposes snapshot as ResourceClaimTracker -func (snapshot *BasicSnapshotStore) ResourceClaims() schedulerframework.ResourceClaimTracker { +func (snapshot *BasicSnapshotStore) ResourceClaims() fwk.ResourceClaimTracker { return snapshot.DraSnapshot().ResourceClaims() } // ResourceSlices exposes snapshot as ResourceSliceLister. -func (snapshot *BasicSnapshotStore) ResourceSlices() schedulerframework.ResourceSliceLister { +func (snapshot *BasicSnapshotStore) ResourceSlices() fwk.ResourceSliceLister { return snapshot.DraSnapshot().ResourceSlices() } // DeviceClasses exposes the snapshot as DeviceClassLister. -func (snapshot *BasicSnapshotStore) DeviceClasses() schedulerframework.DeviceClassLister { +func (snapshot *BasicSnapshotStore) DeviceClasses() fwk.DeviceClassLister { return snapshot.DraSnapshot().DeviceClasses() } +// DeviceClassResolver exposes the snapshot as DeviceClassResolver. +func (snapshot *BasicSnapshotStore) DeviceClassResolver() fwk.DeviceClassResolver { + return snapshot.DraSnapshot().DeviceClassResolver() +} + // List returns the list of nodes in the snapshot. func (snapshot *basicSnapshotStoreNodeLister) List() ([]fwk.NodeInfo, error) { return (*BasicSnapshotStore)(snapshot).getInternalData().listNodeInfos(), nil diff --git a/cluster-autoscaler/simulator/clustersnapshot/store/delta.go b/cluster-autoscaler/simulator/clustersnapshot/store/delta.go index fc3a82ffae42..5a05329f1e82 100644 --- a/cluster-autoscaler/simulator/clustersnapshot/store/delta.go +++ b/cluster-autoscaler/simulator/clustersnapshot/store/delta.go @@ -22,11 +22,12 @@ import ( apiv1 "k8s.io/api/core/v1" "k8s.io/autoscaler/cluster-autoscaler/simulator/clustersnapshot" + csisnapshot "k8s.io/autoscaler/cluster-autoscaler/simulator/csi/snapshot" drasnapshot "k8s.io/autoscaler/cluster-autoscaler/simulator/dynamicresources/snapshot" "k8s.io/client-go/util/workqueue" "k8s.io/klog/v2" fwk "k8s.io/kube-scheduler/framework" - schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" + intreeschedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" ) // DeltaSnapshotStore is an implementation of ClusterSnapshotStore optimized for typical Cluster Autoscaler usage - (fork, add stuff, revert), repeated many times per loop. @@ -54,6 +55,7 @@ import ( type DeltaSnapshotStore struct { data *internalDeltaSnapshotData draSnapshot *drasnapshot.Snapshot + csiSnapshot *csisnapshot.Snapshot parallelism int } @@ -148,7 +150,7 @@ func (data *internalDeltaSnapshotData) buildNodeInfoList() []fwk.NodeInfo { } func (data *internalDeltaSnapshotData) addNode(node *apiv1.Node) (fwk.NodeInfo, error) { - nodeInfo := schedulerframework.NewNodeInfo() + nodeInfo := intreeschedulerframework.NewNodeInfo() nodeInfo.SetNode(node) err := data.addNodeInfo(nodeInfo) if err != nil { @@ -248,7 +250,7 @@ func (data *internalDeltaSnapshotData) addPod(pod *apiv1.Pod, nodeName string) e return clustersnapshot.ErrNodeNotFound } - podInfo, _ := schedulerframework.NewPodInfo(pod) + podInfo, _ := intreeschedulerframework.NewPodInfo(pod) ni.AddPodInfo(podInfo) // Maybe consider deleting from the list in the future. Maybe not. @@ -394,27 +396,27 @@ func (snapshot *DeltaSnapshotStore) getNodeInfo(nodeName string) (fwk.NodeInfo, } // NodeInfos returns node lister. -func (snapshot *DeltaSnapshotStore) NodeInfos() schedulerframework.NodeInfoLister { +func (snapshot *DeltaSnapshotStore) NodeInfos() fwk.NodeInfoLister { return (*deltaSnapshotStoreNodeLister)(snapshot) } // StorageInfos returns storage lister -func (snapshot *DeltaSnapshotStore) StorageInfos() schedulerframework.StorageInfoLister { +func (snapshot *DeltaSnapshotStore) StorageInfos() fwk.StorageInfoLister { return (*deltaSnapshotStoreStorageLister)(snapshot) } // ResourceClaims exposes snapshot as ResourceClaimTracker -func (snapshot *DeltaSnapshotStore) ResourceClaims() schedulerframework.ResourceClaimTracker { +func (snapshot *DeltaSnapshotStore) ResourceClaims() fwk.ResourceClaimTracker { return snapshot.DraSnapshot().ResourceClaims() } // ResourceSlices exposes snapshot as ResourceSliceLister. -func (snapshot *DeltaSnapshotStore) ResourceSlices() schedulerframework.ResourceSliceLister { +func (snapshot *DeltaSnapshotStore) ResourceSlices() fwk.ResourceSliceLister { return snapshot.DraSnapshot().ResourceSlices() } // DeviceClasses exposes the snapshot as DeviceClassLister. -func (snapshot *DeltaSnapshotStore) DeviceClasses() schedulerframework.DeviceClassLister { +func (snapshot *DeltaSnapshotStore) DeviceClasses() fwk.DeviceClassLister { return snapshot.DraSnapshot().DeviceClasses() } @@ -432,6 +434,21 @@ func (snapshot *DeltaSnapshotStore) DraSnapshot() *drasnapshot.Snapshot { return snapshot.draSnapshot } +// CsiSnapshot returns the CSI snapshot. +func (snapshot *DeltaSnapshotStore) CsiSnapshot() *csisnapshot.Snapshot { + return snapshot.csiSnapshot +} + +// CSINodes returns the CSI node lister for this snapshot. +func (snapshot *DeltaSnapshotStore) CSINodes() fwk.CSINodeLister { + return snapshot.csiSnapshot.CSINodes() +} + +// DeviceClassResolver exposes the snapshot as DeviceClassResolver. +func (snapshot *DeltaSnapshotStore) DeviceClassResolver() fwk.DeviceClassResolver { + return snapshot.DraSnapshot().DeviceClassResolver() +} + // AddSchedulerNodeInfo adds a NodeInfo. func (snapshot *DeltaSnapshotStore) AddSchedulerNodeInfo(nodeInfo fwk.NodeInfo) error { if _, err := snapshot.data.addNode(nodeInfo.Node()); err != nil { @@ -450,7 +467,7 @@ func (snapshot *DeltaSnapshotStore) setClusterStatePodsSequential(nodeInfos []fw for _, pod := range scheduledPods { if nodeIdx, ok := nodeNameToIdx[pod.Spec.NodeName]; ok { // Can add pod directly. Cache will be cleared afterwards. - podInfo, _ := schedulerframework.NewPodInfo(pod) + podInfo, _ := intreeschedulerframework.NewPodInfo(pod) nodeInfos[nodeIdx].AddPodInfo(podInfo) } } @@ -472,14 +489,14 @@ func (snapshot *DeltaSnapshotStore) setClusterStatePodsParallelized(nodeInfos [] nodeInfo := nodeInfos[nodeIdx] for _, pod := range podsForNode[nodeIdx] { // Can add pod directly. Cache will be cleared afterwards. - podInfo, _ := schedulerframework.NewPodInfo(pod) + podInfo, _ := intreeschedulerframework.NewPodInfo(pod) nodeInfo.AddPodInfo(podInfo) } }) } // SetClusterState sets the cluster state. -func (snapshot *DeltaSnapshotStore) SetClusterState(nodes []*apiv1.Node, scheduledPods []*apiv1.Pod, draSnapshot *drasnapshot.Snapshot) error { +func (snapshot *DeltaSnapshotStore) SetClusterState(nodes []*apiv1.Node, scheduledPods []*apiv1.Pod, draSnapshot *drasnapshot.Snapshot, csiSnapshot *csisnapshot.Snapshot) error { snapshot.clear() nodeNameToIdx := make(map[string]int, len(nodes)) @@ -510,6 +527,12 @@ func (snapshot *DeltaSnapshotStore) SetClusterState(nodes []*apiv1.Node, schedul snapshot.draSnapshot = draSnapshot } + if csiSnapshot == nil { + snapshot.csiSnapshot = csisnapshot.NewEmptySnapshot() + } else { + snapshot.csiSnapshot = csiSnapshot + } + return nil } @@ -538,6 +561,7 @@ func (snapshot *DeltaSnapshotStore) IsPVCUsedByPods(key string) bool { func (snapshot *DeltaSnapshotStore) Fork() { snapshot.data = snapshot.data.fork() snapshot.draSnapshot.Fork() + snapshot.csiSnapshot.Fork() } // Revert reverts snapshot state to moment of forking. @@ -547,6 +571,7 @@ func (snapshot *DeltaSnapshotStore) Revert() { snapshot.data = snapshot.data.baseData } snapshot.draSnapshot.Revert() + snapshot.csiSnapshot.Revert() } // Commit commits changes done after forking. @@ -558,6 +583,7 @@ func (snapshot *DeltaSnapshotStore) Commit() error { } snapshot.data = newData snapshot.draSnapshot.Commit() + snapshot.csiSnapshot.Commit() return nil } @@ -566,4 +592,5 @@ func (snapshot *DeltaSnapshotStore) Commit() error { func (snapshot *DeltaSnapshotStore) clear() { snapshot.data = newInternalDeltaSnapshotData() snapshot.draSnapshot = drasnapshot.NewEmptySnapshot() + snapshot.csiSnapshot = csisnapshot.NewEmptySnapshot() } diff --git a/cluster-autoscaler/simulator/clustersnapshot/store/delta_benchmark_test.go b/cluster-autoscaler/simulator/clustersnapshot/store/delta_benchmark_test.go index 0d426b7e39b1..7f317323b9cd 100644 --- a/cluster-autoscaler/simulator/clustersnapshot/store/delta_benchmark_test.go +++ b/cluster-autoscaler/simulator/clustersnapshot/store/delta_benchmark_test.go @@ -48,7 +48,7 @@ func BenchmarkBuildNodeInfoList(b *testing.B) { b.Run(fmt.Sprintf("fork add 1000 to %d", tc.nodeCount), func(b *testing.B) { nodes := clustersnapshot.CreateTestNodes(tc.nodeCount + 1000) deltaStore := NewDeltaSnapshotStore(16) - if err := deltaStore.SetClusterState(nodes[:tc.nodeCount], nil, nil); err != nil { + if err := deltaStore.SetClusterState(nodes[:tc.nodeCount], nil, nil, nil); err != nil { assert.NoError(b, err) } deltaStore.Fork() @@ -70,7 +70,7 @@ func BenchmarkBuildNodeInfoList(b *testing.B) { b.Run(fmt.Sprintf("base %d", tc.nodeCount), func(b *testing.B) { nodes := clustersnapshot.CreateTestNodes(tc.nodeCount) deltaStore := NewDeltaSnapshotStore(16) - if err := deltaStore.SetClusterState(nodes, nil, nil); err != nil { + if err := deltaStore.SetClusterState(nodes, nil, nil, nil); err != nil { assert.NoError(b, err) } b.ResetTimer() diff --git a/cluster-autoscaler/simulator/clustersnapshot/test_utils.go b/cluster-autoscaler/simulator/clustersnapshot/test_utils.go index d95a3e6437c4..6473a33f6103 100644 --- a/cluster-autoscaler/simulator/clustersnapshot/test_utils.go +++ b/cluster-autoscaler/simulator/clustersnapshot/test_utils.go @@ -38,7 +38,7 @@ func InitializeClusterSnapshotOrDie( pods []*apiv1.Pod) { var err error - assert.NoError(t, snapshot.SetClusterState(nil, nil, nil)) + assert.NoError(t, snapshot.SetClusterState(nil, nil, nil, nil)) for _, node := range nodes { err = snapshot.AddNodeInfo(framework.NewTestNodeInfo(node)) diff --git a/cluster-autoscaler/simulator/clustersnapshot/testsnapshot/test_snapshot.go b/cluster-autoscaler/simulator/clustersnapshot/testsnapshot/test_snapshot.go index 1eda95dee3ae..37105fdd9b1b 100644 --- a/cluster-autoscaler/simulator/clustersnapshot/testsnapshot/test_snapshot.go +++ b/cluster-autoscaler/simulator/clustersnapshot/testsnapshot/test_snapshot.go @@ -57,5 +57,5 @@ func NewCustomTestSnapshotAndHandle(snapshotStore clustersnapshot.ClusterSnapsho if err != nil { return nil, nil, err } - return predicate.NewPredicateSnapshot(snapshotStore, testFwHandle, true, 1), testFwHandle, nil + return predicate.NewPredicateSnapshot(snapshotStore, testFwHandle, true, 1, false), testFwHandle, nil } diff --git a/cluster-autoscaler/simulator/csi/provider/csinode.go b/cluster-autoscaler/simulator/csi/provider/csinode.go new file mode 100644 index 000000000000..19f50e791258 --- /dev/null +++ b/cluster-autoscaler/simulator/csi/provider/csinode.go @@ -0,0 +1,38 @@ +package provider + +import ( + storagev1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/labels" + csisnapshot "k8s.io/autoscaler/cluster-autoscaler/simulator/csi/snapshot" + informers "k8s.io/client-go/informers" + v1storagelister "k8s.io/client-go/listers/storage/v1" +) + +// Provider provides access to CSI node information for the cluster. +type Provider struct { + csINodesLister v1storagelister.CSINodeLister +} + +// NewCSINodeProvider creates a new Provider with the given CSI node lister. +func NewCSINodeProvider(csINodesLister v1storagelister.CSINodeLister) *Provider { + return &Provider{csINodesLister: csINodesLister} +} + +// NewCSINodeProviderFromInformers creates a new Provider from an informer factory. +func NewCSINodeProviderFromInformers(informerFactory informers.SharedInformerFactory) *Provider { + return NewCSINodeProvider(informerFactory.Storage().V1().CSINodes().Lister()) +} + +func (p *Provider) Snapshot() (*csisnapshot.Snapshot, error) { + csiNodes, err := p.csINodesLister.List(labels.Everything()) + if err != nil { + return nil, err + } + + csiNodesMap := make(map[string]*storagev1.CSINode) + for _, csiNode := range csiNodes { + csiNodesMap[csiNode.Name] = csiNode + } + return csisnapshot.NewSnapshot(csiNodesMap), nil + +} diff --git a/cluster-autoscaler/simulator/csi/snapshot/csinode_lister.go b/cluster-autoscaler/simulator/csi/snapshot/csinode_lister.go new file mode 100644 index 000000000000..f1e5cf92696d --- /dev/null +++ b/cluster-autoscaler/simulator/csi/snapshot/csinode_lister.go @@ -0,0 +1,18 @@ +package snapshot + +import storagev1 "k8s.io/api/storage/v1" + +// SnapshotCSINodeLister provides access to CSI nodes within a snapshot. +type SnapshotCSINodeLister struct { + snapshot *Snapshot +} + +// List returns all CSI nodes in the snapshot. +func (s SnapshotCSINodeLister) List() ([]*storagev1.CSINode, error) { + return s.snapshot.listCSINodes(), nil +} + +// Get retrieves a CSI node by name from the snapshot. +func (s SnapshotCSINodeLister) Get(name string) (*storagev1.CSINode, error) { + return s.snapshot.Get(name) +} diff --git a/cluster-autoscaler/simulator/csi/snapshot/snapshot.go b/cluster-autoscaler/simulator/csi/snapshot/snapshot.go new file mode 100644 index 000000000000..322b12d32d57 --- /dev/null +++ b/cluster-autoscaler/simulator/csi/snapshot/snapshot.go @@ -0,0 +1,120 @@ +package snapshot + +import ( + "fmt" + + storagev1 "k8s.io/api/storage/v1" + "k8s.io/autoscaler/cluster-autoscaler/simulator/common" + "k8s.io/autoscaler/cluster-autoscaler/simulator/framework" + fwk "k8s.io/kube-scheduler/framework" + schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" +) + +// Snapshot represents a snapshot of CSI node information for cluster simulation. +type Snapshot struct { + csiNodes *common.PatchSet[string, *storagev1.CSINode] +} + +// NewSnapshot creates a new Snapshot from a map of CSI nodes. +func NewSnapshot(csiNodes map[string]*storagev1.CSINode) *Snapshot { + csiNdodePatch := common.NewPatchFromMap(csiNodes) + return &Snapshot{ + csiNodes: common.NewPatchSet(csiNdodePatch), + } +} + +func (s *Snapshot) listCSINodes() []*storagev1.CSINode { + csiNodes := s.csiNodes.AsMap() + csiNodesList := make([]*storagev1.CSINode, 0, len(csiNodes)) + for _, csiNode := range csiNodes { + csiNodesList = append(csiNodesList, csiNode) + } + return csiNodesList +} + +// CSINodes returns a CSI node lister for the snapshot. +func (s *Snapshot) CSINodes() fwk.CSINodeLister { + return SnapshotCSINodeLister{snapshot: s} +} + +// AddCSINodes adds a list of CSI nodes to the snapshot. +func (s *Snapshot) AddCSINodes(csiNodes []*storagev1.CSINode) error { + for _, csiNode := range csiNodes { + if _, alreadyInSnapshot := s.csiNodes.FindValue(csiNode.Name); alreadyInSnapshot { + return fmt.Errorf("csi node %s already in snapshot", csiNode.Name) + } + } + + for _, csiNode := range csiNodes { + s.csiNodes.SetCurrent(csiNode.Name, csiNode) + } + + return nil +} + +// Get returns a CSI node from the snapshot by name. +func (s *Snapshot) Get(name string) (*storagev1.CSINode, error) { + csiNode, found := s.csiNodes.FindValue(name) + if !found { + return nil, fmt.Errorf("csi nodes %s not found", name) + } + return csiNode, nil +} + +// WrapSchedulerNodeInfo wraps a scheduler node info with a CSI node. +func (s *Snapshot) WrapSchedulerNodeInfo(schedNodeInfo *schedulerframework.NodeInfo) (*framework.NodeInfo, error) { + csiNode, err := s.Get(schedNodeInfo.Node().Name) + if err != nil { + return nil, err + } + nodeInfo := framework.WrapSchedulerNodeInfo(schedNodeInfo, nil, nil) + nodeInfo.CSINode = csiNode + return nodeInfo, nil +} + +// AddCSINode adds a CSI node to the snapshot. +func (s *Snapshot) AddCSINode(csiNode *storagev1.CSINode) error { + if _, alreadyInSnapshot := s.csiNodes.FindValue(csiNode.Name); alreadyInSnapshot { + return fmt.Errorf("csi node %s already in snapshot", csiNode.Name) + } + + s.csiNodes.SetCurrent(csiNode.Name, csiNode) + return nil +} + +// AddCSINodeInfoToNodeInfo adds a CSI node to the node info. +func (s *Snapshot) AddCSINodeInfoToNodeInfo(nodeInfo *framework.NodeInfo) (*framework.NodeInfo, error) { + csiNode, err := s.Get(nodeInfo.Node().Name) + if err != nil { + return nil, err + } + return nodeInfo.AddCSINode(csiNode), nil +} + +// RemoveCSINode removes a CSI node from the snapshot. +func (s *Snapshot) RemoveCSINode(name string) { + s.csiNodes.DeleteCurrent(name) +} + +// Commit commits the snapshot. +func (s *Snapshot) Commit() { + s.csiNodes.Commit() +} + +// Revert reverts the snapshot. +func (s *Snapshot) Revert() { + s.csiNodes.Revert() +} + +// Fork forks the snapshot. +func (s *Snapshot) Fork() { + s.csiNodes.Fork() +} + +// NewEmptySnapshot creates a new empty snapshot. +func NewEmptySnapshot() *Snapshot { + csiNdodePatch := common.NewPatch[string, *storagev1.CSINode]() + return &Snapshot{ + csiNodes: common.NewPatchSet(csiNdodePatch), + } +} diff --git a/cluster-autoscaler/simulator/drain_test.go b/cluster-autoscaler/simulator/drain_test.go index 3b2faa84539d..8c5a3a6cf4d8 100644 --- a/cluster-autoscaler/simulator/drain_test.go +++ b/cluster-autoscaler/simulator/drain_test.go @@ -806,7 +806,7 @@ func TestGetPodsToMove(t *testing.T) { ssLister, err := kube_util.NewTestStatefulSetLister([]*appsv1.StatefulSet{&statefulset}) assert.NoError(t, err) - registry = kube_util.NewListerRegistry(nil, nil, nil, nil, dsLister, rcLister, jobLister, rsLister, ssLister) + registry = kube_util.NewListerRegistry(nil, nil, nil, nil, nil, dsLister, rcLister, jobLister, rsLister, ssLister) } deleteOptions := options.NodeDeleteOptions{ diff --git a/cluster-autoscaler/simulator/drainability/rules/replicacount/rule_test.go b/cluster-autoscaler/simulator/drainability/rules/replicacount/rule_test.go index 74a55973e345..d3bec6a65c93 100644 --- a/cluster-autoscaler/simulator/drainability/rules/replicacount/rule_test.go +++ b/cluster-autoscaler/simulator/drainability/rules/replicacount/rule_test.go @@ -329,7 +329,7 @@ func TestDrainable(t *testing.T) { ssLister, err := kube_util.NewTestStatefulSetLister([]*appsv1.StatefulSet{&statefulset}) assert.NoError(t, err) - registry := kube_util.NewListerRegistry(nil, nil, nil, nil, dsLister, rcLister, jobLister, rsLister, ssLister) + registry := kube_util.NewListerRegistry(nil, nil, nil, nil, nil, dsLister, rcLister, jobLister, rsLister, ssLister) drainCtx := &drainability.DrainContext{ Listers: registry, diff --git a/cluster-autoscaler/simulator/drainability/rules/replicated/rule_test.go b/cluster-autoscaler/simulator/drainability/rules/replicated/rule_test.go index 1b62b87b5251..c871426ecaa3 100644 --- a/cluster-autoscaler/simulator/drainability/rules/replicated/rule_test.go +++ b/cluster-autoscaler/simulator/drainability/rules/replicated/rule_test.go @@ -225,7 +225,7 @@ func TestDrainable(t *testing.T) { ssLister, err := kube_util.NewTestStatefulSetLister([]*appsv1.StatefulSet{&statefulset}) assert.NoError(t, err) - registry := kube_util.NewListerRegistry(nil, nil, nil, nil, dsLister, rcLister, jobLister, rsLister, ssLister) + registry := kube_util.NewListerRegistry(nil, nil, nil, nil, nil, dsLister, rcLister, jobLister, rsLister, ssLister) drainCtx := &drainability.DrainContext{ Listers: registry, diff --git a/cluster-autoscaler/simulator/dynamicresources/snapshot/snapshot.go b/cluster-autoscaler/simulator/dynamicresources/snapshot/snapshot.go index adb1636d6f58..714edf978ca1 100644 --- a/cluster-autoscaler/simulator/dynamicresources/snapshot/snapshot.go +++ b/cluster-autoscaler/simulator/dynamicresources/snapshot/snapshot.go @@ -29,7 +29,7 @@ import ( resourceclaim "k8s.io/dynamic-resource-allocation/resourceclaim" "k8s.io/klog/v2" fwk "k8s.io/kube-scheduler/framework" - schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" + schedulerframework "k8s.io/kube-scheduler/framework" ) // ResourceClaimId is a unique identifier for a ResourceClaim. @@ -91,6 +91,11 @@ func (s *Snapshot) ResourceClaims() schedulerframework.ResourceClaimTracker { return snapshotClaimTracker{snapshot: s} } +// DeviceClassResolver returns a device class resolver for the snapshot. +func (s *Snapshot) DeviceClassResolver() schedulerframework.DeviceClassResolver { + return newSnapshotDeviceClassResolver(s) +} + // ResourceSlices exposes the Snapshot as schedulerframework.ResourceSliceLister, in order to interact with // the scheduler framework. func (s *Snapshot) ResourceSlices() schedulerframework.ResourceSliceLister { @@ -121,6 +126,22 @@ func (s *Snapshot) WrapSchedulerNodeInfo(schedNodeInfo fwk.NodeInfo) (*framework return framework.WrapSchedulerNodeInfo(schedNodeInfo, nodeSlices, podExtraInfos), nil } +// AddDRAInfo adds DRA information to the node info. +func (s *Snapshot) AddDRAInfo(nodeInfo *framework.NodeInfo) (*framework.NodeInfo, error) { + podExtraInfos := make(map[types.UID]framework.PodExtraInfo, len(nodeInfo.Pods())) + for _, pod := range nodeInfo.Pods() { + podClaims, err := s.PodClaims(pod.Pod) + if err != nil { + return nil, err + } + if len(podClaims) > 0 { + podExtraInfos[pod.Pod.UID] = framework.PodExtraInfo{NeededResourceClaims: podClaims} + } + } + nodeSlices, _ := s.NodeResourceSlices(nodeInfo.Node().Name) + return nodeInfo.AddNodeResourceSlices(nodeSlices).AddPodExtraInfo(podExtraInfos), nil +} + // AddClaims adds additional ResourceClaims to the Snapshot. It can be used e.g. if we need to duplicate a Pod that // owns ResourceClaims. Returns an error if any of the claims is already tracked in the snapshot. func (s *Snapshot) AddClaims(newClaims []*resourceapi.ResourceClaim) error { diff --git a/cluster-autoscaler/simulator/dynamicresources/snapshot/snapshot_claim_tracker_test.go b/cluster-autoscaler/simulator/dynamicresources/snapshot/snapshot_claim_tracker_test.go index 30f6a01c5649..5948415fb0f9 100644 --- a/cluster-autoscaler/simulator/dynamicresources/snapshot/snapshot_claim_tracker_test.go +++ b/cluster-autoscaler/simulator/dynamicresources/snapshot/snapshot_claim_tracker_test.go @@ -28,7 +28,6 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/autoscaler/cluster-autoscaler/utils/test" "k8s.io/dynamic-resource-allocation/structured" - schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" ) var ( @@ -86,7 +85,7 @@ func TestSnapshotClaimTrackerList(t *testing.T) { } { t.Run(tc.testName, func(t *testing.T) { snapshot := NewSnapshot(tc.claims, nil, nil, nil) - var resourceClaimTracker schedulerframework.ResourceClaimTracker = snapshot.ResourceClaims() + resourceClaimTracker := snapshot.ResourceClaims() claims, err := resourceClaimTracker.List() if err != nil { t.Fatalf("snapshotClaimTracker.List(): got unexpected error: %v", err) @@ -132,7 +131,7 @@ func TestSnapshotClaimTrackerGet(t *testing.T) { GetClaimId(claim3): claim3, } snapshot := NewSnapshot(claims, nil, nil, nil) - var resourceClaimTracker schedulerframework.ResourceClaimTracker = snapshot.ResourceClaims() + resourceClaimTracker := snapshot.ResourceClaims() claim, err := resourceClaimTracker.Get(tc.claimNamespace, tc.claimName) if diff := cmp.Diff(tc.wantErr, err, cmpopts.EquateErrors()); diff != "" { t.Fatalf("snapshotClaimTracker.Get(): unexpected error (-want +got): %s", diff) @@ -180,7 +179,7 @@ func TestSnapshotClaimTrackerListAllAllocatedDevices(t *testing.T) { } { t.Run(tc.testName, func(t *testing.T) { snapshot := NewSnapshot(tc.claims, nil, nil, nil) - var resourceClaimTracker schedulerframework.ResourceClaimTracker = snapshot.ResourceClaims() + resourceClaimTracker := snapshot.ResourceClaims() devices, err := resourceClaimTracker.ListAllAllocatedDevices() if err != nil { t.Fatalf("snapshotClaimTracker.ListAllAllocatedDevices(): got unexpected error: %v", err) @@ -224,7 +223,7 @@ func TestSnapshotClaimTrackerSignalClaimPendingAllocation(t *testing.T) { GetClaimId(claim3): claim3, } snapshot := NewSnapshot(claims, nil, nil, nil) - var resourceClaimTracker schedulerframework.ResourceClaimTracker = snapshot.ResourceClaims() + resourceClaimTracker := snapshot.ResourceClaims() err := resourceClaimTracker.SignalClaimPendingAllocation(tc.claimUid, tc.allocatedClaim) if diff := cmp.Diff(tc.wantErr, err, cmpopts.EquateErrors()); diff != "" { diff --git a/cluster-autoscaler/simulator/dynamicresources/snapshot/snapshot_class_lister_test.go b/cluster-autoscaler/simulator/dynamicresources/snapshot/snapshot_class_lister_test.go index ccfe03534231..b61f41cc87c0 100644 --- a/cluster-autoscaler/simulator/dynamicresources/snapshot/snapshot_class_lister_test.go +++ b/cluster-autoscaler/simulator/dynamicresources/snapshot/snapshot_class_lister_test.go @@ -25,7 +25,6 @@ import ( resourceapi "k8s.io/api/resource/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/autoscaler/cluster-autoscaler/utils/test" - schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" ) var ( @@ -52,7 +51,7 @@ func TestSnapshotClassListerList(t *testing.T) { } { t.Run(tc.testName, func(t *testing.T) { snapshot := NewSnapshot(nil, nil, nil, tc.classes) - var deviceClassLister schedulerframework.DeviceClassLister = snapshot.DeviceClasses() + deviceClassLister := snapshot.DeviceClasses() classes, err := deviceClassLister.List() if err != nil { t.Fatalf("snapshotClassLister.List(): got unexpected error: %v", err) @@ -86,7 +85,7 @@ func TestSnapshotClassListerGet(t *testing.T) { t.Run(tc.testName, func(t *testing.T) { classes := map[string]*resourceapi.DeviceClass{"class-1": class1, "class-2": class2, "class-3": class3} snapshot := NewSnapshot(nil, nil, nil, classes) - var deviceClassLister schedulerframework.DeviceClassLister = snapshot.DeviceClasses() + deviceClassLister := snapshot.DeviceClasses() class, err := deviceClassLister.Get(tc.className) if diff := cmp.Diff(tc.wantErr, err, cmpopts.EquateErrors()); diff != "" { t.Fatalf("snapshotClassLister.Get(): unexpected error (-want +got): %s", diff) diff --git a/cluster-autoscaler/simulator/dynamicresources/snapshot/snapshot_class_resolver.go b/cluster-autoscaler/simulator/dynamicresources/snapshot/snapshot_class_resolver.go new file mode 100644 index 000000000000..a7f52afa4541 --- /dev/null +++ b/cluster-autoscaler/simulator/dynamicresources/snapshot/snapshot_class_resolver.go @@ -0,0 +1,39 @@ +package snapshot + +import ( + v1 "k8s.io/api/core/v1" + resourceapi "k8s.io/api/resource/v1" + schedulerframework "k8s.io/kube-scheduler/framework" +) + +type snapshotDeviceClassResolver struct { + // resourceName2class maps extended resource name to device class + resourceName2class map[v1.ResourceName]*resourceapi.DeviceClass +} + +var _ schedulerframework.DeviceClassResolver = &snapshotDeviceClassResolver{} + +func newSnapshotDeviceClassResolver(snapshot *Snapshot) schedulerframework.DeviceClassResolver { + resourceName2class := make(map[v1.ResourceName]*resourceapi.DeviceClass) + for _, class := range snapshot.listDeviceClasses() { + classResourceName := class.Name + extendedResourceName := class.Spec.ExtendedResourceName + if extendedResourceName != nil { + resourceName2class[v1.ResourceName(*extendedResourceName)] = class + } + + // Also add the default extended resource name + defaultResourceName := v1.ResourceName(resourceapi.ResourceDeviceClassPrefix + classResourceName) + resourceName2class[defaultResourceName] = class + } + return snapshotDeviceClassResolver{resourceName2class: resourceName2class} +} + +// GetDeviceClass returns the device class for the given extended resource name +func (s snapshotDeviceClassResolver) GetDeviceClass(resourceName v1.ResourceName) *resourceapi.DeviceClass { + class, found := s.resourceName2class[resourceName] + if !found { + return nil + } + return class +} diff --git a/cluster-autoscaler/simulator/dynamicresources/snapshot/snapshot_slice_lister_test.go b/cluster-autoscaler/simulator/dynamicresources/snapshot/snapshot_slice_lister_test.go index fec516177f2e..0de38fe8d37f 100644 --- a/cluster-autoscaler/simulator/dynamicresources/snapshot/snapshot_slice_lister_test.go +++ b/cluster-autoscaler/simulator/dynamicresources/snapshot/snapshot_slice_lister_test.go @@ -26,7 +26,7 @@ import ( resourceapi "k8s.io/api/resource/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/autoscaler/cluster-autoscaler/utils/test" - schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" + fwk "k8s.io/kube-scheduler/framework" ) func TestSnapshotSliceListerList(t *testing.T) { @@ -78,7 +78,7 @@ func TestSnapshotSliceListerList(t *testing.T) { } { t.Run(tc.testName, func(t *testing.T) { snapshot := NewSnapshot(nil, tc.localSlices, tc.globalSlices, nil) - var resourceSliceLister schedulerframework.ResourceSliceLister = snapshot.ResourceSlices() + var resourceSliceLister fwk.ResourceSliceLister = snapshot.ResourceSlices() slices, err := resourceSliceLister.ListWithDeviceTaintRules() if err != nil { t.Fatalf("snapshotSliceLister.List(): got unexpected error: %v", err) diff --git a/cluster-autoscaler/simulator/framework/delegating_shared_lister.go b/cluster-autoscaler/simulator/framework/delegating_shared_lister.go index 84b452a0a344..19c06fb76d46 100644 --- a/cluster-autoscaler/simulator/framework/delegating_shared_lister.go +++ b/cluster-autoscaler/simulator/framework/delegating_shared_lister.go @@ -19,26 +19,30 @@ package framework import ( "fmt" + v1 "k8s.io/api/core/v1" resourceapi "k8s.io/api/resource/v1" + storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/dynamic-resource-allocation/structured" "k8s.io/klog/v2" fwk "k8s.io/kube-scheduler/framework" - schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" ) // SharedLister groups all interfaces that Cluster Autoscaler needs to implement for integrating with kube-scheduler. type SharedLister interface { - schedulerframework.SharedLister - schedulerframework.SharedDRAManager + fwk.SharedLister + fwk.SharedDRAManager + fwk.CSIManager } -// DelegatingSchedulerSharedLister implements schedulerframework interfaces by passing the logic to a delegate. Delegate can be updated. +// DelegatingSchedulerSharedLister implements fwk interfaces by passing the logic to a delegate. Delegate can be updated. type DelegatingSchedulerSharedLister struct { delegate SharedLister } +var _ SharedLister = &DelegatingSchedulerSharedLister{} + // NewDelegatingSchedulerSharedLister creates new NewDelegatingSchedulerSharedLister func NewDelegatingSchedulerSharedLister() *DelegatingSchedulerSharedLister { return &DelegatingSchedulerSharedLister{ @@ -47,36 +51,46 @@ func NewDelegatingSchedulerSharedLister() *DelegatingSchedulerSharedLister { } // NodeInfos returns a NodeInfoLister. -func (lister *DelegatingSchedulerSharedLister) NodeInfos() schedulerframework.NodeInfoLister { +func (lister *DelegatingSchedulerSharedLister) NodeInfos() fwk.NodeInfoLister { return lister.delegate.NodeInfos() } // StorageInfos returns a StorageInfoLister -func (lister *DelegatingSchedulerSharedLister) StorageInfos() schedulerframework.StorageInfoLister { +func (lister *DelegatingSchedulerSharedLister) StorageInfos() fwk.StorageInfoLister { return lister.delegate.StorageInfos() } // ResourceClaims returns a ResourceClaimTracker. -func (lister *DelegatingSchedulerSharedLister) ResourceClaims() schedulerframework.ResourceClaimTracker { +func (lister *DelegatingSchedulerSharedLister) ResourceClaims() fwk.ResourceClaimTracker { return lister.delegate.ResourceClaims() } // ResourceSlices returns a ResourceSliceLister. -func (lister *DelegatingSchedulerSharedLister) ResourceSlices() schedulerframework.ResourceSliceLister { +func (lister *DelegatingSchedulerSharedLister) ResourceSlices() fwk.ResourceSliceLister { return lister.delegate.ResourceSlices() } // DeviceClasses returns a DeviceClassLister. -func (lister *DelegatingSchedulerSharedLister) DeviceClasses() schedulerframework.DeviceClassLister { +func (lister *DelegatingSchedulerSharedLister) DeviceClasses() fwk.DeviceClassLister { return lister.delegate.DeviceClasses() } +// CSINodes returns a CSINodeLister. +func (lister *DelegatingSchedulerSharedLister) CSINodes() fwk.CSINodeLister { + return lister.delegate.CSINodes() +} + // UpdateDelegate updates the delegate func (lister *DelegatingSchedulerSharedLister) UpdateDelegate(delegate SharedLister) { lister.delegate = delegate } -// ResetDelegate resets delegate to +// DeviceClassResolver returns a device class resolver. +func (lister *DelegatingSchedulerSharedLister) DeviceClassResolver() fwk.DeviceClassResolver { + return lister.delegate.DeviceClassResolver() +} + +// ResetDelegate resets delegate to the unsetSharedListerSingleton. func (lister *DelegatingSchedulerSharedLister) ResetDelegate() { lister.delegate = unsetSharedListerSingleton } @@ -87,6 +101,8 @@ type unsetStorageInfoLister unsetSharedLister type unsetResourceClaimTracker unsetSharedLister type unsetResourceSliceLister unsetSharedLister type unsetDeviceClassLister unsetSharedLister +type unsetCSINodeLister unsetSharedLister +type unsetDeviceClassResolver unsetSharedLister // List always returns an error func (lister *unsetNodeInfoLister) List() ([]fwk.NodeInfo, error) { @@ -112,76 +128,96 @@ func (lister *unsetStorageInfoLister) IsPVCUsedByPods(key string) bool { return false } -func (u unsetResourceClaimTracker) List() ([]*resourceapi.ResourceClaim, error) { +func (u *unsetResourceClaimTracker) List() ([]*resourceapi.ResourceClaim, error) { return nil, fmt.Errorf("lister not set in delegate") } -func (u unsetResourceClaimTracker) Get(namespace, claimName string) (*resourceapi.ResourceClaim, error) { +func (u *unsetResourceClaimTracker) Get(namespace, claimName string) (*resourceapi.ResourceClaim, error) { return nil, fmt.Errorf("lister not set in delegate") } -func (u unsetResourceClaimTracker) ListAllAllocatedDevices() (sets.Set[structured.DeviceID], error) { +func (u *unsetResourceClaimTracker) ListAllAllocatedDevices() (sets.Set[structured.DeviceID], error) { return nil, fmt.Errorf("lister not set in delegate") } -func (u unsetResourceClaimTracker) GatherAllocatedState() (*structured.AllocatedState, error) { +func (u *unsetResourceClaimTracker) GatherAllocatedState() (*structured.AllocatedState, error) { return nil, fmt.Errorf("lister not set in delegate") } -func (u unsetResourceClaimTracker) SignalClaimPendingAllocation(claimUID types.UID, allocatedClaim *resourceapi.ResourceClaim) error { +func (u *unsetResourceClaimTracker) SignalClaimPendingAllocation(claimUID types.UID, allocatedClaim *resourceapi.ResourceClaim) error { return fmt.Errorf("lister not set in delegate") } -func (u unsetResourceClaimTracker) ClaimHasPendingAllocation(claimUID types.UID) bool { +func (u *unsetResourceClaimTracker) ClaimHasPendingAllocation(claimUID types.UID) bool { klog.Errorf("lister not set in delegate") return false } -func (u unsetResourceClaimTracker) RemoveClaimPendingAllocation(claimUID types.UID) (deleted bool) { +func (u *unsetResourceClaimTracker) RemoveClaimPendingAllocation(claimUID types.UID) (deleted bool) { klog.Errorf("lister not set in delegate") return false } -func (u unsetResourceClaimTracker) AssumeClaimAfterAPICall(claim *resourceapi.ResourceClaim) error { +func (u *unsetResourceClaimTracker) AssumeClaimAfterAPICall(claim *resourceapi.ResourceClaim) error { return fmt.Errorf("lister not set in delegate") } -func (u unsetResourceClaimTracker) AssumedClaimRestore(namespace, claimName string) { +func (u *unsetResourceClaimTracker) AssumedClaimRestore(namespace, claimName string) { klog.Errorf("lister not set in delegate") } -func (u unsetResourceSliceLister) ListWithDeviceTaintRules() ([]*resourceapi.ResourceSlice, error) { +func (u *unsetCSINodeLister) List() ([]*storagev1.CSINode, error) { return nil, fmt.Errorf("lister not set in delegate") } -func (u unsetDeviceClassLister) List() ([]*resourceapi.DeviceClass, error) { +func (u *unsetCSINodeLister) Get(name string) (*storagev1.CSINode, error) { return nil, fmt.Errorf("lister not set in delegate") } -func (u unsetDeviceClassLister) Get(className string) (*resourceapi.DeviceClass, error) { +func (u *unsetResourceSliceLister) ListWithDeviceTaintRules() ([]*resourceapi.ResourceSlice, error) { return nil, fmt.Errorf("lister not set in delegate") } +func (u *unsetDeviceClassLister) List() ([]*resourceapi.DeviceClass, error) { + return nil, fmt.Errorf("lister not set in delegate") +} + +func (u *unsetDeviceClassLister) Get(className string) (*resourceapi.DeviceClass, error) { + return nil, fmt.Errorf("lister not set in delegate") +} + +func (u *unsetDeviceClassResolver) GetDeviceClass(resourceName v1.ResourceName) *resourceapi.DeviceClass { + return nil +} + // NodeInfos returns a fake NodeInfoLister which always returns an error -func (lister *unsetSharedLister) NodeInfos() schedulerframework.NodeInfoLister { +func (lister *unsetSharedLister) NodeInfos() fwk.NodeInfoLister { return (*unsetNodeInfoLister)(lister) } // StorageInfos returns a fake StorageInfoLister which always returns an error -func (lister *unsetSharedLister) StorageInfos() schedulerframework.StorageInfoLister { +func (lister *unsetSharedLister) StorageInfos() fwk.StorageInfoLister { return (*unsetStorageInfoLister)(lister) } -func (lister *unsetSharedLister) ResourceClaims() schedulerframework.ResourceClaimTracker { +func (lister *unsetSharedLister) ResourceClaims() fwk.ResourceClaimTracker { return (*unsetResourceClaimTracker)(lister) } -func (lister *unsetSharedLister) ResourceSlices() schedulerframework.ResourceSliceLister { +func (lister *unsetSharedLister) ResourceSlices() fwk.ResourceSliceLister { return (*unsetResourceSliceLister)(lister) } -func (lister *unsetSharedLister) DeviceClasses() schedulerframework.DeviceClassLister { +func (lister *unsetSharedLister) DeviceClasses() fwk.DeviceClassLister { return (*unsetDeviceClassLister)(lister) } +func (lister *unsetSharedLister) CSINodes() fwk.CSINodeLister { + return (*unsetCSINodeLister)(lister) +} + +func (lister *unsetSharedLister) DeviceClassResolver() fwk.DeviceClassResolver { + return (*unsetDeviceClassResolver)(lister) +} + var unsetSharedListerSingleton *unsetSharedLister diff --git a/cluster-autoscaler/simulator/framework/handle.go b/cluster-autoscaler/simulator/framework/handle.go index a981e2bf32b0..d7f2fbf05e6c 100644 --- a/cluster-autoscaler/simulator/framework/handle.go +++ b/cluster-autoscaler/simulator/framework/handle.go @@ -26,6 +26,7 @@ import ( schedulerconfiglatest "k8s.io/kubernetes/pkg/scheduler/apis/config/latest" schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" schedulerplugins "k8s.io/kubernetes/pkg/scheduler/framework/plugins" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodevolumelimits" schedulerframeworkruntime "k8s.io/kubernetes/pkg/scheduler/framework/runtime" schedulermetrics "k8s.io/kubernetes/pkg/scheduler/metrics" ) @@ -41,7 +42,10 @@ type Handle struct { } // NewHandle builds a framework Handle based on the provided informers and scheduler config. -func NewHandle(informerFactory informers.SharedInformerFactory, schedConfig *schedulerconfig.KubeSchedulerConfiguration, draEnabled bool) (*Handle, error) { +func NewHandle(informerFactory informers.SharedInformerFactory, + schedConfig *schedulerconfig.KubeSchedulerConfiguration, + draEnabled bool, + csiEnabled bool) (*Handle, error) { if schedConfig == nil { var err error schedConfig, err = schedulerconfiglatest.Default() @@ -61,7 +65,12 @@ func NewHandle(informerFactory informers.SharedInformerFactory, schedConfig *sch if draEnabled { opts = append(opts, schedulerframeworkruntime.WithSharedDRAManager(sharedLister)) } - + if csiEnabled { + opts = append(opts, schedulerframeworkruntime.WithSharedCSIManager(sharedLister)) + } else { + sharedCSIManager := nodevolumelimits.NewCSIManager(informerFactory.Storage().V1().CSINodes().Lister()) + opts = append(opts, schedulerframeworkruntime.WithSharedCSIManager(sharedCSIManager)) + } initMetricsOnce.Do(func() { schedulermetrics.InitMetrics() }) diff --git a/cluster-autoscaler/simulator/framework/infos.go b/cluster-autoscaler/simulator/framework/infos.go index 68aba50c7a17..71ad8ae5e5c9 100644 --- a/cluster-autoscaler/simulator/framework/infos.go +++ b/cluster-autoscaler/simulator/framework/infos.go @@ -19,6 +19,7 @@ package framework import ( apiv1 "k8s.io/api/core/v1" resourceapi "k8s.io/api/resource/v1" + storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/klog/v2" fwk "k8s.io/kube-scheduler/framework" @@ -54,6 +55,9 @@ type NodeInfo struct { // LocalResourceSlices contains all node-local ResourceSlices exposed by this Node. LocalResourceSlices []*resourceapi.ResourceSlice + + // CSINodes contains all CSINodes exposed by this Node. + CSINode *storagev1.CSINode } // SetNode sets the Node in this NodeInfo @@ -137,6 +141,21 @@ func (n *NodeInfo) ResourceClaims() []*resourceapi.ResourceClaim { return result } +func (n *NodeInfo) AddNodeResourceSlices(slices []*resourceapi.ResourceSlice) *NodeInfo { + n.LocalResourceSlices = slices + return n +} + +func (n *NodeInfo) AddCSINode(csiNode *storagev1.CSINode) *NodeInfo { + n.CSINode = csiNode + return n +} + +func (n *NodeInfo) AddPodExtraInfo(podExtraInfo map[types.UID]PodExtraInfo) *NodeInfo { + n.podsExtraInfo = podExtraInfo + return n +} + // NewNodeInfo returns a new internal NodeInfo from the provided data. func NewNodeInfo(node *apiv1.Node, slices []*resourceapi.ResourceSlice, pods ...*PodInfo) *NodeInfo { result := &NodeInfo{ diff --git a/cluster-autoscaler/simulator/framework/test_utils.go b/cluster-autoscaler/simulator/framework/test_utils.go index ba8489121c2b..ca223ae97622 100644 --- a/cluster-autoscaler/simulator/framework/test_utils.go +++ b/cluster-autoscaler/simulator/framework/test_utils.go @@ -44,7 +44,7 @@ func NewTestFrameworkHandle() (*Handle, error) { if err != nil { return nil, err } - fwHandle, err := NewHandle(informers.NewSharedInformerFactory(clientsetfake.NewSimpleClientset(), 0), defaultConfig, true) + fwHandle, err := NewHandle(informers.NewSharedInformerFactory(clientsetfake.NewSimpleClientset(), 0), defaultConfig, true, true) if err != nil { return nil, err } diff --git a/cluster-autoscaler/simulator/node_info_utils.go b/cluster-autoscaler/simulator/node_info_utils.go index 40251c333b9a..bf69b7f71119 100644 --- a/cluster-autoscaler/simulator/node_info_utils.go +++ b/cluster-autoscaler/simulator/node_info_utils.go @@ -22,6 +22,7 @@ import ( appsv1 "k8s.io/api/apps/v1" apiv1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/uuid" @@ -73,9 +74,14 @@ func SanitizedTemplateNodeInfoFromNodeInfo(example *framework.NodeInfo, nodeGrou if err != nil { return nil, errors.ToAutoscalerError(errors.InternalError, err) } + templateNodeInfo := framework.NewNodeInfo(sanitizedExample.Node(), sanitizedExample.LocalResourceSlices, expectedPods...) + if example.CSINode != nil { + templateNodeInfo.AddCSINode(CreateSanitizedCSINode(example.CSINode, templateNodeInfo)) + } + // No need to sanitize the expected pods again - they either come from sanitizedExample and were sanitized above, // or were added by podsExpectedOnFreshNode and sanitized there. - return framework.NewNodeInfo(sanitizedExample.Node(), sanitizedExample.LocalResourceSlices, expectedPods...), nil + return templateNodeInfo, nil } // SanitizedNodeInfo duplicates the provided template NodeInfo, returning a fresh NodeInfo that can be injected into the cluster snapshot. @@ -127,6 +133,24 @@ func createSanitizedNode(node *apiv1.Node, newName string, taintConfig *taints.T return newNode } +func CreateSanitizedCSINode(csiNode *storagev1.CSINode, templateNodeInfo *framework.NodeInfo) *storagev1.CSINode { + newCSINode := csiNode.DeepCopy() + newCSINode.Name = templateNodeInfo.Node().Name + newCSINode.UID = uuid.NewUUID() + + newCSINode.OwnerReferences = []metav1.OwnerReference{ + { + APIVersion: "v1", + Kind: "Node", + Name: templateNodeInfo.Node().Name, + UID: templateNodeInfo.Node().UID, + }, + } + // TODO: we could add santized nodeID here, but it should not + // be needed for scheduling decisions + return newCSINode +} + func createSanitizedPod(pod *apiv1.Pod, nodeName, nameSuffix string) *apiv1.Pod { sanitizedPod := drautils.SanitizedResourceClaimRefs(pod, nameSuffix) sanitizedPod.UID = uuid.NewUUID() diff --git a/cluster-autoscaler/utils/kubernetes/listers.go b/cluster-autoscaler/utils/kubernetes/listers.go index 72eee8cf2fbb..44d8c31f3ac4 100644 --- a/cluster-autoscaler/utils/kubernetes/listers.go +++ b/cluster-autoscaler/utils/kubernetes/listers.go @@ -21,6 +21,7 @@ import ( apiv1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" + storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/informers" @@ -29,6 +30,7 @@ import ( v1batchlister "k8s.io/client-go/listers/batch/v1" v1lister "k8s.io/client-go/listers/core/v1" v1policylister "k8s.io/client-go/listers/policy/v1" + v1storagelister "k8s.io/client-go/listers/storage/v1" "k8s.io/client-go/tools/cache" podv1 "k8s.io/kubernetes/pkg/api/v1/pod" ) @@ -36,6 +38,7 @@ import ( // ListerRegistry is a registry providing various listers to list pods or nodes matching conditions type ListerRegistry interface { AllNodeLister() NodeLister + AllCSINodeLister() CSINodeLister ReadyNodeLister() NodeLister AllPodLister() PodLister PodDisruptionBudgetLister() PodDisruptionBudgetLister @@ -48,6 +51,7 @@ type ListerRegistry interface { type listerRegistryImpl struct { allNodeLister NodeLister + allCSINodeLister CSINodeLister readyNodeLister NodeLister allPodLister PodLister podDisruptionBudgetLister PodDisruptionBudgetLister @@ -66,12 +70,13 @@ type PodsBySchedulability struct { } // NewListerRegistry returns a registry providing various listers to list pods or nodes matching conditions -func NewListerRegistry(allNode NodeLister, readyNode NodeLister, allPodLister PodLister, podDisruptionBudgetLister PodDisruptionBudgetLister, +func NewListerRegistry(allNode NodeLister, allCSINode CSINodeLister, readyNode NodeLister, allPodLister PodLister, podDisruptionBudgetLister PodDisruptionBudgetLister, daemonSetLister v1appslister.DaemonSetLister, replicationControllerLister v1lister.ReplicationControllerLister, jobLister v1batchlister.JobLister, replicaSetLister v1appslister.ReplicaSetLister, statefulSetLister v1appslister.StatefulSetLister) ListerRegistry { return listerRegistryImpl{ allNodeLister: allNode, + allCSINodeLister: allCSINode, readyNodeLister: readyNode, allPodLister: allPodLister, podDisruptionBudgetLister: podDisruptionBudgetLister, @@ -86,6 +91,7 @@ func NewListerRegistry(allNode NodeLister, readyNode NodeLister, allPodLister Po // NewListerRegistryWithDefaultListers returns a registry filled with listers of the default implementations func NewListerRegistryWithDefaultListers(informerFactory informers.SharedInformerFactory) ListerRegistry { allPodLister := NewAllPodLister(informerFactory.Core().V1().Pods().Lister()) + allCSINodeLister := NewAllCSINodeLister(informerFactory.Storage().V1().CSINodes().Lister()) readyNodeLister := NewReadyNodeLister(informerFactory.Core().V1().Nodes().Lister()) allNodeLister := NewAllNodeLister(informerFactory.Core().V1().Nodes().Lister()) @@ -95,7 +101,7 @@ func NewListerRegistryWithDefaultListers(informerFactory informers.SharedInforme jobLister := informerFactory.Batch().V1().Jobs().Lister() replicaSetLister := informerFactory.Apps().V1().ReplicaSets().Lister() statefulSetLister := informerFactory.Apps().V1().StatefulSets().Lister() - return NewListerRegistry(allNodeLister, readyNodeLister, allPodLister, + return NewListerRegistry(allNodeLister, allCSINodeLister, readyNodeLister, allPodLister, podDisruptionBudgetLister, daemonSetLister, replicationControllerLister, jobLister, replicaSetLister, statefulSetLister) } @@ -110,6 +116,11 @@ func (r listerRegistryImpl) AllNodeLister() NodeLister { return r.allNodeLister } +// AllCSINodeLister returns the AllCSINodeLister registered to this registry +func (r listerRegistryImpl) AllCSINodeLister() CSINodeLister { + return r.allCSINodeLister +} + // ReadyNodeLister returns the ReadyNodeLister registered to this registry func (r listerRegistryImpl) ReadyNodeLister() NodeLister { return r.readyNodeLister @@ -318,6 +329,60 @@ func filterNodes(nodes []*apiv1.Node, predicate func(*apiv1.Node) bool) []*apiv1 return filtered } +type CSINodeLister interface { + List() ([]*storagev1.CSINode, error) + Get(name string) (*storagev1.CSINode, error) +} + +type csiNodeListerImpl struct { + csiNodeLister v1storagelister.CSINodeLister + filter func(*storagev1.CSINode) bool +} + +// NewAllCSINodeLister builds a csi node lister that returns all csi nodes. +func NewAllCSINodeLister(nl v1storagelister.CSINodeLister) CSINodeLister { + return NewCSINodeLister(nl, nil) +} + +// NewCSINodeLister builds a csi node lister. +func NewCSINodeLister(nl v1storagelister.CSINodeLister, filter func(*storagev1.CSINode) bool) CSINodeLister { + return &csiNodeListerImpl{ + csiNodeLister: nl, + filter: filter, + } +} + +func (l *csiNodeListerImpl) Get(name string) (*storagev1.CSINode, error) { + return l.csiNodeLister.Get(name) +} + +// List returns list of csi nodes. +func (l *csiNodeListerImpl) List() ([]*storagev1.CSINode, error) { + var csiNodes []*storagev1.CSINode + var err error + + csiNodes, err = l.csiNodeLister.List(labels.Everything()) + if err != nil { + return []*storagev1.CSINode{}, err + } + + if l.filter != nil { + csiNodes = filterCSINodes(csiNodes, l.filter) + } + + return csiNodes, nil +} + +func filterCSINodes(csinodes []*storagev1.CSINode, predicate func(*storagev1.CSINode) bool) []*storagev1.CSINode { + var filtered []*storagev1.CSINode + for i := range csinodes { + if predicate(csinodes[i]) { + filtered = append(filtered, csinodes[i]) + } + } + return filtered +} + // PodDisruptionBudgetLister lists pod disruption budgets. type PodDisruptionBudgetLister interface { List() ([]*policyv1.PodDisruptionBudget, error)