diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_autodiscovery_test.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_autodiscovery_test.go index 6d0a2db3d273..cc0ddfb68055 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_autodiscovery_test.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_autodiscovery_test.go @@ -204,17 +204,17 @@ func Test_allowedByAutoDiscoverySpec(t *testing.T) { shouldMatch bool }{{ name: "no clustername, namespace, or label selector specified should match any MachineSet", - testSpec: createTestSpec(RandomString(6), RandomString(6), RandomString(6), 1, false, nil, nil), + testSpec: createTestSpec(RandomString(6), RandomString(6), RandomString(6), 1, false, nil, nil, nil), autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{labelSelector: labels.NewSelector()}, shouldMatch: true, }, { name: "no clustername, namespace, or label selector specified should match any MachineDeployment", - testSpec: createTestSpec(RandomString(6), RandomString(6), RandomString(6), 1, true, nil, nil), + testSpec: createTestSpec(RandomString(6), RandomString(6), RandomString(6), 1, true, nil, nil, nil), autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{labelSelector: labels.NewSelector()}, shouldMatch: true, }, { name: "clustername specified does not match MachineSet, namespace matches, no labels specified", - testSpec: createTestSpec("default", RandomString(6), RandomString(6), 1, false, nil, nil), + testSpec: createTestSpec("default", RandomString(6), RandomString(6), 1, false, nil, nil, nil), autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{ clusterName: "foo", namespace: "default", @@ -223,7 +223,7 @@ func Test_allowedByAutoDiscoverySpec(t *testing.T) { shouldMatch: false, }, { name: "clustername specified does not match MachineDeployment, namespace matches, no labels specified", - testSpec: createTestSpec("default", RandomString(6), RandomString(6), 1, true, nil, nil), + testSpec: createTestSpec("default", RandomString(6), RandomString(6), 1, true, nil, nil, nil), autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{ clusterName: "foo", namespace: "default", @@ -232,7 +232,7 @@ func Test_allowedByAutoDiscoverySpec(t *testing.T) { shouldMatch: false, }, { name: "namespace specified does not match MachineSet, clusterName matches, no labels specified", - testSpec: createTestSpec(RandomString(6), "foo", RandomString(6), 1, false, nil, nil), + testSpec: createTestSpec(RandomString(6), "foo", RandomString(6), 1, false, nil, nil, nil), autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{ clusterName: "foo", namespace: "default", @@ -241,7 +241,7 @@ func Test_allowedByAutoDiscoverySpec(t *testing.T) { shouldMatch: false, }, { name: "clustername specified does not match MachineDeployment, namespace matches, no labels specified", - testSpec: createTestSpec(RandomString(6), "foo", RandomString(6), 1, true, nil, nil), + testSpec: createTestSpec(RandomString(6), "foo", RandomString(6), 1, true, nil, nil, nil), autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{ clusterName: "foo", namespace: "default", @@ -250,7 +250,7 @@ func Test_allowedByAutoDiscoverySpec(t *testing.T) { shouldMatch: false, }, { name: "namespace and clusterName matches MachineSet, no labels specified", - testSpec: createTestSpec("default", "foo", RandomString(6), 1, false, nil, nil), + testSpec: createTestSpec("default", "foo", RandomString(6), 1, false, nil, nil, nil), autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{ clusterName: "foo", namespace: "default", @@ -259,7 +259,7 @@ func Test_allowedByAutoDiscoverySpec(t *testing.T) { shouldMatch: true, }, { name: "namespace and clusterName matches MachineDeployment, no labels specified", - testSpec: createTestSpec("default", "foo", RandomString(6), 1, true, nil, nil), + testSpec: createTestSpec("default", "foo", RandomString(6), 1, true, nil, nil, nil), autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{ clusterName: "foo", namespace: "default", @@ -268,7 +268,7 @@ func Test_allowedByAutoDiscoverySpec(t *testing.T) { shouldMatch: true, }, { name: "namespace and clusterName matches MachineSet, does not match label selector", - testSpec: createTestSpec("default", "foo", RandomString(6), 1, false, nil, nil), + testSpec: createTestSpec("default", "foo", RandomString(6), 1, false, nil, nil, nil), autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{ clusterName: "foo", namespace: "default", @@ -277,7 +277,7 @@ func Test_allowedByAutoDiscoverySpec(t *testing.T) { shouldMatch: false, }, { name: "namespace and clusterName matches MachineDeployment, does not match label selector", - testSpec: createTestSpec("default", "foo", RandomString(6), 1, true, nil, nil), + testSpec: createTestSpec("default", "foo", RandomString(6), 1, true, nil, nil, nil), autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{ clusterName: "foo", namespace: "default", @@ -286,7 +286,7 @@ func Test_allowedByAutoDiscoverySpec(t *testing.T) { shouldMatch: false, }, { name: "namespace, clusterName, and label selector matches MachineSet", - testSpec: createTestSpec("default", "foo", RandomString(6), 1, false, nil, nil), + testSpec: createTestSpec("default", "foo", RandomString(6), 1, false, nil, nil, nil), additionalLabels: map[string]string{"color": "green"}, autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{ clusterName: "foo", @@ -296,7 +296,7 @@ func Test_allowedByAutoDiscoverySpec(t *testing.T) { shouldMatch: true, }, { name: "namespace, clusterName, and label selector matches MachineDeployment", - testSpec: createTestSpec("default", "foo", RandomString(6), 1, true, nil, nil), + testSpec: createTestSpec("default", "foo", RandomString(6), 1, true, nil, nil, nil), additionalLabels: map[string]string{"color": "green"}, autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{ clusterName: "foo", diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller_test.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller_test.go index 5515c0e006b3..deef90d0a689 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller_test.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller_test.go @@ -73,6 +73,7 @@ type testSpec struct { namespace string nodeCount int rootIsMachineDeployment bool + nodeInfo map[string]string } const customCAPIGroup = "custom.x-k8s.io" @@ -307,33 +308,33 @@ func mustCreateTestController(t testing.TB, testConfigs ...*testConfig) (*machin } } -func createMachineSetTestConfig(namespace, clusterName, namePrefix string, nodeCount int, annotations map[string]string, capacity map[string]string) *testConfig { - return createTestConfigs(createTestSpecs(namespace, clusterName, namePrefix, 1, nodeCount, false, annotations, capacity)...)[0] +func createMachineSetTestConfig(namespace, clusterName, namePrefix string, nodeCount int, annotations map[string]string, capacity map[string]string, nodeInfo map[string]string) *testConfig { + return createTestConfigs(createTestSpecs(namespace, clusterName, namePrefix, 1, nodeCount, false, annotations, capacity, nodeInfo)...)[0] } -func createMachineSetTestConfigs(namespace, clusterName, namePrefix string, configCount, nodeCount int, annotations map[string]string, capacity map[string]string) []*testConfig { - return createTestConfigs(createTestSpecs(namespace, clusterName, namePrefix, configCount, nodeCount, false, annotations, capacity)...) +func createMachineSetTestConfigs(namespace, clusterName, namePrefix string, configCount, nodeCount int, annotations map[string]string, capacity map[string]string, nodeInfo map[string]string) []*testConfig { + return createTestConfigs(createTestSpecs(namespace, clusterName, namePrefix, configCount, nodeCount, false, annotations, capacity, nodeInfo)...) } -func createMachineDeploymentTestConfig(namespace, clusterName, namePrefix string, nodeCount int, annotations map[string]string, capacity map[string]string) *testConfig { - return createTestConfigs(createTestSpecs(namespace, clusterName, namePrefix, 1, nodeCount, true, annotations, capacity)...)[0] +func createMachineDeploymentTestConfig(namespace, clusterName, namePrefix string, nodeCount int, annotations map[string]string, capacity map[string]string, nodeInfo map[string]string) *testConfig { + return createTestConfigs(createTestSpecs(namespace, clusterName, namePrefix, 1, nodeCount, true, annotations, capacity, nodeInfo)...)[0] } -func createMachineDeploymentTestConfigs(namespace, clusterName, namePrefix string, configCount, nodeCount int, annotations map[string]string, capacity map[string]string) []*testConfig { - return createTestConfigs(createTestSpecs(namespace, clusterName, namePrefix, configCount, nodeCount, true, annotations, capacity)...) +func createMachineDeploymentTestConfigs(namespace, clusterName, namePrefix string, configCount, nodeCount int, annotations map[string]string, capacity map[string]string, nodeInfo map[string]string) []*testConfig { + return createTestConfigs(createTestSpecs(namespace, clusterName, namePrefix, configCount, nodeCount, true, annotations, capacity, nodeInfo)...) } -func createTestSpecs(namespace, clusterName, namePrefix string, scalableResourceCount, nodeCount int, isMachineDeployment bool, annotations map[string]string, capacity map[string]string) []testSpec { +func createTestSpecs(namespace, clusterName, namePrefix string, scalableResourceCount, nodeCount int, isMachineDeployment bool, annotations map[string]string, capacity map[string]string, nodeInfo map[string]string) []testSpec { var specs []testSpec for i := 0; i < scalableResourceCount; i++ { - specs = append(specs, createTestSpec(namespace, clusterName, fmt.Sprintf("%s-%d", namePrefix, i), nodeCount, isMachineDeployment, annotations, capacity)) + specs = append(specs, createTestSpec(namespace, clusterName, fmt.Sprintf("%s-%d", namePrefix, i), nodeCount, isMachineDeployment, annotations, capacity, nodeInfo)) } return specs } -func createTestSpec(namespace, clusterName, name string, nodeCount int, isMachineDeployment bool, annotations map[string]string, capacity map[string]string) testSpec { +func createTestSpec(namespace, clusterName, name string, nodeCount int, isMachineDeployment bool, annotations map[string]string, capacity map[string]string, nodeInfo map[string]string) testSpec { return testSpec{ annotations: annotations, capacity: capacity, @@ -343,6 +344,7 @@ func createTestSpec(namespace, clusterName, name string, nodeCount int, isMachin namespace: namespace, nodeCount: nodeCount, rootIsMachineDeployment: isMachineDeployment, + nodeInfo: nodeInfo, } } @@ -451,9 +453,7 @@ func createTestConfigs(specs ...testSpec) []*testConfig { Kind: config.machineSet.GetKind(), UID: config.machineSet.GetUID(), } - - if spec.capacity != nil { - klog.V(4).Infof("adding capacity to machine template") + if spec.capacity != nil || spec.nodeInfo != nil { config.machineTemplate = &unstructured.Unstructured{ Object: map[string]interface{}{ "apiVersion": "infrastructure.cluster.x-k8s.io/v1beta1", @@ -465,6 +465,10 @@ func createTestConfigs(specs ...testSpec) []*testConfig { }, }, } + } + if spec.capacity != nil { + klog.V(4).Infof("adding capacity to machine template") + if err := unstructured.SetNestedStringMap(config.machineTemplate.Object, spec.capacity, "status", "capacity"); err != nil { panic(err) } @@ -472,6 +476,16 @@ func createTestConfigs(specs ...testSpec) []*testConfig { klog.V(4).Infof("not adding capacity") } + if spec.nodeInfo != nil { + klog.V(4).Infof("adding nodeInfo to machine template") + + if err := unstructured.SetNestedStringMap(config.machineTemplate.Object, spec.nodeInfo, "status", "nodeInfo"); err != nil { + panic(err) + } + } else { + klog.V(4).Infof("not adding nodeInfo") + } + for j := 0; j < spec.nodeCount; j++ { config.nodes[j], config.machines[j] = makeLinkedNodeAndMachine(j, spec.namespace, spec.clusterName, machineOwner, machineSetLabels) } @@ -694,7 +708,7 @@ func TestControllerFindMachine(t *testing.T) { testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{ nodeGroupMinSizeAnnotationKey: "1", nodeGroupMaxSizeAnnotationKey: "10", - }, nil) + }, nil, nil) if tc.name == "" { tc.name = testConfig.machines[0].GetName() } @@ -722,7 +736,7 @@ func TestControllerFindMachineOwner(t *testing.T) { testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{ nodeGroupMinSizeAnnotationKey: "1", nodeGroupMaxSizeAnnotationKey: "10", - }, nil) + }, nil, nil) controller, stop := mustCreateTestController(t, testConfig) defer stop() @@ -771,7 +785,7 @@ func TestControllerFindMachineByProviderID(t *testing.T) { testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{ nodeGroupMinSizeAnnotationKey: "1", nodeGroupMaxSizeAnnotationKey: "10", - }, nil) + }, nil, nil) controller, stop := mustCreateTestController(t, testConfig) defer stop() @@ -833,7 +847,7 @@ func TestControllerFindNodeByNodeName(t *testing.T) { testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{ nodeGroupMinSizeAnnotationKey: "1", nodeGroupMaxSizeAnnotationKey: "10", - }, nil) + }, nil, nil) controller, stop := mustCreateTestController(t, testConfig) defer stop() @@ -933,7 +947,7 @@ func TestControllerListMachinesForScalableResource(t *testing.T) { testConfig1 := createMachineSetTestConfig(namespace, clusterName, RandomString(6), 5, map[string]string{ nodeGroupMinSizeAnnotationKey: "1", nodeGroupMaxSizeAnnotationKey: "10", - }, nil) + }, nil, nil) // Construct a second set of objects and add the machines, // nodes and the additional machineset to the existing set of @@ -942,7 +956,7 @@ func TestControllerListMachinesForScalableResource(t *testing.T) { testConfig2 := createMachineSetTestConfig(namespace, clusterName, RandomString(6), 5, map[string]string{ nodeGroupMinSizeAnnotationKey: "1", nodeGroupMaxSizeAnnotationKey: "10", - }, nil) + }, nil, nil) test(t, testConfig1, testConfig2) }) @@ -953,7 +967,7 @@ func TestControllerListMachinesForScalableResource(t *testing.T) { testConfig1 := createMachineDeploymentTestConfig(namespace, clusterName, RandomString(6), 5, map[string]string{ nodeGroupMinSizeAnnotationKey: "1", nodeGroupMaxSizeAnnotationKey: "10", - }, nil) + }, nil, nil) // Construct a second set of objects and add the machines, // nodes, machineset, and the additional machineset to the existing set of @@ -962,7 +976,7 @@ func TestControllerListMachinesForScalableResource(t *testing.T) { testConfig2 := createMachineDeploymentTestConfig(namespace, clusterName, RandomString(6), 5, map[string]string{ nodeGroupMinSizeAnnotationKey: "1", nodeGroupMaxSizeAnnotationKey: "10", - }, nil) + }, nil, nil) test(t, testConfig1, testConfig2) }) @@ -993,7 +1007,7 @@ func TestControllerLookupNodeGroupForNonExistentNode(t *testing.T) { testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{ nodeGroupMinSizeAnnotationKey: "1", nodeGroupMaxSizeAnnotationKey: "10", - }, nil) + }, nil, nil) test(t, testConfig) }) @@ -1001,7 +1015,7 @@ func TestControllerLookupNodeGroupForNonExistentNode(t *testing.T) { testConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{ nodeGroupMinSizeAnnotationKey: "1", nodeGroupMaxSizeAnnotationKey: "10", - }, nil) + }, nil, nil) test(t, testConfig) }) } @@ -1032,7 +1046,7 @@ func TestControllerNodeGroupForNodeWithMissingMachineOwner(t *testing.T) { testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{ nodeGroupMinSizeAnnotationKey: "1", nodeGroupMaxSizeAnnotationKey: "10", - }, nil) + }, nil, nil) test(t, testConfig) }) @@ -1040,7 +1054,7 @@ func TestControllerNodeGroupForNodeWithMissingMachineOwner(t *testing.T) { testConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{ nodeGroupMinSizeAnnotationKey: "1", nodeGroupMaxSizeAnnotationKey: "10", - }, nil) + }, nil, nil) test(t, testConfig) }) } @@ -1049,7 +1063,7 @@ func TestControllerNodeGroupForNodeWithMissingSetMachineOwner(t *testing.T) { testConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{ nodeGroupMinSizeAnnotationKey: "1", nodeGroupMaxSizeAnnotationKey: "10", - }, nil) + }, nil, nil) controller, stop := mustCreateTestController(t, testConfig) defer stop() @@ -1090,7 +1104,7 @@ func TestControllerNodeGroupForNodeWithPositiveScalingBounds(t *testing.T) { testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{ nodeGroupMinSizeAnnotationKey: "1", nodeGroupMaxSizeAnnotationKey: "1", - }, nil) + }, nil, nil) test(t, testConfig) }) @@ -1098,7 +1112,7 @@ func TestControllerNodeGroupForNodeWithPositiveScalingBounds(t *testing.T) { testConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{ nodeGroupMinSizeAnnotationKey: "1", nodeGroupMaxSizeAnnotationKey: "1", - }, nil) + }, nil, nil) test(t, testConfig) }) } @@ -1130,14 +1144,14 @@ func TestControllerNodeGroups(t *testing.T) { assertNodegroupLen(t, controller, 0) // Test #2: add 5 machineset-based nodegroups - machineSetConfigs := createMachineSetTestConfigs(namespace, clusterName, RandomString(6), 5, 1, annotations, nil) + machineSetConfigs := createMachineSetTestConfigs(namespace, clusterName, RandomString(6), 5, 1, annotations, nil, nil) if err := addTestConfigs(t, controller, machineSetConfigs...); err != nil { t.Fatalf("unexpected error: %v", err) } assertNodegroupLen(t, controller, 5) // Test #2: add 2 machinedeployment-based nodegroups - machineDeploymentConfigs := createMachineDeploymentTestConfigs(namespace, clusterName, RandomString(6), 2, 1, annotations, nil) + machineDeploymentConfigs := createMachineDeploymentTestConfigs(namespace, clusterName, RandomString(6), 2, 1, annotations, nil, nil) if err := addTestConfigs(t, controller, machineDeploymentConfigs...); err != nil { t.Fatalf("unexpected error: %v", err) } @@ -1161,14 +1175,14 @@ func TestControllerNodeGroups(t *testing.T) { } // Test #5: 5 machineset with minSize=maxSize results in a five nodegroup - machineSetConfigs = createMachineSetTestConfigs(namespace, clusterName, RandomString(6), 5, 1, annotations, nil) + machineSetConfigs = createMachineSetTestConfigs(namespace, clusterName, RandomString(6), 5, 1, annotations, nil, nil) if err := addTestConfigs(t, controller, machineSetConfigs...); err != nil { t.Fatalf("unexpected error: %v", err) } assertNodegroupLen(t, controller, 5) // Test #6: add 2 machinedeployment with minSize=maxSize - machineDeploymentConfigs = createMachineDeploymentTestConfigs(namespace, clusterName, RandomString(6), 2, 1, annotations, nil) + machineDeploymentConfigs = createMachineDeploymentTestConfigs(namespace, clusterName, RandomString(6), 2, 1, annotations, nil, nil) if err := addTestConfigs(t, controller, machineDeploymentConfigs...); err != nil { t.Fatalf("unexpected error: %v", err) } @@ -1180,7 +1194,7 @@ func TestControllerNodeGroups(t *testing.T) { } // Test #7: machineset with bad scaling bounds results in an error and no nodegroups - machineSetConfigs = createMachineSetTestConfigs(namespace, clusterName, RandomString(6), 5, 1, annotations, nil) + machineSetConfigs = createMachineSetTestConfigs(namespace, clusterName, RandomString(6), 5, 1, annotations, nil, nil) if err := addTestConfigs(t, controller, machineSetConfigs...); err != nil { t.Fatalf("unexpected error: %v", err) } @@ -1189,7 +1203,7 @@ func TestControllerNodeGroups(t *testing.T) { } // Test #8: machinedeployment with bad scaling bounds results in an error and no nodegroups - machineDeploymentConfigs = createMachineDeploymentTestConfigs(namespace, clusterName, RandomString(6), 2, 1, annotations, nil) + machineDeploymentConfigs = createMachineDeploymentTestConfigs(namespace, clusterName, RandomString(6), 2, 1, annotations, nil, nil) if err := addTestConfigs(t, controller, machineDeploymentConfigs...); err != nil { t.Fatalf("unexpected error: %v", err) } @@ -1253,13 +1267,13 @@ func TestControllerNodeGroupsNodeCount(t *testing.T) { t.Run("MachineSet", func(t *testing.T) { for _, tc := range testCases { - test(t, tc, createMachineSetTestConfigs(RandomString(6), RandomString(6), RandomString(6), tc.nodeGroups, tc.nodesPerGroup, annotations, nil)) + test(t, tc, createMachineSetTestConfigs(RandomString(6), RandomString(6), RandomString(6), tc.nodeGroups, tc.nodesPerGroup, annotations, nil, nil)) } }) t.Run("MachineDeployment", func(t *testing.T) { for _, tc := range testCases { - test(t, tc, createMachineDeploymentTestConfigs(RandomString(6), RandomString(6), RandomString(6), tc.nodeGroups, tc.nodesPerGroup, annotations, nil)) + test(t, tc, createMachineDeploymentTestConfigs(RandomString(6), RandomString(6), RandomString(6), tc.nodeGroups, tc.nodesPerGroup, annotations, nil, nil)) } }) } @@ -1268,7 +1282,7 @@ func TestControllerFindMachineFromNodeAnnotation(t *testing.T) { testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{ nodeGroupMinSizeAnnotationKey: "1", nodeGroupMaxSizeAnnotationKey: "10", - }, nil) + }, nil, nil) controller, stop := mustCreateTestController(t, testConfig) defer stop() @@ -1316,7 +1330,7 @@ func TestControllerMachineSetNodeNamesWithoutLinkage(t *testing.T) { testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 3, map[string]string{ nodeGroupMinSizeAnnotationKey: "1", nodeGroupMaxSizeAnnotationKey: "10", - }, nil) + }, nil, nil) controller, stop := mustCreateTestController(t, testConfig) defer stop() @@ -1365,7 +1379,7 @@ func TestControllerMachineSetNodeNamesUsingProviderID(t *testing.T) { testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 3, map[string]string{ nodeGroupMinSizeAnnotationKey: "1", nodeGroupMaxSizeAnnotationKey: "10", - }, nil) + }, nil, nil) controller, stop := mustCreateTestController(t, testConfig) defer stop() @@ -1404,7 +1418,7 @@ func TestControllerMachineSetNodeNamesUsingStatusNodeRefName(t *testing.T) { testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 3, map[string]string{ nodeGroupMinSizeAnnotationKey: "1", nodeGroupMaxSizeAnnotationKey: "10", - }, nil) + }, nil, nil) controller, stop := mustCreateTestController(t, testConfig) defer stop() @@ -1494,7 +1508,7 @@ func TestControllerGetAPIVersionGroupWithMachineDeployments(t *testing.T) { testConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{ nodeGroupMinSizeAnnotationKey: "1", nodeGroupMaxSizeAnnotationKey: "1", - }, nil) + }, nil, nil) t.Setenv(CAPIGroupEnvVar, customCAPIGroup) testConfig.machineDeployment.SetAPIVersion(fmt.Sprintf("%s/v1beta1", customCAPIGroup)) @@ -1756,7 +1770,7 @@ func Test_machineController_allowedByAutoDiscoverySpecs(t *testing.T) { shouldMatch bool }{{ name: "autodiscovery specs includes permissive spec that should match any MachineSet", - testSpec: createTestSpec(RandomString(6), RandomString(6), RandomString(6), 1, false, nil, nil), + testSpec: createTestSpec(RandomString(6), RandomString(6), RandomString(6), 1, false, nil, nil, nil), autoDiscoverySpecs: []*clusterAPIAutoDiscoveryConfig{ {labelSelector: labels.NewSelector()}, {clusterName: "foo", namespace: "bar", labelSelector: labels.Nothing()}, @@ -1764,7 +1778,7 @@ func Test_machineController_allowedByAutoDiscoverySpecs(t *testing.T) { shouldMatch: true, }, { name: "autodiscovery specs includes permissive spec that should match any MachineDeployment", - testSpec: createTestSpec(RandomString(6), RandomString(6), RandomString(6), 1, true, nil, nil), + testSpec: createTestSpec(RandomString(6), RandomString(6), RandomString(6), 1, true, nil, nil, nil), autoDiscoverySpecs: []*clusterAPIAutoDiscoveryConfig{ {labelSelector: labels.NewSelector()}, {clusterName: "foo", namespace: "bar", labelSelector: labels.Nothing()}, @@ -1772,7 +1786,7 @@ func Test_machineController_allowedByAutoDiscoverySpecs(t *testing.T) { shouldMatch: true, }, { name: "autodiscovery specs includes a restrictive spec that should match specific MachineSet", - testSpec: createTestSpec("default", "foo", RandomString(6), 1, false, nil, nil), + testSpec: createTestSpec("default", "foo", RandomString(6), 1, false, nil, nil, nil), additionalLabels: map[string]string{"color": "green"}, autoDiscoverySpecs: []*clusterAPIAutoDiscoveryConfig{ {clusterName: "foo", namespace: "default", labelSelector: labels.SelectorFromSet(labels.Set{"color": "green"})}, @@ -1781,7 +1795,7 @@ func Test_machineController_allowedByAutoDiscoverySpecs(t *testing.T) { shouldMatch: true, }, { name: "autodiscovery specs includes a restrictive spec that should match specific MachineDeployment", - testSpec: createTestSpec("default", "foo", RandomString(6), 1, true, nil, nil), + testSpec: createTestSpec("default", "foo", RandomString(6), 1, true, nil, nil, nil), additionalLabels: map[string]string{"color": "green"}, autoDiscoverySpecs: []*clusterAPIAutoDiscoveryConfig{ {clusterName: "foo", namespace: "default", labelSelector: labels.SelectorFromSet(labels.Set{"color": "green"})}, @@ -1790,7 +1804,7 @@ func Test_machineController_allowedByAutoDiscoverySpecs(t *testing.T) { shouldMatch: true, }, { name: "autodiscovery specs does not include any specs that should match specific MachineSet", - testSpec: createTestSpec("default", "foo", RandomString(6), 1, false, nil, nil), + testSpec: createTestSpec("default", "foo", RandomString(6), 1, false, nil, nil, nil), additionalLabels: map[string]string{"color": "green"}, autoDiscoverySpecs: []*clusterAPIAutoDiscoveryConfig{ {clusterName: "test", namespace: "default", labelSelector: labels.SelectorFromSet(labels.Set{"color": "blue"})}, @@ -1799,7 +1813,7 @@ func Test_machineController_allowedByAutoDiscoverySpecs(t *testing.T) { shouldMatch: false, }, { name: "autodiscovery specs does not include any specs that should match specific MachineDeployment", - testSpec: createTestSpec("default", "foo", RandomString(6), 1, true, nil, nil), + testSpec: createTestSpec("default", "foo", RandomString(6), 1, true, nil, nil, nil), additionalLabels: map[string]string{"color": "green"}, autoDiscoverySpecs: []*clusterAPIAutoDiscoveryConfig{ {clusterName: "test", namespace: "default", labelSelector: labels.SelectorFromSet(labels.Set{"color": "blue"})}, @@ -1829,9 +1843,9 @@ func Test_machineController_allowedByAutoDiscoverySpecs(t *testing.T) { } func Test_machineController_listScalableResources(t *testing.T) { - uniqueMDConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, nil, nil) + uniqueMDConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, nil, nil, nil) - mdTestConfigs := createMachineDeploymentTestConfigs(RandomString(6), RandomString(6), RandomString(6), 5, 1, nil, nil) + mdTestConfigs := createMachineDeploymentTestConfigs(RandomString(6), RandomString(6), RandomString(6), 5, 1, nil, nil, nil) mdTestConfigs = append(mdTestConfigs, uniqueMDConfig) allMachineDeployments := make([]*unstructured.Unstructured, 0, len(mdTestConfigs)) @@ -1839,9 +1853,9 @@ func Test_machineController_listScalableResources(t *testing.T) { allMachineDeployments = append(allMachineDeployments, mdTestConfigs[i].machineDeployment) } - uniqueMSConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, nil, nil) + uniqueMSConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, nil, nil, nil) - msTestConfigs := createMachineSetTestConfigs(RandomString(6), RandomString(6), RandomString(6), 5, 1, nil, nil) + msTestConfigs := createMachineSetTestConfigs(RandomString(6), RandomString(6), RandomString(6), 5, 1, nil, nil, nil) msTestConfigs = append(msTestConfigs, uniqueMSConfig) allMachineSets := make([]*unstructured.Unstructured, 0, len(msTestConfigs)) @@ -1944,23 +1958,23 @@ func Test_machineController_nodeGroupForNode(t *testing.T) { uniqueMDConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{ nodeGroupMinSizeAnnotationKey: "1", nodeGroupMaxSizeAnnotationKey: "10", - }, nil) + }, nil, nil) mdTestConfigs := createMachineDeploymentTestConfigs(RandomString(6), RandomString(6), RandomString(6), 5, 1, map[string]string{ nodeGroupMinSizeAnnotationKey: "1", nodeGroupMaxSizeAnnotationKey: "10", - }, nil) + }, nil, nil) mdTestConfigs = append(mdTestConfigs, uniqueMDConfig) uniqueMSConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{ nodeGroupMinSizeAnnotationKey: "1", nodeGroupMaxSizeAnnotationKey: "10", - }, nil) + }, nil, nil) msTestConfigs := createMachineSetTestConfigs(RandomString(6), RandomString(6), RandomString(6), 5, 1, map[string]string{ nodeGroupMinSizeAnnotationKey: "1", nodeGroupMaxSizeAnnotationKey: "10", - }, nil) + }, nil, nil) msTestConfigs = append(msTestConfigs, uniqueMSConfig) allTestConfigs := append(mdTestConfigs, msTestConfigs...) @@ -2040,12 +2054,12 @@ func Test_machineController_nodeGroups(t *testing.T) { uniqueMDConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{ nodeGroupMinSizeAnnotationKey: "1", nodeGroupMaxSizeAnnotationKey: "10", - }, nil) + }, nil, nil) mdTestConfigs := createMachineDeploymentTestConfigs(RandomString(6), RandomString(6), RandomString(6), 5, 1, map[string]string{ nodeGroupMinSizeAnnotationKey: "1", nodeGroupMaxSizeAnnotationKey: "10", - }, nil) + }, nil, nil) mdTestConfigs = append(mdTestConfigs, uniqueMDConfig) allMachineDeployments := make([]*unstructured.Unstructured, 0, len(mdTestConfigs)) @@ -2056,12 +2070,12 @@ func Test_machineController_nodeGroups(t *testing.T) { uniqueMSConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, map[string]string{ nodeGroupMinSizeAnnotationKey: "1", nodeGroupMaxSizeAnnotationKey: "10", - }, nil) + }, nil, nil) msTestConfigs := createMachineSetTestConfigs(RandomString(6), RandomString(6), RandomString(6), 5, 1, map[string]string{ nodeGroupMinSizeAnnotationKey: "1", nodeGroupMaxSizeAnnotationKey: "10", - }, nil) + }, nil, nil) msTestConfigs = append(msTestConfigs, uniqueMSConfig) allMachineSets := make([]*unstructured.Unstructured, 0, len(msTestConfigs)) diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup.go index d68a48b89c3f..5e2a1100b77a 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup.go @@ -361,12 +361,17 @@ func (ng *nodegroup) TemplateNodeInfo() (*framework.NodeInfo, error) { }, } + nsi := ng.scalableResource.InstanceSystemInfo() + if nsi != nil { + node.Status.NodeInfo = *nsi + } + node.Status.Capacity = capacity node.Status.Allocatable = capacity node.Status.Conditions = cloudprovider.BuildReadyConditions() node.Spec.Taints = ng.scalableResource.Taints() - node.Labels, err = ng.buildTemplateLabels(nodeName) + node.Labels, err = ng.buildTemplateLabels(nodeName, nsi) if err != nil { return nil, err } @@ -380,8 +385,19 @@ func (ng *nodegroup) TemplateNodeInfo() (*framework.NodeInfo, error) { return nodeInfo, nil } -func (ng *nodegroup) buildTemplateLabels(nodeName string) (map[string]string, error) { - labels := cloudprovider.JoinStringMaps(buildGenericLabels(nodeName), ng.scalableResource.Labels()) +func (ng *nodegroup) buildTemplateLabels(nodeName string, nsi *corev1.NodeSystemInfo) (map[string]string, error) { + nsiLabels := make(map[string]string) + if nsi != nil { + nsiLabels[corev1.LabelArchStable] = nsi.Architecture + nsiLabels[corev1.LabelOSStable] = nsi.OperatingSystem + } + + // The order of priority is: + // - Labels set in existing nodes for not-autoscale-from-zero cases + // - Labels set in the labels capacity annotation of machine template, machine set, and machine deployment. + // - Values in the status.nodeSystemInfo of MachineTemplates + // - Generic/default labels set in the environment of the cluster autoscaler + labels := cloudprovider.JoinStringMaps(buildGenericLabels(nodeName), nsiLabels, ng.scalableResource.Labels()) nodes, err := ng.Nodes() if err != nil { diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup_test.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup_test.go index 2999f8d02d4e..a31a93dbf3f6 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup_test.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup_test.go @@ -214,7 +214,7 @@ func TestNodeGroupNewNodeGroupConstructor(t *testing.T) { t.Run("MachineSet", func(t *testing.T) { for _, tc := range testCases { t.Run(tc.description, func(t *testing.T) { - test(t, tc, createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), tc.nodeCount, tc.annotations, nil)) + test(t, tc, createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), tc.nodeCount, tc.annotations, nil, nil)) }) } }) @@ -222,7 +222,7 @@ func TestNodeGroupNewNodeGroupConstructor(t *testing.T) { t.Run("MachineDeployment", func(t *testing.T) { for _, tc := range testCases { t.Run(tc.description, func(t *testing.T) { - test(t, tc, createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), tc.nodeCount, tc.annotations, nil)) + test(t, tc, createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), tc.nodeCount, tc.annotations, nil, nil)) }) } }) @@ -309,7 +309,7 @@ func TestNodeGroupIncreaseSizeErrors(t *testing.T) { nodeGroupMinSizeAnnotationKey: "1", nodeGroupMaxSizeAnnotationKey: "10", } - test(t, &tc, createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil)) + test(t, &tc, createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil, nil)) }) } }) @@ -321,7 +321,7 @@ func TestNodeGroupIncreaseSizeErrors(t *testing.T) { nodeGroupMinSizeAnnotationKey: "1", nodeGroupMaxSizeAnnotationKey: "10", } - test(t, &tc, createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil)) + test(t, &tc, createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil, nil)) }) } }) @@ -390,7 +390,7 @@ func TestNodeGroupIncreaseSize(t *testing.T) { expected: 4, delta: 1, } - test(t, &tc, createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil)) + test(t, &tc, createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil, nil)) }) t.Run("MachineDeployment", func(t *testing.T) { @@ -400,7 +400,7 @@ func TestNodeGroupIncreaseSize(t *testing.T) { expected: 4, delta: 1, } - test(t, &tc, createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil)) + test(t, &tc, createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil, nil)) }) } @@ -689,13 +689,13 @@ func TestNodeGroupDecreaseTargetSize(t *testing.T) { for _, tc := range testCases { t.Run(tc.description, func(t *testing.T) { - test(t, &tc, createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil)) + test(t, &tc, createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil, nil)) }) } for _, tc := range testCases { t.Run(tc.description, func(t *testing.T) { - test(t, &tc, createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil)) + test(t, &tc, createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil, nil)) }) } } @@ -781,7 +781,7 @@ func TestNodeGroupDecreaseSizeErrors(t *testing.T) { nodeGroupMinSizeAnnotationKey: "1", nodeGroupMaxSizeAnnotationKey: "10", } - test(t, &tc, createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil)) + test(t, &tc, createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil, nil)) }) } }) @@ -793,7 +793,7 @@ func TestNodeGroupDecreaseSizeErrors(t *testing.T) { nodeGroupMinSizeAnnotationKey: "1", nodeGroupMaxSizeAnnotationKey: "10", } - test(t, &tc, createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil)) + test(t, &tc, createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil, nil)) }) } }) @@ -883,6 +883,7 @@ func TestNodeGroupDeleteNodes(t *testing.T) { nodeGroupMaxSizeAnnotationKey: "10", }, nil, + nil, ), ) }) @@ -900,6 +901,7 @@ func TestNodeGroupDeleteNodes(t *testing.T) { nodeGroupMaxSizeAnnotationKey: "10", }, nil, + nil, ), ) }) @@ -972,16 +974,16 @@ func TestNodeGroupMachineSetDeleteNodesWithMismatchedNodes(t *testing.T) { t.Run("MachineSet", func(t *testing.T) { namespace := RandomString(6) clusterName := RandomString(6) - testConfig0 := createMachineSetTestConfigs(namespace, clusterName, RandomString(6), 1, 2, annotations, nil) - testConfig1 := createMachineSetTestConfigs(namespace, clusterName, RandomString(6), 1, 2, annotations, nil) + testConfig0 := createMachineSetTestConfigs(namespace, clusterName, RandomString(6), 1, 2, annotations, nil, nil) + testConfig1 := createMachineSetTestConfigs(namespace, clusterName, RandomString(6), 1, 2, annotations, nil, nil) test(t, 2, append(testConfig0, testConfig1...)) }) t.Run("MachineDeployment", func(t *testing.T) { namespace := RandomString(6) clusterName := RandomString(6) - testConfig0 := createMachineDeploymentTestConfigs(namespace, clusterName, RandomString(6), 1, 2, annotations, nil) - testConfig1 := createMachineDeploymentTestConfigs(namespace, clusterName, RandomString(6), 1, 2, annotations, nil) + testConfig0 := createMachineDeploymentTestConfigs(namespace, clusterName, RandomString(6), 1, 2, annotations, nil, nil) + testConfig1 := createMachineDeploymentTestConfigs(namespace, clusterName, RandomString(6), 1, 2, annotations, nil, nil) test(t, 2, append(testConfig0, testConfig1...)) }) } @@ -1163,6 +1165,7 @@ func TestNodeGroupDeleteNodesTwice(t *testing.T) { nodeGroupMaxSizeAnnotationKey: "10", }, nil, + nil, ), ) }) @@ -1180,6 +1183,7 @@ func TestNodeGroupDeleteNodesTwice(t *testing.T) { nodeGroupMaxSizeAnnotationKey: "10", }, nil, + nil, ), ) }) @@ -1315,6 +1319,7 @@ func TestNodeGroupDeleteNodesSequential(t *testing.T) { nodeGroupMaxSizeAnnotationKey: "10", }, nil, + nil, ), ) }) @@ -1332,6 +1337,7 @@ func TestNodeGroupDeleteNodesSequential(t *testing.T) { nodeGroupMaxSizeAnnotationKey: "10", }, nil, + nil, ), ) }) @@ -1418,6 +1424,7 @@ func TestNodeGroupWithFailedMachine(t *testing.T) { nodeGroupMaxSizeAnnotationKey: "10", }, nil, + nil, ), ) }) @@ -1435,6 +1442,7 @@ func TestNodeGroupWithFailedMachine(t *testing.T) { nodeGroupMaxSizeAnnotationKey: "10", }, nil, + nil, ), ) }) @@ -1667,6 +1675,7 @@ func TestNodeGroupTemplateNodeInfo(t *testing.T) { 10, cloudprovider.JoinStringMaps(enableScaleAnnotations, tc.nodeGroupAnnotations), nil, + nil, ), tc.config, ) @@ -1682,6 +1691,7 @@ func TestNodeGroupTemplateNodeInfo(t *testing.T) { 10, cloudprovider.JoinStringMaps(enableScaleAnnotations, tc.nodeGroupAnnotations), nil, + nil, ), tc.config, ) @@ -1792,6 +1802,7 @@ func TestNodeGroupGetOptions(t *testing.T) { 10, cloudprovider.JoinStringMaps(enableScaleAnnotations, annotations), nil, + nil, ), c.expected, ) @@ -1807,6 +1818,7 @@ func TestNodeGroupGetOptions(t *testing.T) { 10, cloudprovider.JoinStringMaps(enableScaleAnnotations, annotations), nil, + nil, ), c.expected, ) @@ -2009,6 +2021,7 @@ func TestNodeGroupNodesInstancesStatus(t *testing.T) { tc.nodeCount, annotations, nil, + nil, ), ) }) @@ -2028,6 +2041,7 @@ func TestNodeGroupNodesInstancesStatus(t *testing.T) { tc.nodeCount, annotations, nil, + nil, ), ) }) diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_provider_test.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_provider_test.go index 47e58b6bfcdb..4e8b40737416 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_provider_test.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_provider_test.go @@ -110,7 +110,7 @@ func BenchmarkNodeGroups(b *testing.B) { controller, stop := mustCreateTestController(b) defer stop() - machineSetConfigs := createMachineSetTestConfigs("namespace", "", RandomString(6), 100, 1, annotations, nil) + machineSetConfigs := createMachineSetTestConfigs("namespace", "", RandomString(6), 100, 1, annotations, nil, nil) if err := addTestConfigs(b, controller, machineSetConfigs...); err != nil { b.Fatalf("unexpected error: %v", err) } diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_unstructured.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_unstructured.go index 46aceddeeb18..48ed6fde1cee 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_unstructured.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_unstructured.go @@ -23,6 +23,7 @@ import ( "path" "strconv" "strings" + "sync" "time" "github.com/pkg/errors" @@ -42,6 +43,8 @@ import ( type unstructuredScalableResource struct { controller *machineController unstructured *unstructured.Unstructured + infraObj *unstructured.Unstructured + infraMutex sync.RWMutex maxSize int minSize int autoscalingOptions map[string]string @@ -321,6 +324,17 @@ func (r unstructuredScalableResource) InstanceCapacity() (map[corev1.ResourceNam return capacity, nil } +// InstanceSystemInfo sets the nodeSystemInfo from the infrastructure reference resource. +// If the infrastructure reference resource is not found, returns nil. +func (r unstructuredScalableResource) InstanceSystemInfo() *apiv1.NodeSystemInfo { + infraObj, err := r.readInfrastructureReferenceResource() + if err != nil || infraObj == nil { + return nil + } + nsiObj := systemInfoFromInfrastructureObject(infraObj) + return &nsiObj +} + func (r unstructuredScalableResource) InstanceResourceSlices(nodeName string) ([]*resourceapi.ResourceSlice, error) { var result []*resourceapi.ResourceSlice driver := r.InstanceDRADriver() @@ -392,6 +406,17 @@ func (r unstructuredScalableResource) InstanceDRADriver() string { } func (r unstructuredScalableResource) readInfrastructureReferenceResource() (*unstructured.Unstructured, error) { + // Cache w/ lazy loading of the infrastructure reference resource. + r.infraMutex.RLock() + if r.infraObj != nil { + defer r.infraMutex.RUnlock() + return r.infraObj, nil + } + r.infraMutex.RUnlock() + + r.infraMutex.Lock() + defer r.infraMutex.Unlock() + obKind := r.unstructured.GetKind() obName := r.unstructured.GetName() @@ -442,6 +467,8 @@ func (r unstructuredScalableResource) readInfrastructureReferenceResource() (*un return nil, err } + r.infraObj = infra + return infra, nil } @@ -479,6 +506,25 @@ func resourceCapacityFromInfrastructureObject(infraobj *unstructured.Unstructure return capacity } +func systemInfoFromInfrastructureObject(infraobj *unstructured.Unstructured) apiv1.NodeSystemInfo { + nsi := apiv1.NodeSystemInfo{} + infransi, found, err := unstructured.NestedStringMap(infraobj.Object, "status", "nodeInfo") + if !found || err != nil { + return nsi + } + + for k, v := range infransi { + switch k { + case "architecture": + nsi.Architecture = v + case "operatingSystem": + nsi.OperatingSystem = v + } + } + + return nsi +} + // adapted from https://github.com/kubernetes/kubernetes/blob/release-1.25/pkg/util/taints/taints.go#L39 func parseTaint(st string) (apiv1.Taint, error) { var taint apiv1.Taint diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_unstructured_test.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_unstructured_test.go index e54ec563ad24..908a0eb83a55 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_unstructured_test.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_unstructured_test.go @@ -33,9 +33,14 @@ import ( ) const ( - cpuStatusKey = "cpu" - memoryStatusKey = "memory" - nvidiaGpuStatusKey = "nvidia.com/gpu" + cpuStatusKey = "cpu" + memoryStatusKey = "memory" + nvidiaGpuStatusKey = "nvidia.com/gpu" + architectureStatusKey = "architecture" + operatingSystemStatusKey = "operatingSystem" + + arm64 = "arm64" + linux = "linux" ) func TestSetSize(t *testing.T) { @@ -127,6 +132,7 @@ func TestSetSize(t *testing.T) { nodeGroupMaxSizeAnnotationKey: "10", }, nil, + nil, )) }) @@ -140,6 +146,7 @@ func TestSetSize(t *testing.T) { nodeGroupMaxSizeAnnotationKey: "10", }, nil, + nil, )) }) } @@ -244,11 +251,11 @@ func TestReplicas(t *testing.T) { } t.Run("MachineSet", func(t *testing.T) { - test(t, createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), initialReplicas, nil, nil)) + test(t, createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), initialReplicas, nil, nil, nil)) }) t.Run("MachineDeployment", func(t *testing.T) { - test(t, createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), initialReplicas, nil, nil)) + test(t, createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), initialReplicas, nil, nil, nil)) }) } @@ -304,6 +311,7 @@ func TestSetSizeAndReplicas(t *testing.T) { nodeGroupMaxSizeAnnotationKey: "10", }, nil, + nil, )) }) @@ -317,6 +325,7 @@ func TestSetSizeAndReplicas(t *testing.T) { nodeGroupMaxSizeAnnotationKey: "10", }, nil, + nil, )) }) } @@ -423,12 +432,12 @@ func TestAnnotations(t *testing.T) { } t.Run("MachineSet", func(t *testing.T) { - testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, annotations, nil) + testConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, annotations, nil, nil) test(t, testConfig, testConfig.machineSet) }) t.Run("MachineDeployment", func(t *testing.T) { - testConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, annotations, nil) + testConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, annotations, nil, nil) test(t, testConfig, testConfig.machineDeployment) }) } @@ -533,7 +542,7 @@ func TestCanScaleFromZero(t *testing.T) { for _, tc := range testConfigs { testname := fmt.Sprintf("MachineSet %s", tc.name) t.Run(testname, func(t *testing.T) { - msTestConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, tc.annotations, tc.capacity) + msTestConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, tc.annotations, tc.capacity, nil) controller, stop := mustCreateTestController(t, msTestConfig) defer stop() @@ -554,7 +563,7 @@ func TestCanScaleFromZero(t *testing.T) { for _, tc := range testConfigs { testname := fmt.Sprintf("MachineDeployment %s", tc.name) t.Run(testname, func(t *testing.T) { - msTestConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, tc.annotations, tc.capacity) + msTestConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, tc.annotations, tc.capacity, nil) controller, stop := mustCreateTestController(t, msTestConfig) defer stop() @@ -572,3 +581,89 @@ func TestCanScaleFromZero(t *testing.T) { }) } } + +func TestInstanceSystemInfo(t *testing.T) { + // use a constant capacity as that's necessary for the business logic to consider the resource scalable + capacity := map[string]string{ + cpuStatusKey: "1", + memoryStatusKey: "4G", + } + testConfigs := []struct { + name string + nodeInfo map[string]string + expectedArch string + expectedOS string + }{ + { + "with no architecture or operating system in machine template's status' nodeInfo, the system info is empty", + map[string]string{}, + "", + "", + }, + { + "with architecture in machine template's status' nodeInfo, the system info is filled in the scalable resource", + map[string]string{ + architectureStatusKey: arm64, + }, + arm64, + "", + }, + { + "with operating system in machine template's status' nodeInfo, the system info is filled in the scalable resource", + map[string]string{ + operatingSystemStatusKey: linux, + }, + "", + linux, + }, + } + + for _, tc := range testConfigs { + testname := fmt.Sprintf("MachineSet %s", tc.name) + t.Run(testname, func(t *testing.T) { + msTestConfig := createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, nil, capacity, tc.nodeInfo) + controller, stop := mustCreateTestController(t, msTestConfig) + defer stop() + + testResource := msTestConfig.machineSet + + sr, err := newUnstructuredScalableResource(controller, testResource) + if err != nil { + t.Fatal(err) + } + + sysInfo := sr.InstanceSystemInfo() + if sysInfo.Architecture != tc.expectedArch { + t.Errorf("expected architecture %s, got %s", tc.nodeInfo[architectureStatusKey], sysInfo.Architecture) + } + if sysInfo.OperatingSystem != tc.expectedOS { + t.Errorf("expected operating system %s, got %s", tc.nodeInfo[operatingSystemStatusKey], sysInfo.OperatingSystem) + } + }) + } + + for _, tc := range testConfigs { + testname := fmt.Sprintf("MachineDeployment %s", tc.name) + t.Run(testname, func(t *testing.T) { + msTestConfig := createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), 1, nil, capacity, tc.nodeInfo) + controller, stop := mustCreateTestController(t, msTestConfig) + defer stop() + + testResource := msTestConfig.machineDeployment + + sr, err := newUnstructuredScalableResource(controller, testResource) + if err != nil { + t.Fatal(err) + } + + sysInfo := sr.InstanceSystemInfo() + if sysInfo.Architecture != tc.expectedArch { + t.Errorf("expected architecture %s, got %s", tc.nodeInfo[architectureStatusKey], sysInfo.Architecture) + } + + if sysInfo.OperatingSystem != tc.expectedOS { + t.Errorf("expected operating system %s, got %s", tc.nodeInfo[operatingSystemStatusKey], sysInfo.OperatingSystem) + } + }) + } +}