[cluster-autoscaler-release-1.32] Process apiGroup in capi provider (#8454)

* Add process with apiGroup in capi provider

* Replace capi v1alpha3 with v1beta2 in test cases

* Add detailed error messages

* remove unused package

---------

Co-authored-by: Jun Wang <[email protected]>
Co-authored-by: Jack Francis <[email protected]>

Author: 3 people

cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller.go

@@ -81,6 +81,7 @@ type machineController struct {
 	nodeInformer                cache.SharedIndexInformer
 	managementClient            dynamic.Interface
 	managementScaleClient       scale.ScalesGetter
+	managementDiscoveryClient   discovery.DiscoveryInterface
 	machineSetResource          schema.GroupVersionResource
 	machineResource             schema.GroupVersionResource
 	machinePoolResource         schema.GroupVersionResource
@@ -422,7 +423,7 @@ func newMachineController(
 	managementInformerFactory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(managementClient, 0, namespaceToWatch(autoDiscoverySpecs), nil)
 
 	CAPIGroup := getCAPIGroup()
-	CAPIVersion, err := getAPIGroupPreferredVersion(managementDiscoveryClient, CAPIGroup)
+	CAPIVersion, err := getCAPIGroupPreferredVersion(managementDiscoveryClient, CAPIGroup)
 	if err != nil {
 		return nil, fmt.Errorf("could not find preferred version for CAPI group %q: %v", CAPIGroup, err)
 	}
@@ -526,6 +527,7 @@ func newMachineController(
 		nodeInformer:                nodeInformer,
 		managementClient:            managementClient,
 		managementScaleClient:       managementScaleClient,
+		managementDiscoveryClient:   managementDiscoveryClient,
 		machineSetResource:          gvrMachineSet,
 		machinePoolResource:         gvrMachinePool,
 		machinePoolsAvailable:       machinePoolsAvailable,
@@ -551,11 +553,15 @@ func groupVersionHasResource(client discovery.DiscoveryInterface, groupVersion,
 	return false, nil
 }
 
-func getAPIGroupPreferredVersion(client discovery.DiscoveryInterface, APIGroup string) (string, error) {
+func getCAPIGroupPreferredVersion(client discovery.DiscoveryInterface, APIGroup string) (string, error) {
 	if version := os.Getenv(CAPIVersionEnvVar); version != "" {
 		return version, nil
 	}
 
+	return getAPIGroupPreferredVersion(client, APIGroup)
+}
+
+func getAPIGroupPreferredVersion(client discovery.DiscoveryInterface, APIGroup string) (string, error) {
 	groupList, err := client.ServerGroups()
 	if err != nil {
 		return "", fmt.Errorf("failed to get ServerGroups: %v", err)

cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller_test.go

@@ -105,13 +105,13 @@ func mustCreateTestController(t testing.TB, testConfigs ...*testConfig) (*machin
 	dynamicClientset := fakedynamic.NewSimpleDynamicClientWithCustomListKinds(
 		runtime.NewScheme(),
 		map[schema.GroupVersionResource]string{
-			{Group: "cluster.x-k8s.io", Version: "v1alpha3", Resource: "machinedeployments"}:             "kindList",
-			{Group: "cluster.x-k8s.io", Version: "v1alpha3", Resource: "machines"}:                       "kindList",
-			{Group: "cluster.x-k8s.io", Version: "v1alpha3", Resource: "machinesets"}:                    "kindList",
-			{Group: "cluster.x-k8s.io", Version: "v1alpha3", Resource: "machinepools"}:                   "kindList",
 			{Group: "cluster.x-k8s.io", Version: "v1beta1", Resource: "machinedeployments"}:              "kindList",
 			{Group: "cluster.x-k8s.io", Version: "v1beta1", Resource: "machines"}:                        "kindList",
 			{Group: "cluster.x-k8s.io", Version: "v1beta1", Resource: "machinesets"}:                     "kindList",
+			{Group: "cluster.x-k8s.io", Version: "v1beta2", Resource: "machinedeployments"}:              "kindList",
+			{Group: "cluster.x-k8s.io", Version: "v1beta2", Resource: "machines"}:                        "kindList",
+			{Group: "cluster.x-k8s.io", Version: "v1beta2", Resource: "machinesets"}:                     "kindList",
+			{Group: "cluster.x-k8s.io", Version: "v1beta2", Resource: "machinepools"}:                    "kindList",
 			{Group: "custom.x-k8s.io", Version: "v1beta1", Resource: "machinepools"}:                     "kindList",
 			{Group: "custom.x-k8s.io", Version: "v1beta1", Resource: "machinedeployments"}:               "kindList",
 			{Group: "custom.x-k8s.io", Version: "v1beta1", Resource: "machines"}:                         "kindList",
@@ -149,7 +149,7 @@ func mustCreateTestController(t testing.TB, testConfigs ...*testConfig) (*machin
 					},
 				},
 				{
-					GroupVersion: fmt.Sprintf("%s/v1alpha3", defaultCAPIGroup),
+					GroupVersion: fmt.Sprintf("%s/v1beta2", defaultCAPIGroup),
 					APIResources: []metav1.APIResource{
 						{
 							Name: resourceNameMachineDeployment,
@@ -189,7 +189,7 @@ func mustCreateTestController(t testing.TB, testConfigs ...*testConfig) (*machin
 
 		gvr := schema.GroupVersionResource{
 			Group:    action.GetResource().Group,
-			Version:  "v1alpha3",
+			Version:  "v1beta2",
 			Resource: resource,
 		}
 
@@ -331,7 +331,7 @@ func createTestConfigs(specs ...testSpec) []*testConfig {
 		config.machineSet = &unstructured.Unstructured{
 			Object: map[string]interface{}{
 				"kind":       machineSetKind,
-				"apiVersion": "cluster.x-k8s.io/v1alpha3",
+				"apiVersion": "cluster.x-k8s.io/v1beta2",
 				"metadata": map[string]interface{}{
 					"name":      spec.machineSetName,
 					"namespace": spec.namespace,
@@ -369,7 +369,7 @@ func createTestConfigs(specs ...testSpec) []*testConfig {
 			config.machineDeployment = &unstructured.Unstructured{
 				Object: map[string]interface{}{
 					"kind":       machineDeploymentKind,
-					"apiVersion": "cluster.x-k8s.io/v1alpha3",
+					"apiVersion": "cluster.x-k8s.io/v1beta2",
 					"metadata": map[string]interface{}{
 						"name":      spec.machineDeploymentName,
 						"namespace": spec.namespace,
@@ -381,9 +381,9 @@ func createTestConfigs(specs ...testSpec) []*testConfig {
 						"template": map[string]interface{}{
 							"spec": map[string]interface{}{
 								"infrastructureRef": map[string]interface{}{
-									"apiVersion": "infrastructure.cluster.x-k8s.io/v1beta1",
-									"kind":       machineTemplateKind,
-									"name":       "TestMachineTemplate",
+									"apiGroup": "infrastructure.cluster.x-k8s.io",
+									"kind":     machineTemplateKind,
+									"name":     "TestMachineTemplate",
 								},
 							},
 						},
@@ -471,7 +471,7 @@ func makeLinkedNodeAndMachine(i int, namespace, clusterName string, owner metav1
 	machine := &unstructured.Unstructured{
 		Object: map[string]interface{}{
 			"kind":       machineKind,
-			"apiVersion": "cluster.x-k8s.io/v1alpha3",
+			"apiVersion": "cluster.x-k8s.io/v1beta2",
 			"metadata": map[string]interface{}{
 				"name":      fmt.Sprintf("%s-%s-machine-%d", namespace, owner.Name, i),
 				"namespace": namespace,
@@ -1528,7 +1528,7 @@ func TestGetAPIGroupPreferredVersion(t *testing.T) {
 		{
 			description:      "find version for default API group",
 			APIGroup:         defaultCAPIGroup,
-			preferredVersion: "v1alpha3",
+			preferredVersion: "v1beta2",
 			envVar:           "",
 			error:            false,
 		},
@@ -1562,7 +1562,7 @@ func TestGetAPIGroupPreferredVersion(t *testing.T) {
 					GroupVersion: fmt.Sprintf("%s/v1beta1", customCAPIGroup),
 				},
 				{
-					GroupVersion: fmt.Sprintf("%s/v1alpha3", defaultCAPIGroup),
+					GroupVersion: fmt.Sprintf("%s/v1beta2", defaultCAPIGroup),
 				},
 				{
 					GroupVersion: fmt.Sprintf("%s/%s", customCAPIGroup, customVersion),
@@ -1573,7 +1573,7 @@ func TestGetAPIGroupPreferredVersion(t *testing.T) {
 	for _, tc := range testCases {
 		t.Run(tc.description, func(t *testing.T) {
 			t.Setenv(CAPIVersionEnvVar, tc.envVar)
-			version, err := getAPIGroupPreferredVersion(discoveryClient, tc.APIGroup)
+			version, err := getCAPIGroupPreferredVersion(discoveryClient, tc.APIGroup)
 			if (err != nil) != tc.error {
 				t.Errorf("expected to have error: %t. Had an error: %t", tc.error, err != nil)
 			}
@@ -1595,14 +1595,14 @@ func TestGroupVersionHasResource(t *testing.T) {
 		{
 			description:  "true when it finds resource",
 			resourceName: resourceNameMachineDeployment,
-			APIGroup:     fmt.Sprintf("%s/v1alpha3", defaultCAPIGroup),
+			APIGroup:     fmt.Sprintf("%s/v1beta2", defaultCAPIGroup),
 			expected:     true,
 			error:        false,
 		},
 		{
 			description:  "false when it does not find resource",
 			resourceName: "resourceDoesNotExist",
-			APIGroup:     fmt.Sprintf("%s/v1alpha3", defaultCAPIGroup),
+			APIGroup:     fmt.Sprintf("%s/v1beta2", defaultCAPIGroup),
 			expected:     false,
 			error:        false,
 		},
@@ -1619,7 +1619,7 @@ func TestGroupVersionHasResource(t *testing.T) {
 		Fake: &clientgotesting.Fake{
 			Resources: []*metav1.APIResourceList{
 				{
-					GroupVersion: fmt.Sprintf("%s/v1alpha3", defaultCAPIGroup),
+					GroupVersion: fmt.Sprintf("%s/v1beta2", defaultCAPIGroup),
 					APIResources: []metav1.APIResource{
 						{
 							Name: resourceNameMachineDeployment,

cluster-autoscaler/cloudprovider/clusterapi/clusterapi_unstructured.go

@@ -322,22 +322,44 @@ func (r unstructuredScalableResource) InstanceMaxPodsCapacityAnnotation() (resou
 }
 
 func (r unstructuredScalableResource) readInfrastructureReferenceResource() (*unstructured.Unstructured, error) {
+	obKind := r.unstructured.GetKind()
+	obName := r.unstructured.GetName()
+
 	infraref, found, err := unstructured.NestedStringMap(r.unstructured.Object, "spec", "template", "spec", "infrastructureRef")
 	if !found || err != nil {
 		return nil, nil
 	}
 
-	apiversion, ok := infraref["apiVersion"]
-	if !ok {
-		return nil, nil
+	var apiversion string
+
+	apiGroup, ok := infraref["apiGroup"]
+	if ok {
+		if apiversion, err = getAPIGroupPreferredVersion(r.controller.managementDiscoveryClient, apiGroup); err != nil {
+			klog.V(4).Infof("Unable to read preferred version from api group %s, error: %v", apiGroup, err)
+			return nil, err
+		}
+		apiversion = fmt.Sprintf("%s/%s", apiGroup, apiversion)
+	} else {
+		// Fall back to ObjectReference in capi v1beta1
+		apiversion, ok = infraref["apiVersion"]
+		if !ok {
+			info := fmt.Sprintf("Missing apiVersion from %s %s's InfrastructureReference", obKind, obName)
+			klog.V(4).Info(info)
+			return nil, errors.New(info)
+		}
 	}
+
 	kind, ok := infraref["kind"]
 	if !ok {
-		return nil, nil
+		info := fmt.Sprintf("Missing kind from %s %s's InfrastructureReference", obKind, obName)
+		klog.V(4).Info(info)
+		return nil, errors.New(info)
 	}
 	name, ok := infraref["name"]
 	if !ok {
-		return nil, nil
+		info := fmt.Sprintf("Missing name from %s %s's InfrastructureReference", obKind, obName)
+		klog.V(4).Info(info)
+		return nil, errors.New(info)
 	}
 	// kind needs to be lower case and plural
 	kind = fmt.Sprintf("%ss", strings.ToLower(kind))