feat: use the node VM customAttributes to set the nodes providerID if it exists (#231)

In CCM node controller, if the below category is set in the node VM, set
the "providerID" value with the category value, instead of the current
vmUUID value. Only when the category does not exist with the VM, use the
current Node.status.nodeInfo.SystemUUID to set the "providerID".
ahvMetro-<clusterName>-<vmName> : <vmUUID>
The providerID use the format of "nutanix://<vmUUID>".

Author: abhay-nutanix

internal/testing/mock/constants.go

@@ -34,9 +34,11 @@ const (
 	MockVMNamePoweredOnClusterCategories = "mock-vm-poweredon-cluster-categories"
 	MockVMNameDpOffload                  = "mock-vm-dp-offload"
 	MockVMNameSecondaryIPs               = "mock-vm-secondary-ips"
+	MockVMNameCustomProviderID           = "mock-vm-custom-provider-id"
 
-	MockSecondaryIP1 = "2.2.2.2"
-	MockSecondaryIP2 = "3.3.3.3"
+	MockSecondaryIP1     = "2.2.2.2"
+	MockSecondaryIP2     = "3.3.3.3"
+	MockCustomProviderID = "custom-provider-uuid-1234"
 
 	MockNodeNameVMNotExisting = "mock-node-no-vm-exists"
 	MockNodeNameNoSystemUUID  = "mock-node-no-system-uuid"
@@ -65,6 +67,7 @@ const (
 	MockVMPoweredOnClusterCategoriesUUID = "00000000-0000-0000-0000-000000000105"
 	MockVMDpOffloadUUID                  = "00000000-0000-0000-0000-000000000106"
 	MockVMSecondaryIPsUUID               = "00000000-0000-0000-0000-000000000107"
+	MockVMCustomProviderIDUUID           = "00000000-0000-0000-0000-000000000108"
 	MockCategoryRegionUUID               = "00000000-0000-0000-0000-000000000200"
 	MockCategoryZoneUUID                 = "00000000-0000-0000-0000-000000000201"
 )

internal/testing/mock/helpers.go

@@ -175,6 +175,49 @@ func getDefaultVMWithSecondaryIPs(vmName string, vmUUID string, cluster *cluster
 	return vm
 }
 
+func getDefaultVMWithCustomAttributes(vmName string, vmUUID string, cluster *clusterModels.Cluster, host *clusterModels.Host, customAttributes []string) *vmmModels.Vm {
+	nic := vmmModels.NewNic()
+	nicNetInfo := vmmModels.NewVirtualEthernetNicNetworkInfo()
+
+	nicNetInfo.Ipv4Config = vmmModels.NewIpv4Config()
+	nicNetInfo.Ipv4Config.IpAddress = &vmmCommonModels.IPv4Address{
+		Value: ptr.To(MockIP),
+	}
+
+	nicNetInfo.Ipv4Info = vmmModels.NewIpv4Info()
+	nicNetInfo.Ipv4Info.LearnedIpAddresses = []vmmCommonModels.IPv4Address{
+		{
+			Value: ptr.To(MockIP),
+		},
+	}
+
+	err := nic.SetNicNetworkInfo(*nicNetInfo)
+	if err != nil {
+		fmt.Printf("error setting nic network info: %+v\n", err)
+		return nil
+	}
+
+	vm := &vmmModels.Vm{
+		ExtId:            ptr.To(vmUUID),
+		Categories:       make([]vmmModels.CategoryReference, 0),
+		PowerState:       vmmModels.POWERSTATE_ON.Ref(),
+		Name:             ptr.To(vmName),
+		CustomAttributes: customAttributes,
+		Cluster: &vmmModels.ClusterReference{
+			ExtId: cluster.ExtId,
+		},
+		Nics: []vmmModels.Nic{
+			*nic,
+		},
+	}
+	if host != nil {
+		vm.Host = &vmmModels.HostReference{
+			ExtId: host.ExtId,
+		}
+	}
+	return vm
+}
+
 func getDefaultCluster(clusterName string, clusterUUID string) *clusterModels.Cluster {
 	cluster := clusterModels.NewCluster()
 	cluster.ExtId = ptr.To(clusterUUID)

internal/testing/mock/mock_environment.go

@@ -181,6 +181,18 @@ func CreateMockEnvironment(ctx context.Context, kClient *fake.Clientset) (*MockE
 		return nil, err
 	}
 
+	customProviderIDVM := getDefaultVMWithCustomAttributes(
+		MockVMNameCustomProviderID,
+		MockVMCustomProviderIDUUID,
+		cluster,
+		host,
+		[]string{"providerID:" + MockCustomProviderID, "otherKey:otherValue"},
+	)
+	customProviderIDNode, err := createNodeForVM(ctx, kClient, customProviderIDVM)
+	if err != nil {
+		return nil, err
+	}
+
 	return &MockEnvironment{
 		managedMockMachines: map[string]*vmmModels.Vm{
 			*poweredOnVM.ExtId:                  poweredOnVM,
@@ -191,6 +203,7 @@ func CreateMockEnvironment(ctx context.Context, kClient *fake.Clientset) (*MockE
 			*filteredAddressesVM.ExtId:          filteredAddressesVM,
 			*dpOffloadVM.ExtId:                  dpOffloadVM,
 			*secondaryIPsVM.ExtId:               secondaryIPsVM,
+			*customProviderIDVM.ExtId:           customProviderIDVM,
 		},
 		managedMockClusters: map[string]*clusterModels.Cluster{
 			*cluster.ExtId:           cluster,
@@ -215,6 +228,7 @@ func CreateMockEnvironment(ctx context.Context, kClient *fake.Clientset) (*MockE
 			MockVMNameFilteredNodeAddresses:      filteredAddressesNode,
 			MockVMNameDpOffload:                  dpOffloadNode,
 			MockVMNameSecondaryIPs:               secondaryIPsNode,
+			MockVMNameCustomProviderID:           customProviderIDNode,
 		},
 		vmNameToExtId: map[string]string{
 			MockVMNamePoweredOn:                  *poweredOnVM.ExtId,
@@ -225,6 +239,7 @@ func CreateMockEnvironment(ctx context.Context, kClient *fake.Clientset) (*MockE
 			MockVMNamePoweredOnClusterCategories: *poweredOnVMClusterCategories.ExtId,
 			MockVMNameDpOffload:                  *dpOffloadVM.ExtId,
 			MockVMNameSecondaryIPs:               *secondaryIPsVM.ExtId,
+			MockVMNameCustomProviderID:           *customProviderIDVM.ExtId,
 		},
 	}, nil
 }

pkg/provider/manager.go

@@ -123,15 +123,17 @@ func (n *nutanixManager) getInstanceMetadata(ctx context.Context, node *v1.Node)
 		return nil, err
 	}
 
-	providerID, err := n.generateProviderID(ctx, vmUUID)
+	nClient, err := n.nutanixClient.Get()
 	if err != nil {
 		return nil, err
 	}
-	nClient, err := n.nutanixClient.Get()
+	vm, err := nClient.GetVM(ctx, vmUUID)
 	if err != nil {
 		return nil, err
 	}
-	vm, err := nClient.GetVM(ctx, vmUUID)
+
+	// Generate providerID from VM (checks customAttributes first, then falls back to vmUUID)
+	providerID, err := n.generateProviderIDFromVM(ctx, vm)
 	if err != nil {
 		return nil, err
 	}
@@ -322,6 +324,39 @@ func (n *nutanixManager) generateProviderID(ctx context.Context, vmUUID string)
 	return fmt.Sprintf("%s://%s", constants.ProviderName, strings.ToLower(vmUUID)), nil
 }
 
+// generateProviderIDFromVM generates the providerID for a node from the VM object.
+// It first checks if the VM's customAttributes field has "providerID:<UUID>".
+// If yes, it uses this UUID value to set the providerID.
+// If not, it falls back to using the vmUUID (backward compatible).
+func (n *nutanixManager) generateProviderIDFromVM(ctx context.Context, vm *vmmModels.Vm) (string, error) {
+	if vm == nil {
+		return "", fmt.Errorf("VM cannot be nil when generating nutanix provider ID for node")
+	}
+
+	// Check if customAttributes contains providerID
+	if vm.CustomAttributes != nil {
+		for _, attr := range vm.CustomAttributes {
+			// customAttributes are in the format "key:value"
+			parts := strings.SplitN(attr, ":", 2)
+			if len(parts) == 2 && strings.ToLower(strings.TrimSpace(parts[0])) == "providerid" {
+				customProviderID := strings.TrimSpace(parts[1])
+				if customProviderID != "" {
+					klog.V(2).Infof("Using custom providerID from customAttributes: %s", customProviderID) //nolint:typecheck
+					return fmt.Sprintf("%s://%s", constants.ProviderName, customProviderID), nil
+				}
+			}
+		}
+	}
+
+	// Fallback to using vmUUID
+	if vm.ExtId == nil || *vm.ExtId == "" {
+		return "", fmt.Errorf("VM ExtId cannot be empty when generating nutanix provider ID for node")
+	}
+
+	klog.V(2).Infof("Using VM ExtId as providerID: %s", *vm.ExtId) //nolint:typecheck
+	return fmt.Sprintf("%s://%s", constants.ProviderName, strings.ToLower(*vm.ExtId)), nil
+}
+
 func (n *nutanixManager) isNodeAddressesSet(node *v1.Node) bool {
 	if node == nil {
 		return false

pkg/provider/manager_test.go

@@ -268,6 +268,59 @@ var _ = Describe("Test Manager", func() { // nolint:typecheck
 		})
 	})
 
+	Context("Test generateProviderIDFromVM", func() {
+		It("should fail if VM is nil", func() { // nolint:typecheck
+			_, err := m.generateProviderIDFromVM(ctx, nil)
+			Expect(err).Should(HaveOccurred())
+		})
+
+		It("should return providerID from VM ExtId when no customAttributes", func() { // nolint:typecheck
+			vm := mockEnvironment.GetVM(ctx, mock.MockVMNamePoweredOn)
+			Expect(vm).ToNot(BeNil())
+			providerID, err := m.generateProviderIDFromVM(ctx, vm)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(providerID).To(Equal(fmt.Sprintf("nutanix://%s", *vm.ExtId)))
+		})
+
+		It("should return providerID from customAttributes when present", func() { // nolint:typecheck
+			vm := mockEnvironment.GetVM(ctx, mock.MockVMNameCustomProviderID)
+			Expect(vm).ToNot(BeNil())
+			providerID, err := m.generateProviderIDFromVM(ctx, vm)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(providerID).To(Equal(fmt.Sprintf("nutanix://%s", mock.MockCustomProviderID)))
+		})
+
+		It("should handle customAttributes with spaces around providerID", func() { // nolint:typecheck
+			vm := mockEnvironment.GetVM(ctx, mock.MockVMNamePoweredOn)
+			Expect(vm).ToNot(BeNil())
+			// Add customAttributes with spaces
+			vm.CustomAttributes = []string{" providerID : test-uuid-with-spaces ", "otherKey:value"}
+			providerID, err := m.generateProviderIDFromVM(ctx, vm)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(providerID).To(Equal("nutanix://test-uuid-with-spaces"))
+		})
+
+		It("should fallback to ExtId when customAttributes has no providerID", func() { // nolint:typecheck
+			vm := mockEnvironment.GetVM(ctx, mock.MockVMNamePoweredOn)
+			Expect(vm).ToNot(BeNil())
+			// Add customAttributes without providerID
+			vm.CustomAttributes = []string{"someKey:someValue", "anotherKey:anotherValue"}
+			providerID, err := m.generateProviderIDFromVM(ctx, vm)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(providerID).To(Equal(fmt.Sprintf("nutanix://%s", *vm.ExtId)))
+		})
+
+		It("should fallback to ExtId when providerID value is empty", func() { // nolint:typecheck
+			vm := mockEnvironment.GetVM(ctx, mock.MockVMNamePoweredOn)
+			Expect(vm).ToNot(BeNil())
+			// Add customAttributes with empty providerID value
+			vm.CustomAttributes = []string{"providerID:", "otherKey:value"}
+			providerID, err := m.generateProviderIDFromVM(ctx, vm)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(providerID).To(Equal(fmt.Sprintf("nutanix://%s", *vm.ExtId)))
+		})
+	})
+
 	Context("Test getTopologyInfoFromVM", func() {
 		It("should fail if vm is empty", func() { // nolint:typecheck
 			err := m.getTopologyInfoFromVM(ctx, nClient, nil, &config.TopologyInfo{})