Fetch internal IP from dhcp server (#750)

Author: Karthik-K-N

cloud/scope/powervs_machine.go

@@ -33,6 +33,7 @@ import (
 	powerVSUtils "github.com/ppc64le-cloud/powervs-utils"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/cache"
 	"k8s.io/klog/v2/klogr"
 	"k8s.io/utils/pointer"
 	capiv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1"
@@ -58,6 +59,7 @@ type PowerVSMachineScopeParams struct {
 	IBMPowerVSCluster *infrav1beta1.IBMPowerVSCluster
 	IBMPowerVSMachine *infrav1beta1.IBMPowerVSMachine
 	IBMPowerVSImage   *infrav1beta1.IBMPowerVSImage
+	DHCPIPCacheStore  cache.Store
 }
 
 // PowerVSMachineScope defines a scope defined around a Power VS Machine.
@@ -72,6 +74,7 @@ type PowerVSMachineScope struct {
 	IBMPowerVSCluster *infrav1beta1.IBMPowerVSCluster
 	IBMPowerVSMachine *infrav1beta1.IBMPowerVSMachine
 	IBMPowerVSImage   *infrav1beta1.IBMPowerVSImage
+	DHCPIPCacheStore  cache.Store
 }
 
 // NewPowerVSMachineScope creates a new PowerVSMachineScope from the supplied parameters.
@@ -152,7 +155,7 @@ func NewPowerVSMachineScope(params PowerVSMachineScopeParams) (scope *PowerVSMac
 	scope.SetRegion(region)
 	scope.SetZone(*res.RegionID)
 
-	options := powervs.ServiceOptions{
+	serviceOptions := powervs.ServiceOptions{
 		IBMPIOptions: &ibmpisession.IBMPIOptions{
 			Debug:       params.Logger.V(DEBUGLEVEL).Enabled(),
 			UserAccount: account,
@@ -161,13 +164,13 @@ func NewPowerVSMachineScope(params PowerVSMachineScopeParams) (scope *PowerVSMac
 		},
 		CloudInstanceID: m.Spec.ServiceInstanceID,
 	}
-	c, err := powervs.NewService(options)
+	c, err := powervs.NewService(serviceOptions)
 	if err != nil {
 		err = fmt.Errorf("failed to create PowerVS service")
 		return
 	}
 	scope.IBMPowerVSClient = c
-
+	scope.DHCPIPCacheStore = params.DHCPIPCacheStore
 	return scope, nil
 }
 
@@ -417,6 +420,97 @@ func (m *PowerVSMachineScope) SetAddresses(instance *models.PVMInstance) {
 		}
 	}
 	m.IBMPowerVSMachine.Status.Addresses = addresses
+	if len(addresses) > 2 {
+		// If the address length is more than 2 means either NodeInternalIP or NodeExternalIP is updated so return
+		return
+	}
+	// In this case there is no IP found under instance.Networks, So try to fetch the IP from cache or DHCP server
+	// Look for DHCP IP from the cache
+	obj, exists, err := m.DHCPIPCacheStore.GetByKey(*instance.ServerName)
+	if err != nil {
+		m.Error(err, "failed to fetch the DHCP IP address from cache store", "VM", *instance.ServerName)
+	}
+	if exists {
+		m.Info("found IP for VM from DHCP cache", "IP", obj.(powervs.VMip).IP, "VM", *instance.ServerName)
+		addresses = append(addresses, corev1.NodeAddress{
+			Type:    corev1.NodeInternalIP,
+			Address: obj.(powervs.VMip).IP,
+		})
+		m.IBMPowerVSMachine.Status.Addresses = addresses
+		return
+	}
+	// Fetch the VM network ID
+	networkID, err := getNetworkID(m.IBMPowerVSMachine.Spec.Network, m)
+	if err != nil {
+		m.Error(err, "failed to fetch network id from network resource", "VM", *instance.ServerName)
+		return
+	}
+	// Fetch the details of the network attached to the VM
+	var pvmNetwork *models.PVMInstanceNetwork
+	for _, network := range instance.Networks {
+		if network.NetworkID == *networkID {
+			pvmNetwork = network
+			m.Info("found network attached to VM", "Network ID", network.NetworkID, "VM", *instance.ServerName)
+		}
+	}
+	if pvmNetwork == nil {
+		m.Info("failed to get network attached to VM", "VM", *instance.ServerName, "Network ID", *networkID)
+		return
+	}
+	// Get all the DHCP servers
+	dhcpServer, err := m.IBMPowerVSClient.GetAllDHCPServers()
+	if err != nil {
+		m.Error(err, "failed to get DHCP server")
+		return
+	}
+	// Get the Details of DHCP server associated with the network
+	var dhcpServerDetails *models.DHCPServerDetail
+	for _, server := range dhcpServer {
+		if *server.Network.ID == *networkID {
+			m.Info("found DHCP server for network", "DHCP server ID", *server.ID, "network ID", *networkID)
+			dhcpServerDetails, err = m.IBMPowerVSClient.GetDHCPServer(*server.ID)
+			if err != nil {
+				m.Error(err, "failed to get DHCP server details", "DHCP server ID", *server.ID)
+				return
+			}
+			break
+		}
+	}
+	if dhcpServerDetails == nil {
+		errStr := fmt.Errorf("DHCP server details is nil")
+		m.Error(errStr, "DHCP server associated with network is nil", "Network ID", *networkID)
+		return
+	}
+
+	// Fetch the VM IP using VM's mac from DHCP server lease
+	var internalIP *string
+	for _, lease := range dhcpServerDetails.Leases {
+		if *lease.InstanceMacAddress == pvmNetwork.MacAddress {
+			m.Info("found internal ip for VM from DHCP lease", "IP", *lease.InstanceIP, "VM", *instance.ServerName)
+			internalIP = lease.InstanceIP
+			break
+		}
+	}
+	if internalIP == nil {
+		errStr := fmt.Errorf("internal IP is nil")
+		m.Error(errStr, "failed to get internal IP, DHCP lease not found for VM with MAC in DHCP network", "VM", *instance.ServerName,
+			"MAC", pvmNetwork.MacAddress, "DHCP server ID", *dhcpServerDetails.ID)
+		return
+	}
+	m.Info("found internal IP for VM from DHCP lease", "IP", *internalIP, "VM", *instance.ServerName)
+	addresses = append(addresses, corev1.NodeAddress{
+		Type:    corev1.NodeInternalIP,
+		Address: *internalIP,
+	})
+	// Update the cache with the ip and VM name
+	err = m.DHCPIPCacheStore.Add(powervs.VMip{
+		Name: *instance.ServerName,
+		IP:   *internalIP,
+	})
+	if err != nil {
+		m.Error(err, "failed to update the DHCP cache store with the IP", "VM", *instance.ServerName, "IP", *internalIP)
+	}
+	m.IBMPowerVSMachine.Status.Addresses = addresses
 }
 
 // SetInstanceState will set the state for the machine.

controllers/ibmpowervsmachine_controller.go

@@ -25,6 +25,7 @@ import (
 	"github.com/pkg/errors"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/record"
 	capiv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	capierrors "sigs.k8s.io/cluster-api/errors"
@@ -36,6 +37,7 @@ import (
 
 	infrav1beta1 "sigs.k8s.io/cluster-api-provider-ibmcloud/api/v1beta1"
 	"sigs.k8s.io/cluster-api-provider-ibmcloud/cloud/scope"
+	"sigs.k8s.io/cluster-api-provider-ibmcloud/pkg/cloud/services/powervs"
 	capibmrecord "sigs.k8s.io/cluster-api-provider-ibmcloud/pkg/record"
 )
 
@@ -47,6 +49,13 @@ type IBMPowerVSMachineReconciler struct {
 	Scheme   *runtime.Scheme
 }
 
+// dhcpCacheStore is a cache store to hold the Power VS VM DHCP IP.
+var dhcpCacheStore cache.Store
+
+func init() {
+	dhcpCacheStore = powervs.InitialiseDHCPCacheStore()
+}
+
 // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=ibmpowervsmachines,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=ibmpowervsmachines/status,verbs=get;update;patch
 
@@ -114,6 +123,7 @@ func (r *IBMPowerVSMachineReconciler) Reconcile(ctx context.Context, req ctrl.Re
 		Machine:           machine,
 		IBMPowerVSMachine: ibmPowerVSMachine,
 		IBMPowerVSImage:   ibmPowerVSImage,
+		DHCPIPCacheStore:  dhcpCacheStore,
 	})
 	if err != nil {
 		return ctrl.Result{}, errors.Errorf("failed to create scope: %+v", err)
@@ -160,7 +170,11 @@ func (r *IBMPowerVSMachineReconciler) reconcileDelete(scope *scope.PowerVSMachin
 		scope.Info("error deleting IBMPowerVSMachine")
 		return ctrl.Result{}, errors.Wrapf(err, "error deleting IBMPowerVSMachine %s/%s", scope.IBMPowerVSMachine.Namespace, scope.IBMPowerVSMachine.Name)
 	}
-
+	// Remove the cached VM IP
+	err := scope.DHCPIPCacheStore.Delete(powervs.VMip{Name: scope.IBMPowerVSMachine.Name})
+	if err != nil {
+		scope.Error(err, "failed to delete the VM entry from DHCP cache store", "VM", scope.IBMPowerVSMachine.Name)
+	}
 	return ctrl.Result{}, nil
 }
 

pkg/cloud/services/powervs/caching.go

@@ -0,0 +1,45 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package powervs
+
+import (
+	"time"
+
+	"k8s.io/client-go/tools/cache"
+)
+
+// CacheTTL is duration of time to store the vm ip in cache
+// Currently the default sync period is 10 minutes that means every 10 minutes
+// there will be a reconciliation, So setting cache timeout to 20 minutes so the cache updates will happen
+// once in 2 reconciliations.
+const CacheTTL = time.Duration(20) * time.Minute
+
+// VMip holds the vm name and corresponding dhcp ip used to cache the dhcp ip.
+type VMip struct {
+	Name string
+	IP   string
+}
+
+// CacheKeyFunc defines the key function required in TTLStore.
+func CacheKeyFunc(obj interface{}) (string, error) {
+	return obj.(VMip).Name, nil
+}
+
+// InitialiseDHCPCacheStore returns a new cache store.
+func InitialiseDHCPCacheStore() cache.Store {
+	return cache.NewTTLStore(CacheKeyFunc, CacheTTL)
+}

pkg/cloud/services/powervs/mock/powervs_generated.go

@@ -37,4 +37,6 @@ type PowerVS interface {
 	GetCosImages(id string) (*models.Job, error)
 	GetJob(id string) (*models.Job, error)
 	DeleteJob(id string) error
+	GetAllDHCPServers() (models.DHCPServers, error)
+	GetDHCPServer(id string) (*models.DHCPServerDetail, error)
 }

pkg/cloud/services/powervs/powervs.go

@@ -36,6 +36,7 @@ type Service struct {
 	networkClient  *instance.IBMPINetworkClient
 	imageClient    *instance.IBMPIImageClient
 	jobClient      *instance.IBMPIJobClient
+	dhcpClient     *instance.IBMPIDhcpClient
 }
 
 // ServiceOptions holds the PowerVS Service Options specific information.
@@ -110,6 +111,16 @@ func (s *Service) GetAllNetwork() (*models.Networks, error) {
 	return s.networkClient.GetAll()
 }
 
+// GetAllDHCPServers returns all the DHCP servers in the Power VS service instance.
+func (s *Service) GetAllDHCPServers() (models.DHCPServers, error) {
+	return s.dhcpClient.GetAll()
+}
+
+// GetDHCPServer returns the details for DHCP server associated with id.
+func (s *Service) GetDHCPServer(id string) (*models.DHCPServerDetail, error) {
+	return s.dhcpClient.Get(id)
+}
+
 // NewService returns a new service for the Power VS api client.
 func NewService(options ServiceOptions) (PowerVS, error) {
 	auth, err := authenticator.GetAuthenticator()
@@ -130,5 +141,6 @@ func NewService(options ServiceOptions) (PowerVS, error) {
 		networkClient:  instance.NewIBMPINetworkClient(ctx, session, options.CloudInstanceID),
 		imageClient:    instance.NewIBMPIImageClient(ctx, session, options.CloudInstanceID),
 		jobClient:      instance.NewIBMPIJobClient(ctx, session, options.CloudInstanceID),
+		dhcpClient:     instance.NewIBMPIDhcpClient(ctx, session, options.CloudInstanceID),
 	}, nil
 }