Updating ACS error custom counter after every ACS operation that returns an error.

Author: jweite-amazon

pkg/cloud/affinity_groups.go

@@ -45,6 +45,7 @@ func (c *client) FetchAffinityGroup(group *AffinityGroup) (reterr error) {
 		affinityGroup, count, err := c.cs.AffinityGroup.GetAffinityGroupByID(group.ID)
 		if err != nil {
 			// handle via multierr
+			c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 			return err
 		} else if count > 1 {
 			// handle via creating a new error.
@@ -59,6 +60,7 @@ func (c *client) FetchAffinityGroup(group *AffinityGroup) (reterr error) {
 		affinityGroup, count, err := c.cs.AffinityGroup.GetAffinityGroupByName(group.Name)
 		if err != nil {
 			// handle via multierr
+			c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 			return err
 		} else if count > 1 {
 			// handle via creating a new error.
@@ -78,6 +80,7 @@ func (c *client) GetOrCreateAffinityGroup(group *AffinityGroup) (retErr error) {
 		p.SetName(group.Name)
 		resp, err := c.cs.AffinityGroup.CreateAffinityGroup(p)
 		if err != nil {
+			c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 			return err
 		}
 		group.ID = resp.Id
@@ -90,6 +93,7 @@ func (c *client) DeleteAffinityGroup(group *AffinityGroup) (retErr error) {
 	setIfNotEmpty(group.ID, p.SetId)
 	setIfNotEmpty(group.Name, p.SetName)
 	_, retErr = c.cs.AffinityGroup.DeleteAffinityGroup(p)
+	c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(retErr)
 	return retErr
 }
 
@@ -98,6 +102,7 @@ type affinityGroups []AffinityGroup
 func (c *client) getCurrentAffinityGroups(csMachine *infrav1.CloudStackMachine) (affinityGroups, error) {
 	// Start by fetching VM details which includes an array of currently associated affinity groups.
 	if virtM, count, err := c.cs.VirtualMachine.GetVirtualMachineByID(*csMachine.Spec.InstanceID); err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return nil, err
 	} else if count > 1 {
 		return nil, errors.Errorf("found more than one VM for ID: %s", *csMachine.Spec.InstanceID)
@@ -149,15 +154,18 @@ func (c *client) stopAndModifyAffinityGroups(csMachine *infrav1.CloudStackMachin
 
 	p1 := c.cs.VirtualMachine.NewStopVirtualMachineParams(string(*csMachine.Spec.InstanceID))
 	if _, err := c.cs.VirtualMachine.StopVirtualMachine(p1); err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return err
 	}
 
 	if _, err := c.cs.AffinityGroup.UpdateVMAffinityGroup(agp); err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return err
 	}
 
 	p2 := c.cs.VirtualMachine.NewStartVirtualMachineParams(string(*csMachine.Spec.InstanceID))
 	_, err := c.cs.VirtualMachine.StartVirtualMachine(p2)
+	c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 	return err
 }
 

pkg/cloud/instance.go

@@ -56,6 +56,7 @@ func (c *client) ResolveVMInstanceDetails(csMachine *infrav1.CloudStackMachine)
 	if csMachine.Spec.InstanceID != nil {
 		vmResp, count, err := c.cs.VirtualMachine.GetVirtualMachinesMetricByID(*csMachine.Spec.InstanceID)
 		if err != nil && !strings.Contains(strings.ToLower(err.Error()), "no match found") {
+			c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 			return err
 		} else if count > 1 {
 			return fmt.Errorf("found more than one VM Instance with ID %s", *csMachine.Spec.InstanceID)
@@ -69,6 +70,7 @@ func (c *client) ResolveVMInstanceDetails(csMachine *infrav1.CloudStackMachine)
 	if csMachine.Name != "" {
 		vmResp, count, err := c.cs.VirtualMachine.GetVirtualMachinesMetricByName(csMachine.Name) // add opts usage
 		if err != nil && !strings.Contains(strings.ToLower(err.Error()), "no match") {
+			c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 			return err
 		} else if count > 1 {
 			return fmt.Errorf("found more than one VM Instance with name %s", csMachine.Name)
@@ -84,6 +86,7 @@ func (c *client) ResolveServiceOffering(csMachine *infrav1.CloudStackMachine) (o
 	if len(csMachine.Spec.Offering.ID) > 0 {
 		csOffering, count, err := c.cs.ServiceOffering.GetServiceOfferingByID(csMachine.Spec.Offering.ID)
 		if err != nil {
+			c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 			return "", multierror.Append(retErr, errors.Wrapf(
 				err, "could not get Service Offering by ID %s", csMachine.Spec.Offering.ID))
 		} else if count != 1 {
@@ -99,6 +102,7 @@ func (c *client) ResolveServiceOffering(csMachine *infrav1.CloudStackMachine) (o
 	}
 	offeringID, count, err := c.cs.ServiceOffering.GetServiceOfferingID(csMachine.Spec.Offering.Name)
 	if err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return "", multierror.Append(retErr, errors.Wrapf(
 			err, "could not get Service Offering ID from %s", csMachine.Spec.Offering.Name))
 	} else if count != 1 {
@@ -116,6 +120,7 @@ func (c *client) ResolveTemplate(
 	if len(csMachine.Spec.Template.ID) > 0 {
 		csTemplate, count, err := c.cs.Template.GetTemplateByID(csMachine.Spec.Template.ID, "executable")
 		if err != nil {
+			c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 			return "", multierror.Append(retErr, errors.Wrapf(
 				err, "could not get Template by ID %s", csMachine.Spec.Template.ID))
 		} else if count != 1 {
@@ -131,6 +136,7 @@ func (c *client) ResolveTemplate(
 	}
 	templateID, count, err := c.cs.Template.GetTemplateID(csMachine.Spec.Template.Name, "executable", zoneID)
 	if err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return "", multierror.Append(retErr, errors.Wrapf(
 			err, "could not get Template ID from %s", csMachine.Spec.Template.Name))
 	} else if count != 1 {
@@ -148,6 +154,7 @@ func (c *client) ResolveDiskOffering(csMachine *infrav1.CloudStackMachine) (disk
 	if len(csMachine.Spec.DiskOffering.Name) > 0 {
 		diskID, count, err := c.cs.DiskOffering.GetDiskOfferingID(csMachine.Spec.DiskOffering.Name)
 		if err != nil {
+			c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 			return "", multierror.Append(retErr, errors.Wrapf(
 				err, "could not get DiskOffering ID from %s", csMachine.Spec.DiskOffering.Name))
 		} else if count != 1 {
@@ -174,6 +181,7 @@ func (c *client) ResolveDiskOffering(csMachine *infrav1.CloudStackMachine) (disk
 func verifyDiskoffering(csMachine *infrav1.CloudStackMachine, c *client, diskOfferingID string, retErr error) (string, error) {
 	csDiskOffering, count, err := c.cs.DiskOffering.GetDiskOfferingByID(diskOfferingID)
 	if err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return "", multierror.Append(retErr, errors.Wrapf(
 			err, "could not get DiskOffering by ID %s", diskOfferingID))
 	} else if count != 1 {
@@ -255,6 +263,8 @@ func (c *client) GetOrCreateVMInstance(
 
 	deployVMResp, err := c.cs.VirtualMachine.DeployVirtualMachine(p)
 	if err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
+
 		// Just because an error was returned doesn't mean a (failed) VM wasn't created and will need to be dealt with.
 		// Regretfully the deployVMResp may be nil, so we need to get the VM ID with a separate query, so we
 		// can return it to the caller, so they can clean it up.
@@ -265,6 +275,8 @@ func (c *client) GetOrCreateVMInstance(
 		listVirtualMachineParams.SetName(csMachine.Name)
 		if listVirtualMachinesResponse, err2 := c.cs.VirtualMachine.ListVirtualMachines(listVirtualMachineParams); err2 == nil && listVirtualMachinesResponse.Count > 0 {
 			csMachine.Spec.InstanceID = pointer.StringPtr(listVirtualMachinesResponse.VirtualMachines[0].Id)
+		} else {
+			c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err2)
 		}
 		return err
 	}
@@ -290,6 +302,7 @@ func (c *client) DestroyVMInstance(csMachine *infrav1.CloudStackMachine) error {
 		// VM doesn't exist. Success...
 		return nil
 	} else if err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return err
 	}
 
@@ -316,6 +329,7 @@ func (c *client) listVMInstanceDatadiskVolumeIDs(instanceID string) ([]string, e
 
 	listVOLResp, err := c.csAsync.Volume.ListVolumes(p)
 	if err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return nil, err
 	}
 

pkg/cloud/isolated_network.go

@@ -44,6 +44,7 @@ type IsoNetworkIface interface {
 func (c *client) getOfferingID() (string, error) {
 	offeringID, count, retErr := c.cs.NetworkOffering.GetNetworkOfferingID(NetOffering)
 	if retErr != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(retErr)
 		return "", retErr
 	} else if count != 1 {
 		return "", errors.New("found more than one network offering")
@@ -60,7 +61,6 @@ func (c *client) AssociatePublicIPAddress(
 	// Check specified IP address is available or get an unused one if not specified.
 	publicAddress, err := c.GetPublicIP(zone, isoNet, csCluster)
 	if err != nil {
-		c.customMetrics.IncrementAcsReconciliationErrors(err)
 		return errors.Wrapf(err, "fetching a public IP address")
 	}
 	isoNet.Spec.ControlPlaneEndpoint.Host = publicAddress.Ipaddress
@@ -77,6 +77,7 @@ func (c *client) AssociatePublicIPAddress(
 	p.SetIpaddress(isoNet.Spec.ControlPlaneEndpoint.Host)
 	p.SetNetworkid(isoNet.Spec.ID)
 	if _, err := c.cs.Address.AssociateIpAddress(p); err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return errors.Wrapf(err,
 			"associating public IP address with ID %s to network with ID %s",
 			publicAddress.Id, isoNet.Spec.ID)
@@ -102,6 +103,7 @@ func (c *client) CreateIsolatedNetwork(zone *capcv1.CloudStackZone, isoNet *capc
 	p := c.cs.Network.NewCreateNetworkParams(isoNet.Spec.Name, isoNet.Spec.Name, offeringID, zone.Spec.ID)
 	resp, err := c.cs.Network.CreateNetwork(p)
 	if err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return errors.Wrapf(err, "creating network with name %s", isoNet.Spec.Name)
 	}
 	isoNet.Spec.ID = resp.Id
@@ -115,6 +117,7 @@ func (c *client) OpenFirewallRules(isoNet *capcv1.CloudStackIsolatedNetwork) (re
 	if retErr != nil && strings.Contains(strings.ToLower(retErr.Error()), "there is already") { // Already a firewall rule here.
 		retErr = nil
 	}
+	c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(retErr)
 	return retErr
 }
 
@@ -132,6 +135,7 @@ func (c *client) GetPublicIP(
 	setIfNotEmpty(ip, p.SetIpaddress)
 	publicAddresses, err := c.cs.Address.ListPublicIpAddresses(p)
 	if err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return nil, err
 	} else if ip != "" && publicAddresses.Count == 1 { // Endpoint specified and IP found.
 		// Ignore already allocated here since the IP was specified.
@@ -157,6 +161,7 @@ func (c *client) GetPublicIP(
 func (c *client) GetIsolatedNetwork(isoNet *capcv1.CloudStackIsolatedNetwork) (retErr error) {
 	netDetails, count, err := c.cs.Network.GetNetworkByName(isoNet.Spec.Name)
 	if err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		retErr = multierror.Append(retErr, errors.Wrapf(err, "could not get Network ID from %s", isoNet.Spec.Name))
 	} else if count != 1 {
 		retErr = multierror.Append(retErr, errors.Errorf(
@@ -168,6 +173,7 @@ func (c *client) GetIsolatedNetwork(isoNet *capcv1.CloudStackIsolatedNetwork) (r
 
 	netDetails, count, err = c.cs.Network.GetNetworkByID(isoNet.Spec.ID)
 	if err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return multierror.Append(retErr, errors.Wrapf(err, "could not get Network by ID %s", isoNet.Spec.ID))
 	} else if count != 1 {
 		return multierror.Append(retErr, errors.Errorf("expected 1 Network with UUID %s, but got %d", isoNet.Spec.ID, count))
@@ -186,6 +192,7 @@ func (c *client) ResolveLoadBalancerRuleDetails(
 	p.SetPublicipid(isoNet.Status.PublicIPID)
 	loadBalancerRules, err := c.cs.LoadBalancer.ListLoadBalancerRules(p)
 	if err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return errors.Wrap(err, "listing load balancer rules")
 	}
 	for _, rule := range loadBalancerRules.LoadBalancerRules {
@@ -230,6 +237,7 @@ func (c *client) GetOrCreateLoadBalancerRule(
 	p.SetProtocol(NetworkProtocolTCP)
 	resp, err := c.cs.LoadBalancer.CreateLoadBalancerRule(p)
 	if err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return err
 	}
 	isoNet.Status.LBRuleID = resp.Id
@@ -279,6 +287,7 @@ func (c *client) AssignVMToLoadBalancerRule(isoNet *capcv1.CloudStackIsolatedNet
 	lbRuleInstances, retErr := c.cs.LoadBalancer.ListLoadBalancerRuleInstances(
 		c.cs.LoadBalancer.NewListLoadBalancerRuleInstancesParams(isoNet.Status.LBRuleID))
 	if retErr != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(retErr)
 		return retErr
 	}
 	for _, instance := range lbRuleInstances.LoadBalancerRuleInstances {
@@ -291,12 +300,14 @@ func (c *client) AssignVMToLoadBalancerRule(isoNet *capcv1.CloudStackIsolatedNet
 	p := c.cs.LoadBalancer.NewAssignToLoadBalancerRuleParams(isoNet.Status.LBRuleID)
 	p.SetVirtualmachineids([]string{instanceID})
 	_, retErr = c.cs.LoadBalancer.AssignToLoadBalancerRule(p)
+	c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(retErr)
 	return retErr
 }
 
 // DeleteNetwork deletes an isolated network.
 func (c *client) DeleteNetwork(net capcv1.Network) error {
 	_, err := c.cs.Network.DeleteNetwork(c.cs.Network.NewDeleteNetworkParams(net.ID))
+	c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 	return errors.Wrapf(err, "deleting network with id %s", net.ID)
 }
 
@@ -351,6 +362,7 @@ func (c *client) DisassociatePublicIPAddressIfNotInUse(isoNet *capcv1.CloudStack
 	if tagsAllowDisposal, err := c.DoClusterTagsAllowDisposal(ResourceTypeIPAddress, isoNet.Status.PublicIPID); err != nil {
 		return err
 	} else if publicIP, _, err := c.cs.Address.GetPublicIpAddressByID(isoNet.Status.PublicIPID); err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return err
 	} else if publicIP == nil || publicIP.Issourcenat { // Can't disassociate an address if it's the source NAT address.
 		return nil
@@ -370,5 +382,6 @@ func (c *client) DisassociatePublicIPAddress(isoNet *capcv1.CloudStackIsolatedNe
 
 	p := c.cs.Address.NewDisassociateIpAddressParams(isoNet.Status.PublicIPID)
 	_, retErr = c.cs.Address.DisassociateIpAddress(p)
+	c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(retErr)
 	return retErr
 }

pkg/cloud/network.go

@@ -50,6 +50,7 @@ func (c *client) ResolveNetwork(net *capcv1.Network) (retErr error) {
 	netName := net.Name
 	netDetails, count, err := c.cs.Network.GetNetworkByName(netName)
 	if err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		retErr = multierror.Append(retErr, errors.Wrapf(err, "could not get Network ID from %s", netName))
 	} else if count != 1 {
 		retErr = multierror.Append(retErr, errors.Errorf(
@@ -65,6 +66,7 @@ func (c *client) ResolveNetwork(net *capcv1.Network) (retErr error) {
 	if err != nil {
 		return multierror.Append(retErr, errors.Wrapf(err, "could not get Network by ID %s", net.ID))
 	} else if count != 1 {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return multierror.Append(retErr, errors.Errorf("expected 1 Network with UUID %s, but got %d", net.ID, count))
 	}
 	net.Name = netDetails.Name

pkg/cloud/tags.go

@@ -119,6 +119,7 @@ func (c *client) DoClusterTagsAllowDisposal(resourceType ResourceType, resourceI
 func (c *client) AddTags(resourceType ResourceType, resourceID string, tags map[string]string) error {
 	p := c.cs.Resourcetags.NewCreateTagsParams([]string{resourceID}, string(resourceType), tags)
 	_, err := c.cs.Resourcetags.CreateTags(p)
+	c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 	return ignoreAlreadyPresentErrors(err, resourceType, resourceID)
 }
 
@@ -130,6 +131,7 @@ func (c *client) GetTags(resourceType ResourceType, resourceID string) (map[stri
 	p.SetListall(true)
 	listTagResponse, err := c.cs.Resourcetags.ListTags(p)
 	if err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return nil, err
 	}
 	tags := make(map[string]string, listTagResponse.Count)
@@ -146,8 +148,10 @@ func (c *client) DeleteTags(resourceType ResourceType, resourceID string, tagsTo
 		p := c.cs.Resourcetags.NewDeleteTagsParams([]string{resourceID}, string(resourceType))
 		p.SetTags(tagsToDelete)
 		if _, err1 := c.cs.Resourcetags.DeleteTags(p); err1 != nil { // Error in deletion attempt. Check for tag.
+			c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err1)
 			currTag := map[string]string{tagkey: tagval}
 			if tags, err2 := c.GetTags(resourceType, resourceID); len(tags) != 0 {
+				c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err2)
 				if _, foundTag := tags[tagkey]; foundTag {
 					return errors.Wrapf(multierror.Append(err1, err2),
 						"could not remove tag %s from %s with ID %s", currTag, resourceType, resourceID)

pkg/cloud/user_credentials.go

@@ -91,6 +91,7 @@ func (c *client) ResolveDomain(domain *Domain) error {
 
 	resp, retErr := c.cs.Domain.ListDomains(p)
 	if retErr != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(retErr)
 		return retErr
 	}
 
@@ -139,6 +140,7 @@ func (c *client) ResolveAccount(account *Account) error {
 	setIfNotEmpty(account.Name, p.SetName)
 	resp, retErr := c.cs.Account.ListAccounts(p)
 	if retErr != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(retErr)
 		return retErr
 	} else if resp.Count == 0 {
 		return errors.Errorf("could not find account %s", account.Name)
@@ -164,6 +166,7 @@ func (c *client) ResolveUser(user *User) error {
 	p.SetListall(true)
 	resp, err := c.cs.User.ListUsers(p)
 	if err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return err
 	} else if resp.Count != 1 {
 		return errors.Errorf("expected 1 User with username %s but got %d", user.Name, resp.Count)
@@ -185,6 +188,7 @@ func (c *client) ResolveUserKeys(user *User) error {
 	p := c.cs.User.NewGetUserKeysParams(user.ID)
 	resp, err := c.cs.User.GetUserKeys(p)
 	if err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return errors.Errorf("error encountered when resolving user api keys for user %s", user.Name)
 	}
 	user.APIKey = resp.Apikey
@@ -207,6 +211,7 @@ func (c *client) GetUserWithKeys(user *User) (bool, error) {
 	p.SetListall(true)
 	resp, err := c.cs.User.ListUsers(p)
 	if err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return false, err
 	}