Merge branch 'kubernetes-sigs:main' into main

Author: mrog

Makefile

@@ -1,6 +1,10 @@
 # Image URL to use all building/pushing image targets
-IMG ?= public.ecr.aws/a4z9h2b1/cluster-api-provider-capc:latest
-IMG_LOCAL ?= localhost:5000/cluster-api-provider-cloudstack:latest
+STAGING_REGISTRY := gcr.io/k8s-staging-capi-cloudstack
+REGISTRY ?= $(STAGING_REGISTRY)
+IMAGE_NAME ?= capi-cloudstack-controller
+TAG ?= dev
+IMG ?= $(REGISTRY)/$(IMAGE_NAME):$(TAG)
+IMG_LOCAL ?= localhost:5000/$(IMAGE_NAME):$(TAG)
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -45,7 +49,7 @@ help: ## Display this help.
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 
  MANIFEST_GEN_INPUTS=$(shell find ./api ./controllers -type f -name "*test*" -prune -o -name "*zz_generated*" -prune -o -print)
- 
+
 # Using a flag file here as config output is too complicated to be a target.
 # The following triggers manifest building if $(IMG) differs from that found in config/default/manager_image_patch.yaml.
 $(shell	grep -qs "$(IMG)" config/default/manager_image_patch_edited.yaml || rm -f config/.flag.mk)
@@ -58,7 +62,7 @@ config/.flag.mk: bin/controller-gen $(MANIFEST_GEN_INPUTS)
 
 .PHONY: release-manifests
 RELEASE_MANIFEST_TARGETS=$(RELEASE_DIR)/infrastructure-components.yaml $(RELEASE_DIR)/metadata.yaml
-RELEASE_MANIFEST_INPUTS=bin/kustomize config/.flag.mk $(shell find config) 
+RELEASE_MANIFEST_INPUTS=bin/kustomize config/.flag.mk $(shell find config)
 release-manifests: $(RELEASE_MANIFEST_TARGETS) ## Create kustomized release manifest in $RELEASE_DIR (defaults to out).
 $(RELEASE_DIR)/%: $(RELEASE_MANIFEST_INPUTS)
 	@mkdir -p $(RELEASE_DIR)
@@ -146,7 +150,7 @@ bin/golangci-lint: ## Install golangci-lint to bin.
 bin/staticcheck: ## Install staticcheck to bin.
 	GOBIN=$(PROJECT_DIR)/bin go install honnef.co/go/tools/cmd/[email protected]
 bin/ginkgo: bin/ginkgo_v1 bin/ginkgo_v2 ## Install ginkgo to bin.
-bin/ginkgo_v2: 
+bin/ginkgo_v2:
 	GOBIN=$(PROJECT_DIR)/bin go install github.com/onsi/ginkgo/v2/[email protected]
 	mv $(PROJECT_DIR)/bin/ginkgo $(PROJECT_DIR)/bin/ginkgo_v2
 bin/ginkgo_v1:
@@ -180,15 +184,15 @@ test: generate-mocks lint bin/ginkgo bin/kubectl bin/kube-apiserver bin/etcd ##
 	@# The following is a slightly funky way to make sure the ginkgo statements are removed regardless the test results.
 	@ginkgo_v2 --label-filter="!integ" --cover -coverprofile cover.out --covermode=atomic -v ./api/... ./controllers/... ./pkg/...; EXIT_STATUS=$$?;\
 		./hack/testing_ginkgo_recover_statements.sh --remove; exit $$EXIT_STATUS
-	
+
 .PHONY: generate-mocks
 generate-mocks: bin/mockgen generate-deepcopy pkg/mocks/mock_client.go $(shell find ./pkg/mocks -type f -name "mock*.go") ## Generate mocks needed for testing. Primarily mocks of the cloud package.
 pkg/mocks/mock%.go: $(shell find ./pkg/cloud -type f -name "*test*" -prune -o -print)
 	go generate ./...
 
 ##@ Tilt
 
-.PHONY: tilt-up 
+.PHONY: tilt-up
 tilt-up: cluster-api kind-cluster cluster-api/tilt-settings.json manifests cloud-config ## Setup and run tilt for development.
 	export CLOUDSTACK_B64ENCODED_SECRET=$$(base64 -w0 -i cloud-config 2>/dev/null || base64 -b 0 -i cloud-config) && cd cluster-api && tilt up
 

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/onsi/gomega v1.19.0
 	github.com/pkg/errors v0.9.1
 	github.com/smallfish/simpleyaml v0.1.0
+	github.com/prometheus/client_golang v1.11.0
 	github.com/spf13/pflag v1.0.5
 	gopkg.in/ini.v1 v1.63.2
 	k8s.io/api v0.23.0

pkg/cloud/affinity_groups.go

@@ -45,6 +45,7 @@ func (c *client) FetchAffinityGroup(group *AffinityGroup) (reterr error) {
 		affinityGroup, count, err := c.cs.AffinityGroup.GetAffinityGroupByID(group.ID)
 		if err != nil {
 			// handle via multierr
+			c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 			return err
 		} else if count > 1 {
 			// handle via creating a new error.
@@ -59,6 +60,7 @@ func (c *client) FetchAffinityGroup(group *AffinityGroup) (reterr error) {
 		affinityGroup, count, err := c.cs.AffinityGroup.GetAffinityGroupByName(group.Name)
 		if err != nil {
 			// handle via multierr
+			c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 			return err
 		} else if count > 1 {
 			// handle via creating a new error.
@@ -78,6 +80,7 @@ func (c *client) GetOrCreateAffinityGroup(group *AffinityGroup) (retErr error) {
 		p.SetName(group.Name)
 		resp, err := c.cs.AffinityGroup.CreateAffinityGroup(p)
 		if err != nil {
+			c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 			return err
 		}
 		group.ID = resp.Id
@@ -90,6 +93,7 @@ func (c *client) DeleteAffinityGroup(group *AffinityGroup) (retErr error) {
 	setIfNotEmpty(group.ID, p.SetId)
 	setIfNotEmpty(group.Name, p.SetName)
 	_, retErr = c.cs.AffinityGroup.DeleteAffinityGroup(p)
+	c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(retErr)
 	return retErr
 }
 
@@ -98,6 +102,7 @@ type affinityGroups []AffinityGroup
 func (c *client) getCurrentAffinityGroups(csMachine *infrav1.CloudStackMachine) (affinityGroups, error) {
 	// Start by fetching VM details which includes an array of currently associated affinity groups.
 	if virtM, count, err := c.cs.VirtualMachine.GetVirtualMachineByID(*csMachine.Spec.InstanceID); err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return nil, err
 	} else if count > 1 {
 		return nil, errors.Errorf("found more than one VM for ID: %s", *csMachine.Spec.InstanceID)
@@ -149,15 +154,18 @@ func (c *client) stopAndModifyAffinityGroups(csMachine *infrav1.CloudStackMachin
 
 	p1 := c.cs.VirtualMachine.NewStopVirtualMachineParams(string(*csMachine.Spec.InstanceID))
 	if _, err := c.cs.VirtualMachine.StopVirtualMachine(p1); err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return err
 	}
 
 	if _, err := c.cs.AffinityGroup.UpdateVMAffinityGroup(agp); err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return err
 	}
 
 	p2 := c.cs.VirtualMachine.NewStartVirtualMachineParams(string(*csMachine.Spec.InstanceID))
 	_, err := c.cs.VirtualMachine.StartVirtualMachine(p2)
+	c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 	return err
 }
 

pkg/cloud/affinity_groups_test.go

@@ -88,7 +88,8 @@ var _ = Describe("AffinityGroup Unit Tests", func() {
 			// Make the created VM go away quickly by force stopping it.
 			p := realCSClient.VirtualMachine.NewStopVirtualMachineParams(*dummies.CSMachine1.Spec.InstanceID)
 			p.SetForced(true)
-			Ω(realCSClient.VirtualMachine.StopVirtualMachine(p)).Should(Succeed())
+			_, err := realCSClient.VirtualMachine.StopVirtualMachine(p)
+			Ω(err).ShouldNot(HaveOccurred())
 		})
 
 		It("Creates and deletes an affinity group.", func() {

pkg/cloud/client.go

@@ -17,6 +17,7 @@ limitations under the License.
 package cloud
 
 import (
+	"sigs.k8s.io/cluster-api-provider-cloudstack/pkg/metrics"
 	"strings"
 
 	"github.com/apache/cloudstack-go/v2/cloudstack"
@@ -40,9 +41,10 @@ type Client interface {
 }
 
 type client struct {
-	cs      *cloudstack.CloudStackClient
-	csAsync *cloudstack.CloudStackClient
-	config  Config
+	cs            *cloudstack.CloudStackClient
+	csAsync       *cloudstack.CloudStackClient
+	config        Config
+	customMetrics metrics.ACSCustomMetrics
 }
 
 // cloud-config ini structure.
@@ -68,6 +70,7 @@ func NewClient(ccPath string) (Client, error) {
 	// comments for more details
 	c.cs = cloudstack.NewAsyncClient(c.config.APIURL, c.config.APIKey, c.config.SecretKey, c.config.VerifySSL)
 	c.csAsync = cloudstack.NewClient(c.config.APIURL, c.config.APIKey, c.config.SecretKey, c.config.VerifySSL)
+	c.customMetrics = metrics.NewCustomMetrics()
 
 	_, err := c.cs.APIDiscovery.ListApis(c.cs.APIDiscovery.NewListApisParams())
 	if err != nil && strings.Contains(strings.ToLower(err.Error()), "i/o timeout") {
@@ -91,6 +94,7 @@ func (origC *client) NewClientFromSpec(cfg Config) (Client, error) {
 	// comments for more details
 	newC.cs = cloudstack.NewAsyncClient(newC.config.APIURL, newC.config.APIKey, newC.config.SecretKey, newC.config.VerifySSL)
 	newC.csAsync = cloudstack.NewClient(newC.config.APIURL, newC.config.APIKey, newC.config.SecretKey, newC.config.VerifySSL)
+	newC.customMetrics = metrics.NewCustomMetrics()
 
 	_, err := newC.cs.APIDiscovery.ListApis(newC.cs.APIDiscovery.NewListApisParams())
 	if err != nil && strings.Contains(strings.ToLower(err.Error()), "i/o timeout") {
@@ -100,6 +104,6 @@ func (origC *client) NewClientFromSpec(cfg Config) (Client, error) {
 }
 
 func NewClientFromCSAPIClient(cs *cloudstack.CloudStackClient) Client {
-	c := &client{cs: cs, csAsync: cs}
+	c := &client{cs: cs, csAsync: cs, customMetrics: metrics.NewCustomMetrics()}
 	return c
 }

pkg/cloud/instance.go

@@ -56,6 +56,7 @@ func (c *client) ResolveVMInstanceDetails(csMachine *infrav1.CloudStackMachine)
 	if csMachine.Spec.InstanceID != nil {
 		vmResp, count, err := c.cs.VirtualMachine.GetVirtualMachinesMetricByID(*csMachine.Spec.InstanceID)
 		if err != nil && !strings.Contains(strings.ToLower(err.Error()), "no match found") {
+			c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 			return err
 		} else if count > 1 {
 			return fmt.Errorf("found more than one VM Instance with ID %s", *csMachine.Spec.InstanceID)
@@ -69,6 +70,7 @@ func (c *client) ResolveVMInstanceDetails(csMachine *infrav1.CloudStackMachine)
 	if csMachine.Name != "" {
 		vmResp, count, err := c.cs.VirtualMachine.GetVirtualMachinesMetricByName(csMachine.Name) // add opts usage
 		if err != nil && !strings.Contains(strings.ToLower(err.Error()), "no match") {
+			c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 			return err
 		} else if count > 1 {
 			return fmt.Errorf("found more than one VM Instance with name %s", csMachine.Name)
@@ -84,6 +86,7 @@ func (c *client) ResolveServiceOffering(csMachine *infrav1.CloudStackMachine) (o
 	if len(csMachine.Spec.Offering.ID) > 0 {
 		csOffering, count, err := c.cs.ServiceOffering.GetServiceOfferingByID(csMachine.Spec.Offering.ID)
 		if err != nil {
+			c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 			return "", multierror.Append(retErr, errors.Wrapf(
 				err, "could not get Service Offering by ID %s", csMachine.Spec.Offering.ID))
 		} else if count != 1 {
@@ -99,6 +102,7 @@ func (c *client) ResolveServiceOffering(csMachine *infrav1.CloudStackMachine) (o
 	}
 	offeringID, count, err := c.cs.ServiceOffering.GetServiceOfferingID(csMachine.Spec.Offering.Name)
 	if err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return "", multierror.Append(retErr, errors.Wrapf(
 			err, "could not get Service Offering ID from %s", csMachine.Spec.Offering.Name))
 	} else if count != 1 {
@@ -116,6 +120,7 @@ func (c *client) ResolveTemplate(
 	if len(csMachine.Spec.Template.ID) > 0 {
 		csTemplate, count, err := c.cs.Template.GetTemplateByID(csMachine.Spec.Template.ID, "executable")
 		if err != nil {
+			c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 			return "", multierror.Append(retErr, errors.Wrapf(
 				err, "could not get Template by ID %s", csMachine.Spec.Template.ID))
 		} else if count != 1 {
@@ -131,6 +136,7 @@ func (c *client) ResolveTemplate(
 	}
 	templateID, count, err := c.cs.Template.GetTemplateID(csMachine.Spec.Template.Name, "executable", zoneID)
 	if err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return "", multierror.Append(retErr, errors.Wrapf(
 			err, "could not get Template ID from %s", csMachine.Spec.Template.Name))
 	} else if count != 1 {
@@ -148,6 +154,7 @@ func (c *client) ResolveDiskOffering(csMachine *infrav1.CloudStackMachine) (disk
 	if len(csMachine.Spec.DiskOffering.Name) > 0 {
 		diskID, count, err := c.cs.DiskOffering.GetDiskOfferingID(csMachine.Spec.DiskOffering.Name)
 		if err != nil {
+			c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 			return "", multierror.Append(retErr, errors.Wrapf(
 				err, "could not get DiskOffering ID from %s", csMachine.Spec.DiskOffering.Name))
 		} else if count != 1 {
@@ -174,6 +181,7 @@ func (c *client) ResolveDiskOffering(csMachine *infrav1.CloudStackMachine) (disk
 func verifyDiskoffering(csMachine *infrav1.CloudStackMachine, c *client, diskOfferingID string, retErr error) (string, error) {
 	csDiskOffering, count, err := c.cs.DiskOffering.GetDiskOfferingByID(diskOfferingID)
 	if err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return "", multierror.Append(retErr, errors.Wrapf(
 			err, "could not get DiskOffering by ID %s", diskOfferingID))
 	} else if count != 1 {
@@ -255,6 +263,8 @@ func (c *client) GetOrCreateVMInstance(
 
 	deployVMResp, err := c.cs.VirtualMachine.DeployVirtualMachine(p)
 	if err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
+
 		// Just because an error was returned doesn't mean a (failed) VM wasn't created and will need to be dealt with.
 		// Regretfully the deployVMResp may be nil, so we need to get the VM ID with a separate query, so we
 		// can return it to the caller, so they can clean it up.
@@ -265,6 +275,8 @@ func (c *client) GetOrCreateVMInstance(
 		listVirtualMachineParams.SetName(csMachine.Name)
 		if listVirtualMachinesResponse, err2 := c.cs.VirtualMachine.ListVirtualMachines(listVirtualMachineParams); err2 == nil && listVirtualMachinesResponse.Count > 0 {
 			csMachine.Spec.InstanceID = pointer.StringPtr(listVirtualMachinesResponse.VirtualMachines[0].Id)
+		} else {
+			c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err2)
 		}
 		return err
 	}
@@ -290,6 +302,7 @@ func (c *client) DestroyVMInstance(csMachine *infrav1.CloudStackMachine) error {
 		// VM doesn't exist. Success...
 		return nil
 	} else if err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return err
 	}
 
@@ -316,6 +329,7 @@ func (c *client) listVMInstanceDatadiskVolumeIDs(instanceID string) ([]string, e
 
 	listVOLResp, err := c.csAsync.Volume.ListVolumes(p)
 	if err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return nil, err
 	}
 

pkg/cloud/isolated_network.go

@@ -44,6 +44,7 @@ type IsoNetworkIface interface {
 func (c *client) getOfferingID() (string, error) {
 	offeringID, count, retErr := c.cs.NetworkOffering.GetNetworkOfferingID(NetOffering)
 	if retErr != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(retErr)
 		return "", retErr
 	} else if count != 1 {
 		return "", errors.New("found more than one network offering")
@@ -76,6 +77,7 @@ func (c *client) AssociatePublicIPAddress(
 	p.SetIpaddress(isoNet.Spec.ControlPlaneEndpoint.Host)
 	p.SetNetworkid(isoNet.Spec.ID)
 	if _, err := c.cs.Address.AssociateIpAddress(p); err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return errors.Wrapf(err,
 			"associating public IP address with ID %s to network with ID %s",
 			publicAddress.Id, isoNet.Spec.ID)
@@ -101,6 +103,7 @@ func (c *client) CreateIsolatedNetwork(zone *capcv1.CloudStackZone, isoNet *capc
 	p := c.cs.Network.NewCreateNetworkParams(isoNet.Spec.Name, isoNet.Spec.Name, offeringID, zone.Spec.ID)
 	resp, err := c.cs.Network.CreateNetwork(p)
 	if err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return errors.Wrapf(err, "creating network with name %s", isoNet.Spec.Name)
 	}
 	isoNet.Spec.ID = resp.Id
@@ -114,6 +117,7 @@ func (c *client) OpenFirewallRules(isoNet *capcv1.CloudStackIsolatedNetwork) (re
 	if retErr != nil && strings.Contains(strings.ToLower(retErr.Error()), "there is already") { // Already a firewall rule here.
 		retErr = nil
 	}
+	c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(retErr)
 	return retErr
 }
 
@@ -131,6 +135,7 @@ func (c *client) GetPublicIP(
 	setIfNotEmpty(ip, p.SetIpaddress)
 	publicAddresses, err := c.cs.Address.ListPublicIpAddresses(p)
 	if err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return nil, err
 	} else if ip != "" && publicAddresses.Count == 1 { // Endpoint specified and IP found.
 		// Ignore already allocated here since the IP was specified.
@@ -156,6 +161,7 @@ func (c *client) GetPublicIP(
 func (c *client) GetIsolatedNetwork(isoNet *capcv1.CloudStackIsolatedNetwork) (retErr error) {
 	netDetails, count, err := c.cs.Network.GetNetworkByName(isoNet.Spec.Name)
 	if err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		retErr = multierror.Append(retErr, errors.Wrapf(err, "could not get Network ID from %s", isoNet.Spec.Name))
 	} else if count != 1 {
 		retErr = multierror.Append(retErr, errors.Errorf(
@@ -167,6 +173,7 @@ func (c *client) GetIsolatedNetwork(isoNet *capcv1.CloudStackIsolatedNetwork) (r
 
 	netDetails, count, err = c.cs.Network.GetNetworkByID(isoNet.Spec.ID)
 	if err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return multierror.Append(retErr, errors.Wrapf(err, "could not get Network by ID %s", isoNet.Spec.ID))
 	} else if count != 1 {
 		return multierror.Append(retErr, errors.Errorf("expected 1 Network with UUID %s, but got %d", isoNet.Spec.ID, count))
@@ -185,6 +192,7 @@ func (c *client) ResolveLoadBalancerRuleDetails(
 	p.SetPublicipid(isoNet.Status.PublicIPID)
 	loadBalancerRules, err := c.cs.LoadBalancer.ListLoadBalancerRules(p)
 	if err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return errors.Wrap(err, "listing load balancer rules")
 	}
 	for _, rule := range loadBalancerRules.LoadBalancerRules {
@@ -229,6 +237,7 @@ func (c *client) GetOrCreateLoadBalancerRule(
 	p.SetProtocol(NetworkProtocolTCP)
 	resp, err := c.cs.LoadBalancer.CreateLoadBalancerRule(p)
 	if err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return err
 	}
 	isoNet.Status.LBRuleID = resp.Id
@@ -278,6 +287,7 @@ func (c *client) AssignVMToLoadBalancerRule(isoNet *capcv1.CloudStackIsolatedNet
 	lbRuleInstances, retErr := c.cs.LoadBalancer.ListLoadBalancerRuleInstances(
 		c.cs.LoadBalancer.NewListLoadBalancerRuleInstancesParams(isoNet.Status.LBRuleID))
 	if retErr != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(retErr)
 		return retErr
 	}
 	for _, instance := range lbRuleInstances.LoadBalancerRuleInstances {
@@ -290,12 +300,14 @@ func (c *client) AssignVMToLoadBalancerRule(isoNet *capcv1.CloudStackIsolatedNet
 	p := c.cs.LoadBalancer.NewAssignToLoadBalancerRuleParams(isoNet.Status.LBRuleID)
 	p.SetVirtualmachineids([]string{instanceID})
 	_, retErr = c.cs.LoadBalancer.AssignToLoadBalancerRule(p)
+	c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(retErr)
 	return retErr
 }
 
 // DeleteNetwork deletes an isolated network.
 func (c *client) DeleteNetwork(net capcv1.Network) error {
 	_, err := c.cs.Network.DeleteNetwork(c.cs.Network.NewDeleteNetworkParams(net.ID))
+	c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 	return errors.Wrapf(err, "deleting network with id %s", net.ID)
 }
 
@@ -350,6 +362,7 @@ func (c *client) DisassociatePublicIPAddressIfNotInUse(isoNet *capcv1.CloudStack
 	if tagsAllowDisposal, err := c.DoClusterTagsAllowDisposal(ResourceTypeIPAddress, isoNet.Status.PublicIPID); err != nil {
 		return err
 	} else if publicIP, _, err := c.cs.Address.GetPublicIpAddressByID(isoNet.Status.PublicIPID); err != nil {
+		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)
 		return err
 	} else if publicIP == nil || publicIP.Issourcenat { // Can't disassociate an address if it's the source NAT address.
 		return nil
@@ -369,5 +382,6 @@ func (c *client) DisassociatePublicIPAddress(isoNet *capcv1.CloudStackIsolatedNe
 
 	p := c.cs.Address.NewDisassociateIpAddressParams(isoNet.Status.PublicIPID)
 	_, retErr = c.cs.Address.DisassociateIpAddress(p)
+	c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(retErr)
 	return retErr
 }