Custom Metrics for ACS API calls. One metric updater implemented in isolated_network.go. Draft for feedback.

Author: jweite-amazon

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/onsi/gomega v1.19.0
 	github.com/pkg/errors v0.9.1
 	github.com/smallfish/simpleyaml v0.1.0
+	github.com/prometheus/client_golang v1.11.0
 	github.com/spf13/pflag v1.0.5
 	gopkg.in/ini.v1 v1.63.2
 	k8s.io/api v0.23.0

pkg/cloud/client.go

@@ -17,6 +17,7 @@ limitations under the License.
 package cloud
 
 import (
+	"sigs.k8s.io/cluster-api-provider-cloudstack/pkg/metrics"
 	"strings"
 
 	"github.com/apache/cloudstack-go/v2/cloudstack"
@@ -40,9 +41,10 @@ type Client interface {
 }
 
 type client struct {
-	cs      *cloudstack.CloudStackClient
-	csAsync *cloudstack.CloudStackClient
-	config  Config
+	cs            *cloudstack.CloudStackClient
+	csAsync       *cloudstack.CloudStackClient
+	config        Config
+	customMetrics metrics.AcsCustomMetrics
 }
 
 // cloud-config ini structure.
@@ -68,6 +70,7 @@ func NewClient(ccPath string) (Client, error) {
 	// comments for more details
 	c.cs = cloudstack.NewAsyncClient(c.config.APIURL, c.config.APIKey, c.config.SecretKey, c.config.VerifySSL)
 	c.csAsync = cloudstack.NewClient(c.config.APIURL, c.config.APIKey, c.config.SecretKey, c.config.VerifySSL)
+	c.customMetrics = metrics.NewCustomMetrics()
 
 	_, err := c.cs.APIDiscovery.ListApis(c.cs.APIDiscovery.NewListApisParams())
 	if err != nil && strings.Contains(strings.ToLower(err.Error()), "i/o timeout") {

pkg/cloud/isolated_network.go

@@ -60,6 +60,7 @@ func (c *client) AssociatePublicIPAddress(
 	// Check specified IP address is available or get an unused one if not specified.
 	publicAddress, err := c.GetPublicIP(zone, isoNet, csCluster)
 	if err != nil {
+		c.customMetrics.IncrementAcsReconciliationErrors(err)
 		return errors.Wrapf(err, "fetching a public IP address")
 	}
 	isoNet.Spec.ControlPlaneEndpoint.Host = publicAddress.Ipaddress

pkg/metrics/metrics.go

@@ -0,0 +1,60 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+//Package metrics implements custom metrics for CAPC
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"regexp"
+	crtlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
+)
+
+// AcsCustomMetrics encapsulates all CloudStack custom metrics defined for the controller.
+type AcsCustomMetrics struct {
+	acsReconciliationErrorCount *prometheus.CounterVec
+	errorCodeRegexp             *regexp.Regexp
+}
+
+// NewCustomMetrics constructs an AcsCustomMetrics with all desired CloudStack custom metrics and any supporting resources.
+func NewCustomMetrics() AcsCustomMetrics {
+	customMetrics := AcsCustomMetrics{}
+	customMetrics.acsReconciliationErrorCount = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "acs_reconciliation_errors",
+			Help: "Count of reconciliation errors caused by ACS issues, bucketed by error code",
+		},
+		[]string{"acs_error_code"},
+	)
+	crtlmetrics.Registry.MustRegister(customMetrics.acsReconciliationErrorCount)
+
+	// ACS standard error messages of the form "CloudStack API error 431 (CSExceptionErrorCode: 9999):..."
+	//  This regexp is used to extract CSExceptionCodes from the message.
+	customMetrics.errorCodeRegexp, _ = regexp.Compile(".+CSExceptionErrorCode: ([0-9]+).+")
+
+	return customMetrics
+}
+
+// IncrementAcsReconciliationErrors accepts a CloudStack error message and increments the custom
+// acs_reconciliation_errors counter, labeled with the error code if present in the error message.
+func (m *AcsCustomMetrics) IncrementAcsReconciliationErrors(acsError error) {
+	matches := m.errorCodeRegexp.FindStringSubmatch(acsError.Error())
+	if len(matches) > 1 {
+		m.acsReconciliationErrorCount.WithLabelValues(matches[1]).Inc()
+	} else {
+		m.acsReconciliationErrorCount.WithLabelValues("No error code").Inc()
+	}
+}