Add support for oci cli metrics (#268)

* Add support for OCI Client metrics

Author: shyamradhakrishnan

cloud/metrics/http_request_dispatcher_wrapper.go

@@ -0,0 +1,58 @@
+/*
+Copyright (c) 2023 Oracle and/or its affiliates.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/oracle/oci-go-sdk/v65/common"
+)
+
+// HttpRequestDispatcherWrapper is a wrapper around standard common.HTTPRequestDispatcher to handle
+// metrics
+type HttpRequestDispatcherWrapper struct {
+	dispatcher common.HTTPRequestDispatcher
+	region     string
+}
+
+// Do is wrapper implementation of common.HTTPRequestDispatcher Do method
+func (wrapper HttpRequestDispatcherWrapper) Do(req *http.Request) (*http.Response, error) {
+	t := time.Now()
+	resp, err := wrapper.dispatcher.Do(req)
+	defer func() {
+		// taken from https://docs.oracle.com/en-us/iaas/Content/API/Concepts/usingapi.htm
+		// a URL consists of a version string and then a resource
+		urlSplit := strings.Split(req.URL.Path, "/")
+		if len(urlSplit) < 2 {
+			return
+		}
+		resource := urlSplit[2]
+		IncRequestCounter(err, resource, req.Method, wrapper.region, resp)
+		ObserverRequestDuration(resource, req.Method, wrapper.region, time.Since(t))
+	}()
+	return resp, err
+}
+
+// NewHttpRequestDispatcherWrapper creates a new instance of HttpRequestDispatcherWrapper
+func NewHttpRequestDispatcherWrapper(dispatcher common.HTTPRequestDispatcher, region string) HttpRequestDispatcherWrapper {
+	return HttpRequestDispatcherWrapper{
+		dispatcher: dispatcher,
+		region:     region,
+	}
+}

cloud/metrics/metrics.go

@@ -0,0 +1,89 @@
+/*
+Copyright (c) 2023 Oracle and/or its affiliates.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
+)
+
+type verb string
+
+const (
+	SubSystemOCI     = "oci"
+	OCIRequestsTotal = "requests_total"
+	Duration         = "request_duration"
+	Resource         = "resource"
+	StatusCode       = "status_code"
+	Operation        = "operation"
+
+	Region        = "region"
+	Get    string = "get"
+	List   string = "list"
+	Create string = "create"
+	Update string = "update"
+	Delete string = "delete"
+)
+
+var (
+	ociRequestCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Subsystem: SubSystemOCI,
+			Name:      OCIRequestsTotal,
+			Help:      "OCI API requests total.",
+		},
+		[]string{Resource, StatusCode, Operation, Region},
+	)
+	ociRequestDurationSeconds = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Subsystem: SubSystemOCI,
+		Name:      Duration,
+		Help:      "Duration/Latency of HTTP requests to OCI",
+	}, []string{Resource, Operation, Region})
+)
+
+// IncRequestCounter increments the request count metric for the given resource.
+// Unknown errors from request dispatcher will have response code of 999
+func IncRequestCounter(err error, resource string, operation string, region string, response *http.Response) {
+	statusCode := 999
+	if err == nil {
+		statusCode = response.StatusCode
+	}
+	ociRequestCounter.With(prometheus.Labels{
+		Resource:   resource,
+		Operation:  operation,
+		StatusCode: strconv.Itoa(statusCode),
+		Region:     region,
+	}).Inc()
+}
+
+// ObserverRequestDuration observes the request duration for the partcular OCI request
+func ObserverRequestDuration(resource string, operation string, region string, duration time.Duration) {
+	ociRequestDurationSeconds.With(prometheus.Labels{
+		Resource:  resource,
+		Operation: operation,
+		Region:    region,
+	}).Observe(duration.Seconds())
+}
+
+func init() {
+	metrics.Registry.MustRegister(ociRequestCounter)
+	metrics.Registry.MustRegister(ociRequestDurationSeconds)
+}

cloud/scope/clients.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/go-logr/logr"
 	"github.com/oracle/cluster-api-provider-oci/api/v1beta2"
+	"github.com/oracle/cluster-api-provider-oci/cloud/metrics"
 	"github.com/oracle/cluster-api-provider-oci/cloud/services/base"
 	"github.com/oracle/cluster-api-provider-oci/cloud/services/compute"
 	"github.com/oracle/cluster-api-provider-oci/cloud/services/computemanagement"
@@ -133,7 +134,7 @@ func (c *ClientProvider) GetRegion() (string, error) {
 }
 
 func (c *ClientProvider) createClients(region string) (OCIClients, error) {
-	vcnClient, err := c.createVncClient(region, c.ociAuthConfigProvider, c.Logger)
+	vcnClient, err := c.createVcnClient(region, c.ociAuthConfigProvider, c.Logger)
 	if err != nil {
 		return OCIClients{}, err
 	}
@@ -182,16 +183,18 @@ func (c *ClientProvider) createClients(region string) (OCIClients, error) {
 	}, err
 }
 
-func (c *ClientProvider) createVncClient(region string, ociAuthConfigProvider common.ConfigurationProvider, logger *logr.Logger) (*core.VirtualNetworkClient, error) {
+func (c *ClientProvider) createVcnClient(region string, ociAuthConfigProvider common.ConfigurationProvider, logger *logr.Logger) (*core.VirtualNetworkClient, error) {
 	vcnClient, err := core.NewVirtualNetworkClientWithConfigurationProvider(ociAuthConfigProvider)
 	if err != nil {
 		logger.Error(err, "unable to create OCI VCN Client")
 		return nil, err
 	}
 	vcnClient.SetRegion(region)
+	dispatcher := vcnClient.HTTPClient
+	vcnClient.HTTPClient = metrics.NewHttpRequestDispatcherWrapper(dispatcher, region)
 
 	if c.certOverride != nil {
-		if client, ok := vcnClient.HTTPClient.(*http.Client); ok {
+		if client, ok := dispatcher.(*http.Client); ok {
 			err = c.setCerts(client)
 			if err != nil {
 				logger.Error(err, "unable to create OCI VCN Client")
@@ -218,9 +221,11 @@ func (c *ClientProvider) createNLbClient(region string, ociAuthConfigProvider co
 		return nil, err
 	}
 	nlbClient.SetRegion(region)
+	dispatcher := nlbClient.HTTPClient
+	nlbClient.HTTPClient = metrics.NewHttpRequestDispatcherWrapper(dispatcher, region)
 
 	if c.certOverride != nil {
-		if client, ok := nlbClient.HTTPClient.(*http.Client); ok {
+		if client, ok := dispatcher.(*http.Client); ok {
 			err = c.setCerts(client)
 			if err != nil {
 				logger.Error(err, "unable to create OCI NetworkLoadBalancer Client")
@@ -246,9 +251,11 @@ func (c *ClientProvider) createLBClient(region string, ociAuthConfigProvider com
 		return nil, err
 	}
 	lbClient.SetRegion(region)
+	dispatcher := lbClient.HTTPClient
+	lbClient.HTTPClient = metrics.NewHttpRequestDispatcherWrapper(dispatcher, region)
 
 	if c.certOverride != nil {
-		if client, ok := lbClient.HTTPClient.(*http.Client); ok {
+		if client, ok := dispatcher.(*http.Client); ok {
 			err = c.setCerts(client)
 			if err != nil {
 				logger.Error(err, "unable to create OCI Loadbalancer Client")
@@ -274,9 +281,11 @@ func (c *ClientProvider) createIdentityClient(region string, ociAuthConfigProvid
 		return nil, err
 	}
 	identityClt.SetRegion(region)
+	dispatcher := identityClt.HTTPClient
+	identityClt.HTTPClient = metrics.NewHttpRequestDispatcherWrapper(dispatcher, region)
 
 	if c.certOverride != nil {
-		if client, ok := identityClt.HTTPClient.(*http.Client); ok {
+		if client, ok := dispatcher.(*http.Client); ok {
 			err = c.setCerts(client)
 			if err != nil {
 				logger.Error(err, "unable to create OCI Identity Client")
@@ -295,16 +304,18 @@ func (c *ClientProvider) createIdentityClient(region string, ociAuthConfigProvid
 	return &identityClt, nil
 }
 
-func (c *ClientProvider) createComputeClient(region string, ociAuthConfigProvider common.ConfigurationProvider, logger *logr.Logger) (*core.ComputeClient, error) {
+func (c *ClientProvider) createComputeClient(region string, ociAuthConfigProvider common.ConfigurationProvider, logger *logr.Logger) (compute.ComputeClient, error) {
 	computeClient, err := core.NewComputeClientWithConfigurationProvider(ociAuthConfigProvider)
 	if err != nil {
 		logger.Error(err, "unable to create OCI Compute Client")
 		return nil, err
 	}
 	computeClient.SetRegion(region)
+	dispatcher := computeClient.HTTPClient
+	computeClient.HTTPClient = metrics.NewHttpRequestDispatcherWrapper(dispatcher, region)
 
 	if c.certOverride != nil {
-		if client, ok := computeClient.HTTPClient.(*http.Client); ok {
+		if client, ok := dispatcher.(*http.Client); ok {
 			err = c.setCerts(client)
 			if err != nil {
 				logger.Error(err, "unable to create OCI Compute Client")
@@ -330,9 +341,11 @@ func (c *ClientProvider) createComputeManagementClient(region string, ociAuthCon
 		return nil, err
 	}
 	computeManagementClient.SetRegion(region)
+	dispatcher := computeManagementClient.HTTPClient
+	computeManagementClient.HTTPClient = metrics.NewHttpRequestDispatcherWrapper(dispatcher, region)
 
 	if c.certOverride != nil {
-		if client, ok := computeManagementClient.HTTPClient.(*http.Client); ok {
+		if client, ok := dispatcher.(*http.Client); ok {
 			err = c.setCerts(client)
 			if err != nil {
 				logger.Error(err, "unable to create OCI Compute Management Client")
@@ -358,9 +371,11 @@ func (c *ClientProvider) createContainerEngineClient(region string, ociAuthConfi
 		return nil, err
 	}
 	containerEngineClt.SetRegion(region)
+	dispatcher := containerEngineClt.HTTPClient
+	containerEngineClt.HTTPClient = metrics.NewHttpRequestDispatcherWrapper(dispatcher, region)
 
 	if c.certOverride != nil {
-		if client, ok := containerEngineClt.HTTPClient.(*http.Client); ok {
+		if client, ok := dispatcher.(*http.Client); ok {
 			err = c.setCerts(client)
 			if err != nil {
 				logger.Error(err, "unable to create OCI Container Engine Client")

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/onsi/gomega v1.27.5
 	github.com/oracle/oci-go-sdk/v65 v65.29.0
 	github.com/pkg/errors v0.9.1
+	github.com/prometheus/client_golang v1.15.1
 	github.com/spf13/pflag v1.0.5
 	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/api v0.26.1
@@ -36,7 +37,7 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver v3.5.1+incompatible // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/coredns/caddy v1.1.0 // indirect
 	github.com/coredns/corefile-migration v1.0.20 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
@@ -76,7 +77,7 @@ require (
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
@@ -87,10 +88,10 @@ require (
 	github.com/opencontainers/image-spec v1.0.2 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.6 // indirect
-	github.com/prometheus/client_golang v1.14.0 // indirect
+	github.com/prometheus/client_golang v1.15.1 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
-	github.com/prometheus/common v0.37.0 // indirect
-	github.com/prometheus/procfs v0.8.0 // indirect
+	github.com/prometheus/common v0.42.0 // indirect
+	github.com/prometheus/procfs v0.9.0 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/sony/gobreaker v0.5.0 // indirect
@@ -116,7 +117,7 @@ require (
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect