add support for openstack api metrics (kubernetes#2077)

Author: zetaab

cmd/cinder-csi-plugin/main.go

@@ -32,10 +32,11 @@ import (
 )
 
 var (
-	endpoint    string
-	nodeID      string
-	cloudconfig []string
-	cluster     string
+	endpoint     string
+	nodeID       string
+	cloudconfig  []string
+	cluster      string
+	httpEndpoint string
 )
 
 func main() {
@@ -88,7 +89,7 @@ func main() {
 	}
 
 	cmd.PersistentFlags().StringVar(&cluster, "cluster", "", "The identifier of the cluster that the plugin is running in.")
-
+	cmd.PersistentFlags().StringVar(&httpEndpoint, "http-endpoint", "", "The TCP network address where the HTTP server for diagnostics, including metrics and leader election health check, will listen (example: `:8080`). The default is empty string, which means the server is disabled.")
 	openstack.AddExtraFlags(pflag.CommandLine)
 
 	code := cli.Run(cmd)
@@ -99,7 +100,7 @@ func handle() {
 
 	// Initialize cloud
 	d := cinder.NewDriver(endpoint, cluster)
-	openstack.InitOpenStackProvider(cloudconfig)
+	openstack.InitOpenStackProvider(cloudconfig, httpEndpoint)
 	cloud, err := openstack.GetOpenStackProvider()
 	if err != nil {
 		klog.Warningf("Failed to GetOpenStackProvider: %v", err)

pkg/csi/cinder/openstack/openstack.go

@@ -18,6 +18,7 @@ package openstack
 
 import (
 	"fmt"
+	"net/http"
 	"os"
 
 	"github.com/gophercloud/gophercloud"
@@ -28,7 +29,9 @@ import (
 	"github.com/spf13/pflag"
 	gcfg "gopkg.in/gcfg.v1"
 	"k8s.io/cloud-provider-openstack/pkg/client"
+	"k8s.io/cloud-provider-openstack/pkg/metrics"
 	"k8s.io/cloud-provider-openstack/pkg/util/metadata"
+	"k8s.io/component-base/metrics/legacyregistry"
 	"k8s.io/klog/v2"
 )
 
@@ -131,7 +134,20 @@ const defaultMaxVolAttachLimit int64 = 256
 var OsInstance IOpenStack
 var configFiles = []string{"/etc/cloud.conf"}
 
-func InitOpenStackProvider(cfgFiles []string) {
+func InitOpenStackProvider(cfgFiles []string, httpEndpoint string) {
+	metrics.RegisterMetrics("cinder-csi")
+	if httpEndpoint != "" {
+		mux := http.NewServeMux()
+		mux.Handle("/metrics", legacyregistry.HandlerWithReset())
+		go func() {
+			err := http.ListenAndServe(httpEndpoint, mux)
+			if err != nil {
+				klog.Fatalf("failed to listen & serve metrics from %q: %v", httpEndpoint, err)
+			}
+			klog.Infof("metrics available in %q", httpEndpoint)
+		}()
+	}
+
 	configFiles = cfgFiles
 	klog.V(2).Infof("InitOpenStackProvider configFiles: %s", configFiles)
 }

pkg/csi/cinder/openstack/openstack_instances.go

@@ -2,12 +2,14 @@ package openstack
 
 import (
 	"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
+	"k8s.io/cloud-provider-openstack/pkg/metrics"
 )
 
 // GetInstanceByID returns server with specified instanceID
 func (os *OpenStack) GetInstanceByID(instanceID string) (*servers.Server, error) {
+	mc := metrics.NewMetricContext("server", "get")
 	server, err := servers.Get(os.compute, instanceID).Extract()
-	if err != nil {
+	if mc.ObserveRequest(err) != nil {
 		return nil, err
 	}
 	return server, nil

pkg/csi/cinder/openstack/openstack_snapshots.go

@@ -27,6 +27,7 @@ import (
 	"github.com/gophercloud/gophercloud/openstack/blockstorage/v3/snapshots"
 	"github.com/gophercloud/gophercloud/pagination"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/cloud-provider-openstack/pkg/metrics"
 	"k8s.io/klog/v2"
 )
 
@@ -67,9 +68,9 @@ func (os *OpenStack) CreateSnapshot(name, volID string, tags *map[string]string)
 		opts.Metadata = *tags
 	}
 	// TODO: Do some check before really call openstack API on the input
-
+	mc := metrics.NewMetricContext("snapshot", "create")
 	snap, err := snapshots.Create(os.blockstorage, opts).Extract()
-	if err != nil {
+	if mc.ObserveRequest(err) != nil {
 		return &snapshots.Snapshot{}, err
 	}
 	// There's little value in rewrapping these gophercloud types into yet another abstraction/type, instead just
@@ -103,6 +104,7 @@ func (os *OpenStack) ListSnapshots(filters map[string]string) ([]snapshots.Snaps
 			klog.V(3).Infof("Not a valid filter key %s", key)
 		}
 	}
+	mc := metrics.NewMetricContext("snapshot", "list")
 	err := snapshots.List(os.blockstorage, opts).EachPage(func(page pagination.Page) (bool, error) {
 		var err error
 
@@ -126,7 +128,7 @@ func (os *OpenStack) ListSnapshots(filters map[string]string) ([]snapshots.Snaps
 
 		return false, nil
 	})
-	if err != nil {
+	if mc.ObserveRequest(err) != nil {
 		return nil, nextPageToken, err
 	}
 
@@ -135,17 +137,19 @@ func (os *OpenStack) ListSnapshots(filters map[string]string) ([]snapshots.Snaps
 
 // DeleteSnapshot issues a request to delete the Snapshot with the specified ID from the Cinder backend
 func (os *OpenStack) DeleteSnapshot(snapID string) error {
+	mc := metrics.NewMetricContext("snapshot", "delete")
 	err := snapshots.Delete(os.blockstorage, snapID).ExtractErr()
-	if err != nil {
+	if mc.ObserveRequest(err) != nil {
 		klog.Errorf("Failed to delete snapshot: %v", err)
 	}
 	return err
 }
 
 // GetSnapshotByID returns snapshot details by id
 func (os *OpenStack) GetSnapshotByID(snapshotID string) (*snapshots.Snapshot, error) {
+	mc := metrics.NewMetricContext("snapshot", "get")
 	s, err := snapshots.Get(os.blockstorage, snapshotID).Extract()
-	if err != nil {
+	if mc.ObserveRequest(err) != nil {
 		klog.Errorf("Failed to get snapshot: %v", err)
 		return nil, err
 	}

pkg/csi/cinder/openstack/openstack_volumes.go

@@ -27,6 +27,7 @@ import (
 	"github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/volumeattach"
 	"github.com/gophercloud/gophercloud/pagination"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/cloud-provider-openstack/pkg/metrics"
 	cpoerrors "k8s.io/cloud-provider-openstack/pkg/util/errors"
 
 	"k8s.io/klog/v2"
@@ -65,8 +66,9 @@ func (os *OpenStack) CreateVolume(name string, size int, vtype, availability str
 		opts.Metadata = *tags
 	}
 
+	mc := metrics.NewMetricContext("volume", "create")
 	vol, err := volumes.Create(os.blockstorage, opts).Extract()
-	if err != nil {
+	if mc.ObserveRequest(err) != nil {
 		return nil, err
 	}
 
@@ -79,6 +81,7 @@ func (os *OpenStack) ListVolumes(limit int, startingToken string) ([]volumes.Vol
 	var vols []volumes.Volume
 
 	opts := volumes.ListOpts{Limit: limit, Marker: startingToken}
+	mc := metrics.NewMetricContext("volume", "list")
 	err := volumes.List(os.blockstorage, opts).EachPage(func(page pagination.Page) (bool, error) {
 		var err error
 
@@ -102,7 +105,7 @@ func (os *OpenStack) ListVolumes(limit int, startingToken string) ([]volumes.Vol
 
 		return false, nil
 	})
-	if err != nil {
+	if mc.ObserveRequest(err) != nil {
 		return nil, nextPageToken, err
 	}
 
@@ -125,8 +128,9 @@ func (os *OpenStack) GetVolumesByName(n string) ([]volumes.Volume, error) {
 	}
 
 	opts := volumes.ListOpts{Name: n}
+	mc := metrics.NewMetricContext("volume", "list")
 	pages, err := volumes.List(blockstorageClient, opts).AllPages()
-	if err != nil {
+	if mc.ObserveRequest(err) != nil {
 		return nil, err
 	}
 
@@ -148,15 +152,16 @@ func (os *OpenStack) DeleteVolume(volumeID string) error {
 		return fmt.Errorf("Cannot delete the volume %q, it's still attached to a node", volumeID)
 	}
 
+	mc := metrics.NewMetricContext("volume", "delete")
 	err = volumes.Delete(os.blockstorage, volumeID, nil).ExtractErr()
-	return err
+	return mc.ObserveRequest(err)
 }
 
 // GetVolume retrieves Volume by its ID.
 func (os *OpenStack) GetVolume(volumeID string) (*volumes.Volume, error) {
-
+	mc := metrics.NewMetricContext("volume", "get")
 	vol, err := volumes.Get(os.blockstorage, volumeID).Extract()
-	if err != nil {
+	if mc.ObserveRequest(err) != nil {
 		return nil, err
 	}
 
@@ -189,11 +194,12 @@ func (os *OpenStack) AttachVolume(instanceID, volumeID string) (string, error) {
 		computeServiceClient.Microversion = "2.60"
 	}
 
+	mc := metrics.NewMetricContext("volume", "attach")
 	_, err = volumeattach.Create(computeServiceClient, instanceID, &volumeattach.CreateOpts{
 		VolumeID: volume.ID,
 	}).Extract()
 
-	if err != nil {
+	if mc.ObserveRequest(err) != nil {
 		return "", fmt.Errorf("failed to attach %s volume to %s compute: %v", volumeID, instanceID, err)
 	}
 
@@ -276,8 +282,9 @@ func (os *OpenStack) DetachVolume(instanceID, volumeID string) error {
 	// Incase volume is of type multiattach, it could be attached to more than one instance
 	for _, att := range volume.Attachments {
 		if att.ServerID == instanceID {
+			mc := metrics.NewMetricContext("volume", "detach")
 			err = volumeattach.Delete(os.compute, instanceID, volume.ID).ExtractErr()
-			if err != nil {
+			if mc.ObserveRequest(err) != nil {
 				return fmt.Errorf("failed to detach volume %s from compute %s : %v", volume.ID, instanceID, err)
 			}
 			klog.V(2).Infof("Successfully detached volume: %s from compute: %s", volume.ID, instanceID)
@@ -357,9 +364,11 @@ func (os *OpenStack) ExpandVolume(volumeID string, status string, newSize int) e
 		// https://docs.openstack.org/cinder/latest/contributor/api_microversion_history.html#id40
 		blockstorageClient.Microversion = "3.42"
 
-		return volumeexpand.ExtendSize(blockstorageClient, volumeID, extendOpts).ExtractErr()
+		mc := metrics.NewMetricContext("volume", "expand")
+		return mc.ObserveRequest(volumeexpand.ExtendSize(blockstorageClient, volumeID, extendOpts).ExtractErr())
 	case VolumeAvailableStatus:
-		return volumeexpand.ExtendSize(os.blockstorage, volumeID, extendOpts).ExtractErr()
+		mc := metrics.NewMetricContext("volume", "expand")
+		return mc.ObserveRequest(volumeexpand.ExtendSize(os.blockstorage, volumeID, extendOpts).ExtractErr())
 	}
 
 	// cinder volume can not be expanded when volume status is not volumeInUseStatus or not volumeAvailableStatus

pkg/metrics/metrics.go

@@ -59,7 +59,9 @@ func (mc *MetricContext) Observe(om *OpenstackMetrics, err error) error {
 	return err
 }
 
-func RegisterMetrics() {
+func RegisterMetrics(component string) {
 	doRegisterAPIMetrics()
-	doRegisterOccmMetrics()
+	if component == "occm" {
+		doRegisterOccmMetrics()
+	}
 }

pkg/openstack/openstack.go

@@ -160,7 +160,7 @@ type Config struct {
 }
 
 func init() {
-	metrics.RegisterMetrics()
+	metrics.RegisterMetrics("occm")
 
 	cloudprovider.RegisterCloudProvider(ProviderName, func(config io.Reader) (cloudprovider.Interface, error) {
 		cfg, err := ReadConfig(config)