feature: monitor bucket formance and resource status (#22552)

Co-authored-by: Qiu Jian <[email protected]>

Author: swordqiu

cmd/climc/shell/compute/buckets.go

@@ -93,12 +93,12 @@ require (
 	k8s.io/cri-api v0.22.17
 	k8s.io/klog/v2 v2.20.0
 	moul.io/http2curl/v2 v2.3.0
-	yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250515074509-c7dba5d8b5e2
-	yunion.io/x/executor v0.0.0-20241205080005-48f5b1212256
+	yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250519035927-2b74cec67090
+	yunion.io/x/executor v0.0.0-20250518005516-5402e9e0bed0
 	yunion.io/x/jsonutils v1.0.1-0.20250507052344-1abcf4f443b1
 	yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91
 	yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900
-	yunion.io/x/pkg v1.10.4-0.20250513085810-efeaaca81a0e
+	yunion.io/x/pkg v1.10.4-0.20250519013345-54017bf6c1f0
 	yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1
 	yunion.io/x/sqlchemy v1.1.3-0.20250513031856-ce9f71063b3a
 	yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c

go.mod

@@ -1376,10 +1376,10 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
-yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250515074509-c7dba5d8b5e2 h1:Zrqat1z8qiSqZXitZlUOs1uD4xN+pHXkrWir1vYBXr8=
-yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250515074509-c7dba5d8b5e2/go.mod h1:FXxAEbdNfWXX9gjME3K2nJhkydHY5EKEUZb+RLEzVwQ=
-yunion.io/x/executor v0.0.0-20241205080005-48f5b1212256 h1:kLKQ6zbgPDQflRwoHFAjxNChcbhXIFgsUVLkJwiXu/8=
-yunion.io/x/executor v0.0.0-20241205080005-48f5b1212256/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
+yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250519035927-2b74cec67090 h1:E/I9CkZumtHaEBweSYc9bq5rFZ25i+s17bAZskXLGDc=
+yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250519035927-2b74cec67090/go.mod h1:FXxAEbdNfWXX9gjME3K2nJhkydHY5EKEUZb+RLEzVwQ=
+yunion.io/x/executor v0.0.0-20250518005516-5402e9e0bed0 h1:msG4SiDSVU7CrXH06WuHlNEZXIooTcmNbfrIGHuIHBU=
+yunion.io/x/executor v0.0.0-20250518005516-5402e9e0bed0/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
 yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
 yunion.io/x/jsonutils v1.0.1-0.20250507052344-1abcf4f443b1 h1:/+THlvf/MvgCW+7KeCDCr33e81KSRa5JmdZ1IIyLOXQ=
 yunion.io/x/jsonutils v1.0.1-0.20250507052344-1abcf4f443b1/go.mod h1:VK4Z93dgiKgAijcSqbMKmGaBMJuHulR16Hz4K015ZPo=
@@ -1391,8 +1391,8 @@ yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900 h1:Hu/4ERvoWaN6aiFs4h4/yvVB
 yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900/go.mod h1:0vLkNEhlmA64HViPBAnSTUMrx5QP1CLsxXmxDKQ80tc=
 yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
-yunion.io/x/pkg v1.10.4-0.20250513085810-efeaaca81a0e h1:OiBxJR18bq5fFK/IZWYRX+5HTMs2KgQJe0a3BZfdIVU=
-yunion.io/x/pkg v1.10.4-0.20250513085810-efeaaca81a0e/go.mod h1:0Bwxqd9MA3ACi119/l02FprY/o9gHahmYC2bsSbnVpM=
+yunion.io/x/pkg v1.10.4-0.20250519013345-54017bf6c1f0 h1:iKWkBMKazSijYNhOaSh4qBuIu+PmXYhEAMNwrxaXL4Q=
+yunion.io/x/pkg v1.10.4-0.20250519013345-54017bf6c1f0/go.mod h1:0Bwxqd9MA3ACi119/l02FprY/o9gHahmYC2bsSbnVpM=
 yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1 h1:1KJ3YYinydPHpDEQRXdr/T8SYcKZ5Er+m489H+PnaQ4=
 yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
 yunion.io/x/sqlchemy v1.1.3-0.20250513031856-ce9f71063b3a h1:mDW1VyYJxZ4ORITZGDWacUiBNyqX6rMObnQk77NvUOg=

go.sum

@@ -15,28 +15,5 @@
 package handler
 
 import (
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/ansible"
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/cloudevent"
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/cloudid"
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/cloudnet"
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/cloudproxy"
-	modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute"
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/devtool"
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/etcd"
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/identity"
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/image"
-	"yunion.io/x/onecloud/pkg/mcclient/modules/k8s"
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/logger"
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/monitor"
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/notify"
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/quota"
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/scheduledtask"
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/scheduler"
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/webconsole"
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/yunionconf"
+	_ "yunion.io/x/onecloud/pkg/mcclient/modules/loader"
 )
-
-func init() {
-	modules.InitUsages()
-	modules.Usages.RegisterManager(modules.UsageManagerK8s, k8s.Usages)
-}

pkg/apigateway/handler/init.go

@@ -17,6 +17,7 @@ package compute
 import (
 	"net/http"
 	"reflect"
+	"time"
 
 	"yunion.io/x/cloudmux/pkg/apis/compute"
 	"yunion.io/x/cloudmux/pkg/cloudprovider"
@@ -53,6 +54,8 @@ type BucketCreateInput struct {
 	CloudproviderResourceInput
 
 	StorageClass string `json:"storage_class"`
+
+	EnablePerfMon *bool `json:"enable_perf_mon"`
 }
 
 type BucketDetails struct {
@@ -66,28 +69,28 @@ type BucketDetails struct {
 	AccessUrls []cloudprovider.SBucketAccessUrl `json:"access_urls"`
 }
 
-func (self BucketDetails) GetMetricTags() map[string]string {
+func (bucket BucketDetails) GetMetricTags() map[string]string {
 	ret := map[string]string{
-		"id":             self.Id,
-		"brand":          self.Brand,
-		"cloudregion":    self.Cloudregion,
-		"cloudregion_id": self.CloudregionId,
-		"domain_id":      self.DomainId,
-		"oss_id":         self.Id,
-		"oss_name":       self.Name,
-		"project_domain": self.ProjectDomain,
-		"region_ext_id":  self.RegionExtId,
-		"status":         self.Status,
-		"tenant":         self.Project,
-		"tenant_id":      self.ProjectId,
-		"account":        self.Account,
-		"account_id":     self.AccountId,
-		"external_id":    self.ExternalId,
+		"id":             bucket.Id,
+		"brand":          bucket.Brand,
+		"cloudregion":    bucket.Cloudregion,
+		"cloudregion_id": bucket.CloudregionId,
+		"domain_id":      bucket.DomainId,
+		"oss_id":         bucket.Id,
+		"oss_name":       bucket.Name,
+		"project_domain": bucket.ProjectDomain,
+		"region_ext_id":  bucket.RegionExtId,
+		"status":         bucket.Status,
+		"tenant":         bucket.Project,
+		"tenant_id":      bucket.ProjectId,
+		"account":        bucket.Account,
+		"account_id":     bucket.AccountId,
+		"external_id":    bucket.ExternalId,
 	}
-	return AppendMetricTags(ret, self.MetadataResourceInfo, self.ProjectizedResourceInfo)
+	return AppendMetricTags(ret, bucket.MetadataResourceInfo, bucket.ProjectizedResourceInfo)
 }
 
-func (self BucketDetails) GetMetricPairs() map[string]string {
+func (bucket BucketDetails) GetMetricPairs() map[string]string {
 	ret := map[string]string{}
 	return ret
 }
@@ -150,6 +153,8 @@ type BucketSyncstatusInput struct {
 
 type BucketUpdateInput struct {
 	apis.SharableVirtualResourceBaseUpdateInput
+
+	EnablePerfMon *bool `json:"enable_perf_mon"`
 }
 
 type BucketPerformTempUrlInput struct {
@@ -370,3 +375,29 @@ func init() {
 		return &SBackupStorageAccessInfo{}
 	})
 }
+
+type BucketProbeResult struct {
+	UploadTime   time.Duration
+	DownloadTime time.Duration
+	DeleteTime   time.Duration
+}
+
+func (result BucketProbeResult) UploadDelayMs() float64 {
+	return float64(result.UploadTime) / float64(time.Millisecond)
+}
+
+func (result BucketProbeResult) DownloadDelayMs() float64 {
+	return float64(result.DownloadTime) / float64(time.Millisecond)
+}
+
+func (result BucketProbeResult) DeleteDelayMs() float64 {
+	return float64(result.DeleteTime) / float64(time.Millisecond)
+}
+
+func (result BucketProbeResult) UploadThroughputMbps(sizeMBytes int) float64 {
+	return float64(sizeMBytes) * 8 / float64(result.UploadTime.Seconds())
+}
+
+func (result BucketProbeResult) DownloadThroughputMbps(sizeMBytes int) float64 {
+	return float64(sizeMBytes) * 8 / float64(result.DownloadTime.Seconds())
+}

pkg/apis/compute/bucket.go

@@ -0,0 +1,177 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package misc
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+
+	computeapi "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudmon/options"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	computemodules "yunion.io/x/onecloud/pkg/mcclient/modules/compute"
+	baseoptions "yunion.io/x/onecloud/pkg/mcclient/options"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+)
+
+func BucketProbe(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
+	if options.Options.EnableBucketProbeDebug {
+		log.Debugf("BucketProbe start")
+	}
+	if !options.Options.EnableBucketProbe {
+		if options.Options.EnableBucketProbeDebug {
+			log.Debugf("BucketProbe is disabled")
+		}
+		return
+	}
+
+	sess := auth.GetSession(ctx, userCred, options.Options.Region)
+
+	metrics, err := gatherBucketMetrics(ctx, sess)
+	if err != nil {
+		log.Errorf("BucketProbe gatherBucketMetrics failed: %s", err)
+		return
+	}
+
+	err = sendMetrics(sess, metrics, "telegraf")
+	if err != nil {
+		log.Errorf("StatusProbe SendMetrics error: %s", err)
+	}
+}
+
+func gatherBucketMetrics(ctx context.Context, sess *mcclient.ClientSession) ([]influxdb.SMetricData, error) {
+	allMetrics := []influxdb.SMetricData{}
+
+	params := baseoptions.BaseListOptions{}
+	params.Scope = "max"
+	limit := 1000
+	params.Limit = &limit
+	params.Filter = []string{
+		"enable_perf_mon.equals(1)",
+	}
+	boolTrue := true
+	params.Details = &boolTrue
+
+	total := -1
+	offset := 0
+	for total < 0 || offset < total {
+		params.Offset = &offset
+		results, err := computemodules.Buckets.List(sess, jsonutils.Marshal(params))
+		if err != nil {
+			return nil, errors.Wrap(err, "computemodules.Buckets.List")
+		}
+		total = results.Total
+		offset = results.Offset + len(results.Data)
+
+		for _, bucket := range results.Data {
+			bucketDetails := computeapi.BucketDetails{}
+			err = bucket.Unmarshal(&bucketDetails)
+			if err != nil {
+				log.Errorf("BucketProbe failed: %s", err)
+				continue
+			}
+
+			metrics, err := probeBucketStats(ctx, sess, &bucketDetails)
+			if err != nil {
+				log.Errorf("BucketProbe failed: %s", err)
+				continue
+			}
+			allMetrics = append(allMetrics, metrics...)
+		}
+	}
+
+	return allMetrics, nil
+}
+
+func probeBucketStats(ctx context.Context, sess *mcclient.ClientSession, bucketDetails *computeapi.BucketDetails) ([]influxdb.SMetricData, error) {
+	bucket, err := computemodules.GetIBucket(ctx, sess, bucketDetails)
+	if err != nil {
+		return nil, errors.Wrap(err, "getIBucket")
+	}
+
+	resultDelay, err := computemodules.ProbeBucketStats(ctx, bucket, options.Options.BucketProbeTestKey, 0)
+	if err != nil {
+		return nil, errors.Wrap(err, "doProbeBucketStats zero")
+	}
+
+	resultRate, err := computemodules.ProbeBucketStats(ctx, bucket, options.Options.BucketProbeTestKey, int64(options.Options.BucketProbeTestSizeMb)*1024*1024)
+	if err != nil {
+		return nil, errors.Wrap(err, "doProbeBucketStats with payload")
+	}
+
+	metricTags := []influxdb.SKeyValue{}
+	for k, v := range bucketDetails.GetMetricTags() {
+		if len(v) == 0 {
+			continue
+		}
+		metricTags = append(metricTags, influxdb.SKeyValue{
+			Key:   k,
+			Value: v,
+		})
+	}
+
+	metrics := []influxdb.SKeyValue{}
+	for k, v := range bucketDetails.GetMetricTags() {
+		if len(v) == 0 {
+			continue
+		}
+		metrics = append(metrics, influxdb.SKeyValue{
+			Key:   k,
+			Value: v,
+		})
+	}
+
+	metrics = append(metrics,
+		influxdb.SKeyValue{
+			Key:   "upload_delay_ms",
+			Value: fmt.Sprintf("%f", resultDelay.UploadDelayMs()),
+		},
+		influxdb.SKeyValue{
+			Key:   "download_delay_ms",
+			Value: fmt.Sprintf("%f", resultDelay.DownloadDelayMs()),
+		},
+		influxdb.SKeyValue{
+			Key:   "delete_delay_ms",
+			Value: fmt.Sprintf("%f", resultDelay.DeleteDelayMs()),
+		},
+		influxdb.SKeyValue{
+			Key:   "upload_rate_mbps",
+			Value: fmt.Sprintf("%f", resultRate.UploadThroughputMbps(options.Options.BucketProbeTestSizeMb)),
+		},
+		influxdb.SKeyValue{
+			Key:   "download_rate_mbps",
+			Value: fmt.Sprintf("%f", resultRate.DownloadThroughputMbps(options.Options.BucketProbeTestSizeMb)),
+		},
+	)
+
+	if options.Options.EnableBucketProbeDebug {
+		log.Debugf("BucketProbe for bucket %s metrics: %s", bucketDetails.Name, jsonutils.Marshal(metrics))
+	}
+
+	return []influxdb.SMetricData{
+		{
+			Name:      "bucket_perf",
+			Tags:      metricTags,
+			Metrics:   metrics,
+			Timestamp: time.Now(),
+		},
+	}, nil
+}

pkg/cloudmon/misc/bucketprobe.go

@@ -110,9 +110,11 @@ func getNetworkAddrMap(s *mcclient.ClientSession, netId string) (map[string]api.
 		return nil, errors.Wrap(err, "GetSpecific addresses")
 	}
 	addrList := make([]api.SNetworkUsedAddress, 0)
-	err = addrListJson.Unmarshal(&addrList, "addresses")
-	if err != nil {
-		return nil, errors.Wrap(err, "Unmarshal addreses")
+	if addrListJson.Contains("addresses") {
+		err = addrListJson.Unmarshal(&addrList, "addresses")
+		if err != nil {
+			return nil, errors.Wrap(err, "Unmarshal addreses")
+		}
 	}
 	addrMap := make(map[string]api.SNetworkUsedAddress)
 	for i := range addrList {