fix(monitor): add service metric monitor (#22226)

Author: ioito

pkg/apis/monitor/metric.go

@@ -32,6 +32,10 @@ const (
 	METRIC_RES_TYPE_ELB          = "elb"
 	METRIC_RES_TYPE_K8S          = "k8s"
 	METRIC_RES_TYPE_CONTAINER    = "container"
+	METRIC_RES_TYPE_WORKER       = "worker"
+	METRIC_RES_TYPE_HTTP_REQUEST = "http_request"
+	METRIC_RES_TYPE_DB_STATS     = "db_stats"
+	METRIC_RES_TYPE_PROCESS      = "process"
 
 	//ext is prefix！
 	METRIC_RES_TYPE_JENKINS      = "ext_jenkins"
@@ -52,8 +56,9 @@ const (
 	METRIC_UNIT_MB      = "Mb"
 	METRIC_UNIT_NULL    = "NULL"
 
-	METRIC_DATABASE_TELE  = "telegraf"
-	METRIC_DATABASE_METER = "meter_db"
+	METRIC_DATABASE_TELE   = "telegraf"
+	METRIC_DATABASE_METER  = "meter_db"
+	METRIC_DATABASE_SYSTEM = "system"
 )
 
 var (

pkg/cloudmon/misc/system.go

@@ -26,14 +26,17 @@ import (
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/gotypes"
 	"yunion.io/x/pkg/util/httputils"
+	"yunion.io/x/pkg/util/version"
 	"yunion.io/x/pkg/utils"
 
 	"yunion.io/x/onecloud/pkg/apis"
+	compute_api "yunion.io/x/onecloud/pkg/apis/compute"
 	api "yunion.io/x/onecloud/pkg/apis/identity"
 	"yunion.io/x/onecloud/pkg/cloudcommon/tsdb"
 	"yunion.io/x/onecloud/pkg/cloudmon/options"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	"yunion.io/x/onecloud/pkg/mcclient/modules/compute"
 	"yunion.io/x/onecloud/pkg/mcclient/modules/identity"
 	"yunion.io/x/onecloud/pkg/util/influxdb"
 )
@@ -67,7 +70,13 @@ func CollectServiceMetrics(ctx context.Context, userCred mcclient.TokenCredentia
 		return
 	}
 	s := auth.GetAdminSession(ctx, options.Options.CommonOptions.Region)
-	err := func() error {
+	urls, err := tsdb.GetDefaultServiceSourceURLs(s, options.Options.SessionEndpointType)
+	if err != nil {
+		return
+	}
+
+	tk := auth.AdminCredential().GetTokenString()
+	err = func() error {
 		endpoints, err := getEndpoints(ctx, s)
 		if err != nil {
 			return errors.Wrapf(err, "getEndpoints")
@@ -83,7 +92,6 @@ func CollectServiceMetrics(ctx context.Context, userCred mcclient.TokenCredentia
 				continue
 			}
 			url := httputils.JoinPath(ep.Url, "version")
-			tk := auth.AdminCredential().GetTokenString()
 			hdr := http.Header{}
 			hdr.Set("X-Auth-Token", tk)
 			resp, err := httputils.Request(
@@ -106,15 +114,48 @@ func CollectServiceMetrics(ctx context.Context, userCred mcclient.TokenCredentia
 			}
 			metrics = append(metrics, part...)
 		}
-		urls, err := tsdb.GetDefaultServiceSourceURLs(s, options.Options.SessionEndpointType)
-		if err != nil {
-			return errors.Wrap(err, "GetServiceURLs")
-		}
 		return influxdb.SendMetrics(urls, SYSTEM_METRIC_DATABASE, metrics, false)
 	}()
 	if err != nil {
 		log.Errorf("collect service metric error: %v", err)
 	}
+	params := compute_api.HostListInput{}
+	limit := 20
+	params.Limit = &limit
+	params.Brand = []string{compute_api.CLOUD_PROVIDER_ONECLOUD}
+	params.Scope = "system"
+	params.Status = []string{compute_api.HOST_STATUS_RUNNING}
+	details := false
+	params.Details = &details
+	hosts := []compute_api.HostDetails{}
+	for {
+		offset := len(hosts)
+		params.Offset = &offset
+		resp, err := compute.Hosts.List(s, jsonutils.Marshal(params))
+		if err != nil {
+			return
+		}
+		part := []compute_api.HostDetails{}
+		err = jsonutils.Update(&part, resp.Data)
+		if err != nil {
+			return
+		}
+		hosts = append(hosts, part...)
+		if len(hosts) >= resp.Total {
+			break
+		}
+	}
+	metrics := []influxdb.SMetricData{}
+	for _, host := range hosts {
+		service := fmt.Sprintf("host-%s", host.Name)
+		part, err := collectWorkerMetrics(ctx, host.ManagerUri, service, version.GetShortString(), tk)
+		if err != nil {
+			log.Errorf("collect host %s metric error: %v", service, err)
+			continue
+		}
+		metrics = append(metrics, part...)
+	}
+	influxdb.SendMetrics(urls, SYSTEM_METRIC_DATABASE, metrics, false)
 }
 
 func collectStatsMetrics(ctx context.Context, ep api.EndpointDetails, version, token string) ([]influxdb.SMetricData, error) {
@@ -257,8 +298,8 @@ func collectStatsMetrics(ctx context.Context, ep api.EndpointDetails, version, t
 	return result, nil
 }
 
-func collectWorkerMetrics(ctx context.Context, ep api.EndpointDetails, version, token string) ([]influxdb.SMetricData, error) {
-	statsUrl := httputils.JoinPath(ep.Url, "worker_stats")
+func collectWorkerMetrics(ctx context.Context, url, service, version, token string) ([]influxdb.SMetricData, error) {
+	statsUrl := httputils.JoinPath(url, "worker_stats")
 	hdr := http.Header{}
 	hdr.Set("X-Auth-Token", token)
 	_, ret, err := httputils.JSONRequest(
@@ -301,7 +342,7 @@ func collectWorkerMetrics(ctx context.Context, ep api.EndpointDetails, version,
 				},
 				{
 					Key:   "service",
-					Value: ep.ServiceName,
+					Value: service,
 				},
 				{
 					Key:   "worker_name",
@@ -484,7 +525,7 @@ func collectServiceMetrics(ctx context.Context, ep api.EndpointDetails, version,
 		errs = append(errs, err)
 	}
 	ret = append(ret, stats...)
-	worker, err := collectWorkerMetrics(ctx, ep, version, token)
+	worker, err := collectWorkerMetrics(ctx, ep.Url, ep.ServiceName, version, token)
 	if err != nil {
 		errs = append(errs, err)
 	}

pkg/monitor/dbinit/measurements/all.go

@@ -31,6 +31,11 @@ var All = []SMeasurement{
 	system,
 	vasmi,
 
+	worker,
+	serviceHttpCode,
+	serviceProcessStats,
+	dbStats,
+
 	vmCpu,
 	vmMem,
 	vmDisk,

pkg/monitor/dbinit/measurements/db_state.go

@@ -0,0 +1,55 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package measurements
+
+import "yunion.io/x/onecloud/pkg/apis/monitor"
+
+var dbStats = SMeasurement{
+	Context: []SMonitorContext{
+		{
+			"db_stats", "Database Stats",
+			monitor.METRIC_RES_TYPE_DB_STATS, monitor.METRIC_DATABASE_SYSTEM,
+		},
+	},
+	Metrics: []SMetric{
+		{
+			"idle", "Database Idle", monitor.METRIC_UNIT_NULL,
+		},
+		{
+			"in_use", "Database InUse", monitor.METRIC_UNIT_NULL,
+		},
+		{
+			"max_idle_closed", "Database max idle closed", monitor.METRIC_UNIT_NULL,
+		},
+		{
+			"max_idle_time_closed", "Database max idle time closed", monitor.METRIC_UNIT_NULL,
+		},
+		{
+			"max_lifetime_closed", "Database max lifetime closed", monitor.METRIC_UNIT_NULL,
+		},
+		{
+			"max_open_connections", "Database max open connections", monitor.METRIC_UNIT_NULL,
+		},
+		{
+			"open_connections", "Database open connections", monitor.METRIC_UNIT_NULL,
+		},
+		{
+			"wait_count", "Database wait count", monitor.METRIC_UNIT_NULL,
+		},
+		{
+			"wait_duration", "Database wait duration", monitor.METRIC_UNIT_NULL,
+		},
+	},
+}

pkg/monitor/dbinit/measurements/service.go

@@ -0,0 +1,40 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package measurements
+
+import "yunion.io/x/onecloud/pkg/apis/monitor"
+
+var worker = SMeasurement{
+	Context: []SMonitorContext{
+		{
+			"worker", "Worker queue",
+			monitor.METRIC_RES_TYPE_WORKER, monitor.METRIC_DATABASE_SYSTEM,
+		},
+	},
+	Metrics: []SMetric{
+		{
+			"active_worker_cnt", "Active Worker Count", monitor.METRIC_UNIT_NULL,
+		},
+		{
+			"max_worker_count", "Max Worker Count", monitor.METRIC_UNIT_NULL,
+		},
+		{
+			"detach_worker_cnt", "Detach worker Count", monitor.METRIC_UNIT_NULL,
+		},
+		{
+			"queue_cnt", "Worker Queue Count", monitor.METRIC_UNIT_NULL,
+		},
+	},
+}

pkg/monitor/dbinit/measurements/service_http_code.go

@@ -0,0 +1,46 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package measurements
+
+import "yunion.io/x/onecloud/pkg/apis/monitor"
+
+var serviceHttpCode = SMeasurement{
+	Context: []SMonitorContext{
+		{
+			"http_request", "HTTP Request hit",
+			monitor.METRIC_RES_TYPE_HTTP_REQUEST, monitor.METRIC_DATABASE_SYSTEM,
+		},
+	},
+	Metrics: []SMetric{
+		{
+			"duration.2xx", "http code 2xxx duration", monitor.METRIC_UNIT_NULL,
+		},
+		{
+			"duration.4xx", "http code 4xxx duration", monitor.METRIC_UNIT_NULL,
+		},
+		{
+			"duration.5xx", "http code 5xxx duration", monitor.METRIC_UNIT_NULL,
+		},
+		{
+			"hit.2xx", "http code 2xxx hit", monitor.METRIC_UNIT_NULL,
+		},
+		{
+			"hit.4xx", "http code 4xxx hit", monitor.METRIC_UNIT_NULL,
+		},
+		{
+			"hit.5xx", "http code 5xxx hit", monitor.METRIC_UNIT_NULL,
+		},
+	},
+}

pkg/monitor/dbinit/measurements/service_process_stats.go

@@ -0,0 +1,40 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package measurements
+
+import "yunion.io/x/onecloud/pkg/apis/monitor"
+
+var serviceProcessStats = SMeasurement{
+	Context: []SMonitorContext{
+		{
+			"process", "Service process stats",
+			monitor.METRIC_RES_TYPE_PROCESS, monitor.METRIC_DATABASE_SYSTEM,
+		},
+	},
+	Metrics: []SMetric{
+		{
+			"cpu_percent", "CPU percent", monitor.METRIC_UNIT_NULL,
+		},
+		{
+			"mem_percent", "Memory percent", monitor.METRIC_UNIT_NULL,
+		},
+		{
+			"mem_size", "Memory size", monitor.METRIC_UNIT_NULL,
+		},
+		{
+			"goroutine_num", "Goroutine num", monitor.METRIC_UNIT_NULL,
+		},
+	},
+}