Merge pull request #57 from BattleL/dev/support-service-role

Dev/support service role

Author: zero3233

.gitignore

@@ -2,4 +2,6 @@
 /.idea
 build/*
 qcloud.yml
-.DS_Store
+.DS_Store
+*mock.go
+/bin

README.md

@@ -82,26 +82,27 @@ products:
 credential:
   access_key: <YOUR_ACCESS_KEY>                  // 必须, 云API的SecretId
   access_secret: <YOUR_ACCESS_SECRET>            // 必须, 云API的SecretKey
+  role: <YOUR_SERVICE_ROLE>                      // 可选，可在云上CVM/TKE中使用，没有access_key和access_secret时会使用role申请临时密钥
   region: <REGION>                               // 必须, 实例所在区域信息
 
-rate_limit: 15                                   // 腾讯云监控拉取指标数据限制, 官方默认限制最大20qps 
+rate_limit: 15                                   // 腾讯云监控拉取指标数据限制, 官方默认限制最大20qps
 
 
-// 整个产品纬度配置, 每个产品一个item                                              
+// 整个产品纬度配置, 每个产品一个item
 products:
   - namespace: QCE/CMONGO                        // 必须, 产品命名空间; QCE前缀可自定义,CMONGO产品名不区分大小写, 可用别名
-    all_metrics: true                            // 常用, 推荐开启, 导出支持的所有指标 
-    all_instances: true                          // 常用, 推荐开启, 导出该region下的所有实例 
+    all_metrics: true                            // 常用, 推荐开启, 导出支持的所有指标
+    all_instances: true                          // 常用, 推荐开启, 导出该region下的所有实例
     extra_labels: [InstanceName,Zone]            // 可选, 将实例的字段作为指标的lables导出
     only_include_metrics: [Inserts]              // 可选, 只导出这些指标, 配置时all_metrics失效
     exclude_metrics: [Reads]                     // 可选, 不导出这些指标
     instance_filters:                            // 可选, 在all_instances开启情况下, 根据每个实例的字段进行过滤
       - ProjectId: 1
-        Status: 1                        
+        Status: 1
     only_include_instances: [cmgo-xxxxxxxx]      // 可选, 只导出这些实例id, 配置时all_instances失效
     exclude_instances: [cmgo-xxxxxxxx]           // 可选, 不导出这些实例id
     custom_query_dimensions:                     // 可选, 不常用, 自定义指标查询条件, 配置时all_instances,only_include_instances,exclude_instances失效, 用于不支持按实例纬度查询的指标
-     - target: cmgo-xxxxxxxx
+      - target: cmgo-xxxxxxxx
     statistics_types: [avg]                      // 可选, 拉取N个数据点, 再进行max、min、avg、last计算, 默认last取最新值
     period_seconds: 60                           // 可选, 指标统计周期, 默认自动获取指标支持的最小统计周期
     range_seconds: 300                           // 可选, 选取时间范围, 开始时间=now-range_seconds, 结束时间=now
@@ -112,33 +113,33 @@ products:
 
 // 单个指标纬度配置, 每个指标一个item
 metrics:
-  - tc_namespace: QCE/CMONGO                     // 产品命名空间, 同namespace       
+  - tc_namespace: QCE/CMONGO                     // 产品命名空间, 同namespace
     tc_metric_name: Inserts                      // 云监控定义的指标名
     tc_metric_rename: Inserts                    // 导出指标的显示名
     tc_metric_name_type: 1                       // 可选，导出指标的名字格式化类型, 1=大写转小写加下划线, 2=转小写; 默认1
     tc_labels: [InstanceName]                    // 可选, 将实例的字段作为指标的lables导出
     tc_filters:                                  // 可选, 根据每个实例的字段进行过滤, 否则默认导出region下所有实例
       - ProjectId: 1
-        Status: 1                             
+        Status: 1
     tc_myself_dimensions:                        // 可选, 同custom_query_dimensions
     tc_statistics: [Avg]                         // 可选, 同statistics_types
     period_seconds: 60                           // 可选, 同period_seconds
     range_seconds: 300                           // 可选, 同range_seconds
     delay_seconds: 60                            // 可选, 同delay_seconds
-
 ```
 特殊说明:
 1. **custom_query_dimensions**  
-每个实例的纬度字段信息, 可从对应的云监控产品指标文档查询, 如mongo支持的纬度字段信息可由[云监控指标详情](https://cloud.tencent.com/document/product/248/45104#%E5%90%84%E7%BB%B4%E5%BA%A6%E5%AF%B9%E5%BA%94%E5%8F%82%E6%95%B0%E6%80%BB%E8%A7%88) 查询 
+   每个实例的纬度字段信息, 可从对应的云监控产品指标文档查询, 如mongo支持的纬度字段信息可由[云监控指标详情](https://cloud.tencent.com/document/product/248/45104#%E5%90%84%E7%BB%B4%E5%BA%A6%E5%AF%B9%E5%BA%94%E5%8F%82%E6%95%B0%E6%80%BB%E8%A7%88) 查询
 2. **extra_labels**  
-每个导出metric的labels还额外上报实例对应的字段信息, 实例可选的字段列表可从对应产品文档查询, 如mongo实例支持的字段可从[实例查询api文档](https://cloud.tencent.com/document/product/240/38568) 获取, 目前只支持str、int类型的字段
+   每个导出metric的labels还额外上报实例对应的字段信息, 实例可选的字段列表可从对应产品文档查询, 如mongo实例支持的字段可从[实例查询api文档](https://cloud.tencent.com/document/product/240/38568) 获取, 目前只支持str、int类型的字段
 3. **period_seconds**  
-每个指标支持的时间纬度统计, 一般支持60、300秒等, 具体可由对应产品的云监控产品指标文档查询, 如mongo可由[指标元数据查询](https://cloud.tencent.com/document/product/248/30351) , 假如不配置, 使用默认值(60), 假如该指标不支持60, 则自动使用该指标支持的最小值  
+   每个指标支持的时间纬度统计, 一般支持60、300秒等, 具体可由对应产品的云监控产品指标文档查询, 如mongo可由[指标元数据查询](https://cloud.tencent.com/document/product/248/30351) , 假如不配置, 使用默认值(60), 假如该指标不支持60, 则自动使用该指标支持的最小值
 4. **credential**  
-SecretId、SecretKey、Region可由环境变量获取
+   SecretId、SecretKey、Region可由环境变量获取
 ```bash
 export TENCENTCLOUD_SECRET_ID="YOUR_ACCESS_KEY"
 export TENCENTCLOUD_SECRET_KEY="YOUR_ACCESS_SECRET"
+export TENCENTCLOUD_SERVICE_ROLE = "YOUR_SERVICE_ROLE"
 export TENCENTCLOUD_REGION="REGION"
 ```
 
@@ -156,7 +157,7 @@ export TENCENTCLOUD_REGION="REGION"
 --log.level|日志级别|info
 
 
-## 五、qcloud.yml样例  
+## 五、qcloud.yml样例
 在git的configs里有支持产品的配置模版样例
 
 

cmd/qcloud-exporter/qcloud_exporter.go

@@ -4,9 +4,14 @@ import (
 	"fmt"
 	"net/http"
 	"os"
+	"time"
 
-	"github.com/go-kit/kit/log"
-	"github.com/go-kit/kit/log/level"
+	"github.com/tencentyun/tencentcloud-exporter/pkg/cachedtransactiongather"
+
+	"github.com/tencentyun/tencentcloud-exporter/pkg/common"
+
+	"github.com/go-kit/log"
+	"github.com/go-kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/prometheus/common/promlog"
@@ -17,7 +22,9 @@ import (
 	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
 
-func newHandler(c *config.TencentConfig, includeExporterMetrics bool, maxRequests int, logger log.Logger) (*http.Handler, error) {
+func newHandler(cred common.CredentialIface, c *config.TencentConfig,
+	includeExporterMetrics bool, maxRequests int, logger log.Logger) (*http.Handler, error) {
+
 	exporterMetricsRegistry := prometheus.NewRegistry()
 	if includeExporterMetrics {
 		exporterMetricsRegistry.MustRegister(
@@ -26,7 +33,7 @@ func newHandler(c *config.TencentConfig, includeExporterMetrics bool, maxRequest
 		)
 	}
 
-	nc, err := collector.NewTcMonitorCollector(c, logger)
+	nc, err := collector.NewTcMonitorCollector(cred, c, logger)
 	if err != nil {
 		return nil, fmt.Errorf("couldn't create collector: %s", err)
 	}
@@ -35,15 +42,24 @@ func newHandler(c *config.TencentConfig, includeExporterMetrics bool, maxRequest
 	if err := r.Register(nc); err != nil {
 		return nil, fmt.Errorf("couldn't register tencent cloud monitor collector: %s", err)
 	}
+	var handler http.Handler
+	gatherers := prometheus.Gatherers{exporterMetricsRegistry, r}
+	opts := promhttp.HandlerOpts{
+		ErrorHandling:       promhttp.ContinueOnError,
+		MaxRequestsInFlight: maxRequests,
+		Registry:            exporterMetricsRegistry,
+	}
+	if c.CacheInterval <= 0 {
+		handler = promhttp.HandlerFor(gatherers, opts)
+	} else {
+		handler = promhttp.HandlerForTransactional(
+			cachedtransactiongather.NewCachedTransactionGather(
+				prometheus.ToTransactionalGatherer(gatherers),
+				time.Duration(c.CacheInterval)*time.Second, logger,
+			), opts,
+		)
+	}
 
-	handler := promhttp.HandlerFor(
-		prometheus.Gatherers{exporterMetricsRegistry, r},
-		promhttp.HandlerOpts{
-			ErrorHandling:       promhttp.ContinueOnError,
-			MaxRequestsInFlight: maxRequests,
-			Registry:            exporterMetricsRegistry,
-		},
-	)
 	if includeExporterMetrics {
 		handler = promhttp.InstrumentMetricHandler(
 			exporterMetricsRegistry, handler,
@@ -94,7 +110,27 @@ func main() {
 		level.Info(logger).Log("msg", "Load config ok")
 	}
 
-	handler, err := newHandler(tencentConfig, *enableExporterMetrics, *maxRequests, logger)
+	cred := &common.Credential{}
+	if tencentConfig.Credential.Role != "" {
+		var err error
+		cred, err = common.NewCredential(tencentConfig.Credential.Role)
+		if err != nil {
+			level.Error(logger).Log("msg", "init cred error", "err", err)
+			panic(err)
+		}
+		go func() {
+			err := cred.Refresh()
+			if err != nil {
+				level.Error(logger).Log("msg", "cred refresh error", "err", err)
+				panic(err)
+			}
+		}()
+	} else {
+		cred.SecretId = tencentConfig.Credential.AccessKey
+		cred.SecretKey = tencentConfig.Credential.SecretKey
+	}
+
+	handler, err := newHandler(cred, tencentConfig, *enableExporterMetrics, *maxRequests, logger)
 	if err != nil {
 		level.Error(logger).Log("msg", "Create handler fail", "err", err)
 		os.Exit(1)

go.mod

@@ -3,18 +3,19 @@ module github.com/tencentyun/tencentcloud-exporter
 go 1.12
 
 require (
-	github.com/go-kit/kit v0.9.0
+	github.com/go-kit/log v0.2.0
 	github.com/golang/mock v1.4.4
-	github.com/prometheus/client_golang v1.5.1
-	github.com/prometheus/common v0.9.1
+	github.com/prometheus/client_golang v1.12.2-0.20220630150036-810fcb46abcd
+	github.com/prometheus/client_model v0.2.0
+	github.com/prometheus/common v0.35.0
 	github.com/stretchr/testify v1.6.1
 	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cbs v1.0.334
 	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cdb v1.0.413
-	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cdn v1.0.423
+	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cdn v1.0.437
 	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/ckafka v1.0.334
 	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/clb v1.0.334
 	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cmq v1.0.334
-	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.430
+	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.437
 	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cvm v1.0.334
 	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cynosdb v1.0.413
 	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dc v1.0.334
@@ -34,6 +35,6 @@ require (
 	github.com/tencentyun/cos-go-sdk-v5 v0.7.35
 	golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
-	gopkg.in/yaml.v2 v2.2.5
+	gopkg.in/yaml.v2 v2.4.0
 
 )