Skip to content

Commit 489091c

Browse files
BupycHukBupycHuk
authored
PMM-9870 fix collstats indexSizes metrics. (#953)
* PMM-9870 fix collstats indexSizes metrics. * PMM-9870 fix collstats indexSizes metrics. * PMM-9870 fix collstats indexDetails metrics. * PMM-9870 fix linter. * PMM-9870 improve help. * PMM-9870 fix test and linters.
1 parent 09af8cd commit 489091c

10 files changed

+134
-63
lines changed

‎exporter/collstats_collector.go‎

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -99,23 +99,16 @@ func (d *collstatsCollector) collect(ch chan<- prometheus.Metric) {
9999

100100
aggregation := bson.D{
101101
{
102-
Key: "$collStats", Value: bson.M{
102+
Key: "$collStats",
103+
Value: bson.M{
103104
// TODO: PMM-9568 : Add support to handle histogram metrics
104105
"latencyStats": bson.M{"histograms": false},
105106
"storageStats": bson.M{"scale": 1},
106107
},
107108
},
108109
}
109-
project := bson.D{
110-
{
111-
Key: "$project", Value: bson.M{
112-
"storageStats.wiredTiger": 0,
113-
"storageStats.indexDetails": 0,
114-
},
115-
},
116-
}
117110

118-
cursor, err := client.Database(database).Collection(collection).Aggregate(d.ctx, mongo.Pipeline{aggregation, project})
111+
cursor, err := client.Database(database).Collection(collection).Aggregate(d.ctx, mongo.Pipeline{aggregation})
119112
if err != nil {
120113
logger.Errorf("cannot get $collstats cursor for collection %s.%s: %s", database, collection, err)
121114

‎exporter/collstats_collector_test.go‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -53,21 +53,27 @@ func TestCollStatsCollector(t *testing.T) {
5353
ti := labelsGetterMock{}
5454

5555
collection := []string{"testdb.testcol_00", "testdb.testcol_01", "testdb.testcol_02"}
56-
c := newCollectionStatsCollector(ctx, client, logrus.New(), false, false, ti, collection)
56+
logger := logrus.New()
57+
c := newCollectionStatsCollector(ctx, client, logger, false, false, ti, collection)
5758

5859
// The last \n at the end of this string is important
5960
expected := strings.NewReader(`
60-
# HELP mongodb_collstats_latencyStats_commands_latency collstats.latencyStats.commands.
61+
# HELP mongodb_collstats_latencyStats_commands_latency collstats.latencyStats.commands.latency
6162
# TYPE mongodb_collstats_latencyStats_commands_latency untyped
6263
mongodb_collstats_latencyStats_commands_latency{collection="testcol_00",database="testdb"} 0
6364
mongodb_collstats_latencyStats_commands_latency{collection="testcol_01",database="testdb"} 0
6465
mongodb_collstats_latencyStats_commands_latency{collection="testcol_02",database="testdb"} 0
65-
# HELP mongodb_collstats_latencyStats_transactions_ops collstats.latencyStats.transactions.
66+
# HELP mongodb_collstats_latencyStats_transactions_ops collstats.latencyStats.transactions.ops
6667
# TYPE mongodb_collstats_latencyStats_transactions_ops untyped
6768
mongodb_collstats_latencyStats_transactions_ops{collection="testcol_00",database="testdb"} 0
6869
mongodb_collstats_latencyStats_transactions_ops{collection="testcol_01",database="testdb"} 0
6970
mongodb_collstats_latencyStats_transactions_ops{collection="testcol_02",database="testdb"} 0
70-
# HELP mongodb_collstats_storageStats_capped collstats.storageStats.
71+
# HELP mongodb_collstats_storageStats_indexSizes collstats.storageStats.indexSizes
72+
# TYPE mongodb_collstats_storageStats_indexSizes untyped
73+
mongodb_collstats_storageStats_indexSizes{collection="testcol_00",database="testdb",index_name="_id_"} 4096
74+
mongodb_collstats_storageStats_indexSizes{collection="testcol_01",database="testdb",index_name="_id_"} 4096
75+
mongodb_collstats_storageStats_indexSizes{collection="testcol_02",database="testdb",index_name="_id_"} 4096
76+
# HELP mongodb_collstats_storageStats_capped collstats.storageStats.capped
7177
# TYPE mongodb_collstats_storageStats_capped untyped
7278
mongodb_collstats_storageStats_capped{collection="testcol_00",database="testdb"} 0
7379
mongodb_collstats_storageStats_capped{collection="testcol_01",database="testdb"} 0
@@ -81,6 +87,7 @@ mongodb_collstats_storageStats_capped{collection="testcol_02",database="testdb"}
8187
filter := []string{
8288
"mongodb_collstats_latencyStats_commands_latency",
8389
"mongodb_collstats_storageStats_capped",
90+
"mongodb_collstats_storageStats_indexSizes",
8491
"mongodb_collstats_latencyStats_transactions_ops",
8592
}
8693
err := testutil.CollectAndCompare(c, expected, filter...)

‎exporter/common.go‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,10 @@ func listCollections(ctx context.Context, client *mongo.Client, database string,
7979
//
8080
// - exclude: List of databases to be excluded. Useful to ignore system databases.
8181
func databases(ctx context.Context, client *mongo.Client, filterInNamespaces []string, exclude []string) ([]string, error) {
82-
opts := &options.ListDatabasesOptions{NameOnly: pointer.ToBool(true), AuthorizedDatabases: pointer.ToBool(true)}
82+
opts := &options.ListDatabasesOptions{
83+
NameOnly: pointer.ToBool(true),
84+
AuthorizedDatabases: pointer.ToBool(true),
85+
}
8386

8487
filter := bson.D{}
8588

@@ -100,35 +103,34 @@ func databases(ctx context.Context, client *mongo.Client, filterInNamespaces []s
100103
}
101104

102105
func makeExcludeFilter(exclude []string) *primitive.E {
103-
filterExpressions := []bson.D{}
106+
if len(exclude) == 0 {
107+
return nil
108+
}
109+
110+
filterExpressions := make([]bson.D, 0, len(exclude))
104111
for _, dbname := range exclude {
105112
filterExpressions = append(filterExpressions,
106113
bson.D{{Key: "name", Value: bson.D{{Key: "$ne", Value: dbname}}}},
107114
)
108115
}
109116

110-
if len(filterExpressions) == 0 {
111-
return nil
112-
}
113-
114117
return &primitive.E{Key: "$and", Value: filterExpressions}
115118
}
116119

117120
func makeDBsFilter(filterInNamespaces []string) *primitive.E {
118-
filterExpressions := []bson.D{}
119-
120121
nss := removeEmptyStrings(filterInNamespaces)
122+
if len(nss) == 0 {
123+
return nil
124+
}
125+
126+
filterExpressions := make([]bson.D, 0, len(nss))
121127
for _, namespace := range nss {
122128
parts := strings.Split(namespace, ".")
123129
filterExpressions = append(filterExpressions,
124130
bson.D{{Key: "name", Value: bson.D{{Key: "$eq", Value: parts[0]}}}},
125131
)
126132
}
127133

128-
if len(filterExpressions) == 0 {
129-
return nil
130-
}
131-
132134
return &primitive.E{Key: "$or", Value: filterExpressions}
133135
}
134136

‎exporter/dbstats_collector_test.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -58,15 +58,16 @@ func TestDBStatsCollector(t *testing.T) {
5858

5959
ti := labelsGetterMock{}
6060

61-
c := newDBStatsCollector(ctx, client, logrus.New(), false, ti, []string{dbName}, false)
61+
logger := logrus.New()
62+
c := newDBStatsCollector(ctx, client, logger, false, ti, []string{dbName}, false)
6263
expected := strings.NewReader(`
63-
# HELP mongodb_dbstats_collections dbstats.
64+
# HELP mongodb_dbstats_collections dbstats.collections
6465
# TYPE mongodb_dbstats_collections untyped
6566
mongodb_dbstats_collections{database="testdb"} 3
66-
# HELP mongodb_dbstats_indexes dbstats.
67+
# HELP mongodb_dbstats_indexes dbstats.indexes
6768
# TYPE mongodb_dbstats_indexes untyped
6869
mongodb_dbstats_indexes{database="testdb"} 3
69-
# HELP mongodb_dbstats_objects dbstats.
70+
# HELP mongodb_dbstats_objects dbstats.objects
7071
# TYPE mongodb_dbstats_objects untyped
7172
mongodb_dbstats_objects{database="testdb"} 30` + "\n")
7273

‎exporter/diagnostic_data_collector_test.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,10 +58,10 @@ func TestDiagnosticDataCollector(t *testing.T) {
5858

5959
// The last \n at the end of this string is important
6060
expectedString := fmt.Sprintf(`
61-
# HELP mongodb_oplog_stats_wt_btree_fixed_record_size %s.btree.
61+
# HELP mongodb_oplog_stats_wt_btree_fixed_record_size %s.btree.fixed-record size
6262
# TYPE mongodb_oplog_stats_wt_btree_fixed_record_size untyped
6363
mongodb_oplog_stats_wt_btree_fixed_record_size 0
64-
# HELP mongodb_oplog_stats_wt_transaction_update_conflicts %s.transaction.
64+
# HELP mongodb_oplog_stats_wt_transaction_update_conflicts %s.transaction.update conflicts
6565
# TYPE mongodb_oplog_stats_wt_transaction_update_conflicts untyped
6666
mongodb_oplog_stats_wt_transaction_update_conflicts 0`, prefix, prefix)
6767
expected := strings.NewReader(expectedString + "\n")
@@ -211,6 +211,7 @@ func TestAllDiagnosticDataCollectorMetrics(t *testing.T) {
211211
client := tu.DefaultTestClient(ctx, t)
212212

213213
logger := logrus.New()
214+
logger.SetLevel(logrus.DebugLevel)
214215
ti := newTopologyInfo(ctx, client, logger)
215216

216217
dbBuildInfo, err := retrieveMongoDBBuildInfo(ctx, client, logger.WithField("component", "test"))

‎exporter/indexstats_collector_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ func TestIndexStatsCollector(t *testing.T) {
6868

6969
// The last \n at the end of this string is important
7070
expected := strings.NewReader(`
71-
# HELP mongodb_indexstats_accesses_ops indexstats.accesses.
71+
# HELP mongodb_indexstats_accesses_ops indexstats.accesses.ops
7272
# TYPE mongodb_indexstats_accesses_ops untyped
7373
mongodb_indexstats_accesses_ops{collection="testcol_00",database="testdb",key_name="_id_"} 0
7474
mongodb_indexstats_accesses_ops{collection="testcol_00",database="testdb",key_name="idx_01"} 0
@@ -118,7 +118,7 @@ func TestDescendingIndexOverride(t *testing.T) {
118118

119119
// The last \n at the end of this string is important
120120
expected := strings.NewReader(`
121-
# HELP mongodb_indexstats_accesses_ops indexstats.accesses.
121+
# HELP mongodb_indexstats_accesses_ops indexstats.accesses.ops
122122
# TYPE mongodb_indexstats_accesses_ops untyped
123123
mongodb_indexstats_accesses_ops{collection="testcol_00",database="testdb",key_name="_id_"} 0
124124
mongodb_indexstats_accesses_ops{collection="testcol_00",database="testdb",key_name="f1_1"} 0

‎exporter/metrics.go‎

Lines changed: 84 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -93,24 +93,68 @@ var (
9393
// mongodb_ss_opcounters{legacy_op_type="command"} 67923
9494
//
9595
nodeToPDMetrics = map[string]string{
96-
"collStats.storageStats.indexDetails.": "index_name",
97-
"globalLock.activeQueue.": "count_type",
98-
"globalLock.locks.": "lock_type",
99-
"serverStatus.asserts.": "assert_type",
100-
"serverStatus.connections.": "conn_type",
101-
"serverStatus.globalLock.currentQueue.": "count_type",
102-
"serverStatus.metrics.commands.": "cmd_name",
103-
"serverStatus.metrics.cursor.open.": "csr_type",
104-
"serverStatus.metrics.document.": "doc_op_type",
105-
"serverStatus.opLatencies.": "op_type",
106-
"serverStatus.opReadConcernCounters.": "concern_type",
107-
"serverStatus.opcounters.": "legacy_op_type",
108-
"serverStatus.opcountersRepl.": "legacy_op_type",
109-
"serverStatus.transactions.commitTypes.": "commit_type",
110-
"serverStatus.wiredTiger.concurrentTransactions.": "txn_rw_type",
111-
"serverStatus.queues.execution.": "txn_rw_type",
112-
"serverStatus.wiredTiger.perf.": "perf_bucket",
113-
"systemMetrics.disks.": "device_name",
96+
"collStats.storageStats.indexDetails.": "index_name",
97+
"globalLock.activeQueue.": "count_type",
98+
"globalLock.locks.": "lock_type",
99+
"serverStatus.asserts.": "assert_type",
100+
"serverStatus.connections.": "conn_type",
101+
"serverStatus.globalLock.currentQueue.": "count_type",
102+
"serverStatus.metrics.commands.": "cmd_name",
103+
"serverStatus.metrics.cursor.open.": "csr_type",
104+
"serverStatus.metrics.document.": "doc_op_type",
105+
"serverStatus.opLatencies.": "op_type",
106+
"serverStatus.opReadConcernCounters.": "concern_type",
107+
"serverStatus.opcounters.": "legacy_op_type",
108+
"serverStatus.opcountersRepl.": "legacy_op_type",
109+
"serverStatus.transactions.commitTypes.": "commit_type",
110+
"serverStatus.wiredTiger.concurrentTransactions.": "txn_rw_type",
111+
"serverStatus.queues.execution.": "txn_rw_type",
112+
"serverStatus.wiredTiger.perf.": "perf_bucket",
113+
"systemMetrics.disks.": "device_name",
114+
"collstats.storageStats.indexSizes.": "index_name",
115+
"config.transactions.stats.storageStats.indexSizes.": "index_name",
116+
"config.image_collection.stats.storageStats.indexSizes.": "index_name",
117+
}
118+
119+
// This map is used to add labels to some specific metrics.
120+
// The difference from the case above that it works with middle nodes in the structure.
121+
// For example, the fields under the storageStats.indexDetails. structure have this
122+
// signature:
123+
//
124+
// "storageStats": primitive.M{
125+
// "indexDetails": primitive.M{
126+
// "_id_": primitive.M{
127+
// "LSM": primitive.M{
128+
// "bloom filter false positives": int32(0),
129+
// "bloom filter hits": int32(0),
130+
// "bloom filter misses": int32(0),
131+
// ...
132+
// },
133+
// "block-manager": primitive.M{
134+
// "allocations requiring file extension": int32(0),
135+
// ...
136+
// },
137+
// ...
138+
// },
139+
// "name_1": primitive.M{
140+
// ...
141+
// },
142+
// ...
143+
// },
144+
// },
145+
//
146+
// Applying the renaming rules, storageStats will become storageStats but instead of having metrics
147+
// with the form storageStats.indexDetails.<index_name>.<metric_name> where index_name is each one of
148+
// the fields inside the structure (_id_, name_1, etc), those keys will become labels for the same
149+
// metric name. The label name is defined as the value for each metric name in the map and the value
150+
// the label will have is the field name in the structure. Example.
151+
//
152+
// mongodb_storageStats_indexDetails_index_name_LSM_bloom_filter_false_positives{index_name="_id_"} 0
153+
keyNodesToLabels = map[string]string{
154+
"storageStats.indexDetails.": "index_name",
155+
"config.image_collection.stats.storageStats.indexDetails.": "index_name",
156+
"config.transactions.stats.storageStats.indexDetails.": "index_name",
157+
"collstats.storageStats.indexDetails.": "index_name",
114158
}
115159

116160
// Regular expressions used to make the metric name Prometheus-compatible
@@ -236,8 +280,11 @@ func rawToPrometheusMetric(rm *rawMetric) (prometheus.Metric, error) {
236280
// by prometheus. For first level metrics, there is no prefix so we should use the metric name or
237281
// the help would be empty.
238282
func metricHelp(prefix, name string) string {
283+
if _, ok := nodeToPDMetrics[prefix]; ok {
284+
return strings.TrimSuffix(prefix, ".")
285+
}
239286
if prefix != "" {
240-
return prefix
287+
return prefix + name
241288
}
242289

243290
return name
@@ -251,17 +298,29 @@ func makeMetrics(prefix string, m bson.M, labels map[string]string, compatibleMo
251298
}
252299

253300
for k, val := range m {
301+
nextPrefix := prefix + k
302+
303+
l := make(map[string]string)
304+
if label, ok := keyNodesToLabels[prefix]; ok {
305+
for k, v := range labels {
306+
l[k] = v
307+
}
308+
l[label] = k
309+
nextPrefix = prefix + label
310+
} else {
311+
l = labels
312+
}
254313
switch v := val.(type) {
255314
case bson.M:
256-
res = append(res, makeMetrics(prefix+k, v, labels, compatibleMode)...)
315+
res = append(res, makeMetrics(nextPrefix, v, l, compatibleMode)...)
257316
case map[string]interface{}:
258-
res = append(res, makeMetrics(prefix+k, v, labels, compatibleMode)...)
317+
res = append(res, makeMetrics(nextPrefix, v, l, compatibleMode)...)
259318
case primitive.A:
260-
res = append(res, processSlice(prefix, k, v, labels, compatibleMode)...)
319+
res = append(res, processSlice(nextPrefix, v, l, compatibleMode)...)
261320
case []interface{}:
262321
continue
263322
default:
264-
rm, err := makeRawMetric(prefix, k, v, labels)
323+
rm, err := makeRawMetric(prefix, k, v, l)
265324
if err != nil {
266325
invalidMetric := prometheus.NewInvalidMetric(prometheus.NewInvalidDesc(err), err)
267326
res = append(res, invalidMetric)
@@ -302,7 +361,7 @@ func makeMetrics(prefix string, m bson.M, labels map[string]string, compatibleMo
302361

303362
// Extract maps from arrays. Only some structures like replicasets have arrays of members
304363
// and each member is represented by a map[string]interface{}.
305-
func processSlice(prefix, k string, v []interface{}, commonLabels map[string]string, compatibleMode bool) []prometheus.Metric {
364+
func processSlice(prefix string, v []interface{}, commonLabels map[string]string, compatibleMode bool) []prometheus.Metric {
306365
metrics := make([]prometheus.Metric, 0)
307366
labels := make(map[string]string)
308367
for name, value := range commonLabels {
@@ -332,7 +391,7 @@ func processSlice(prefix, k string, v []interface{}, commonLabels map[string]str
332391
labels["member_idx"] = host
333392
}
334393

335-
metrics = append(metrics, makeMetrics(prefix+k, s, labels, compatibleMode)...)
394+
metrics = append(metrics, makeMetrics(prefix, s, labels, compatibleMode)...)
336395
}
337396

338397
return metrics

‎exporter/profile_status_collector_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ func TestProfileCollector(t *testing.T) {
5252
c := newProfileCollector(ctx, client, logrus.New(), false, ti, 30)
5353

5454
expected := strings.NewReader(`
55-
# HELP mongodb_profile_slow_query_count profile_slow_query.
55+
# HELP mongodb_profile_slow_query_count profile_slow_query.count
5656
# TYPE mongodb_profile_slow_query_count counter
5757
mongodb_profile_slow_query_count{database="admin"} 0
5858
mongodb_profile_slow_query_count{database="config"} 0

‎exporter/replset_config_collector_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ func TestReplsetConfigCollector(t *testing.T) {
4040

4141
// The last \n at the end of this string is important
4242
expected := strings.NewReader(`
43-
# HELP mongodb_rs_cfg_protocolVersion rs_cfg.
43+
# HELP mongodb_rs_cfg_protocolVersion rs_cfg.protocolVersion
4444
# TYPE mongodb_rs_cfg_protocolVersion untyped
4545
mongodb_rs_cfg_protocolVersion 1` + "\n")
4646
// Filter metrics for 2 reasons:

‎main.go‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -151,11 +151,19 @@ func buildExporter(opts GlobalFlags, uri string, log *logrus.Logger) *exporter.E
151151
nodeName = uriParsed.Host
152152
}
153153

154+
collStatsNamespaces := []string{}
155+
if opts.CollStatsNamespaces != "" {
156+
collStatsNamespaces = strings.Split(opts.CollStatsNamespaces, ",")
157+
}
158+
indexStatsCollections := []string{}
159+
if opts.IndexStatsCollections != "" {
160+
indexStatsCollections = strings.Split(opts.IndexStatsCollections, ",")
161+
}
154162
exporterOpts := &exporter.Opts{
155-
CollStatsNamespaces: strings.Split(opts.CollStatsNamespaces, ","),
163+
CollStatsNamespaces: collStatsNamespaces,
156164
CompatibleMode: opts.CompatibleMode,
157165
DiscoveringMode: opts.DiscoveringMode,
158-
IndexStatsCollections: strings.Split(opts.IndexStatsCollections, ","),
166+
IndexStatsCollections: indexStatsCollections,
159167
Logger: log,
160168
URI: uri,
161169
NodeName: nodeName,

0 commit comments

Comments
 (0)