Skip to content

Commit 9e7bf83

Browse files
BupycHukBupycHuk
committed
PMM-9870 fix collstats indexSizes metrics.
1 parent 6907552 commit 9e7bf83

File tree

4 files changed

+109
-48
lines changed

4 files changed

+109
-48
lines changed

‎exporter/collstats_collector.go‎

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -99,23 +99,16 @@ func (d *collstatsCollector) collect(ch chan<- prometheus.Metric) {
9999

100100
aggregation := bson.D{
101101
{
102-
Key: "$collStats", Value: bson.M{
102+
Key: "$collStats",
103+
Value: bson.M{
103104
// TODO: PMM-9568 : Add support to handle histogram metrics
104105
"latencyStats": bson.M{"histograms": false},
105106
"storageStats": bson.M{"scale": 1},
106107
},
107108
},
108109
}
109-
project := bson.D{
110-
{
111-
Key: "$project", Value: bson.M{
112-
"storageStats.wiredTiger": 0,
113-
"storageStats.indexDetails": 0,
114-
},
115-
},
116-
}
117110

118-
cursor, err := client.Database(database).Collection(collection).Aggregate(d.ctx, mongo.Pipeline{aggregation, project})
111+
cursor, err := client.Database(database).Collection(collection).Aggregate(d.ctx, mongo.Pipeline{aggregation})
119112
if err != nil {
120113
logger.Errorf("cannot get $collstats cursor for collection %s.%s: %s", database, collection, err)
121114

‎exporter/common.go‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,10 @@ func listCollections(ctx context.Context, client *mongo.Client, database string,
7979
//
8080
// - exclude: List of databases to be excluded. Useful to ignore system databases.
8181
func databases(ctx context.Context, client *mongo.Client, filterInNamespaces []string, exclude []string) ([]string, error) {
82-
opts := &options.ListDatabasesOptions{NameOnly: pointer.ToBool(true), AuthorizedDatabases: pointer.ToBool(true)}
82+
opts := &options.ListDatabasesOptions{
83+
NameOnly: pointer.ToBool(true),
84+
AuthorizedDatabases: pointer.ToBool(true),
85+
}
8386

8487
filter := bson.D{}
8588

@@ -100,35 +103,33 @@ func databases(ctx context.Context, client *mongo.Client, filterInNamespaces []s
100103
}
101104

102105
func makeExcludeFilter(exclude []string) *primitive.E {
103-
filterExpressions := []bson.D{}
106+
if len(exclude) == 0 {
107+
return nil
108+
}
109+
var filterExpressions []bson.D
104110
for _, dbname := range exclude {
105111
filterExpressions = append(filterExpressions,
106112
bson.D{{Key: "name", Value: bson.D{{Key: "$ne", Value: dbname}}}},
107113
)
108114
}
109115

110-
if len(filterExpressions) == 0 {
111-
return nil
112-
}
113-
114116
return &primitive.E{Key: "$and", Value: filterExpressions}
115117
}
116118

117119
func makeDBsFilter(filterInNamespaces []string) *primitive.E {
118120
filterExpressions := []bson.D{}
119121

120122
nss := removeEmptyStrings(filterInNamespaces)
123+
if len(nss) == 0 {
124+
return nil
125+
}
121126
for _, namespace := range nss {
122127
parts := strings.Split(namespace, ".")
123128
filterExpressions = append(filterExpressions,
124129
bson.D{{Key: "name", Value: bson.D{{Key: "$eq", Value: parts[0]}}}},
125130
)
126131
}
127132

128-
if len(filterExpressions) == 0 {
129-
return nil
130-
}
131-
132133
return &primitive.E{Key: "$or", Value: filterExpressions}
133134
}
134135

‎exporter/metrics.go‎

Lines changed: 85 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -93,25 +93,69 @@ var (
9393
// mongodb_ss_opcounters{legacy_op_type="command"} 67923
9494
//
9595
nodeToPDMetrics = map[string]string{
96-
"collStats.storageStats.indexDetails.": "index_name",
97-
"globalLock.activeQueue.": "count_type",
98-
"globalLock.locks.": "lock_type",
99-
"serverStatus.asserts.": "assert_type",
100-
"serverStatus.connections.": "conn_type",
101-
"serverStatus.globalLock.currentQueue.": "count_type",
102-
"serverStatus.metrics.commands.": "cmd_name",
103-
"serverStatus.metrics.cursor.open.": "csr_type",
104-
"serverStatus.metrics.document.": "doc_op_type",
105-
"serverStatus.opLatencies.": "op_type",
106-
"serverStatus.opReadConcernCounters.": "concern_type",
107-
"serverStatus.opcounters.": "legacy_op_type",
108-
"serverStatus.opcountersRepl.": "legacy_op_type",
109-
"serverStatus.transactions.commitTypes.": "commit_type",
110-
"serverStatus.wiredTiger.concurrentTransactions.": "txn_rw_type",
111-
"serverStatus.queues.execution.": "txn_rw_type",
112-
"serverStatus.wiredTiger.perf.": "perf_bucket",
113-
"systemMetrics.disks.": "device_name",
114-
"collstats.storageStats.indexSizes.": "index_name",
96+
"collStats.storageStats.indexDetails.": "index_name",
97+
"globalLock.activeQueue.": "count_type",
98+
"globalLock.locks.": "lock_type",
99+
"serverStatus.asserts.": "assert_type",
100+
"serverStatus.connections.": "conn_type",
101+
"serverStatus.globalLock.currentQueue.": "count_type",
102+
"serverStatus.metrics.commands.": "cmd_name",
103+
"serverStatus.metrics.cursor.open.": "csr_type",
104+
"serverStatus.metrics.document.": "doc_op_type",
105+
"serverStatus.opLatencies.": "op_type",
106+
"serverStatus.opReadConcernCounters.": "concern_type",
107+
"serverStatus.opcounters.": "legacy_op_type",
108+
"serverStatus.opcountersRepl.": "legacy_op_type",
109+
"serverStatus.transactions.commitTypes.": "commit_type",
110+
"serverStatus.wiredTiger.concurrentTransactions.": "txn_rw_type",
111+
"serverStatus.queues.execution.": "txn_rw_type",
112+
"serverStatus.wiredTiger.perf.": "perf_bucket",
113+
"systemMetrics.disks.": "device_name",
114+
"collstats.storageStats.indexSizes.": "index_name",
115+
"config.transactions.stats.storageStats.indexSizes.": "index_name",
116+
"config.image_collection.stats.storageStats.indexSizes.": "index_name",
117+
}
118+
119+
// This map is used to add labels to some specific metrics.
120+
// The difference from the case above that it works with middle nodes in the structure.
121+
// For example, the fields under the storageStats.indexDetails. structure have this
122+
// signature:
123+
//
124+
// "storageStats": primitive.M{
125+
// "indexDetails": primitive.M{
126+
// "_id_": primitive.M{
127+
// "LSM": primitive.M{
128+
// "bloom filter false positives": int32(0),
129+
// "bloom filter hits": int32(0),
130+
// "bloom filter misses": int32(0),
131+
// ...
132+
// },
133+
// "block-manager": primitive.M{
134+
// "allocations requiring file extension": int32(0),
135+
// ...
136+
// },
137+
// ...
138+
// },
139+
// "name_1": primitive.M{
140+
// ...
141+
// },
142+
// ...
143+
// },
144+
// },
145+
//
146+
// Applying the renaming rules, storageStats will become storageStats but instead of having metrics
147+
// with the form storageStats.indexDetails.<index_name>.<metric_name> where index_name is each one of
148+
// the fields inside the structure (_id_, name_1, etc), those keys will become labels for the same
149+
// metric name. The label name is defined as the value for each metric name in the map and the value
150+
// the label will have is the field name in the structure. Example.
151+
//
152+
// mongodb_storageStats_indexDetails_index_name_LSM_bloom_filter_false_positives{index_name="_id_"} 0
153+
keyNodesToLabels = map[string]string{
154+
"storageStats.indexDetails.": "index_name",
155+
"config.image_collection.stats.storageStats.indexDetails.": "index_name",
156+
"config.transactions.stats.storageStats.indexDetails.": "index_name",
157+
"config.image_collection.stats.storageStats.indexSizes.": "index_name",
158+
"collstats.storageStats.indexDetails.": "index_name",
115159
}
116160

117161
// Regular expressions used to make the metric name Prometheus-compatible
@@ -237,9 +281,12 @@ func rawToPrometheusMetric(rm *rawMetric) (prometheus.Metric, error) {
237281
// by prometheus. For first level metrics, there is no prefix so we should use the metric name or
238282
// the help would be empty.
239283
func metricHelp(prefix, name string) string {
240-
if prefix != "" {
284+
if _, ok := nodeToPDMetrics[prefix]; ok {
241285
return prefix
242286
}
287+
if prefix != "" {
288+
return prefix + name
289+
}
243290

244291
return name
245292
}
@@ -252,17 +299,29 @@ func makeMetrics(prefix string, m bson.M, labels map[string]string, compatibleMo
252299
}
253300

254301
for k, val := range m {
302+
nextPrefix := prefix + k
303+
304+
var l = make(map[string]string)
305+
if label, ok := keyNodesToLabels[prefix]; ok {
306+
for k, v := range labels {
307+
l[k] = v
308+
}
309+
l[label] = k
310+
nextPrefix = prefix + label
311+
} else {
312+
l = labels
313+
}
255314
switch v := val.(type) {
256315
case bson.M:
257-
res = append(res, makeMetrics(prefix+k, v, labels, compatibleMode)...)
316+
res = append(res, makeMetrics(nextPrefix, v, l, compatibleMode)...)
258317
case map[string]interface{}:
259-
res = append(res, makeMetrics(prefix+k, v, labels, compatibleMode)...)
318+
res = append(res, makeMetrics(nextPrefix, v, l, compatibleMode)...)
260319
case primitive.A:
261-
res = append(res, processSlice(prefix, k, v, labels, compatibleMode)...)
320+
res = append(res, processSlice(nextPrefix, v, l, compatibleMode)...)
262321
case []interface{}:
263322
continue
264323
default:
265-
rm, err := makeRawMetric(prefix, k, v, labels)
324+
rm, err := makeRawMetric(prefix, k, v, l)
266325
if err != nil {
267326
invalidMetric := prometheus.NewInvalidMetric(prometheus.NewInvalidDesc(err), err)
268327
res = append(res, invalidMetric)
@@ -303,7 +362,7 @@ func makeMetrics(prefix string, m bson.M, labels map[string]string, compatibleMo
303362

304363
// Extract maps from arrays. Only some structures like replicasets have arrays of members
305364
// and each member is represented by a map[string]interface{}.
306-
func processSlice(prefix, k string, v []interface{}, commonLabels map[string]string, compatibleMode bool) []prometheus.Metric {
365+
func processSlice(prefix string, v []interface{}, commonLabels map[string]string, compatibleMode bool) []prometheus.Metric {
307366
metrics := make([]prometheus.Metric, 0)
308367
labels := make(map[string]string)
309368
for name, value := range commonLabels {
@@ -330,7 +389,7 @@ func processSlice(prefix, k string, v []interface{}, commonLabels map[string]str
330389
labels["member_state"] = state
331390
}
332391

333-
metrics = append(metrics, makeMetrics(prefix+k, s, labels, compatibleMode)...)
392+
metrics = append(metrics, makeMetrics(prefix, s, labels, compatibleMode)...)
334393
}
335394

336395
return metrics

‎main.go‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -150,11 +150,19 @@ func buildExporter(opts GlobalFlags, uri string, log *logrus.Logger) *exporter.E
150150
nodeName = uriParsed.Host
151151
}
152152

153+
collStatsNamespaces := []string{}
154+
if opts.CollStatsNamespaces != "" {
155+
collStatsNamespaces = strings.Split(opts.CollStatsNamespaces, ",")
156+
}
157+
indexStatsCollections := []string{}
158+
if opts.IndexStatsCollections != "" {
159+
indexStatsCollections = strings.Split(opts.IndexStatsCollections, ",")
160+
}
153161
exporterOpts := &exporter.Opts{
154-
CollStatsNamespaces: strings.Split(opts.CollStatsNamespaces, ","),
162+
CollStatsNamespaces: collStatsNamespaces,
155163
CompatibleMode: opts.CompatibleMode,
156164
DiscoveringMode: opts.DiscoveringMode,
157-
IndexStatsCollections: strings.Split(opts.IndexStatsCollections, ","),
165+
IndexStatsCollections: indexStatsCollections,
158166
Logger: log,
159167
URI: uri,
160168
NodeName: nodeName,

0 commit comments

Comments
 (0)