Merge branch 'main' into replSetGetConfigCollector

Author: hiroshi

CHANGELOG

@@ -1,3 +1,7 @@
+Release 0.20.8
+Improvement PMM-7424: Add dbStats metric
+Improvement         : Add support for compatible mode for dbStats metric to provide old metrics
+
 Release 0.20.7 released 2021-08-10
 
 Improvement         : Upgraded dependencies

README.md

@@ -41,6 +41,7 @@ Currently, these metric sources are implemented:
 |\-\-disable.diagnosticdata|Disable collecting metrics from getDiagnosticData||
 |\-\-disable.replicasetstatus|Disable collecting metrics from replSetGetStatus||
 |\-\-disable.replicasetconfig|Disable collecting metrics from replSetGetConfig||
+|\-\-disable.dbstats|Disable collecting metrics from dbStats||
 |--version|Show version and exit|
 
  ### Build the exporter

exporter/dbstats_collector.go

@@ -0,0 +1,75 @@
+// mongodb_exporter
+// Copyright (C) 2017 Percona LLC
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+package exporter
+
+import (
+	"context"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sirupsen/logrus"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+)
+
+type dbstatsCollector struct {
+	ctx            context.Context
+	client         *mongo.Client
+	compatibleMode bool
+	logger         *logrus.Logger
+	topologyInfo   labelsGetter
+}
+
+func (d *dbstatsCollector) Describe(ch chan<- *prometheus.Desc) {
+	prometheus.DescribeByCollect(d, ch)
+}
+
+func (d *dbstatsCollector) Collect(ch chan<- prometheus.Metric) {
+	// List all databases names
+	dbNames, err := d.client.ListDatabaseNames(d.ctx, bson.M{})
+	if err != nil {
+		d.logger.Errorf("Failed to get database names: %s", err)
+		return
+	}
+	d.logger.Debugf("getting stats for databases: %v", dbNames)
+	for _, db := range dbNames {
+		var dbStats bson.M
+		cmd := bson.D{{Key: "dbStats", Value: 1}, {Key: "scale", Value: 1}}
+		r := d.client.Database(db).RunCommand(d.ctx, cmd)
+		err := r.Decode(&dbStats)
+		if err != nil {
+			d.logger.Errorf("Failed to get $dbstats for database %s: %s", db, err)
+			continue
+		}
+
+		d.logger.Debugf("$dbStats metrics for %s", db)
+		debugResult(d.logger, dbStats)
+
+		prefix := "dbstats"
+
+		labels := d.topologyInfo.baseLabels()
+
+		// Since all dbstats will have the same fields, we need to use a label
+		// to differentiate metrics between different databases.
+		labels["database"] = db
+
+		for _, metric := range makeMetrics(prefix, dbStats, labels, d.compatibleMode) {
+			ch <- metric
+		}
+	}
+}
+
+var _ prometheus.Collector = (*dbstatsCollector)(nil)

exporter/dbstats_collector_test.go

@@ -0,0 +1,101 @@
+// mongodb_exporter
+// Copyright (C) 2017 Percona LLC
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+package exporter
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/percona/exporter_shared/helpers"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+	"go.mongodb.org/mongo-driver/bson"
+
+	"github.com/percona/mongodb_exporter/internal/tu"
+)
+
+func TestDBStatsCollector(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+
+	client := tu.DefaultTestClient(ctx, t)
+
+	database := client.Database("testdb")
+	database.Drop(ctx) //nolint
+
+	defer func() {
+		err := database.Drop(ctx)
+		assert.NoError(t, err)
+	}()
+
+	for i := 0; i < 3; i++ {
+		coll := fmt.Sprintf("testcol_%02d", i)
+		for j := 0; j < 10; j++ {
+			_, err := database.Collection(coll).InsertOne(ctx, bson.M{"f1": j, "f2": "2"})
+			assert.NoError(t, err)
+		}
+	}
+
+	ti := labelsGetterMock{}
+
+	c := &dbstatsCollector{
+		client:       client,
+		logger:       logrus.New(),
+		topologyInfo: ti,
+	}
+
+	expected := []string{
+		"# HELP mongodb_dbstats_collections dbstats.",
+		"# TYPE mongodb_dbstats_collections untyped",
+		"mongodb_dbstats_collections{database=\"testdb\"} 3",
+		"# HELP mongodb_dbstats_dataSize dbstats.",
+		"# TYPE mongodb_dbstats_dataSize untyped",
+		"mongodb_dbstats_dataSize{database=\"testdb\"} 1200",
+		"# HELP mongodb_dbstats_indexSize dbstats.",
+		"# TYPE mongodb_dbstats_indexSize untyped",
+		"mongodb_dbstats_indexSize{database=\"testdb\"} 12288",
+		"# HELP mongodb_dbstats_indexes dbstats.",
+		"# TYPE mongodb_dbstats_indexes untyped",
+		"mongodb_dbstats_indexes{database=\"testdb\"} 3",
+		"# HELP mongodb_dbstats_objects dbstats.",
+		"# TYPE mongodb_dbstats_objects untyped",
+		"mongodb_dbstats_objects{database=\"testdb\"} 30",
+	}
+
+	metrics := helpers.CollectMetrics(c)
+	actualMetrics := helpers.ReadMetrics(metrics)
+	filters := []string{
+		"mongodb_dbstats_collections",
+		"mongodb_dbstats_dataSize",
+		"mongodb_dbstats_indexSize",
+		"mongodb_dbstats_indexes",
+		"mongodb_dbstats_objects",
+	}
+	labels := map[string]string{
+		"database": "testdb",
+	}
+	actualMetrics = filterMetricsWithLabels(actualMetrics,
+		filters,
+		labels)
+	actualLines := helpers.Format(helpers.WriteMetrics(actualMetrics))
+	sort.Strings(actualLines)
+	sort.Strings(expected)
+	assert.Equal(t, expected, actualLines)
+}

exporter/exporter.go

@@ -55,6 +55,7 @@ type Opts struct {
 	DisableDiagnosticData   bool
 	DisableReplicasetStatus bool
 	DisableReplicasetConfig bool
+	DisableDBStats          bool
 }
 
 var (
@@ -148,6 +149,17 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol
 		registry.MustRegister(&ddc)
 	}
 
+	if !e.opts.DisableDBStats {
+		cc := dbstatsCollector{
+			ctx:            ctx,
+			client:         client,
+			compatibleMode: e.opts.CompatibleMode,
+			logger:         e.opts.Logger,
+			topologyInfo:   topologyInfo,
+		}
+		registry.MustRegister(&cc)
+	}
+
 	// replSetGetStatus is not supported through mongos
 	if !e.opts.DisableReplicasetStatus && nodeType != typeMongos {
 		rsgsc := replSetGetStatusCollector{

exporter/utils_test.go

@@ -35,3 +35,27 @@ func getMetricNames(lines []string) map[string]bool {
 
 	return names
 }
+
+func filterMetricsWithLabels(metrics []*helpers.Metric, filters []string, labels map[string]string) []*helpers.Metric {
+	res := make([]*helpers.Metric, 0, len(metrics))
+	for _, m := range metrics {
+		for _, filterName := range filters {
+			if m.Name == filterName {
+				validMetric := true
+				for labelKey, labelValue := range labels {
+					if m.Labels[labelKey] != labelValue {
+						validMetric = false
+
+						break
+					}
+				}
+				if validMetric {
+					res = append(res, m)
+				}
+
+				break
+			}
+		}
+	}
+	return res
+}

exporter/v1_compatibility.go

@@ -592,6 +592,26 @@ func conversions() []conversion {
 			oldName: "mongodb_ss_wt_cache_maximum_bytes_configured",
 			newName: "mongodb_mongod_wiredtiger_cache_max_bytes",
 		},
+		{
+			oldName: "mongodb_mongod_db_collections_total",
+			newName: "mongodb_dbstats_collections",
+		},
+		{
+			oldName: "mongodb_mongod_db_data_size_bytes",
+			newName: "mongodb_dbstats_dataSize",
+		},
+		{
+			oldName: "mongodb_mongod_db_index_size_bytes",
+			newName: "mongodb_dbstats_indexSize",
+		},
+		{
+			oldName: "mongodb_mongod_db_indexes_total",
+			newName: "mongodb_dbstats_indexes",
+		},
+		{
+			oldName: "mongodb_mongod_db_objects_total",
+			newName: "mongodb_dbstats_objects",
+		},
 	}
 }
 

main.go

@@ -48,6 +48,7 @@ type GlobalFlags struct {
 	DisableDiagnosticData   bool `name:"disable.diagnosticdata" help:"Disable collecting metrics from getDiagnosticData"`
 	DisableReplicasetStatus bool `name:"disable.replicasetstatus" help:"Disable collecting metrics from replSetGetStatus"`
 	DisableReplicasetConfig bool `name:"disable.replicasetconfig" help:"Disable collecting metrics from replSetGetConfig"`
+	DisableDBStats          bool `name:"disable.dbstats" help:"Disable collecting metrics from dbStats"`
 
 	DiscoveringMode bool `name:"discovering-mode" help:"Enable autodiscover collections"`
 	CompatibleMode  bool `name:"compatible-mode" help:"Enable old mongodb-exporter compatible metrics"`
@@ -119,6 +120,7 @@ func buildExporter(opts GlobalFlags) (*exporter.Exporter, error) {
 		DisableReplicasetStatus: opts.DisableReplicasetStatus,
 		DisableReplicasetConfig: opts.DisableReplicasetConfig,
 		DirectConnect:           opts.DirectConnect,
+		DisableDBStats:          opts.DisableDBStats,
 	}
 
 	e, err := exporter.New(exporterOpts)