PMM-7424 Add dbStatsCollector (#329)

* https://jira.percona.com/browse/PMM-7424 Add dbStatsCollector

* Run make format

* Add dbstats_ prefix

* Update Changelog

Co-authored-by: JiriCtvrtka <[email protected]>

Author: kanakagrawal

CHANGELOG

@@ -2,6 +2,7 @@ Release 0.20.7 released 2021-08-10
 
 Improvement         : Upgraded dependencies
 Fixed error PMM-6877: replsetGetStatus error on mongos
+Improvement PMM-7424: Add dbStats metric
 
 Release 0.20.6 released 2021-06-26
 

README.md

@@ -40,6 +40,7 @@ Currently, these metric sources are implemented:
 |\-\-log.level|Only log messages with the given severity or above. Valid levels: [debug, info, warn, error]|\-\-log.level="error"|
 |\-\-disable.diagnosticdata|Disable collecting metrics from getDiagnosticData||
 |\-\-disable.replicasetstatus|Disable collecting metrics from replSetGetStatus||
+|\-\-disable.dbstats|Disable collecting metrics from dbStats||
 |--version|Show version and exit|
 
  ### Build the exporter

exporter/dbstats_collector.go

@@ -0,0 +1,75 @@
+// mongodb_exporter
+// Copyright (C) 2017 Percona LLC
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+package exporter
+
+import (
+	"context"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sirupsen/logrus"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+)
+
+type dbstatsCollector struct {
+	ctx            context.Context
+	client         *mongo.Client
+	compatibleMode bool
+	logger         *logrus.Logger
+	topologyInfo   labelsGetter
+}
+
+func (d *dbstatsCollector) Describe(ch chan<- *prometheus.Desc) {
+	prometheus.DescribeByCollect(d, ch)
+}
+
+func (d *dbstatsCollector) Collect(ch chan<- prometheus.Metric) {
+	// List all databases names
+	dbNames, err := d.client.ListDatabaseNames(d.ctx, bson.M{})
+	if err != nil {
+		d.logger.Errorf("Failed to get database names: %s", err)
+		return
+	}
+	d.logger.Debugf("getting stats for databases: %v", dbNames)
+	for _, db := range dbNames {
+		var dbStats bson.M
+		cmd := bson.D{{Key: "dbStats", Value: 1}, {Key: "scale", Value: 1}}
+		r := d.client.Database(db).RunCommand(d.ctx, cmd)
+		err := r.Decode(&dbStats)
+		if err != nil {
+			d.logger.Errorf("Failed to get $dbstats for database %s: %s", db, err)
+			continue
+		}
+
+		d.logger.Debugf("$dbStats metrics for %s", db)
+		debugResult(d.logger, dbStats)
+
+		// Since all dbstats will have the same fields, we need to use a metric prefix (db)
+		// to differentiate metrics between different databases. Labels are being set only to make it easier
+		// to filter
+		prefix := "dbstats_" + db
+
+		labels := d.topologyInfo.baseLabels()
+		labels["database"] = db
+
+		for _, metric := range makeMetrics(prefix, dbStats, d.topologyInfo.baseLabels(), d.compatibleMode) {
+			ch <- metric
+		}
+	}
+}
+
+var _ prometheus.Collector = (*dbstatsCollector)(nil)

exporter/dbstats_collector_test.go

@@ -0,0 +1,112 @@
+// mongodb_exporter
+// Copyright (C) 2017 Percona LLC
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+package exporter
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+	"go.mongodb.org/mongo-driver/bson"
+
+	"github.com/percona/mongodb_exporter/internal/tu"
+)
+
+func TestDBStatsCollector(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+
+	client := tu.DefaultTestClient(ctx, t)
+
+	database := client.Database("testdb")
+	database.Drop(ctx) //nolint
+
+	defer func() {
+		err := database.Drop(ctx)
+		assert.NoError(t, err)
+	}()
+
+	for i := 0; i < 3; i++ {
+		coll := fmt.Sprintf("testcol_%02d", i)
+		for j := 0; j < 10; j++ {
+			_, err := database.Collection(coll).InsertOne(ctx, bson.M{"f1": j, "f2": "2"})
+			assert.NoError(t, err)
+		}
+	}
+
+	ti := labelsGetterMock{}
+
+	c := &dbstatsCollector{
+		client:       client,
+		logger:       logrus.New(),
+		topologyInfo: ti,
+	}
+
+	// The last \n at the end of this string is important
+	expected := strings.NewReader(`
+# HELP mongodb_dbstats_testdb_avgObjSize dbstats_testdb.
+# TYPE mongodb_dbstats_testdb_avgObjSize untyped
+mongodb_dbstats_testdb_avgObjSize 40
+# HELP mongodb_dbstats_testdb_collections dbstats_testdb.
+# TYPE mongodb_dbstats_testdb_collections untyped
+mongodb_dbstats_testdb_collections 3
+# HELP mongodb_dbstats_testdb_dataSize dbstats_testdb.
+# TYPE mongodb_dbstats_testdb_dataSize untyped
+mongodb_dbstats_testdb_dataSize 1200
+# HELP mongodb_dbstats_testdb_indexSize dbstats_testdb.
+# TYPE mongodb_dbstats_testdb_indexSize untyped
+mongodb_dbstats_testdb_indexSize 12288
+# HELP mongodb_dbstats_testdb_indexes dbstats_testdb.
+# TYPE mongodb_dbstats_testdb_indexes untyped
+mongodb_dbstats_testdb_indexes 3
+# HELP mongodb_dbstats_testdb_objects dbstats_testdb.
+# TYPE mongodb_dbstats_testdb_objects untyped
+mongodb_dbstats_testdb_objects 30
+# HELP mongodb_dbstats_testdb_ok dbstats_testdb.
+# TYPE mongodb_dbstats_testdb_ok untyped
+mongodb_dbstats_testdb_ok 1
+# HELP mongodb_dbstats_testdb_storageSize dbstats_testdb.
+# TYPE mongodb_dbstats_testdb_storageSize untyped
+mongodb_dbstats_testdb_storageSize 12288
+# HELP mongodb_dbstats_testdb_views dbstats_testdb.
+# TYPE mongodb_dbstats_testdb_views untyped
+mongodb_dbstats_testdb_views 0` +
+		"\n")
+
+	// Filter metrics for 2 reasons:
+	// 1. The result is huge
+	// 2. We need to check against know values. Don't use metrics that return counters like uptime
+	//    or counters like the number of transactions because they won't return a known value to compare
+	filter := []string{
+		"mongodb_dbstats_testdb_avgObjSize",
+		"mongodb_dbstats_testdb_collections",
+		"mongodb_dbstats_testdb_dataSize",
+		"mongodb_dbstats_testdb_indexSize",
+		"mongodb_dbstats_testdb_indexes",
+		"mongodb_dbstats_testdb_objects",
+		"mongodb_dbstats_testdb_views",
+		"mongodb_dbstats_testdb_storageSize",
+		"mongodb_dbstats_testdb_ok",
+	}
+	err := testutil.CollectAndCompare(c, expected, filter...)
+	assert.NoError(t, err)
+}

exporter/exporter.go

@@ -54,6 +54,7 @@ type Opts struct {
 	Logger                  *logrus.Logger
 	DisableDiagnosticData   bool
 	DisableReplicasetStatus bool
+	DisableDBStats          bool
 }
 
 var (
@@ -147,6 +148,17 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol
 		registry.MustRegister(&ddc)
 	}
 
+	if !e.opts.DisableDBStats {
+		cc := dbstatsCollector{
+			ctx:            ctx,
+			client:         client,
+			compatibleMode: e.opts.CompatibleMode,
+			logger:         e.opts.Logger,
+			topologyInfo:   topologyInfo,
+		}
+		registry.MustRegister(&cc)
+	}
+
 	// replSetGetStatus is not supported through mongos
 	if !e.opts.DisableReplicasetStatus && nodeType != typeMongos {
 		rsgsc := replSetGetStatusCollector{

main.go

@@ -47,6 +47,7 @@ type GlobalFlags struct {
 
 	DisableDiagnosticData   bool `name:"disable.diagnosticdata" help:"Disable collecting metrics from getDiagnosticData"`
 	DisableReplicasetStatus bool `name:"disable.replicasetstatus" help:"Disable collecting metrics from replSetGetStatus"`
+	DisableDBStats          bool `name:"disable.dbstats" help:"Disable collecting metrics from dbStats"`
 
 	DiscoveringMode bool `name:"discovering-mode" help:"Enable autodiscover collections"`
 	CompatibleMode  bool `name:"compatible-mode" help:"Enable old mongodb-exporter compatible metrics"`
@@ -117,6 +118,7 @@ func buildExporter(opts GlobalFlags) (*exporter.Exporter, error) {
 		DisableDiagnosticData:   opts.DisableDiagnosticData,
 		DisableReplicasetStatus: opts.DisableReplicasetStatus,
 		DirectConnect:           opts.DirectConnect,
+		DisableDBStats:          opts.DisableDBStats,
 	}
 
 	e, err := exporter.New(exporterOpts)