Collector for working with data from the system profile (#710)

* Collector for working with data from the system profile

* add description

* fix: tab instead of spaces

---------

Co-authored-by: Alex Tymchuk <[email protected]>

Author: mikle7771

README.md

@@ -106,6 +106,16 @@ HELP mongodb_mongod_wiredtiger_log_bytes_total mongodb_mongod_wiredtiger_log_byt
 # TYPE mongodb_mongod_wiredtiger_log_bytes_total untyped
 mongodb_mongod_wiredtiger_log_bytes_total{type="unwritten"} 2.6208e+06
 ```
+#### Enabling profile metrics gathering
+`--collector.profile` 
+To collect metrics, you need to enable the profiler in [MongoDB](https://www.mongodb.com/docs/manual/tutorial/manage-the-database-profiler/):
+Usage example: `db.setProfilingLevel(2)`
+
+|Level|Description|
+|-----|-----------|
+|0| The profiler is off and does not collect any data. This is the default profiler level.|
+|1| The profiler collects data for operations that take longer than the value of `slowms` or that match a filter.<br> When a filter is set: <ul><li> The `slowms` and `sampleRate` options are not used for profiling.</li><li>The profiler only captures operations that match the filter.</li></ul>
+|2|The profiler collects data for all operations.|
 
 #### Cluster role labels
 The exporter sets some topology labels in all metrics.

REFERENCE.md

@@ -24,5 +24,7 @@
 |--collector.collstats|Enable collecting metrics from $collStats|
 |--collect-all|Enable all collectors. Same as specifying all --collector.\<name\>|
 |--collector.collstats-limit=0|Disable collstats, dbstats, topmetrics and indexstats collector if there are more than \<n\> collections. 0=No limit|
+|--collector.profile-time-ts=30|Set time for scrape slow queries| This interval must be synchronized with the Prometheus scrape interval|
+|--collector.profile|Enable collecting metrics from profile|
 |--metrics.overridedescendingindex| Enable descending index name override to replace -1 with _DESC ||
 |--version|Show version and exit|

exporter/exporter.go

@@ -59,6 +59,7 @@ type Opts struct {
 	DisableDefaultRegistry bool
 	DiscoveringMode        bool
 	GlobalConnPool         bool
+	ProfileTimeTS          int
 
 	CollectAll               bool
 	EnableDBStats            bool
@@ -69,6 +70,7 @@ type Opts struct {
 	EnableTopMetrics         bool
 	EnableIndexStats         bool
 	EnableCollStats          bool
+	EnableProfile            bool
 
 	EnableOverrideDescendingIndex bool
 
@@ -172,6 +174,7 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol
 		e.opts.EnableReplicasetStatus = true
 		e.opts.EnableIndexStats = true
 		e.opts.EnableCurrentopMetrics = true
+		e.opts.EnableProfile = true
 	}
 
 	// arbiter only have isMaster privileges
@@ -183,6 +186,7 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol
 		e.opts.EnableReplicasetStatus = false
 		e.opts.EnableIndexStats = false
 		e.opts.EnableCurrentopMetrics = false
+		e.opts.EnableProfile = false
 	}
 
 	// If we manually set the collection names we want or auto discovery is set.
@@ -219,6 +223,12 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol
 		registry.MustRegister(coc)
 	}
 
+	if e.opts.EnableProfile && nodeType != typeMongos && limitsOk && requestOpts.EnableProfile && e.opts.ProfileTimeTS != 0 {
+		pc := newProfileCollector(ctx, client, e.opts.Logger,
+			e.opts.CompatibleMode, topologyInfo, e.opts.ProfileTimeTS)
+		registry.MustRegister(pc)
+	}
+
 	if e.opts.EnableTopMetrics && nodeType != typeMongos && limitsOk && requestOpts.EnableTopMetrics {
 		tc := newTopCollector(ctx, client, e.opts.Logger,
 			e.opts.CompatibleMode, topologyInfo)
@@ -303,6 +313,8 @@ func (e *Exporter) Handler() http.Handler {
 				requestOpts.EnableIndexStats = true
 			case "collstats":
 				requestOpts.EnableCollStats = true
+			case "profile":
+				requestOpts.EnableProfile = true
 			}
 		}
 

exporter/profile_status_collector.go

@@ -0,0 +1,96 @@
+// mongodb_exporter
+// Copyright (C) 2017 Percona LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package exporter
+
+import (
+	"context"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sirupsen/logrus"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"go.mongodb.org/mongo-driver/mongo"
+)
+
+type profileCollector struct {
+	ctx            context.Context
+	base           *baseCollector
+	compatibleMode bool
+	topologyInfo   labelsGetter
+	profiletimets  int
+}
+
+// newProfileCollector creates a collector for being processed queries.
+func newProfileCollector(ctx context.Context, client *mongo.Client, logger *logrus.Logger,
+	compatible bool, topology labelsGetter, profileTimeTS int,
+) *profileCollector {
+	return &profileCollector{
+		ctx:            ctx,
+		base:           newBaseCollector(client, logger),
+		compatibleMode: compatible,
+		topologyInfo:   topology,
+		profiletimets:  profileTimeTS,
+	}
+}
+
+func (d *profileCollector) Describe(ch chan<- *prometheus.Desc) {
+	d.base.Describe(d.ctx, ch, d.collect)
+}
+
+func (d *profileCollector) Collect(ch chan<- prometheus.Metric) {
+	d.base.Collect(ch)
+}
+
+func (d *profileCollector) collect(ch chan<- prometheus.Metric) {
+	defer measureCollectTime(ch, "mongodb", "profile")()
+
+	logger := d.base.logger
+	client := d.base.client
+	timeScrape := d.profiletimets
+
+	databases, err := databases(d.ctx, client, nil, nil)
+	if err != nil {
+		errors.Wrap(err, "cannot get the database names list")
+		return
+	}
+
+	// Now time + '--collector.profile-time-ts'
+	ts := primitive.NewDateTimeFromTime(time.Now().Add(-time.Duration(time.Second * time.Duration(timeScrape))))
+
+	labels := d.topologyInfo.baseLabels()
+
+	// Get all slow queries from all databases
+	cmd := bson.M{"ts": bson.M{"$gte": ts}}
+	for _, db := range databases {
+		res, err := client.Database(db).Collection("system.profile").CountDocuments(d.ctx, cmd)
+		if err != nil {
+			errors.Wrapf(err, "cannot read system.profile")
+			break
+		}
+		labels["database"] = db
+
+		m := primitive.M{"count": res}
+
+		logger.Debug("profile response from MongoDB:")
+		debugResult(logger, primitive.M{db: m})
+
+		for _, metric := range makeMetrics("profile_slow_query", m, labels, d.compatibleMode) {
+			ch <- metric
+		}
+	}
+}

exporter/profile_status_collector_test.go

@@ -0,0 +1,69 @@
+// mongodb_exporter
+// Copyright (C) 2017 Percona LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package exporter
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+	"go.mongodb.org/mongo-driver/bson"
+
+	"github.com/percona/mongodb_exporter/internal/tu"
+)
+
+func TestProfileCollector(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+
+	client := tu.DefaultTestClient(ctx, t)
+
+	database := client.Database("testdb")
+	database.Drop(ctx) //nolint
+
+	defer func() {
+		err := database.Drop(ctx)
+		assert.NoError(t, err)
+	}()
+
+	// Enable database profiler https://www.mongodb.com/docs/manual/tutorial/manage-the-database-profiler/
+	cmd := bson.M{"profile": 2}
+	_ = database.RunCommand(ctx, cmd)
+
+	ti := labelsGetterMock{}
+
+	c := newProfileCollector(ctx, client, logrus.New(), false, ti, 30)
+
+	expected := strings.NewReader(`
+	# HELP mongodb_profile_slow_query_count profile_slow_query.
+	# TYPE mongodb_profile_slow_query_count counter
+	mongodb_profile_slow_query_count{database="admin"} 0
+	mongodb_profile_slow_query_count{database="config"} 0
+	mongodb_profile_slow_query_count{database="local"} 0
+	mongodb_profile_slow_query_count{database="testdb"} 0` +
+		"\n")
+
+	filter := []string{
+		"mongodb_profile_slow_query_count",
+	}
+
+	err := testutil.CollectAndCompare(c, expected, filter...)
+	assert.NoError(t, err)
+}

main.go

@@ -54,13 +54,16 @@ type GlobalFlags struct {
 	EnableCurrentopMetrics   bool `name:"collector.currentopmetrics" help:"Enable collecting metrics currentop admin command"`
 	EnableIndexStats         bool `name:"collector.indexstats" help:"Enable collecting metrics from $indexStats"`
 	EnableCollStats          bool `name:"collector.collstats" help:"Enable collecting metrics from $collStats"`
+	EnableProfile            bool `name:"collector.profile" help:"Enable collecting metrics from profile"`
 
 	EnableOverrideDescendingIndex bool `name:"metrics.overridedescendingindex" help:"Enable descending index name override to replace -1 with _DESC"`
 
 	CollectAll bool `name:"collect-all" help:"Enable all collectors. Same as specifying all --collector.<name>"`
 
 	CollStatsLimit int `name:"collector.collstats-limit" help:"Disable collstats, dbstats, topmetrics and indexstats collector if there are more than <n> collections. 0=No limit" default:"0"`
 
+	ProfileTimeTS int `name:"collector.profile-time-ts" help:"Set time for scrape slow queries." default:"30"`
+
 	DiscoveringMode bool `name:"discovering-mode" help:"Enable autodiscover collections" negatable:""`
 	CompatibleMode  bool `name:"compatible-mode" help:"Enable old mongodb-exporter compatible metrics" negatable:""`
 	Version         bool `name:"version" help:"Show version and exit"`
@@ -146,11 +149,13 @@ func buildExporter(opts GlobalFlags) *exporter.Exporter {
 		EnableDBStatsFreeStorage: opts.EnableDBStatsFreeStorage,
 		EnableIndexStats:         opts.EnableIndexStats,
 		EnableCollStats:          opts.EnableCollStats,
+		EnableProfile:            opts.EnableProfile,
 
 		EnableOverrideDescendingIndex: opts.EnableOverrideDescendingIndex,
 
 		CollStatsLimit: opts.CollStatsLimit,
 		CollectAll:     opts.CollectAll,
+		ProfileTimeTS:  opts.ProfileTimeTS,
 	}
 
 	e := exporter.New(exporterOpts)