Skip to content

Commit c099cc1

Browse files
authored
Merge branch 'main' into PMM-7-use-image-from-env-for-encrypted
2 parents 8eb2882 + 4cbba0f commit c099cc1

File tree

11 files changed

+540
-8
lines changed

11 files changed

+540
-8
lines changed

README.md

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,14 +71,25 @@ Connecting user should have sufficient rights to query needed stats:
7171
More info about roles in MongoDB [documentation](https://docs.mongodb.com/manual/reference/built-in-roles/#mongodb-authrole-clusterMonitor).
7272

7373
#### Example
74-
```
74+
```sh
7575
mongodb_exporter_linux_amd64/mongodb_exporter --mongodb.uri=mongodb://127.0.0.1:17001
7676
```
7777

78+
#### MongoDB Authentication
79+
You can supply the mongodb user/password direct in the `--mongodb.uri=` like `--mongodb.uri=mongodb://user:[email protected]:17001`, you can also supply the mongodb user/password with `--mongodb.user=`, `--mongodb.password=`
80+
but the user and password info will be leaked via `ps` or `top` command, for security issue, you can use `MONGODB_USER` and `MONGODB_PASSWORD` env variable to set user/password for given uri
81+
```sh
82+
MONGODB_USER=XXX MONGODB_PASSWORD=YYY mongodb_exporter_linux_amd64/mongodb_exporter --mongodb.uri=mongodb://127.0.0.1:17001 --mongodb.collstats-colls=db1.c1,db2.c2
83+
# or
84+
export MONGODB_USER=XXX
85+
export MONGODB_PASSWORD=YYY
86+
mongodb_exporter_linux_amd64/mongodb_exporter --mongodb.uri=mongodb://127.0.0.1:17001 --mongodb.collstats-colls=db1.c1,db2.c2
87+
```
88+
7889
#### Enabling collstats metrics gathering
7990
`--mongodb.collstats-colls` receives a list of databases and collections to monitor using collstats.
8091
Usage example: `--mongodb.collstats-colls=database1.collection1,database2.collection2`
81-
```
92+
```sh
8293
mongodb_exporter_linux_amd64/mongodb_exporter --mongodb.uri=mongodb://127.0.0.1:17001 --mongodb.collstats-colls=db1.c1,db2.c2
8394
```
8495
#### Enabling compatibility mode.
@@ -95,6 +106,16 @@ HELP mongodb_mongod_wiredtiger_log_bytes_total mongodb_mongod_wiredtiger_log_byt
95106
# TYPE mongodb_mongod_wiredtiger_log_bytes_total untyped
96107
mongodb_mongod_wiredtiger_log_bytes_total{type="unwritten"} 2.6208e+06
97108
```
109+
#### Enabling profile metrics gathering
110+
`--collector.profile`
111+
To collect metrics, you need to enable the profiler in [MongoDB](https://www.mongodb.com/docs/manual/tutorial/manage-the-database-profiler/):
112+
Usage example: `db.setProfilingLevel(2)`
113+
114+
|Level|Description|
115+
|-----|-----------|
116+
|0| The profiler is off and does not collect any data. This is the default profiler level.|
117+
|1| The profiler collects data for operations that take longer than the value of `slowms` or that match a filter.<br> When a filter is set: <ul><li> The `slowms` and `sampleRate` options are not used for profiling.</li><li>The profiler only captures operations that match the filter.</li></ul>
118+
|2|The profiler collects data for all operations.|
98119

99120
#### Cluster role labels
100121
The exporter sets some topology labels in all metrics.

REFERENCE.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,12 @@
1919
|--collector.replicasetstatus|Enable collecting metrics from replSetGetStatus|
2020
|--collector.dbstats|Enable collecting metrics from dbStats||
2121
|--collector.topmetrics|Enable collecting metrics from top admin command|
22+
|--collector.currentopmetrics|Enable collecting metrics from currentop admin command|
2223
|--collector.indexstats|Enable collecting metrics from $indexStats|
2324
|--collector.collstats|Enable collecting metrics from $collStats|
2425
|--collect-all|Enable all collectors. Same as specifying all --collector.\<name\>|
2526
|--collector.collstats-limit=0|Disable collstats, dbstats, topmetrics and indexstats collector if there are more than \<n\> collections. 0=No limit|
27+
|--collector.profile-time-ts=30|Set time for scrape slow queries| This interval must be synchronized with the Prometheus scrape interval|
28+
|--collector.profile|Enable collecting metrics from profile|
2629
|--metrics.overridedescendingindex| Enable descending index name override to replace -1 with _DESC ||
2730
|--version|Show version and exit|

exporter/currentop_collector.go

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
// mongodb_exporter
2+
// Copyright (C) 2017 Percona LLC
3+
//
4+
// Licensed under the Apache License, Version 2.0 (the "License");
5+
// you may not use this file except in compliance with the License.
6+
// You may obtain a copy of the License at
7+
//
8+
// http://www.apache.org/licenses/LICENSE-2.0
9+
//
10+
// Unless required by applicable law or agreed to in writing, software
11+
// distributed under the License is distributed on an "AS IS" BASIS,
12+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
// See the License for the specific language governing permissions and
14+
// limitations under the License.
15+
16+
package exporter
17+
18+
import (
19+
"context"
20+
"strconv"
21+
22+
"github.com/pkg/errors"
23+
"github.com/prometheus/client_golang/prometheus"
24+
"github.com/sirupsen/logrus"
25+
"go.mongodb.org/mongo-driver/bson"
26+
"go.mongodb.org/mongo-driver/bson/primitive"
27+
"go.mongodb.org/mongo-driver/mongo"
28+
)
29+
30+
type currentopCollector struct {
31+
ctx context.Context
32+
base *baseCollector
33+
compatibleMode bool
34+
topologyInfo labelsGetter
35+
}
36+
37+
var ErrInvalidOrMissingInprogEntry = errors.New("invalid or missing inprog entry in currentop results")
38+
39+
// newCurrentopCollector creates a collector for being processed queries.
40+
func newCurrentopCollector(ctx context.Context, client *mongo.Client, logger *logrus.Logger,
41+
compatible bool, topology labelsGetter,
42+
) *currentopCollector {
43+
return &currentopCollector{
44+
ctx: ctx,
45+
base: newBaseCollector(client, logger),
46+
compatibleMode: compatible,
47+
topologyInfo: topology,
48+
}
49+
}
50+
51+
func (d *currentopCollector) Describe(ch chan<- *prometheus.Desc) {
52+
d.base.Describe(d.ctx, ch, d.collect)
53+
}
54+
55+
func (d *currentopCollector) Collect(ch chan<- prometheus.Metric) {
56+
d.base.Collect(ch)
57+
}
58+
59+
func (d *currentopCollector) collect(ch chan<- prometheus.Metric) {
60+
defer measureCollectTime(ch, "mongodb", "currentop")()
61+
62+
logger := d.base.logger
63+
client := d.base.client
64+
65+
// Get all requests that are being processed except system requests (admin and local).
66+
cmd := bson.D{
67+
{Key: "currentOp", Value: true},
68+
{Key: "active", Value: true},
69+
{Key: "microsecs_running", Value: bson.D{
70+
{Key: "$exists", Value: true},
71+
}},
72+
{Key: "op", Value: bson.D{{Key: "$ne", Value: ""}}},
73+
{Key: "ns", Value: bson.D{
74+
{Key: "$ne", Value: ""},
75+
{Key: "$not", Value: bson.D{{Key: "$regex", Value: "^admin.*|^local.*"}}},
76+
}},
77+
}
78+
res := client.Database("admin").RunCommand(d.ctx, cmd)
79+
80+
var r primitive.M
81+
if err := res.Decode(&r); err != nil {
82+
ch <- prometheus.NewInvalidMetric(prometheus.NewInvalidDesc(err), err)
83+
return
84+
}
85+
86+
logger.Debug("currentop response from MongoDB:")
87+
debugResult(logger, r)
88+
89+
inprog, ok := r["inprog"].(primitive.A)
90+
91+
if !ok {
92+
ch <- prometheus.NewInvalidMetric(prometheus.NewInvalidDesc(ErrInvalidOrMissingInprogEntry),
93+
ErrInvalidOrMissingInprogEntry)
94+
}
95+
96+
for _, bsonMap := range inprog {
97+
98+
bsonMapElement, ok := bsonMap.(primitive.M)
99+
if !ok {
100+
logger.Errorf("Invalid type primitive.M assertion for bsonMap: %T", bsonMapElement)
101+
continue
102+
}
103+
opid, ok := bsonMapElement["opid"].(int32)
104+
if !ok {
105+
logger.Errorf("Invalid type int32 assertion for 'opid': %T", bsonMapElement)
106+
continue
107+
}
108+
namespace, ok := bsonMapElement["ns"].(string)
109+
if !ok {
110+
logger.Errorf("Invalid type string assertion for 'ns': %T", bsonMapElement)
111+
continue
112+
}
113+
db, collection := splitNamespace(namespace)
114+
op, ok := bsonMapElement["op"].(string)
115+
if !ok {
116+
logger.Errorf("Invalid type string assertion for 'op': %T", bsonMapElement)
117+
continue
118+
}
119+
desc, ok := bsonMapElement["desc"].(string)
120+
if !ok {
121+
logger.Errorf("Invalid type string assertion for 'desc': %T", bsonMapElement)
122+
continue
123+
}
124+
microsecs_running, ok := bsonMapElement["microsecs_running"].(int64)
125+
if !ok {
126+
logger.Errorf("Invalid type int64 assertion for 'microsecs_running': %T", bsonMapElement)
127+
continue
128+
}
129+
130+
labels := d.topologyInfo.baseLabels()
131+
labels["opid"] = strconv.Itoa(int(opid))
132+
labels["op"] = op
133+
labels["desc"] = desc
134+
labels["database"] = db
135+
labels["collection"] = collection
136+
labels["ns"] = namespace
137+
138+
m := primitive.M{"uptime": microsecs_running}
139+
140+
for _, metric := range makeMetrics("currentop_query", m, labels, d.compatibleMode) {
141+
ch <- metric
142+
}
143+
}
144+
}
Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
// mongodb_exporter
2+
// Copyright (C) 2017 Percona LLC
3+
//
4+
// Licensed under the Apache License, Version 2.0 (the "License");
5+
// you may not use this file except in compliance with the License.
6+
// You may obtain a copy of the License at
7+
//
8+
// http://www.apache.org/licenses/LICENSE-2.0
9+
//
10+
// Unless required by applicable law or agreed to in writing, software
11+
// distributed under the License is distributed on an "AS IS" BASIS,
12+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
// See the License for the specific language governing permissions and
14+
// limitations under the License.
15+
16+
package exporter
17+
18+
import (
19+
"context"
20+
"fmt"
21+
"sync"
22+
"testing"
23+
"time"
24+
25+
"github.com/prometheus/client_golang/prometheus/testutil"
26+
"github.com/sirupsen/logrus"
27+
"github.com/stretchr/testify/assert"
28+
"go.mongodb.org/mongo-driver/bson"
29+
30+
"github.com/percona/mongodb_exporter/internal/tu"
31+
)
32+
33+
func TestCurrentopCollector(t *testing.T) {
34+
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
35+
defer cancel()
36+
37+
var wg sync.WaitGroup
38+
39+
client := tu.DefaultTestClient(ctx, t)
40+
41+
database := client.Database("testdb")
42+
database.Drop(ctx)
43+
44+
defer func() {
45+
err := database.Drop(ctx)
46+
assert.NoError(t, err)
47+
}()
48+
wg.Add(1)
49+
go func() {
50+
defer wg.Done()
51+
for i := 0; i < 3; i++ {
52+
coll := fmt.Sprintf("testcol_%02d", i)
53+
_, err := database.Collection(coll).InsertOne(ctx, bson.M{"f1": 1, "f2": "2"})
54+
assert.NoError(t, err)
55+
}
56+
}()
57+
58+
ti := labelsGetterMock{}
59+
60+
c := newCurrentopCollector(ctx, client, logrus.New(), false, ti)
61+
62+
// Filter metrics by reason:
63+
// 1. The result will be different on different hardware
64+
// 2. Can't check labels like 'decs' and 'opid' because they don't return a known value for comparison
65+
// It looks like:
66+
// # HELP mongodb_currentop_query_uptime currentop_query.
67+
// # TYPE mongodb_currentop_query_uptime untyped
68+
// mongodb_currentop_query_uptime{collection="testcol_00",database="testdb",decs="conn6365",ns="testdb.testcol_00",op="insert",opid="448307"} 2524
69+
70+
filter := []string{
71+
"mongodb_currentop_query_uptime",
72+
}
73+
74+
count := testutil.CollectAndCount(c, filter...)
75+
assert.True(t, count > 0)
76+
wg.Wait()
77+
}

exporter/exporter.go

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,15 +59,18 @@ type Opts struct {
5959
DisableDefaultRegistry bool
6060
DiscoveringMode bool
6161
GlobalConnPool bool
62+
ProfileTimeTS int
6263

6364
CollectAll bool
6465
EnableDBStats bool
6566
EnableDBStatsFreeStorage bool
6667
EnableDiagnosticData bool
6768
EnableReplicasetStatus bool
69+
EnableCurrentopMetrics bool
6870
EnableTopMetrics bool
6971
EnableIndexStats bool
7072
EnableCollStats bool
73+
EnableProfile bool
7174

7275
EnableOverrideDescendingIndex bool
7376

@@ -170,6 +173,8 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol
170173
e.opts.EnableTopMetrics = true
171174
e.opts.EnableReplicasetStatus = true
172175
e.opts.EnableIndexStats = true
176+
e.opts.EnableCurrentopMetrics = true
177+
e.opts.EnableProfile = true
173178
}
174179

175180
// arbiter only have isMaster privileges
@@ -180,6 +185,8 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol
180185
e.opts.EnableTopMetrics = false
181186
e.opts.EnableReplicasetStatus = false
182187
e.opts.EnableIndexStats = false
188+
e.opts.EnableCurrentopMetrics = false
189+
e.opts.EnableProfile = false
183190
}
184191

185192
// If we manually set the collection names we want or auto discovery is set.
@@ -210,6 +217,18 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol
210217
registry.MustRegister(cc)
211218
}
212219

220+
if e.opts.EnableCurrentopMetrics && nodeType != typeMongos && limitsOk && requestOpts.EnableCurrentopMetrics {
221+
coc := newCurrentopCollector(ctx, client, e.opts.Logger,
222+
e.opts.CompatibleMode, topologyInfo)
223+
registry.MustRegister(coc)
224+
}
225+
226+
if e.opts.EnableProfile && nodeType != typeMongos && limitsOk && requestOpts.EnableProfile && e.opts.ProfileTimeTS != 0 {
227+
pc := newProfileCollector(ctx, client, e.opts.Logger,
228+
e.opts.CompatibleMode, topologyInfo, e.opts.ProfileTimeTS)
229+
registry.MustRegister(pc)
230+
}
231+
213232
if e.opts.EnableTopMetrics && nodeType != typeMongos && limitsOk && requestOpts.EnableTopMetrics {
214233
tc := newTopCollector(ctx, client, e.opts.Logger,
215234
e.opts.CompatibleMode, topologyInfo)
@@ -288,10 +307,14 @@ func (e *Exporter) Handler() http.Handler {
288307
requestOpts.EnableDBStats = true
289308
case "topmetrics":
290309
requestOpts.EnableTopMetrics = true
310+
case "currentopmetrics":
311+
requestOpts.EnableCurrentopMetrics = true
291312
case "indexstats":
292313
requestOpts.EnableIndexStats = true
293314
case "collstats":
294315
requestOpts.EnableCollStats = true
316+
case "profile":
317+
requestOpts.EnableProfile = true
295318
}
296319
}
297320

exporter/metrics.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -208,7 +208,9 @@ func asFloat64(value interface{}) (*float64, error) {
208208
f = v
209209
case primitive.DateTime:
210210
f = float64(v)
211-
case primitive.A, primitive.ObjectID, primitive.Timestamp, primitive.Binary, string, []uint8, time.Time:
211+
case primitive.Timestamp:
212+
f = float64(v.T)
213+
case primitive.A, primitive.ObjectID, primitive.Binary, string, []uint8, time.Time:
212214
return nil, nil
213215
default:
214216
return nil, errors.Wrapf(errCannotHandleType, "%T", v)

exporter/metrics_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,7 @@ func TestMakeRawMetric(t *testing.T) {
141141
{value: float32(1.23), wantVal: pointer.ToFloat64(float64(float32(1.23)))},
142142
{value: float64(1.23), wantVal: pointer.ToFloat64(1.23)},
143143
{value: primitive.A{}, wantVal: nil},
144-
{value: primitive.Timestamp{}, wantVal: nil},
144+
{value: primitive.Timestamp{T: 123, I: 456}, wantVal: pointer.ToFloat64(123)},
145145
{value: "zapp", wantVal: nil},
146146
{value: []byte{}, wantVal: nil},
147147
{value: time.Date(2020, 6, 15, 0, 0, 0, 0, time.UTC), wantVal: nil},

0 commit comments

Comments
 (0)