Merge branch 'main' into PMM-7-use-image-from-env-for-encrypted

Author: BupycHuk

.github/pull_request_template.md

@@ -1,12 +1,15 @@
 [PMM-XXXX](https://jira.percona.com/browse/PMM-XXXX) (optional, if ticket reported)
 
-- [ ] Links to other linked pull requests (optional).
+- [ ] Links to related pull requests (optional).
 
 ---
 
-- [ ] Tests passed.
-- [ ] Fix conflicts with target branch.
-- [ ] Update jira ticket description if needed.
-- [ ] Attach screenshots/console output to confirm new behavior to jira ticket, if applicable.
+Below we provide a basic checklist of things that would make it a good PR:
+- Make sure to sign the CLA (Contributor License Agreement).
+- Make sure all tests pass.
+- Keep current with the target branch and fix conflicts if necessary.
+- Update jira ticket description if necessary.
+- Attach screenshots and/or console output to the jira ticket to confirm new behavior, if applicable.
+- Leave notes to the reviewers if you need to focus their attention on something specific.
 
 Once all checks pass and the code is ready for review, please add `pmm-review-exporters` team as the reviewer. That would assign people from the review team automatically. Report any issues on our [Forum](https://forums.percona.com).

.github/workflows/release.yml

@@ -45,7 +45,7 @@ jobs:
       - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v5
         with:
-          version: latest
+          version: v1.18.2
           args: release --rm-dist
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

CHANGELOG

@@ -1,3 +1,14 @@
+Release 0.40.0
+
+- Multi-target support
+- Update license to Apache 2.0
+- Add option --collector.dbstatsfreestorage to receive freeStorage stats from dbstats
+- New Current Op Metrics collector
+- New collector to collect data from the system profile
+- Add support for timestamp metrics with seconds precision
+- Update Go version
+- Update dependencies
+
 Release 0.39.0
 
 - Fixed issue with connection to authenticated MongoDB Arbiter

README.md

@@ -86,6 +86,17 @@ export MONGODB_PASSWORD=YYY
 mongodb_exporter_linux_amd64/mongodb_exporter --mongodb.uri=mongodb://127.0.0.1:17001 --mongodb.collstats-colls=db1.c1,db2.c2
 ```
 
+#### Multi-target support
+You can run the exporter specifying multiple URIs, devided by a comma in --mongodb.uri option or MONGODB_URI environment variable in order to monitor multiple mongodb instances with the a single mongodb_exporter instance.
+```sh
+--mongodb.uri=mongodb://user:[email protected]:27017/admin,mongodb://user2:[email protected]:27018/admin
+```
+In this case you can use the **/scrape** endpoint with the **target** parameter to retreive the specified tartget's metrics.  When querying the data you can use just mongodb://host:port in the targer parameter without other parameters and, of course without host credentials
+```sh
+GET /scrape?target=mongodb://127.0.0.1:27018
+```
+
+
 #### Enabling collstats metrics gathering
 `--mongodb.collstats-colls` receives a list of databases and collections to monitor using collstats.
 Usage example: `--mongodb.collstats-colls=database1.collection1,database2.collection2`

REFERENCE.md

@@ -14,6 +14,7 @@
 |--web.listen-address|Address to listen on for web interface and telemetry|--web.listen-address=":9216"|
 |--web.telemetry-path|Metrics expose path|--web.telemetry-path="/metrics"|
 |--web.config|Path to the file having Prometheus TLS config for basic auth|--web.config=STRING|
+|--web.timeout-offset|Offset to subtract from the timeout in seconds|--web.timeout-offset=1|
 |--log.level|Only log messages with the given severity or above. Valid levels: [debug, info, warn, error, fatal]|--log.level="error"|
 |--collector.diagnosticdata|Enable collecting metrics from getDiagnosticData|
 |--collector.replicasetstatus|Enable collecting metrics from replSetGetStatus|
@@ -27,4 +28,4 @@
 |--collector.profile-time-ts=30|Set time for scrape slow queries| This interval must be synchronized with the Prometheus scrape interval|
 |--collector.profile|Enable collecting metrics from profile|
 |--metrics.overridedescendingindex| Enable descending index name override to replace -1 with _DESC ||
-|--version|Show version and exit|
+|--version|Show version and exit|

exporter/base_collector.go

@@ -43,7 +43,10 @@ func newBaseCollector(client *mongo.Client, logger *logrus.Logger) *baseCollecto
 func (d *baseCollector) Describe(ctx context.Context, ch chan<- *prometheus.Desc, collect func(mCh chan<- prometheus.Metric)) {
 	select {
 	case <-ctx.Done():
-		return
+		// don't interrupt, let mongodb_up metric to be registered if on timeout we still don't have client connected
+		if d.client != nil {
+			return
+		}
 	default:
 	}
 

exporter/exporter.go

@@ -21,15 +21,12 @@ import (
 	"fmt"
 	"net/http"
 	_ "net/http/pprof"
-	"os"
 	"strconv"
 	"sync"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"github.com/prometheus/common/promlog"
-	"github.com/prometheus/exporter-toolkit/web"
 	"github.com/sirupsen/logrus"
 	"go.mongodb.org/mongo-driver/mongo"
 
@@ -38,12 +35,10 @@ import (
 
 // Exporter holds Exporter methods and attributes.
 type Exporter struct {
-	path                  string
 	client                *mongo.Client
 	clientMu              sync.Mutex
 	logger                *logrus.Logger
 	opts                  *Opts
-	webListenAddress      string
 	lock                  *sync.Mutex
 	totalCollectionsCount int
 }
@@ -56,10 +51,12 @@ type Opts struct {
 	CollStatsLimit         int
 	CompatibleMode         bool
 	DirectConnect          bool
+	ConnectTimeoutMS       int
 	DisableDefaultRegistry bool
 	DiscoveringMode        bool
 	GlobalConnPool         bool
 	ProfileTimeTS          int
+	TimeoutOffset          int
 
 	CollectAll               bool
 	EnableDBStats            bool
@@ -76,10 +73,8 @@ type Opts struct {
 
 	IndexStatsCollections []string
 	Logger                *logrus.Logger
-	Path                  string
-	URI                   string
-	WebListenAddress      string
-	TLSConfigPath         string
+
+	URI string
 }
 
 var (
@@ -103,16 +98,9 @@ func New(opts *Opts) *Exporter {
 
 	ctx := context.Background()
 
-	if opts.Path == "" {
-		opts.Logger.Warn("Web telemetry path \"\" invalid, falling back to \"/\" instead")
-		opts.Path = "/"
-	}
-
 	exp := &Exporter{
-		path:                  opts.Path,
 		logger:                opts.Logger,
 		opts:                  opts,
-		webListenAddress:      opts.WebListenAddress,
 		lock:                  &sync.Mutex{},
 		totalCollectionsCount: -1, // Not calculated yet. waiting the db connection.
 	}
@@ -257,7 +245,7 @@ func (e *Exporter) getClient(ctx context.Context) (*mongo.Client, error) {
 			return e.client, nil
 		}
 
-		client, err := connect(context.Background(), e.opts.URI, e.opts.DirectConnect)
+		client, err := connect(context.Background(), e.opts)
 		if err != nil {
 			return nil, err
 		}
@@ -267,7 +255,7 @@ func (e *Exporter) getClient(ctx context.Context) (*mongo.Client, error) {
 	}
 
 	// !e.opts.GlobalConnPool: create new client for every scrape.
-	client, err := connect(ctx, e.opts.URI, e.opts.DirectConnect)
+	client, err := connect(ctx, e.opts)
 	if err != nil {
 		return nil, err
 	}
@@ -284,6 +272,7 @@ func (e *Exporter) Handler() http.Handler {
 		if err != nil {
 			seconds = 10
 		}
+		seconds -= e.opts.TimeoutOffset
 
 		var client *mongo.Client
 		ctx, cancel := context.WithTimeout(r.Context(), time.Duration(seconds)*time.Second)
@@ -350,14 +339,15 @@ func (e *Exporter) Handler() http.Handler {
 			gatherers = append(gatherers, prometheus.DefaultGatherer)
 		}
 
+		var ti *topologyInfo
 		if client != nil {
 			// Topology can change between requests, so we need to get it every time.
-			ti := newTopologyInfo(ctx, client, e.logger)
-
-			registry := e.makeRegistry(ctx, client, ti, requestOpts)
-			gatherers = append(gatherers, registry)
+			ti = newTopologyInfo(ctx, client, e.logger)
 		}
 
+		registry := e.makeRegistry(ctx, client, ti, requestOpts)
+		gatherers = append(gatherers, registry)
+
 		// Delegate http serving to Prometheus client library, which will call collector.Collect.
 		h := promhttp.HandlerFor(gatherers, promhttp.HandlerOpts{
 			ErrorHandling: promhttp.ContinueOnError,
@@ -368,41 +358,21 @@ func (e *Exporter) Handler() http.Handler {
 	})
 }
 
-// Run starts the exporter.
-func (e *Exporter) Run() {
-	mux := http.DefaultServeMux
-	mux.Handle(e.path, e.Handler())
-	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
-		w.Write([]byte(`<html>
-            <head><title>MongoDB Exporter</title></head>
-            <body>
-            <h1>MongoDB Exporter</h1>
-            <p><a href='/metrics'>Metrics</a></p>
-            </body>
-            </html>`))
-	})
-
-	server := &http.Server{
-		Handler: mux,
-	}
-	flags := &web.FlagConfig{
-		WebListenAddresses: &[]string{e.webListenAddress},
-		WebConfigFile:      &e.opts.TLSConfigPath,
-	}
-	if err := web.ListenAndServe(server, flags, promlog.New(&promlog.Config{})); err != nil {
-		e.logger.Errorf("error starting server: %v", err)
-		os.Exit(1)
-	}
-}
-
-func connect(ctx context.Context, dsn string, directConnect bool) (*mongo.Client, error) {
-	clientOpts, err := dsn_fix.ClientOptionsForDSN(dsn)
+func connect(ctx context.Context, opts *Opts) (*mongo.Client, error) {
+	clientOpts, err := dsn_fix.ClientOptionsForDSN(opts.URI)
 	if err != nil {
 		return nil, fmt.Errorf("invalid dsn: %w", err)
 	}
-	clientOpts.SetDirect(directConnect)
+
+	clientOpts.SetDirect(opts.DirectConnect)
 	clientOpts.SetAppName("mongodb_exporter")
 
+	if clientOpts.ConnectTimeout == nil {
+		connectTimeout := time.Duration(opts.ConnectTimeoutMS) * time.Millisecond
+		clientOpts.SetConnectTimeout(connectTimeout)
+		clientOpts.SetServerSelectionTimeout(connectTimeout)
+	}
+
 	client, err := mongo.Connect(ctx, clientOpts)
 	if err != nil {
 		return nil, fmt.Errorf("invalid MongoDB options: %w", err)

exporter/exporter_test.go

@@ -19,11 +19,15 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
+	"strings"
 	"sync"
 	"testing"
 
+	"github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 
@@ -61,8 +65,11 @@ func TestConnect(t *testing.T) {
 
 	t.Run("Connect without SSL", func(t *testing.T) {
 		for name, port := range ports {
-			dsn := fmt.Sprintf("mongodb://%s:%s/admin", hostname, port)
-			client, err := connect(ctx, dsn, true)
+			exporterOpts := &Opts{
+				URI:           fmt.Sprintf("mongodb://%s/admin", net.JoinHostPort(hostname, port)),
+				DirectConnect: true,
+			}
+			client, err := connect(ctx, exporterOpts)
 			assert.NoError(t, err, name)
 			err = client.Disconnect(ctx)
 			assert.NoError(t, err, name)
@@ -167,17 +174,17 @@ func TestMongoS(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		dsn := fmt.Sprintf("mongodb://%s:%s/admin", hostname, test.port)
-		client, err := connect(ctx, dsn, true)
-		assert.NoError(t, err)
-
 		exporterOpts := &Opts{
 			Logger:                 logrus.New(),
-			URI:                    dsn,
+			URI:                    fmt.Sprintf("mongodb://%s/admin", net.JoinHostPort(hostname, test.port)),
+			DirectConnect:          true,
 			GlobalConnPool:         false,
 			EnableReplicasetStatus: true,
 		}
 
+		client, err := connect(ctx, exporterOpts)
+		assert.NoError(t, err)
+
 		e := New(exporterOpts)
 
 		rsgsc := newReplicationSetStatusCollector(ctx, client, e.opts.Logger,
@@ -195,17 +202,17 @@ func TestMongoS(t *testing.T) {
 func TestMongoUp(t *testing.T) {
 	ctx := context.Background()
 
-	dsn := "mongodb://127.0.0.1:123456/admin"
-	client, err := connect(ctx, dsn, true)
-	assert.Error(t, err)
-
 	exporterOpts := &Opts{
 		Logger:         logrus.New(),
-		URI:            dsn,
+		URI:            "mongodb://127.0.0.1:123456/admin",
+		DirectConnect:  true,
 		GlobalConnPool: false,
 		CollectAll:     true,
 	}
 
+	client, err := connect(ctx, exporterOpts)
+	assert.Error(t, err)
+
 	e := New(exporterOpts)
 
 	gc := newGeneralCollector(ctx, client, e.opts.Logger)
@@ -215,3 +222,52 @@ func TestMongoUp(t *testing.T) {
 	res := r.Unregister(gc)
 	assert.Equal(t, true, res)
 }
+
+func TestMongoUpMetric(t *testing.T) {
+	ctx := context.Background()
+
+	type testcase struct {
+		URI  string
+		Want int
+	}
+
+	testCases := []testcase{
+		{URI: "mongodb://127.0.0.1:12345/admin", Want: 0},
+		{URI: fmt.Sprintf("mongodb://127.0.0.1:%s/admin", tu.GetenvDefault("TEST_MONGODB_STANDALONE_PORT", "27017")), Want: 1},
+	}
+
+	for _, tc := range testCases {
+		exporterOpts := &Opts{
+			Logger:           logrus.New(),
+			URI:              tc.URI,
+			ConnectTimeoutMS: 200,
+			DirectConnect:    true,
+			GlobalConnPool:   false,
+			CollectAll:       true,
+		}
+
+		client, err := connect(ctx, exporterOpts)
+		if tc.Want == 1 {
+			assert.NoError(t, err, "Must be able to connect to %s", tc.URI)
+		} else {
+			assert.Error(t, err, "Must be unable to connect to %s", tc.URI)
+		}
+
+		e := New(exporterOpts)
+		gc := newGeneralCollector(ctx, client, e.opts.Logger)
+		r := e.makeRegistry(ctx, client, new(labelsGetterMock), *e.opts)
+
+		expected := strings.NewReader(`
+		# HELP mongodb_up Whether MongoDB is up.
+		# TYPE mongodb_up gauge
+		mongodb_up ` + strconv.Itoa(tc.Want) + "\n")
+		filter := []string{
+			"mongodb_up",
+		}
+		err = testutil.CollectAndCompare(gc, expected, filter...)
+		assert.NoError(t, err, "mongodb_up metric should be %d", tc.Want)
+
+		res := r.Unregister(gc)
+		assert.Equal(t, true, res)
+	}
+}