diff --git a/README.md b/README.md index 8abe7670..37fc328d 100644 --- a/README.md +++ b/README.md @@ -199,6 +199,31 @@ The labels are: - rs_state: Replicaset state is an integer from `getDiagnosticData()` -> `replSetGetStatus.myState`. Check [the official documentation](https://docs.mongodb.com/manual/reference/replica-states/) for details on replicaset status values. +#### Prometheus Configuration to Scrape Multiple MongoDB Hosts +The Prometheus documentation [provides](https://prometheus.io/docs/guides/multi-target-exporter/) a good example of multi-target exporters. + +To use `mongodb_exporter` in multi-target mode, you can use the `/scrape` endpoint with the `target` parameter. + +You can optionally specify initial URIs using `--mongodb.uri` (or `MONGODB_URI`) to preload a set of MongoDB instances, but it is not required. Additional targets can still be queried dynamically via `/scrape?target=...`. + +This allows combining static and dynamic target discovery in a flexible way. +``` +scrape_configs: + - job_name: 'mongodb_exporter_targets' + metrics_path: /scrape + static_configs: + - targets: + - mongodb://mongo-host1:27017 + - mongo-host2:27017 + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: <>:9216 +``` + ## Usage Reference See the [Reference Guide](REFERENCE.md) for details on using the exporter. diff --git a/exporter/build_uri.go b/exporter/build_uri.go new file mode 100644 index 00000000..758684c7 --- /dev/null +++ b/exporter/build_uri.go @@ -0,0 +1,130 @@ +// mongodb_exporter +// Copyright (C) 2025 Percona LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package exporter + +import ( + "fmt" + "log" + "log/slog" + "net/url" + "regexp" + "strings" +) + +func ParseURIList(uriList []string, logger *slog.Logger, splitCluster bool) []string { //nolint:gocognit,cyclop + var URIs []string + + // If server URI is prefixed with mongodb scheme string, then every next URI in + // line not prefixed with mongodb scheme string is a part of cluster. Otherwise, + // treat it as a standalone server + realURI := "" + matchRegexp := regexp.MustCompile(`^mongodb(\+srv)?://`) + for _, URI := range uriList { + matches := matchRegexp.FindStringSubmatch(URI) + if matches != nil { + if realURI != "" { + // Add the previous host buffer to the url list as we met the scheme part + URIs = append(URIs, realURI) + realURI = "" + } + if matches[1] == "" { + realURI = URI + } else { + // There can be only one host in SRV connection string + if splitCluster { + // In splitCluster mode we get srv connection string from SRV recors + URI = GetSeedListFromSRV(URI, logger) + } + URIs = append(URIs, URI) + } + } else { + if realURI == "" { + URIs = append(URIs, "mongodb://"+URI) + } else { + realURI += "," + URI + } + } + } + if realURI != "" { + URIs = append(URIs, realURI) + } + + if splitCluster { + // In this mode we split cluster strings into separate targets + separateURIs := []string{} + for _, hosturl := range URIs { + urlParsed, err := url.Parse(hosturl) + if err != nil { + log.Fatalf("Failed to parse URI %s: %v", hosturl, err) + } + for _, host := range strings.Split(urlParsed.Host, ",") { + targetURI := "mongodb://" + if urlParsed.User != nil { + targetURI += urlParsed.User.String() + "@" + } + targetURI += host + if urlParsed.Path != "" { + targetURI += urlParsed.Path + } + if urlParsed.RawQuery != "" { + targetURI += "?" + urlParsed.RawQuery + } + separateURIs = append(separateURIs, targetURI) + } + } + return separateURIs + } + return URIs +} + +// buildURIManually builds the URI manually by checking if the user and password are supplied +func buildURIManually(uri string, user string, password string) string { + uriArray := strings.SplitN(uri, "://", 2) //nolint:mnd + prefix := uriArray[0] + "://" + uri = uriArray[1] + + // IF user@pass not contained in uri AND custom user and pass supplied in arguments + // DO concat a new uri with user and pass arguments value + if !strings.Contains(uri, "@") && user != "" && password != "" { + // add user and pass to the uri + uri = fmt.Sprintf("%s:%s@%s", user, password, uri) + } + + // add back prefix after adding the user and pass + uri = prefix + uri + + return uri +} + +func BuildURI(uri string, user string, password string) string { + defaultPrefix := "mongodb://" // default prefix + + if !strings.HasPrefix(uri, defaultPrefix) && !strings.HasPrefix(uri, "mongodb+srv://") { + uri = defaultPrefix + uri + } + parsedURI, err := url.Parse(uri) + if err != nil { + // PMM generates URI with escaped path to socket file, so url.Parse fails + // in this case we build URI manually + return buildURIManually(uri, user, password) + } + + if parsedURI.User == nil && user != "" && password != "" { + parsedURI.User = url.UserPassword(user, password) + } + + return parsedURI.String() +} diff --git a/exporter/exporter.go b/exporter/exporter.go index d69ddff0..1684a05a 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -84,6 +84,8 @@ type Opts struct { URI string NodeName string + User string + Password string } var ( diff --git a/exporter/multi_target_test.go b/exporter/multi_target_test.go index 8278591d..68779fd1 100644 --- a/exporter/multi_target_test.go +++ b/exporter/multi_target_test.go @@ -22,6 +22,7 @@ import ( "net/http" "net/http/httptest" "regexp" + "sync" "testing" "github.com/prometheus/common/promslog" @@ -69,9 +70,21 @@ func TestMultiTarget(t *testing.T) { "mongodb_up{cluster_role=\"\"} 0\n", } + exportersCache := make(map[string]*Exporter) + var cacheMutex sync.Mutex + + for _, e := range exporters { + cacheMutex.Lock() + exportersCache[e.opts.URI] = e + cacheMutex.Unlock() + } + // Test all targets for sn, opt := range opts { - assert.HTTPBodyContains(t, multiTargetHandler(serverMap), "GET", fmt.Sprintf("?target=%s", opt.URI), nil, expected[sn]) + t.Run(fmt.Sprintf("target_%d", sn), func(t *testing.T) { + handler := multiTargetHandler(serverMap, opt, exportersCache, &cacheMutex, log) + assert.HTTPBodyContains(t, handler, "GET", fmt.Sprintf("?target=%s", opt.URI), nil, expected[sn]) + }) } } diff --git a/exporter/server.go b/exporter/server.go index e7c4f5cd..7ea587e5 100644 --- a/exporter/server.go +++ b/exporter/server.go @@ -22,7 +22,7 @@ import ( "net/url" "os" "strconv" - "strings" + "sync" "time" "github.com/prometheus/client_golang/prometheus" @@ -44,18 +44,48 @@ type ServerOpts struct { } // RunWebServer runs the main web-server -func RunWebServer(opts *ServerOpts, exporters []*Exporter, log *slog.Logger) { +func RunWebServer(opts *ServerOpts, exporters []*Exporter, exporterOpts *Opts, log *slog.Logger) { mux := http.NewServeMux() + serverMap := buildServerMap(exporters, log) + + exportersCache := make(map[string]*Exporter) + var cacheMutex sync.Mutex - if len(exporters) == 0 { - panic("No exporters were built. You must specify --mongodb.uri command argument or MONGODB_URI environment variable") + // Prefill cache with existing exporters + for _, exp := range exporters { + cacheMutex.Lock() + cacheKey := exp.opts.URI + exportersCache[cacheKey] = exp + cacheMutex.Unlock() } - serverMap := buildServerMap(exporters, log) + mux.HandleFunc(opts.Path, func(w http.ResponseWriter, r *http.Request) { + targetHost := r.URL.Query().Get("target") + + if targetHost == "" { + if len(exporters) > 0 { + defaultExporter := exporters[0] + defaultExporter.Handler().ServeHTTP(w, r) + return + } + + cacheMutex.Lock() + defer cacheMutex.Unlock() + for _, exp := range exportersCache { + exp.Handler().ServeHTTP(w, r) + return + } - defaultExporter := exporters[0] - mux.Handle(opts.Path, defaultExporter.Handler()) - mux.HandleFunc(opts.MultiTargetPath, multiTargetHandler(serverMap)) + reg := prometheus.NewRegistry() + h := promhttp.HandlerFor(reg, promhttp.HandlerOpts{}) + h.ServeHTTP(w, r) + return + } + + multiTargetHandler(serverMap, exporterOpts, exportersCache, &cacheMutex, log).ServeHTTP(w, r) + }) + + mux.HandleFunc(opts.MultiTargetPath, multiTargetHandler(serverMap, exporterOpts, exportersCache, &cacheMutex, log)) mux.HandleFunc(opts.OverallTargetPath, OverallTargetsHandler(exporters, log)) mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { @@ -85,21 +115,62 @@ func RunWebServer(opts *ServerOpts, exporters []*Exporter, log *slog.Logger) { } } -func multiTargetHandler(serverMap ServerMap) http.HandlerFunc { +// multiTargetHandler returns a handler that scrapes metrics from a target specified by the 'target' query parameter. +// It validates the URI and caches dynamic exporters by target. +func multiTargetHandler(serverMap ServerMap, exporterOpts *Opts, exportersCache map[string]*Exporter, cacheMutex *sync.Mutex, logger *slog.Logger) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { targetHost := r.URL.Query().Get("target") - if targetHost != "" { - if !strings.HasPrefix(targetHost, "mongodb://") { - targetHost = "mongodb://" + targetHost - } - if uri, err := url.Parse(targetHost); err == nil { - if e, ok := serverMap[uri.Host]; ok { - e.ServeHTTP(w, r) - return - } - } + if targetHost == "" { + logger.Warn("Missing target parameter") + http.Error(w, "Missing target parameter", http.StatusBadRequest) + return } - http.Error(w, "Unable to find target", http.StatusNotFound) + + parsed, err := url.Parse(targetHost) + if err != nil { + logger.Warn("Invalid target parameter", "target", targetHost, "error", err) + http.Error(w, "Invalid target parameter", http.StatusBadRequest) + return + } + + fullURI := targetHost + if parsed.User == nil && exporterOpts.User != "" { + fullURI = BuildURI(targetHost, exporterOpts.User, exporterOpts.Password) + } + + uri, err := url.Parse(fullURI) + if err != nil { + logger.Warn("Invalid full URI", "target", targetHost, "error", err) + http.Error(w, "Invalid target parameter", http.StatusBadRequest) + return + } + + if handler, ok := serverMap[uri.Host]; ok { + logger.Debug("Serving from static serverMap", "host", uri.Host) + handler.ServeHTTP(w, r) + return + } + + cacheMutex.Lock() + exp, ok := exportersCache[fullURI] + cacheMutex.Unlock() + + if !ok { + logger.Info("Creating new exporter for target", "target", targetHost) + opts := *exporterOpts + opts.URI = fullURI + opts.Logger = logger + + exp = New(&opts) + + cacheMutex.Lock() + exportersCache[fullURI] = exp + cacheMutex.Unlock() + } else { + logger.Debug("Serving from cache", "target", targetHost) + } + + exp.Handler().ServeHTTP(w, r) } } diff --git a/main.go b/main.go index 4eae7774..d8944257 100644 --- a/main.go +++ b/main.go @@ -17,11 +17,9 @@ package main import ( "fmt" - "log" "log/slog" "net" "net/url" - "regexp" "strings" "github.com/alecthomas/kong" @@ -119,7 +117,7 @@ func main() { } if len(opts.URI) == 0 { - ctx.Fatalf("No MongoDB hosts were specified. You must specify the host(s) with the --mongodb.uri command argument or the MONGODB_URI environment variable") + ctx.Printf("No MongoDB hosts specified. You can specify the host(s) with the --mongodb.uri command argument or the MONGODB_URI environment variable") } if opts.TimeoutOffset <= 0 { @@ -134,24 +132,13 @@ func main() { WebListenAddress: opts.WebListenAddress, TLSConfigPath: opts.TLSConfigPath, } - exporter.RunWebServer(serverOpts, buildServers(opts, logger), logger) -} -func buildExporter(opts GlobalFlags, uri string, log *slog.Logger) *exporter.Exporter { - uri = buildURI(uri, opts.User, opts.Password) - log.Debug("Connection URI", "uri", uri) + exporterOpts := buildOpts(opts) - uriParsed, _ := url.Parse(uri) - var nodeName string - switch { - case uriParsed == nil: - nodeName = "" - case uriParsed.Port() != "": - nodeName = net.JoinHostPort(uriParsed.Hostname(), uriParsed.Port()) - default: - nodeName = uriParsed.Host - } + exporter.RunWebServer(serverOpts, buildServers(opts, logger, exporterOpts), exporterOpts, logger) +} +func buildOpts(opts GlobalFlags) *exporter.Opts { collStatsNamespaces := []string{} if opts.CollStatsNamespaces != "" { collStatsNamespaces = strings.Split(opts.CollStatsNamespaces, ",") @@ -160,14 +147,12 @@ func buildExporter(opts GlobalFlags, uri string, log *slog.Logger) *exporter.Exp if opts.IndexStatsCollections != "" { indexStatsCollections = strings.Split(opts.IndexStatsCollections, ",") } - exporterOpts := &exporter.Opts{ + + return &exporter.Opts{ CollStatsNamespaces: collStatsNamespaces, CompatibleMode: opts.CompatibleMode, DiscoveringMode: opts.DiscoveringMode, IndexStatsCollections: indexStatsCollections, - Logger: log, - URI: uri, - NodeName: nodeName, GlobalConnPool: opts.GlobalConnPool, DirectConnect: opts.DirectConnect, ConnectTimeoutMS: opts.ConnectTimeoutMS, @@ -195,122 +180,44 @@ func buildExporter(opts GlobalFlags, uri string, log *slog.Logger) *exporter.Exp CollectAll: opts.CollectAll, ProfileTimeTS: opts.ProfileTimeTS, CurrentOpSlowTime: opts.CurrentOpSlowTime, - } - return exporter.New(exporterOpts) -} - -func buildServers(opts GlobalFlags, logger *slog.Logger) []*exporter.Exporter { - URIs := parseURIList(opts.URI, logger, opts.SplitCluster) - servers := make([]*exporter.Exporter, len(URIs)) - for serverIdx := range URIs { - servers[serverIdx] = buildExporter(opts, URIs[serverIdx], logger) + User: opts.User, + Password: opts.Password, } - - return servers } -func parseURIList(uriList []string, logger *slog.Logger, splitCluster bool) []string { //nolint:gocognit,cyclop - var URIs []string - - // If server URI is prefixed with mongodb scheme string, then every next URI in - // line not prefixed with mongodb scheme string is a part of cluster. Otherwise, - // treat it as a standalone server - realURI := "" - matchRegexp := regexp.MustCompile(`^mongodb(\+srv)?://`) - for _, URI := range uriList { - matches := matchRegexp.FindStringSubmatch(URI) - if matches != nil { - if realURI != "" { - // Add the previous host buffer to the url list as we met the scheme part - URIs = append(URIs, realURI) - realURI = "" - } - if matches[1] == "" { - realURI = URI - } else { - // There can be only one host in SRV connection string - if splitCluster { - // In splitCluster mode we get srv connection string from SRV recors - URI = exporter.GetSeedListFromSRV(URI, logger) - } - URIs = append(URIs, URI) - } - } else { - if realURI == "" { - URIs = append(URIs, "mongodb://"+URI) - } else { - realURI += "," + URI - } - } - } - if realURI != "" { - URIs = append(URIs, realURI) - } +func buildExporter(baseOpts *exporter.Opts, uri string, log *slog.Logger) *exporter.Exporter { + uri = exporter.BuildURI(uri, baseOpts.User, baseOpts.Password) + log.Debug("Connection URI", "uri", uri) - if splitCluster { - // In this mode we split cluster strings into separate targets - separateURIs := []string{} - for _, hosturl := range URIs { - urlParsed, err := url.Parse(hosturl) - if err != nil { - log.Fatalf("Failed to parse URI %s: %v", hosturl, err) - } - for _, host := range strings.Split(urlParsed.Host, ",") { - targetURI := "mongodb://" - if urlParsed.User != nil { - targetURI += urlParsed.User.String() + "@" - } - targetURI += host - if urlParsed.Path != "" { - targetURI += urlParsed.Path - } - if urlParsed.RawQuery != "" { - targetURI += "?" + urlParsed.RawQuery - } - separateURIs = append(separateURIs, targetURI) - } + uriParsed, _ := url.Parse(uri) + var nodeName string + if uriParsed != nil { + if uriParsed.Port() != "" { + nodeName = net.JoinHostPort(uriParsed.Hostname(), uriParsed.Port()) + } else { + nodeName = uriParsed.Host } - return separateURIs - } - return URIs -} - -// buildURIManually builds the URI manually by checking if the user and password are supplied -func buildURIManually(uri string, user string, password string) string { - uriArray := strings.SplitN(uri, "://", 2) //nolint:mnd - prefix := uriArray[0] + "://" - uri = uriArray[1] - - // IF user@pass not contained in uri AND custom user and pass supplied in arguments - // DO concat a new uri with user and pass arguments value - if !strings.Contains(uri, "@") && user != "" && password != "" { - // add user and pass to the uri - uri = fmt.Sprintf("%s:%s@%s", user, password, uri) } - // add back prefix after adding the user and pass - uri = prefix + uri + exporterOpts := *baseOpts + exporterOpts.URI = uri + exporterOpts.Logger = log + exporterOpts.NodeName = nodeName - return uri + return exporter.New(&exporterOpts) } -func buildURI(uri string, user string, password string) string { - defaultPrefix := "mongodb://" // default prefix - - if !strings.HasPrefix(uri, defaultPrefix) && !strings.HasPrefix(uri, "mongodb+srv://") { - uri = defaultPrefix + uri - } - parsedURI, err := url.Parse(uri) - if err != nil { - // PMM generates URI with escaped path to socket file, so url.Parse fails - // in this case we build URI manually - return buildURIManually(uri, user, password) +func buildServers(opts GlobalFlags, logger *slog.Logger, baseOpts *exporter.Opts) []*exporter.Exporter { + if len(opts.URI) == 0 { + return []*exporter.Exporter{} } - if parsedURI.User == nil && user != "" && password != "" { - parsedURI.User = url.UserPassword(user, password) + URIs := exporter.ParseURIList(opts.URI, logger, opts.SplitCluster) + servers := make([]*exporter.Exporter, len(URIs)) + for i, uri := range URIs { + servers[i] = buildExporter(baseOpts, uri, logger) } - return parsedURI.String() + return servers } diff --git a/main_test.go b/main_test.go index 3d54f866..8799590d 100644 --- a/main_test.go +++ b/main_test.go @@ -24,6 +24,7 @@ import ( "github.com/prometheus/common/promslog" "github.com/stretchr/testify/assert" + "github.com/percona/mongodb_exporter/exporter" "github.com/percona/mongodb_exporter/internal/tu" ) @@ -56,7 +57,7 @@ func TestParseURIList(t *testing.T) { } logger := promslog.New(&promslog.Config{}) for test, expected := range tests { - actual := parseURIList(strings.Split(test, ","), logger, false) + actual := exporter.ParseURIList(strings.Split(test, ","), logger, false) assert.Equal(t, expected, actual) } } @@ -95,7 +96,7 @@ func TestSplitCluster(t *testing.T) { defer mockdns.UnpatchNet(net.DefaultResolver) for test, expected := range tests { - actual := parseURIList(strings.Split(test, ","), logger, true) + actual := exporter.ParseURIList(strings.Split(test, ","), logger, true) assert.Equal(t, expected, actual) } } @@ -117,7 +118,7 @@ func TestBuildExporter(t *testing.T) { CompatibleMode: true, } log := promslog.New(&promslog.Config{}) - buildExporter(opts, "mongodb://usr:pwd@127.0.0.1/", log) + buildExporter(buildOpts(opts), "mongodb://usr:pwd@127.0.0.1/", log) } func TestBuildURI(t *testing.T) { @@ -272,7 +273,7 @@ func TestBuildURI(t *testing.T) { } for _, tc := range tests { t.Run(tc.situation, func(t *testing.T) { - newURI := buildURI(tc.origin, tc.newUser, tc.newPassword) + newURI := exporter.BuildURI(tc.origin, tc.newUser, tc.newPassword) assert.Equal(t, tc.expect, newURI) }) }