PMM-14141: Respect Prometheus scrape-timeout header (#1107)

* Fix parsing of X-Prometheus-Scrape-Timeout-Seconds header
The previous implementation expected this header to be an integer, but vmagent
 may send it as a float. This caused incorrect handling in `mongodb_exporter`.

Updated to support float values.

* Avoid stale connections by pinging on acquire
The exporter was returning cached MongoDB connections without validating their health. This meant that if a connection had become unhealthy,
subsequent operations would fail after a delay (serverSelectionTimeout etc), causing the entire request to be slower than it should be.

This commit adds  `Ping` check before returning a cached client.

* Update exporter/exporter.go

Co-authored-by: Alex Demidoff <[email protected]>

---------

Co-authored-by: Alex Demidoff <[email protected]>
Co-authored-by: Michael Okoko <[email protected]>

Author: 3 people

exporter/exporter.go

@@ -273,8 +273,13 @@ func (e *Exporter) getClient(ctx context.Context) (*mongo.Client, error) {
 		e.clientMu.Lock()
 		defer e.clientMu.Unlock()
 
-		// If client is already initialized, return it.
+		// If client is already initialized, and Ping is successful -- return it.
 		if e.client != nil {
+			err := e.client.Ping(ctx, nil)
+			if err != nil {
+				return nil, fmt.Errorf("cannot connect to MongoDB: %w", err)
+			}
+
 			return e.client, nil
 		}
 
@@ -300,20 +305,25 @@ func (e *Exporter) getClient(ctx context.Context) (*mongo.Client, error) {
 // run for hooking up custom HTTP servers.
 func (e *Exporter) Handler() http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		seconds, err := strconv.Atoi(r.Header.Get("X-Prometheus-Scrape-Timeout-Seconds"))
-		// To support older ones vmagents.
-		if err != nil {
-			seconds = 10
+		scrapeTimeoutHeader := r.Header.Get("X-Prometheus-Scrape-Timeout-Seconds")
+		seconds := 10.0
+
+		if scrapeTimeoutHeader != "" {
+			if parsedSeconds, err := strconv.ParseFloat(scrapeTimeoutHeader, 64); err == nil {
+				seconds = parsedSeconds
+			} else {
+				e.logger.Info("Invalid X-Prometheus-Scrape-Timeout-Seconds header", "error", err)
+			}
 		}
-		seconds -= e.opts.TimeoutOffset
+		seconds -= float64(e.opts.TimeoutOffset)
 
 		var client *mongo.Client
-		ctx, cancel := context.WithTimeout(r.Context(), time.Duration(seconds)*time.Second)
+		ctx, cancel := context.WithTimeout(r.Context(), time.Duration(seconds*float64(time.Second)))
 		defer cancel()
 
 		requestOpts := GetRequestOpts(r.URL.Query()["collect[]"], e.opts)
 
-		client, err = e.getClient(ctx)
+		client, err := e.getClient(ctx)
 		if err != nil {
 			e.logger.Error("Cannot connect to MongoDB", "error", err)
 		}