[VPA] Add prometheus bearer auth support (#8263)

* Add prometheus bearer auth

Signed-off-by: Yuriy Losev <[email protected]>

* Prefix for prometheus bearer flags

Signed-off-by: Yuriy Losev <[email protected]>

* Update vertical-pod-autoscaler/pkg/recommender/input/history/history_provider_test.go

Co-authored-by: Adrian Moisey <[email protected]>

* Update vertical-pod-autoscaler/pkg/recommender/main.go

Co-authored-by: Adrian Moisey <[email protected]>

* rename the flags and change comment

Signed-off-by: Yuriy Losev <[email protected]>

* Add a test for both methods

Signed-off-by: Yuriy Losev <[email protected]>

---------

Signed-off-by: Yuriy Losev <[email protected]>
Co-authored-by: Adrian Moisey <[email protected]>

Author: yalosev

vertical-pod-autoscaler/docs/flags.md

@@ -105,7 +105,10 @@ This document is auto-generated from the flag definitions in the VPA recommender
 | `pod-recommendation-min-memory-mb` | float |  250 | Minimum memory recommendation for a pod  |
 | `profiling` | int |  | Is debug/pprof endpoenabled |
 | `prometheus-address` | string |  "http://prometheus.monitoring.svc" | Where to reach for Prometheus metrics  |
+| `prometheus-bearer-token` | string |  | The bearer token used in the Prometheus server bearer token auth |
+| `prometheus-bearer-token-file` | string |  | Path to the bearer token file used for authentication by the Prometheus server |
 | `prometheus-cadvisor-job-name` | string |  "kubernetes-cadvisor" | Name of the prometheus job name which scrapes the cAdvisor metrics  |
+| `prometheus-insecure` |  |  | Skip tls verify if https is used in the prometheus-address |
 | `prometheus-query-timeout` | string |  "5m" | How long to wait before killing long queries  |
 | `recommendation-lower-bound-cpu-percentile` | float |  0.5 | CPU usage percentile that will be used for the lower bound on CPU recommendation.  |
 | `recommendation-lower-bound-memory-percentile` | float |  0.5 | Memory usage percentile that will be used for the lower bound on memory recommendation.  |

vertical-pod-autoscaler/pkg/recommender/input/history/history_provider.go

@@ -18,6 +18,7 @@ package history
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
 	"net/http"
 	"sort"
@@ -33,30 +34,73 @@ import (
 	metrics_recommender "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/recommender"
 )
 
-// PrometheusBasicAuthTransport contains the username and password of prometheus server
+// PrometheusBasicAuthTransport injects basic auth headers into HTTP requests.
 type PrometheusBasicAuthTransport struct {
 	Username string
 	Password string
+	Base     http.RoundTripper
 }
 
 // RoundTrip function injects the username and password in the request's basic auth header
 func (t *PrometheusBasicAuthTransport) RoundTrip(req *http.Request) (*http.Response, error) {
-	req.SetBasicAuth(t.Username, t.Password)
-	return http.DefaultTransport.RoundTrip(req)
+	// Use default transport if none specified
+	rt := t.Base
+	if rt == nil {
+		rt = http.DefaultTransport
+	}
+
+	// Clone the request before modification to avoid data races and side effects.
+	// Original http.Request contains shared fields (Header, URL, Body) that are unsafe to modify directly.
+	// Also, RoundTripper interface recommends not to modify the request:
+	//   https://cs.opensource.google/go/go/+/refs/tags/go1.24.4:src/net/http/client.go;l=128-132
+	// Extra materials: https://pkg.go.dev/net/http#Request.Clone (deep copy requirement)
+	//   and https://github.com/golang/go/issues/36095 (concurrency safety discussion)
+	cloned := req.Clone(req.Context())
+	cloned.SetBasicAuth(t.Username, t.Password)
+	return rt.RoundTrip(cloned)
+}
+
+// PrometheusBearerTokenAuthTransport injects bearer token into HTTP requests.
+type PrometheusBearerTokenAuthTransport struct {
+	Token string
+	Base  http.RoundTripper
+}
+
+// RoundTrip function injects the bearer token in the request's Authorization header
+func (bt *PrometheusBearerTokenAuthTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	rt := bt.Base
+	if rt == nil {
+		rt = http.DefaultTransport
+	}
+
+	cloned := req.Clone(req.Context())
+	cloned.Header.Set("Authorization", fmt.Sprintf("Bearer %s", bt.Token))
+	return rt.RoundTrip(cloned)
 }
 
 // PrometheusHistoryProviderConfig allow to select which metrics
 // should be queried to get real resource utilization.
 type PrometheusHistoryProviderConfig struct {
 	Address                                          string
+	Insecure                                         bool
 	QueryTimeout                                     time.Duration
 	HistoryLength, HistoryResolution                 string
 	PodLabelPrefix, PodLabelsMetricName              string
 	PodNamespaceLabel, PodNameLabel                  string
 	CtrNamespaceLabel, CtrPodNameLabel, CtrNameLabel string
 	CadvisorMetricsJobName                           string
 	Namespace                                        string
-	PrometheusBasicAuthTransport
+
+	Authentication PrometheusCredentials
+}
+
+// PrometheusCredentials keeps credentials for Prometheus API. The Username + Password pair is mutually exclusive with
+// the BearerToken field. It's handled in the CLI flags. But if BearerToken is set, it will have priority over the basic auth.
+// If both are empty, no authentication is used.
+type PrometheusCredentials struct {
+	Username    string
+	Password    string
+	BearerToken string
 }
 
 // PodHistory represents history of usage and labels for a given pod.
@@ -90,23 +134,33 @@ type prometheusHistoryProvider struct {
 
 // NewPrometheusHistoryProvider constructs a history provider that gets data from Prometheus.
 func NewPrometheusHistoryProvider(config PrometheusHistoryProviderConfig) (HistoryProvider, error) {
-	promConfig := promapi.Config{
-		Address:      config.Address,
-		RoundTripper: promapi.DefaultRoundTripper,
+	prometheusTransport := promapi.DefaultRoundTripper
+
+	if config.Insecure {
+		prometheusTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
 	}
 
-	if config.Username != "" && config.Password != "" {
-		transport := &PrometheusBasicAuthTransport{
-			Username: config.Username,
-			Password: config.Password,
+	if config.Authentication.BearerToken != "" {
+		prometheusTransport = &PrometheusBearerTokenAuthTransport{
+			Token: config.Authentication.BearerToken,
+			Base:  prometheusTransport,
+		}
+	} else if config.Authentication.Username != "" && config.Authentication.Password != "" {
+		prometheusTransport = &PrometheusBasicAuthTransport{
+			Username: config.Authentication.Username,
+			Password: config.Authentication.Password,
+			Base:     prometheusTransport,
 		}
-		promConfig.RoundTripper = transport
 	}
 
 	roundTripper := metrics_recommender.NewPrometheusRoundTripperCounter(
-		metrics_recommender.NewPrometheusRoundTripperDuration(promConfig.RoundTripper),
+		metrics_recommender.NewPrometheusRoundTripperDuration(prometheusTransport),
 	)
-	promConfig.RoundTripper = roundTripper
+
+	promConfig := promapi.Config{
+		Address:      config.Address,
+		RoundTripper: roundTripper,
+	}
 
 	promClient, err := promapi.NewClient(promConfig)
 	if err != nil {

vertical-pod-autoscaler/pkg/recommender/input/history/history_provider_test.go

@@ -19,6 +19,8 @@ package history
 import (
 	"context"
 	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"testing"
 	"time"
 
@@ -345,3 +347,55 @@ func TestGetLabels(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, histories, map[model.PodID]*PodHistory{podID: podHistory})
 }
+
+func TestPrometheusAuth(t *testing.T) {
+	var capturedRequest *http.Request
+	ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		capturedRequest = r
+		_, _ = w.Write([]byte(`{"status": "success","data": {"resultType": "matrix","result": []}}`))
+	}))
+	defer ts.Close()
+
+	cfg := PrometheusHistoryProviderConfig{
+		Address:           ts.URL,
+		Insecure:          true,
+		HistoryLength:     "8d",
+		HistoryResolution: "30s",
+		QueryTimeout:      30 * time.Second,
+	}
+
+	t.Run("Basic auth", func(t *testing.T) {
+		cfg.Authentication.Username = "user"
+		cfg.Authentication.Password = "password"
+
+		prov, _ := NewPrometheusHistoryProvider(cfg)
+		_, err := prov.GetClusterHistory()
+
+		assert.Nil(t, err)
+		assert.Equal(t, capturedRequest.Header.Get("Authorization"), "Basic dXNlcjpwYXNzd29yZA==") // "user:password"
+	})
+
+	t.Run("Bearer token auth", func(t *testing.T) {
+		cfg.Authentication.BearerToken = "token"
+
+		prov, _ := NewPrometheusHistoryProvider(cfg)
+		_, err := prov.GetClusterHistory()
+
+		assert.Nil(t, err)
+		assert.Equal(t, capturedRequest.Header.Get("Authorization"), "Bearer token")
+	})
+
+	t.Run("Basic auth and Bearer token auth are set at once", func(t *testing.T) {
+		// if both auth methods are set we prefer Bearer token
+		cfg.Authentication.BearerToken = "token"
+		cfg.Authentication.Username = "user"
+		cfg.Authentication.Password = "password"
+
+		prov, _ := NewPrometheusHistoryProvider(cfg)
+		_, err := prov.GetClusterHistory()
+
+		assert.Nil(t, err)
+		assert.NotContains(t, capturedRequest.Header.Get("Authorization"), "Basic")
+		assert.Equal(t, capturedRequest.Header.Get("Authorization"), "Bearer token")
+	})
+}

vertical-pod-autoscaler/pkg/recommender/main.go

@@ -71,20 +71,23 @@ var (
 
 // Prometheus history provider flags
 var (
-	prometheusAddress   = flag.String("prometheus-address", "http://prometheus.monitoring.svc", `Where to reach for Prometheus metrics`)
-	prometheusJobName   = flag.String("prometheus-cadvisor-job-name", "kubernetes-cadvisor", `Name of the prometheus job name which scrapes the cAdvisor metrics`)
-	historyLength       = flag.String("history-length", "8d", `How much time back prometheus have to be queried to get historical metrics`)
-	historyResolution   = flag.String("history-resolution", "1h", `Resolution at which Prometheus is queried for historical metrics`)
-	queryTimeout        = flag.String("prometheus-query-timeout", "5m", `How long to wait before killing long queries`)
-	podLabelPrefix      = flag.String("pod-label-prefix", "pod_label_", `Which prefix to look for pod labels in metrics`)
-	podLabelsMetricName = flag.String("metric-for-pod-labels", "up{job=\"kubernetes-pods\"}", `Which metric to look for pod labels in metrics`)
-	podNamespaceLabel   = flag.String("pod-namespace-label", "kubernetes_namespace", `Label name to look for pod namespaces`)
-	podNameLabel        = flag.String("pod-name-label", "kubernetes_pod_name", `Label name to look for pod names`)
-	ctrNamespaceLabel   = flag.String("container-namespace-label", "namespace", `Label name to look for container namespaces`)
-	ctrPodNameLabel     = flag.String("container-pod-name-label", "pod_name", `Label name to look for container pod names`)
-	ctrNameLabel        = flag.String("container-name-label", "name", `Label name to look for container names`)
-	username            = flag.String("username", "", "The username used in the prometheus server basic auth")
-	password            = flag.String("password", "", "The password used in the prometheus server basic auth")
+	prometheusAddress         = flag.String("prometheus-address", "http://prometheus.monitoring.svc", `Where to reach for Prometheus metrics`)
+	prometheusInsecure        = flag.Bool("prometheus-insecure", false, `Skip tls verify if https is used in the prometheus-address`)
+	prometheusJobName         = flag.String("prometheus-cadvisor-job-name", "kubernetes-cadvisor", `Name of the prometheus job name which scrapes the cAdvisor metrics`)
+	historyLength             = flag.String("history-length", "8d", `How much time back prometheus have to be queried to get historical metrics`)
+	historyResolution         = flag.String("history-resolution", "1h", `Resolution at which Prometheus is queried for historical metrics`)
+	queryTimeout              = flag.String("prometheus-query-timeout", "5m", `How long to wait before killing long queries`)
+	podLabelPrefix            = flag.String("pod-label-prefix", "pod_label_", `Which prefix to look for pod labels in metrics`)
+	podLabelsMetricName       = flag.String("metric-for-pod-labels", "up{job=\"kubernetes-pods\"}", `Which metric to look for pod labels in metrics`)
+	podNamespaceLabel         = flag.String("pod-namespace-label", "kubernetes_namespace", `Label name to look for pod namespaces`)
+	podNameLabel              = flag.String("pod-name-label", "kubernetes_pod_name", `Label name to look for pod names`)
+	ctrNamespaceLabel         = flag.String("container-namespace-label", "namespace", `Label name to look for container namespaces`)
+	ctrPodNameLabel           = flag.String("container-pod-name-label", "pod_name", `Label name to look for container pod names`)
+	ctrNameLabel              = flag.String("container-name-label", "name", `Label name to look for container names`)
+	username                  = flag.String("username", "", "The username used in the prometheus server basic auth")
+	password                  = flag.String("password", "", "The password used in the prometheus server basic auth")
+	prometheusBearerToken     = flag.String("prometheus-bearer-token", "", "The bearer token used in the Prometheus server bearer token auth")
+	prometheusBearerTokenFile = flag.String("prometheus-bearer-token-file", "", "Path to the bearer token file used for authentication by the Prometheus server")
 )
 
 // External metrics provider flags
@@ -149,6 +152,20 @@ func main() {
 		klog.InfoS("DEPRECATION WARNING: The 'min-checkpoints' flag is deprecated and has no effect. It will be removed in a future release.")
 	}
 
+	if *prometheusBearerToken != "" && *prometheusBearerTokenFile != "" && *username != "" {
+		klog.ErrorS(nil, "--bearer-token, --bearer-token-file and --username are mutually exclusive and can't be set together.")
+		klog.FlushAndExit(klog.ExitFlushTimeout, 1)
+	}
+
+	if *prometheusBearerTokenFile != "" {
+		fileContent, err := os.ReadFile(*prometheusBearerTokenFile)
+		if err != nil {
+			klog.ErrorS(err, "Unable to read bearer token file", "filename", *prometheusBearerTokenFile)
+			klog.FlushAndExit(klog.ExitFlushTimeout, 1)
+		}
+		*prometheusBearerToken = strings.TrimSpace(string(fileContent))
+	}
+
 	ctx := context.Background()
 
 	healthCheck := metrics.NewHealthCheck(*metricsFetcherInterval * 5)
@@ -314,6 +331,7 @@ func run(ctx context.Context, healthCheck *metrics.HealthCheck, commonFlag *comm
 	} else {
 		config := history.PrometheusHistoryProviderConfig{
 			Address:                *prometheusAddress,
+			Insecure:               *prometheusInsecure,
 			QueryTimeout:           promQueryTimeout,
 			HistoryLength:          *historyLength,
 			HistoryResolution:      *historyResolution,
@@ -326,9 +344,10 @@ func run(ctx context.Context, healthCheck *metrics.HealthCheck, commonFlag *comm
 			CtrNameLabel:           *ctrNameLabel,
 			CadvisorMetricsJobName: *prometheusJobName,
 			Namespace:              commonFlag.VpaObjectNamespace,
-			PrometheusBasicAuthTransport: history.PrometheusBasicAuthTransport{
-				Username: *username,
-				Password: *password,
+			Authentication: history.PrometheusCredentials{
+				BearerToken: *prometheusBearerToken,
+				Username:    *username,
+				Password:    *password,
 			},
 		}
 		provider, err := history.NewPrometheusHistoryProvider(config)