Support scraping metrics from target running with TLS (#1190)

* Support scraping metrics from target running with TLS

vLLM server can run with TLS and metrics scraping doesn't
work currently in that case.

Signed-off-by: Pierangelo Di Pilato <[email protected]>

* Add '--model-server-metrics-*' flags to Helm values and EPP template

Signed-off-by: Pierangelo Di Pilato <[email protected]>

* Update config/charts/inferencepool/values.yaml

Co-authored-by: Abdullah Gharaibeh <[email protected]>

* Update cmd/epp/runner/runner.go

Co-authored-by: Abdullah Gharaibeh <[email protected]>

---------

Signed-off-by: Pierangelo Di Pilato <[email protected]>
Co-authored-by: Kellen Swain <[email protected]>
Co-authored-by: Abdullah Gharaibeh <[email protected]>

Author: 3 people

cmd/epp/runner/runner.go

@@ -18,8 +18,10 @@ package runner
 
 import (
 	"context"
+	"crypto/tls"
 	"flag"
 	"fmt"
+	"net/http"
 	"net/http/pprof"
 	"os"
 
@@ -136,7 +138,9 @@ var (
 
 	modelServerMetricsPort = flag.Int("modelServerMetricsPort", 0, "Port to scrape metrics from pods. "+
 		"Default value will be set to InferencePool.Spec.TargetPortNumber if not set.")
-	modelServerMetricsPath = flag.String("modelServerMetricsPath", "/metrics", "Path to scrape metrics from pods")
+	modelServerMetricsPath                    = flag.String("modelServerMetricsPath", "/metrics", "Path to scrape metrics from pods")
+	modelServerMetricsScheme                  = flag.String("modelServerMetricsScheme", "http", "Scheme to scrape metrics from pods")
+	modelServerMetricsHttpsInsecureSkipVerify = flag.Bool("modelServerMetricsHttpsInsecureSkipVerify", true, "When using 'https' scheme for 'modelServerMetricsScheme', configure 'InsecureSkipVerify' (default to true)")
 
 	setupLog = ctrl.Log.WithName("setup")
 )
@@ -167,13 +171,15 @@ func (r *Runner) WithSchedulerConfig(schedulerConfig *scheduling.SchedulerConfig
 func bindEnvToFlags() {
 	// map[ENV_VAR]flagName   – add more as needed
 	for env, flg := range map[string]string{
-		"GRPC_PORT":                     "grpcPort",
-		"GRPC_HEALTH_PORT":              "grpcHealthPort",
-		"MODEL_SERVER_METRICS_PORT":     "modelServerMetricsPort",
-		"MODEL_SERVER_METRICS_PATH":     "modelServerMetricsPath",
-		"DESTINATION_ENDPOINT_HINT_KEY": "destinationEndpointHintKey",
-		"POOL_NAME":                     "poolName",
-		"POOL_NAMESPACE":                "poolNamespace",
+		"GRPC_PORT":                                       "grpcPort",
+		"GRPC_HEALTH_PORT":                                "grpcHealthPort",
+		"MODEL_SERVER_METRICS_PORT":                       "modelServerMetricsPort",
+		"MODEL_SERVER_METRICS_PATH":                       "modelServerMetricsPath",
+		"DESTINATION_ENDPOINT_HINT_KEY":                   "destinationEndpointHintKey",
+		"MODEL_SERVER_METRICS_SCHEME":                     "modelServerMetricsScheme",
+		"MODEL_SERVER_METRICS_HTTPS_INSECURE_SKIP_VERIFY": "modelServerMetricsHttpsInsecureSkipVerify",
+		"POOL_NAME":                                       "poolName",
+		"POOL_NAMESPACE":                                  "poolNamespace",
 		// durations & bools work too; flag.Set expects the *string* form
 		"REFRESH_METRICS_INTERVAL": "refreshMetricsInterval",
 		"SECURE_SERVING":           "secureServing",
@@ -231,10 +237,26 @@ func (r *Runner) Run(ctx context.Context) error {
 		return err
 	}
 	verifyMetricMapping(*mapping, setupLog)
+
+	var metricsHttpClient *http.Client
+	if *modelServerMetricsScheme == "https" {
+		metricsHttpClient = &http.Client{
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{
+					InsecureSkipVerify: *modelServerMetricsHttpsInsecureSkipVerify,
+				},
+			},
+		}
+	} else {
+		metricsHttpClient = http.DefaultClient
+	}
+
 	pmf := backendmetrics.NewPodMetricsFactory(&backendmetrics.PodMetricsClientImpl{
-		MetricMapping:          mapping,
-		ModelServerMetricsPort: int32(*modelServerMetricsPort),
-		ModelServerMetricsPath: *modelServerMetricsPath,
+		MetricMapping:            mapping,
+		ModelServerMetricsPort:   int32(*modelServerMetricsPort),
+		ModelServerMetricsPath:   *modelServerMetricsPath,
+		ModelServerMetricsScheme: *modelServerMetricsScheme,
+		Client:                   metricsHttpClient,
 	}, *refreshMetricsInterval)
 
 	datastore := datastore.NewDatastore(ctx, pmf)
@@ -412,6 +434,9 @@ func validateFlags() error {
 	if *configText != "" && *configFile != "" {
 		return fmt.Errorf("both the %q and %q flags can not be set at the same time", "configText", "configFile")
 	}
+	if *modelServerMetricsScheme != "http" && *modelServerMetricsScheme != "https" {
+		return fmt.Errorf("unexpected %q value for %q flag, it can only be set to 'http' or 'https'", *modelServerMetricsScheme, "model-server-metrics-scheme")
+	}
 
 	return nil
 }

config/charts/inferencepool/templates/epp-deployment.yaml

@@ -38,7 +38,10 @@ spec:
         - -configFile
         - "config/{{ .Values.inferenceExtension.pluginsConfigFile }}"
         # https://pkg.go.dev/flag#hdr-Command_line_flag_syntax; space is only for non-bool flags
-        - "-enablePprof={{ .Values.inferenceExtension.enablePprof }}"
+        - "--enablePprof={{ .Values.inferenceExtension.enablePprof }}"
+        - "--modelServerMetricsPath={{ .Values.inferenceExtension.modelServerMetricsPath }}"
+        - "--modelServerMetricsScheme={{ .Values.inferenceExtension.modelServerMetricsScheme }}"
+        - "--modelServerMetricsHttpsInsecureSkipVerify={{ .Values.inferenceExtension.modelServerMetricsHttpsInsecureSkipVerify }}"
         {{- if eq (.Values.inferencePool.modelServerType | default "vllm") "triton-tensorrt-llm" }}
         - -totalQueuedRequestsMetric
         - "nv_trt_llm_request_metrics{request_type=waiting}"

config/charts/inferencepool/values.yaml

@@ -8,6 +8,9 @@ inferenceExtension:
   extProcPort: 9002
   env: {}
   enablePprof: true # Enable pprof handlers for profiling and debugging
+  modelServerMetricsPath: "/metrics"
+  modelServerMetricsScheme: "http"
+  modelServerMetricsHttpsInsecureSkipVerify: true
   # This is the plugins configuration file. 
   pluginsConfigFile: "default-plugins.yaml"
   # pluginsCustomConfig:

pkg/epp/backend/metrics/metrics.go

@@ -37,9 +37,12 @@ const (
 )
 
 type PodMetricsClientImpl struct {
-	MetricMapping          *MetricMapping
-	ModelServerMetricsPort int32
-	ModelServerMetricsPath string
+	MetricMapping            *MetricMapping
+	ModelServerMetricsPort   int32
+	ModelServerMetricsPath   string
+	ModelServerMetricsScheme string
+
+	Client *http.Client
 }
 
 // FetchMetrics fetches metrics from a given pod, clones the existing metrics object and returns an updated one.
@@ -49,7 +52,7 @@ func (p *PodMetricsClientImpl) FetchMetrics(ctx context.Context, pod *backend.Po
 	if err != nil {
 		return nil, fmt.Errorf("failed to create request: %v", err)
 	}
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := p.Client.Do(req)
 	if err != nil {
 		return nil, fmt.Errorf("failed to fetch metrics from %s: %w", pod.NamespacedName, err)
 	}
@@ -73,7 +76,7 @@ func (p *PodMetricsClientImpl) getMetricEndpoint(pod *backend.Pod, targetPortNum
 	if p.ModelServerMetricsPort == 0 {
 		p.ModelServerMetricsPort = targetPortNumber
 	}
-	return fmt.Sprintf("http://%s:%d%s", pod.Address, p.ModelServerMetricsPort, p.ModelServerMetricsPath)
+	return fmt.Sprintf("%s://%s:%d%s", p.ModelServerMetricsScheme, pod.Address, p.ModelServerMetricsPort, p.ModelServerMetricsPath)
 }
 
 // promToPodMetrics updates internal pod metrics with scraped Prometheus metrics.

pkg/epp/backend/metrics/metrics_test.go

@@ -19,6 +19,7 @@ package metrics
 import (
 	"context"
 	"errors"
+	"net/http"
 	"reflect"
 	"strconv"
 	"strings"
@@ -495,7 +496,13 @@ func TestFetchMetrics(t *testing.T) {
 		},
 	}
 	existing := &MetricsState{}
-	p := &PodMetricsClientImpl{ModelServerMetricsPort: 9999, ModelServerMetricsPath: "/metrics"} // No MetricMapping needed for this basic test
+	// No MetricMapping needed for this basic test
+	p := &PodMetricsClientImpl{
+		ModelServerMetricsScheme: "http",
+		ModelServerMetricsPort:   9999,
+		ModelServerMetricsPath:   "/metrics",
+		Client:                   http.DefaultClient,
+	}
 
 	_, err := p.FetchMetrics(ctx, pod, existing, 9999) // Use a port that's unlikely to be in use
 	if err == nil {