feat(metrics): add request prompt, generation, max_tokens and success metrics

Signed-off-by: googs1025 <[email protected]>

Author: googs1025

manifests/config_with_fake.yaml

@@ -7,10 +7,18 @@ time-to-first-token: 2000
 inter-token-latency: 1000
 kv-cache-transfer-latency: 100
 seed: 100100100
-fake-metrics: 
+fake-metrics:
   running-requests: 16
   waiting-requests: 3 
   kv-cache-usage: 0.3
+  request-success-total:
+    stop: 20
+    length: 0
+    tool_calls: 0
+    remote_decode: 0
+  request-prompt-tokens: [ 10, 20, 30, 15 ]
+  request-generation-tokens: [ 50, 60, 40 ]
+  request-params-max-tokens: [ 128, 256, 512 ]
   loras:
   - '{"running":"lora1,lora2","waiting":"lora3","timestamp":1257894567}'
   - '{"running":"lora1,lora3","waiting":"","timestamp":1257894569}'

pkg/common/config.go

@@ -223,6 +223,13 @@ type Metrics struct {
 	// 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.75,
 	// 1.0, 2.5, 5.0, 7.5, 10.0, 20.0, 40.0, 80.0, +Inf
 	TPOTBucketValues []int `yaml:"tpot-buckets-values" json:"tpot-buckets-values"`
+	// RequestPromptTokens RequestGenerationTokens RequestParamsMaxTokens Histogram fake-observation arrays for init.
+	// Each value will be passed to Observe() once at start-up.
+	RequestPromptTokens     []float64 `yaml:"request-prompt-tokens" json:"request-prompt-tokens"`         // prompt-length samples
+	RequestGenerationTokens []float64 `yaml:"request-generation-tokens" json:"request-generation-tokens"` // generation-length samples
+	RequestParamsMaxTokens  []float64 `yaml:"request-params-max-tokens" json:"request-params-max-tokens"` // max_tokens parameter samples
+	// RequestSuccessTotal is the number of successful requests, key: finish-reason (stop, length, etc.).
+	RequestSuccessTotal map[string]int64 `yaml:"request-success-total" json:"request-success-total"`
 }
 
 type LorasMetrics struct {
@@ -501,52 +508,14 @@ func (c *Configuration) validate() error {
 		if c.FakeMetrics.KVCacheUsagePercentage < 0 || c.FakeMetrics.KVCacheUsagePercentage > 1 {
 			return errors.New("fake metrics KV cache usage must be between 0 ans 1")
 		}
-		if c.FakeMetrics.TTFTBucketValues != nil {
-			if len(c.FakeMetrics.TTFTBucketValues) > len(TTFTBucketsBoundaries)+1 {
-				return errors.New("fake time-to-first-token array is too long")
-			}
-			for v := range c.FakeMetrics.TTFTBucketValues {
-				if v < 0 {
-					return errors.New("time-to-first-token fake metrics should contain only non-negative values")
-				}
-			}
-		}
-		if c.FakeMetrics.TPOTBucketValues != nil {
-			if len(c.FakeMetrics.TPOTBucketValues) > len(TPOTBucketsBoundaries)+1 {
-				return errors.New("fake time-per-output-token array is too long")
-			}
-			for v := range c.FakeMetrics.TPOTBucketValues {
-				if v < 0 {
-					return errors.New("time-per-output-token fake metrics should contain only non-negative values")
-				}
-			}
-		}
 	}
 
 	if c.DPSize < 1 || c.DPSize > 8 {
 		return errors.New("data parallel size must be between 1 ans 8")
 	}
-
-	if (c.SSLCertFile == "") != (c.SSLKeyFile == "") {
-		return errors.New("both ssl-certfile and ssl-keyfile must be provided together")
-	}
-
-	if c.SelfSignedCerts && (c.SSLCertFile != "" || c.SSLKeyFile != "") {
-		return errors.New("cannot use both self-signed-certs and explicit ssl-certfile/ssl-keyfile")
-	}
-
-	if c.DatasetPath == "" && c.DatasetURL != "" {
-		return errors.New("dataset-path is required when dataset-url is set")
-	}
-
 	return nil
 }
 
-// SSLEnabled returns true if SSL is enabled either via certificate files or self-signed certificates
-func (c *Configuration) SSLEnabled() bool {
-	return (c.SSLCertFile != "" && c.SSLKeyFile != "") || c.SelfSignedCerts
-}
-
 func (c *Configuration) Copy() (*Configuration, error) {
 	var dst Configuration
 	data, err := json.Marshal(c)
@@ -631,10 +600,6 @@ func ParseCommandParamsAndLoadConfig() (*Configuration, error) {
 	f.Var(&dummyFailureTypes, "failure-types", failureTypesDescription)
 	f.Lookup("failure-types").NoOptDefVal = dummy
 
-	f.StringVar(&config.SSLCertFile, "ssl-certfile", config.SSLCertFile, "Path to SSL certificate file for HTTPS (optional)")
-	f.StringVar(&config.SSLKeyFile, "ssl-keyfile", config.SSLKeyFile, "Path to SSL private key file for HTTPS (optional)")
-	f.BoolVar(&config.SelfSignedCerts, "self-signed-certs", config.SelfSignedCerts, "Enable automatic generation of self-signed certificates for HTTPS")
-
 	// These values were manually parsed above in getParamValueFromArgs, we leave this in order to get these flags in --help
 	var dummyString string
 	f.StringVar(&dummyString, "config", "", "The path to a yaml configuration file. The command line values overwrite the configuration file values")

pkg/common/config_test.go

@@ -205,6 +205,15 @@ var _ = Describe("Simulator configuration", func() {
 		},
 		TTFTBucketValues: []int{10, 20, 30, 10},
 		TPOTBucketValues: []int{0, 0, 10, 20, 30},
+		RequestPromptTokens:     []float64{10, 20, 30, 15},
+		RequestGenerationTokens: []float64{50, 60, 40},
+		RequestParamsMaxTokens:  []float64{128, 256, 512},
+		RequestSuccessTotal: map[string]int64{
+			StopFinishReason:         20,
+			LengthFinishReason:       0,
+			ToolsFinishReason:        0,
+			RemoteDecodeFinishReason: 0,
+		},
 	}
 	test = testCase{
 		name:           "config with fake metrics file",

pkg/llm-d-inference-sim/metrics.go

@@ -20,6 +20,7 @@ package llmdinferencesim
 
 import (
 	"context"
+	"math"
 	"strconv"
 	"strings"
 	"sync"
@@ -65,6 +66,7 @@ func (s *VllmSimulator) createAndRegisterPrometheus() error {
 		return err
 	}
 
+	// not supported for now, reports constant value
 	s.waitingRequests = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Subsystem: "",
@@ -123,6 +125,61 @@ func (s *VllmSimulator) createAndRegisterPrometheus() error {
 		return err
 	}
 
+	s.requestPromptTokens = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Subsystem: "",
+			Name:      "vllm:request_prompt_tokens",
+			Help:      "Number of prefill tokens processed.",
+			Buckets:   build125Buckets(s.config.MaxModelLen),
+		},
+		[]string{vllmapi.PromLabelModelName},
+	)
+	if err := s.registry.Register(s.requestPromptTokens); err != nil {
+		s.logger.Error(err, "Prometheus request_prompt_tokens histogram register failed")
+		return err
+	}
+
+	s.requestGenerationTokens = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Subsystem: "",
+			Name:      "vllm:request_generation_tokens",
+			Help:      "Number of generation tokens processed.",
+			Buckets:   build125Buckets(s.config.MaxModelLen),
+		},
+		[]string{vllmapi.PromLabelModelName},
+	)
+	if err := s.registry.Register(s.requestGenerationTokens); err != nil {
+		s.logger.Error(err, "Prometheus request_generation_tokens histogram register failed")
+		return err
+	}
+
+	s.requestParamsMaxTokens = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Subsystem: "",
+			Name:      "vllm:request_params_max_tokens",
+			Help:      "Histogram of the max_tokens request parameter.",
+			Buckets:   build125Buckets(s.config.MaxModelLen),
+		},
+		[]string{vllmapi.PromLabelModelName},
+	)
+	if err := s.registry.Register(s.requestParamsMaxTokens); err != nil {
+		s.logger.Error(err, "Prometheus request_params_max_tokens histogram register failed")
+		return err
+	}
+
+	s.requestSuccessTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Subsystem: "",
+			Name:      "vllm:request_success_total",
+			Help:      "Count of successfully processed requests.",
+		},
+		[]string{vllmapi.PromLabelModelName, vllmapi.PromLabelFinishReason},
+	)
+	if err := s.registry.Register(s.requestSuccessTotal); err != nil {
+		s.logger.Error(err, "Prometheus request_success_total counter register failed")
+		return err
+	}
+
 	s.setInitialPrometheusMetrics()
 
 	return nil
@@ -132,6 +189,7 @@ func (s *VllmSimulator) createAndRegisterPrometheus() error {
 // the fake metrics if set
 func (s *VllmSimulator) setInitialPrometheusMetrics() {
 	var nRunningReqs, nWaitingReqs, kvCacheUsage float64
+	modelName := s.getDisplayedModelName(s.config.Model)
 	if s.config.FakeMetrics != nil {
 		nRunningReqs = float64(s.config.FakeMetrics.RunningRequests)
 		nWaitingReqs = float64(s.config.FakeMetrics.WaitingRequests)
@@ -144,9 +202,21 @@ func (s *VllmSimulator) setInitialPrometheusMetrics() {
 		if s.config.FakeMetrics.TPOTBucketValues != nil {
 			s.initFakeHistogram(s.tpot, common.TPOTBucketsBoundaries, s.config.FakeMetrics.TPOTBucketValues)
 		}
+		for _, requestPromptToken := range s.config.FakeMetrics.RequestPromptTokens {
+			s.requestPromptTokens.WithLabelValues(modelName).Observe(requestPromptToken)
+		}
+		for _, requestGenerationToken := range s.config.FakeMetrics.RequestGenerationTokens {
+			s.requestGenerationTokens.WithLabelValues(modelName).Observe(requestGenerationToken)
+		}
+		for _, requestParamsMaxToken := range s.config.FakeMetrics.RequestParamsMaxTokens {
+			s.requestParamsMaxTokens.WithLabelValues(modelName).Observe(requestParamsMaxToken)
+		}
+		for reason, requestSuccessTotal := range s.config.FakeMetrics.RequestSuccessTotal {
+			s.requestSuccessTotal.WithLabelValues(modelName, reason).Add(float64(requestSuccessTotal))
+		}
+
 	}
 
-	modelName := s.getDisplayedModelName(s.config.Model)
 	s.runningRequests.WithLabelValues(modelName).Set(nRunningReqs)
 	s.waitingRequests.WithLabelValues(modelName).Set(nWaitingReqs)
 	s.kvCacheUsagePercentage.WithLabelValues(modelName).Set(kvCacheUsage)
@@ -288,6 +358,7 @@ func (s *VllmSimulator) startMetricsUpdaters(ctx context.Context) {
 	go s.kvCacheUsageUpdater(ctx)
 	go s.ttftUpdater(ctx)
 	go s.tpotUpdater(ctx)
+	go s.recordRequestUpdater(ctx)
 }
 
 // waitingRequestsUpdater updates the waiting requests metric by listening on the relevant channel
@@ -396,3 +467,75 @@ func (s *VllmSimulator) decrementLoraRefCount(lora string, theMap *sync.Map) {
 		s.logger.Error(nil, "Zero model reference", "model", lora)
 	}
 }
+
+// recordRequestUpdater listens on requestSuccessChan and drives the Prometheus metric
+// for successfully completed requests.
+func (s *VllmSimulator) recordRequestUpdater(ctx context.Context) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case event := <-s.requestSuccessChan:
+			s.recordRequestMetricsOnSuccess(
+				event.promptTokens,
+				event.generationTokens,
+				event.maxTokens,
+				event.finishReason,
+			)
+		}
+	}
+}
+
+// requestSuccessEvent represents the data associated with a successfully completed request,
+// which is sent through the requestSuccessChan for asynchronous metrics recording.
+type requestSuccessEvent struct {
+	// promptTokens is the number of input (prompt) tokens in the request
+	promptTokens int
+	// generationTokens is the number of generated (output) tokens in the response
+	generationTokens int
+	// maxTokens is the maximum number of tokens allowed for generation (if specified in the request)
+	maxTokens *int64
+	// finishReason indicates why the generation stopped (e.g., "stop", "length", "tool_calls")
+	finishReason string
+}
+
+// recordRequestMetricsOnSuccess records metrics for a successfully completed request
+func (s *VllmSimulator) recordRequestMetricsOnSuccess(promptTokens,
+	generationTokens int, maxTokens *int64, finishReason string) {
+	modelName := s.getDisplayedModelName(s.config.Model)
+	s.requestPromptTokens.WithLabelValues(modelName).Observe(float64(promptTokens))
+	s.requestGenerationTokens.WithLabelValues(modelName).Observe(float64(generationTokens))
+	if maxTokens != nil {
+		s.requestParamsMaxTokens.WithLabelValues(modelName).Observe(float64(*maxTokens))
+	}
+	s.requestSuccessTotal.WithLabelValues(modelName, finishReason).Inc()
+}
+
+// build125Buckets generates histogram buckets in powers of 10 scaled by [1,2,5].
+// This matches vLLM's build_1_2_5_buckets() in metrics.py.
+//
+// Reference: https://github.com/vllm-project/vllm/blob/main/vllm/engine/metrics.py#L175
+func build125Buckets(maxValue int) []float64 {
+	if maxValue <= 0 {
+		return []float64{}
+	}
+	var buckets []float64
+	exponent := 0
+	mantissa := []int{1, 2, 5}
+
+	for {
+		complete := true
+		for _, m := range mantissa {
+			value := m * int(math.Pow10(exponent))
+			if value <= maxValue {
+				buckets = append(buckets, float64(value))
+				complete = false
+			}
+		}
+		if complete {
+			break
+		}
+		exponent++
+	}
+	return buckets
+}