You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
"The gRPC port used for communicating with Envoy proxy")
71
-
grpcHealthPort=flag.Int(
72
-
"grpc-health-port",
73
-
runserver.DefaultGrpcHealthPort,
74
-
"The port used for gRPC liveness and readiness probes")
75
-
metricsPort=flag.Int(
76
-
"metrics-port",
77
-
runserver.DefaultMetricsPort,
78
-
"The metrics port")
79
-
enablePprof=flag.Bool(
80
-
"enable-pprof",
81
-
runserver.DefaultEnablePprof,
82
-
"Enables pprof handlers. Defaults to true. Set to false to disable pprof handlers.")
83
-
poolName=flag.String(
84
-
"pool-name",
85
-
runserver.DefaultPoolName,
86
-
"Name of the InferencePool this Endpoint Picker is associated with.")
87
-
poolGroup=flag.String(
88
-
"pool-group",
89
-
runserver.DefaultPoolGroup,
90
-
"group of the InferencePool this Endpoint Picker is associated with.")
91
-
poolNamespace=flag.String(
92
-
"pool-namespace",
93
-
runserver.DefaultPoolNamespace,
94
-
"Namespace of the InferencePool this Endpoint Picker is associated with.")
95
-
logVerbosity=flag.Int(
96
-
"v",
97
-
logging.DEFAULT,
98
-
"number for the log level verbosity")
99
-
secureServing=flag.Bool(
100
-
"secure-serving",
101
-
runserver.DefaultSecureServing,
102
-
"Enables secure serving. Defaults to true.")
103
-
healthChecking=flag.Bool(
104
-
"health-checking",
105
-
runserver.DefaultHealthChecking,
106
-
"Enables health checking")
107
-
certPath=flag.String(
108
-
"cert-path",
109
-
runserver.DefaultCertPath,
110
-
"The path to the certificate for secure serving. The certificate and private key files "+
111
-
"are assumed to be named tls.crt and tls.key, respectively. If not set, and secureServing is enabled, "+
112
-
"then a self-signed certificate is used.")
67
+
grpcPort=flag.Int("grpc-port", runserver.DefaultGrpcPort, "The gRPC port used for communicating with Envoy proxy")
68
+
grpcHealthPort=flag.Int("grpc-health-port", runserver.DefaultGrpcHealthPort, "The port used for gRPC liveness and readiness probes")
69
+
metricsPort=flag.Int("metrics-port", runserver.DefaultMetricsPort, "The metrics port")
70
+
enablePprof=flag.Bool("enable-pprof", runserver.DefaultEnablePprof, "Enables pprof handlers. Defaults to true. Set to false to disable pprof handlers.")
71
+
poolName=flag.String("pool-name", runserver.DefaultPoolName, "Name of the InferencePool this Endpoint Picker is associated with.")
72
+
poolGroup=flag.String("pool-group", runserver.DefaultPoolGroup, "group of the InferencePool this Endpoint Picker is associated with.")
73
+
poolNamespace=flag.String("pool-namespace", runserver.DefaultPoolNamespace, "Namespace of the InferencePool this Endpoint Picker is associated with.")
74
+
logVerbosity=flag.Int("v", logging.DEFAULT, "number for the log level verbosity")
75
+
secureServing=flag.Bool("secure-serving", runserver.DefaultSecureServing, "Enables secure serving. Defaults to true.")
76
+
healthChecking=flag.Bool("health-checking", runserver.DefaultHealthChecking, "Enables health checking")
77
+
certPath=flag.String("cert-path", runserver.DefaultCertPath, "The path to the certificate for secure serving. The certificate and private key files "+
78
+
"are assumed to be named tls.crt and tls.key, respectively. If not set, and secureServing is enabled, "+
79
+
"then a self-signed certificate is used.")
113
80
// metric flags
114
-
totalQueuedRequestsMetric=flag.String(
115
-
"total-queued-requests-metric",
116
-
runserver.DefaultTotalQueuedRequestsMetric,
117
-
"Prometheus metric for the number of queued requests.")
118
-
kvCacheUsagePercentageMetric=flag.String(
119
-
"kv-cache-usage-percentage-metric",
120
-
runserver.DefaultKvCacheUsagePercentageMetric,
121
-
"Prometheus metric for the fraction of KV-cache blocks currently in use (from 0 to 1).")
81
+
totalQueuedRequestsMetric=flag.String("total-queued-requests-metric", runserver.DefaultTotalQueuedRequestsMetric, "Prometheus metric for the number of queued requests.")
82
+
kvCacheUsagePercentageMetric=flag.String("kv-cache-usage-percentage-metric", runserver.DefaultKvCacheUsagePercentageMetric, "Prometheus metric for the fraction of KV-cache blocks currently in use (from 0 to 1).")
122
83
// LoRA metrics
123
-
loraInfoMetric=flag.String(
124
-
"lora-info-metric",
125
-
runserver.DefaultLoraInfoMetric,
126
-
"Prometheus metric for the LoRA info metrics (must be in vLLM label format).")
127
-
84
+
loraInfoMetric=flag.String("lora-info-metric", runserver.DefaultLoraInfoMetric, "Prometheus metric for the LoRA info metrics (must be in vLLM label format).")
"Duration after which metrics are considered stale. This is used to determine if a pod's metrics are fresh enough.")
86
+
refreshMetricsInterval=flag.Duration("refresh-metrics-interval", runserver.DefaultRefreshMetricsInterval, "interval to refresh metrics")
87
+
refreshPrometheusMetricsInterval=flag.Duration("refresh-prometheus-metrics-interval", runserver.DefaultRefreshPrometheusMetricsInterval, "interval to flush prometheus metrics")
88
+
metricsStalenessThreshold=flag.Duration("metrics-staleness-threshold", runserver.DefaultMetricsStalenessThreshold, "Duration after which metrics are considered stale. This is used to determine if a pod's metrics are fresh enough.")
140
89
// configuration flags
141
-
configFile=flag.String(
142
-
"config-file",
143
-
runserver.DefaultConfigFile,
144
-
"The path to the configuration file")
145
-
configText=flag.String(
146
-
"config-text",
147
-
runserver.DefaultConfigText,
148
-
"The configuration specified as text, in lieu of a file")
90
+
configFile=flag.String("config-file", runserver.DefaultConfigFile, "The path to the configuration file")
91
+
configText=flag.String("config-text", runserver.DefaultConfigText, "The configuration specified as text, in lieu of a file")
149
92
150
93
modelServerMetricsPort=flag.Int("model-server-metrics-port", 0, "Port to scrape metrics from pods. "+
151
94
"Default value will be set to the InferencePool.Spec.TargetPorts[0].Number if not set.")
152
95
modelServerMetricsPath=flag.String("model-server-metrics-path", "/metrics", "Path to scrape metrics from pods")
153
96
modelServerMetricsScheme=flag.String("model-server-metrics-scheme", "http", "Scheme to scrape metrics from pods")
154
97
modelServerMetricsHttpsInsecureSkipVerify=flag.Bool("model-server-metrics-https-insecure-skip-verify", true, "When using 'https' scheme for 'model-server-metrics-scheme', configure 'InsecureSkipVerify' (default to true)")
155
-
haEnableLeaderElection=flag.Bool(
156
-
"ha-enable-leader-election",
157
-
false,
158
-
"Enables leader election for high availability. When enabled, readiness probes will only pass on the leader.")
98
+
haEnableLeaderElection=flag.Bool("ha-enable-leader-election", false, "Enables leader election for high availability. When enabled, readiness probes will only pass on the leader.")
0 commit comments