chore: Ensure EPP flags are configurable via Helm chart

Author: rahulgurnani

config/charts/inferencepool/templates/epp-deployment.yaml

@@ -44,6 +44,20 @@ spec:
         - "--model-server-metrics-path={{ .Values.inferenceExtension.modelServerMetricsPath }}"
         - "--model-server-metrics-scheme={{ .Values.inferenceExtension.modelServerMetricsScheme }}"
         - "--model-server-metrics-https-insecure-skip-verify={{ .Values.inferenceExtension.modelServerMetricsHttpsInsecureSkipVerify }}"
+        - "--model-server-metrics-port={{ .Values.inferenceExtension.modelServerMetricsPort }}"
+        - "--secure-serving={{ .Values.inferenceExtension.secureServing }}"
+        - "--health-checking={{ .Values.inferenceExtension.healthChecking }}"
+        - "--cert-path={{ .Values.inferenceExtension.certPath }}"
+        - "--destination-endpoint-hint-key={{ .Values.inferenceExtension.destinationEndpointHintKey }}"
+        - "--destination-endpoint-hint-metadata-namespace={{ .Values.inferenceExtension.destinationEndpointHintMetadataNamespace }}"
+        - "--fairness-id-header-key={{ .Values.inferenceExtension.fairnessIDHeaderKey }}"
+        - "--total-queued-requests-metric={{ .Values.inferenceExtension.totalQueuedRequestsMetric }}"
+        - "--kv-cache-usage-percentage-metric={{ .Values.inferenceExtension.kvCacheUsagePercentageMetric }}"
+        - "--lora-info-metric={{ .Values.inferenceExtension.loraInfoMetric }}"
+        - "--refresh-metrics-interval={{ .Values.inferenceExtension.refreshMetricsInterval }}"
+        - "--refresh-prometheus-metrics-interval={{ .Values.inferenceExtension.refreshPrometheusMetricsInterval }}"
+        - "--metrics-staleness-threshold={{ .Values.inferenceExtension.metricsStalenessThreshold }}"
+        - "--config-text={{ .Values.inferenceExtension.configText }}"
         {{- if eq (.Values.inferencePool.modelServerType | default "vllm") "triton-tensorrt-llm" }}
         - --total-queued-requests-metric
         - "nv_trt_llm_request_metrics{request_type=waiting}"

config/charts/inferencepool/values.yaml

@@ -7,12 +7,34 @@ inferenceExtension:
     pullPolicy: Always
   extProcPort: 9002
   env: []
-  enablePprof: true # Enable pprof handlers for profiling and debugging
+  enablePprof: true  # Enable pprof handlers for profiling and debugging
   modelServerMetricsPath: "/metrics"
   modelServerMetricsScheme: "http"
   modelServerMetricsHttpsInsecureSkipVerify: true
-  # This is the plugins configuration file. 
+  grpcPort: 9002
+  grpcHealthPort: 9003
+  metricsPort: 9090
+  destinationEndpointHintMetadataNamespace: "envoy.lb"
+  destinationEndpointHintKey: "x-gateway-destination-endpoint"
+  fairnessIDHeaderKey: "x-gateway-inference-fairness-id"
+  poolName: ""
+  poolNamespace: "default"
+  refreshMetricsInterval: "50ms"
+  refreshPrometheusMetricsInterval: "5s"
+  secureServing: true
+  healthChecking: false
+  totalQueuedRequestsMetric: "vllm:num_requests_waiting"
+  kvCacheUsagePercentageMetric: "vllm:gpu_cache_usage_perc"
+  loraInfoMetric: "vllm:lora_requests_info"
+  certPath: ""
+  configFile: ""
+  configText: ""
+  metricsStalenessThreshold: "2s"
+  
   pluginsConfigFile: "default-plugins.yaml"
+  logVerbosity: 1
+  
+  # This is the plugins configuration file.
   # pluginsCustomConfig:
   #   custom-plugins.yaml: |
   #     apiVersion: inference.networking.x-k8s.io/v1alpha1
@@ -34,18 +56,18 @@ inferenceExtension:
   # Example environment variables:
   # env:
   #   KV_CACHE_SCORE_WEIGHT: "1"
-
   # Define additional container ports
+  modelServerMetricsPort: 0
   extraContainerPorts: []
   # Define additional service ports
   extraServicePorts: []
 
 inferencePool:
   targetPortNumber: 8000
   modelServerType: vllm # vllm, triton-tensorrt-llm
-  # modelServers: # REQUIRED
-    # matchLabels: 
-    #   app: vllm-llama3-8b-instruct
+  modelServers:
+    matchLabels:
+      app: vllm-llama3-8b-instruct
 
 provider:
   name: none