build: change epp-config default yamls and image pull policy (#249)

* build: change epp-config default yaml
* build: change imagePullPolicy to IfNotPresent for simulation

Signed-off-by: Kfir Toledo <[email protected]>

---------

Signed-off-by: Kfir Toledo <[email protected]>

Author: kfirtoledo

deploy/components/inference-gateway/deployments.yaml

@@ -19,7 +19,7 @@ spec:
       containers:
       - name: epp
         image: ghcr.io/llm-d/llm-d-inference-scheduler:latest
-        imagePullPolicy: Always
+        imagePullPolicy: IfNotPresent
         args:
         - -poolName
         - "${POOL_NAME}"

deploy/config/epp-config.yaml

@@ -4,10 +4,6 @@ apiVersion: inference.networking.x-k8s.io/v1alpha1
 kind: EndpointPickerConfig
 plugins:
 - type: prefix-cache-scorer
-  parameters:
-    hashBlockSize: 5
-    maxPrefixBlocksToMatch: 256
-    lruCapacityPerServer: 31250
 - type: decode-filter
 - type: max-score-picker
 - type: single-profile-handler
@@ -17,4 +13,4 @@ schedulingProfiles:
   - pluginRef: decode-filter
   - pluginRef: max-score-picker
   - pluginRef: prefix-cache-scorer
-    weight: 50
+    weight: 2

deploy/config/pd-epp-config.yaml

@@ -4,27 +4,20 @@ kind: EndpointPickerConfig
 plugins:
 - type: prefill-header-handler
 - type: prefix-cache-scorer
-  parameters:
-    hashBlockSize: 5
-    maxPrefixBlocksToMatch: 256
-    lruCapacityPerServer: 31250
 - type: prefill-filter
 - type: decode-filter
 - type: max-score-picker
 - type: pd-profile-handler
-  parameters:
-    threshold: 10
-    hashBlockSize: 5
 schedulingProfiles:
 - name: prefill
   plugins:
   - pluginRef: prefill-filter
   - pluginRef: max-score-picker
   - pluginRef: prefix-cache-scorer
-    weight: 50
+    weight: 2
 - name: decode
   plugins:
   - pluginRef: decode-filter
   - pluginRef: max-score-picker
   - pluginRef: prefix-cache-scorer
-    weight: 50
+    weight: 2

deploy/config/sim-epp-config.yaml

@@ -0,0 +1,20 @@
+# Sample EPP configuration for running without P/D
+# with small hash block size for simulation purposes
+apiVersion: inference.networking.x-k8s.io/v1alpha1
+kind: EndpointPickerConfig
+plugins:
+- type: prefix-cache-scorer
+  parameters:
+    hashBlockSize: 5
+    maxPrefixBlocksToMatch: 256
+    lruCapacityPerServer: 31250
+- type: decode-filter
+- type: max-score-picker
+- type: single-profile-handler
+schedulingProfiles:
+- name: default
+  plugins:
+  - pluginRef: decode-filter
+  - pluginRef: max-score-picker
+  - pluginRef: prefix-cache-scorer
+    weight: 2

deploy/config/sim-pd-epp-config.yaml

@@ -0,0 +1,31 @@
+# Sample EPP configuration for tunning with P/D
+# Use with small hash block size for simulation purposes
+apiVersion: inference.networking.x-k8s.io/v1alpha1
+kind: EndpointPickerConfig
+plugins:
+- type: prefill-header-handler
+- type: prefix-cache-scorer
+  parameters:
+    hashBlockSize: 5
+    maxPrefixBlocksToMatch: 256
+    lruCapacityPerServer: 31250
+- type: prefill-filter
+- type: decode-filter
+- type: max-score-picker
+- type: pd-profile-handler
+  parameters:
+    threshold: 10
+    hashBlockSize: 5
+schedulingProfiles:
+- name: prefill
+  plugins:
+  - pluginRef: prefill-filter
+  - pluginRef: max-score-picker
+  - pluginRef: prefix-cache-scorer
+    weight: 2
+- name: decode
+  plugins:
+  - pluginRef: decode-filter
+  - pluginRef: max-score-picker
+  - pluginRef: prefix-cache-scorer
+    weight: 2

deploy/environments/dev/kubernetes-kgateway/patch-deployments.yaml

@@ -8,6 +8,7 @@ spec:
       containers:
       - name: epp
         image: ${EPP_IMAGE}:${EPP_TAG}
+        imagePullPolicy: Always
         args:
         - -poolName
         - ${POOL_NAME}

scripts/kind-dev-env.sh

@@ -58,9 +58,9 @@ export VLLM_REPLICA_COUNT_P="${VLLM_REPLICA_COUNT_P:-1}"
 export VLLM_REPLICA_COUNT_D="${VLLM_REPLICA_COUNT_D:-2}"
 
 if [ "${PD_ENABLED}" != "\"true\"" ]; then
-  DEFAULT_EPP_CONFIG="deploy/config/epp-config.yaml"
+  DEFAULT_EPP_CONFIG="deploy/config/sim-epp-config.yaml"
 else
-  DEFAULT_EPP_CONFIG="deploy/config/pd-epp-config.yaml"
+  DEFAULT_EPP_CONFIG="deploy/config/sim-pd-epp-config.yaml"
 fi
 export EPP_CONFIG="${EPP_CONFIG:-${DEFAULT_EPP_CONFIG}}"
 # ------------------------------------------------------------------------------
@@ -170,7 +170,7 @@ if [ "${PD_ENABLED}" != "\"true\"" ]; then
   KUSTOMIZE_DIR="deploy/environments/dev/kind-istio"
 else
   KUSTOMIZE_DIR="deploy/environments/dev/kind-istio-pd"
-fi 
+fi
 
 kubectl --context ${KUBE_CONTEXT} create configmap epp-config --from-file=epp-config.yaml=${EPP_CONFIG}