[Feat] Include runner and convert flag (#803)

* feat: Include runner and convert flag

Signed-off-by: Shern Shiou Tan <shernshiou@gmail.com>

* chores: Add validation and description of runner and convert

Signed-off-by: Shern Shiou Tan <shernshiou@gmail.com>

* docs: Include runner and convert flag at helm README

Signed-off-by: Shern Shiou Tan <shernshiou@gmail.com>

* feat: Move runner and convert to vLLM Configuration

Signed-off-by: Shern Shiou Tan <shernshiou@gmail.com>

---------

Signed-off-by: Shern Shiou Tan <shernshiou@gmail.com>

Author: shernshiou

helm/README.md

@@ -132,6 +132,8 @@ This table documents all available configuration values for the Production Stack
 | `servingEngineSpec.modelSpec[].vllmConfig.maxNumSeqs` | integer | `256` | Maximum number of sequences to be processed in a single iteration |
 | `servingEngineSpec.modelSpec[].vllmConfig.maxLoras` | integer | `0` | The maximum number of LoRA models to be loaded in a single batch |
 | `servingEngineSpec.modelSpec[].vllmConfig.gpuMemoryUtilization` | number | `0.9` | The fraction of GPU memory to be used for the model executor (0-1) |
+| `servingEngineSpec.modelSpec[].vllmConfig.runner` | string | `""` | The runner type for the model, can be "auto" or "pooling" |
+| `servingEngineSpec.modelSpec[].vllmConfig.convert` | string | `""` | The conversion type for the model, can be "token_embed", "embed", "token_classify", "classify", or "score" |
 | `servingEngineSpec.modelSpec[].vllmConfig.extraArgs` | list | `["--disable-log-requests"]` | Extra command line arguments to pass to vLLM |
 
 #### LMCache Configuration

helm/templates/deployment-vllm-multi.yaml

@@ -168,6 +168,14 @@ spec:
           - "--max_loras"
           - {{ .maxLoras | quote }}
           {{-   end }}
+          {{-   if hasKey . "runner" }}
+          - "--runner"
+          - {{ .runner | quote }}
+          {{-   end }}
+          {{-   if hasKey . "convert" }}
+          - "--convert"
+          - {{ .convert | quote }}
+          {{-   end }}
           {{-   if .extraArgs }}
           {{-     range .extraArgs }}
           - {{ . | quote }}

helm/values.schema.json

@@ -208,6 +208,14 @@
                   "dtype": {
                     "type": "string"
                   },
+                  "runner": {
+                    "type": "string",
+                    "enum": ["auto", "pooling"]
+                  },
+                  "convert": {
+                    "type": "string",
+                    "enum": ["token_embed", "embed", "token_classify", "classify", "score"]
+                  },
                   "extraArgs": {
                     "type": "array",
                     "items": {

helm/values.yaml

@@ -82,6 +82,8 @@ servingEngineSpec:
   #   - maxNumSeqs: (optional, int) Maximum number of sequences to be processed in a single iteration., e.g., 32
   #   - maxLoras: (optional, int) The maximum number of LoRA models to be loaded in a single batch, e.g., 4
   #   - gpuMemoryUtilization: (optional, float) The fraction of GPU memory to be used for the model executor, which can range from 0 to 1. e.g., 0.95
+  #   - runner: (optional, string) The runner type for the model, can be "auto" or "pooling". e.g., "pooling"
+  #   - convert: (optional, string) The conversion type for the model, can be "token_embed", "embed", "token_classify", "classify", or "score". e.g., "embed"
   #   - extraArgs: (optional, list) Extra command line arguments to pass to vLLM, e.g., ["--disable-log-requests"]
   #
   # - lmcacheConfig: (optional, map) The configuration of the LMCache for KV offloading, supported options are: