Skip to content

Commit 7e65477

Browse files
mgoinmgoin
authored
[Bugfix] Allow "None" or "" to be passed to CLI for string args that default to None (#4586)
1 parent 3521ba4 commit 7e65477

File tree

2 files changed

+34
-25
lines changed

2 files changed

+34
-25
lines changed

‎vllm/engine/arg_utils.py‎

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,12 @@
1111
from vllm.utils import str_to_int_tuple
1212

1313

14+
def nullable_str(val: str):
15+
if not val or val == "None":
16+
return None
17+
return val
18+
19+
1420
@dataclass
1521
class EngineArgs:
1622
"""Arguments for vLLM engine."""
@@ -96,7 +102,7 @@ def add_cli_args(
96102
help='Name or path of the huggingface model to use.')
97103
parser.add_argument(
98104
'--tokenizer',
99-
type=str,
105+
type=nullable_str,
100106
default=EngineArgs.tokenizer,
101107
help='Name or path of the huggingface tokenizer to use.')
102108
parser.add_argument(
@@ -105,21 +111,21 @@ def add_cli_args(
105111
help='Skip initialization of tokenizer and detokenizer')
106112
parser.add_argument(
107113
'--revision',
108-
type=str,
114+
type=nullable_str,
109115
default=None,
110116
help='The specific model version to use. It can be a branch '
111117
'name, a tag name, or a commit id. If unspecified, will use '
112118
'the default version.')
113119
parser.add_argument(
114120
'--code-revision',
115-
type=str,
121+
type=nullable_str,
116122
default=None,
117123
help='The specific revision to use for the model code on '
118124
'Hugging Face Hub. It can be a branch name, a tag name, or a '
119125
'commit id. If unspecified, will use the default version.')
120126
parser.add_argument(
121127
'--tokenizer-revision',
122-
type=str,
128+
type=nullable_str,
123129
default=None,
124130
help='The specific tokenizer version to use. It can be a branch '
125131
'name, a tag name, or a commit id. If unspecified, will use '
@@ -136,7 +142,7 @@ def add_cli_args(
136142
action='store_true',
137143
help='Trust remote code from huggingface.')
138144
parser.add_argument('--download-dir',
139-
type=str,
145+
type=nullable_str,
140146
default=EngineArgs.download_dir,
141147
help='Directory to download and load the weights, '
142148
'default to the default cache dir of '
@@ -187,7 +193,7 @@ def add_cli_args(
187193
'supported for common inference criteria.')
188194
parser.add_argument(
189195
'--quantization-param-path',
190-
type=str,
196+
type=nullable_str,
191197
default=None,
192198
help='Path to the JSON file containing the KV cache '
193199
'scaling factors. This should generally be supplied, when '
@@ -304,7 +310,7 @@ def add_cli_args(
304310
# Quantization settings.
305311
parser.add_argument('--quantization',
306312
'-q',
307-
type=str,
313+
type=nullable_str,
308314
choices=[*QUANTIZATION_METHODS, None],
309315
default=EngineArgs.quantization,
310316
help='Method used to quantize the weights. If '
@@ -349,7 +355,7 @@ def add_cli_args(
349355
'asynchronous tokenization. Ignored '
350356
'if tokenizer_pool_size is 0.')
351357
parser.add_argument('--tokenizer-pool-extra-config',
352-
type=str,
358+
type=nullable_str,
353359
default=EngineArgs.tokenizer_pool_extra_config,
354360
help='Extra config for tokenizer pool. '
355361
'This should be a JSON string that will be '
@@ -404,7 +410,7 @@ def add_cli_args(
404410
# Related to Vision-language models such as llava
405411
parser.add_argument(
406412
'--image-input-type',
407-
type=str,
413+
type=nullable_str,
408414
default=None,
409415
choices=[
410416
t.name.lower() for t in VisionLanguageConfig.ImageInputType
@@ -417,7 +423,7 @@ def add_cli_args(
417423
help=('Input id for image token.'))
418424
parser.add_argument(
419425
'--image-input-shape',
420-
type=str,
426+
type=nullable_str,
421427
default=None,
422428
help=('The biggest image input shape (worst for memory footprint) '
423429
'given an input type. Only used for vLLM\'s profile_run.'))
@@ -440,7 +446,7 @@ def add_cli_args(
440446

441447
parser.add_argument(
442448
'--speculative-model',
443-
type=str,
449+
type=nullable_str,
444450
default=EngineArgs.speculative_model,
445451
help=
446452
'The name of the draft model to be used in speculative decoding.')
@@ -454,7 +460,7 @@ def add_cli_args(
454460

455461
parser.add_argument(
456462
'--speculative-max-model-len',
457-
type=str,
463+
type=int,
458464
default=EngineArgs.speculative_max_model_len,
459465
help='The maximum sequence length supported by the '
460466
'draft model. Sequences over this length will skip '
@@ -475,7 +481,7 @@ def add_cli_args(
475481
'decoding.')
476482

477483
parser.add_argument('--model-loader-extra-config',
478-
type=str,
484+
type=nullable_str,
479485
default=EngineArgs.model_loader_extra_config,
480486
help='Extra config for model loader. '
481487
'This will be passed to the model loader '

‎vllm/entrypoints/openai/cli_args.py‎

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
import json
99
import ssl
1010

11-
from vllm.engine.arg_utils import AsyncEngineArgs
11+
from vllm.engine.arg_utils import AsyncEngineArgs, nullable_str
1212
from vllm.entrypoints.openai.serving_engine import LoRAModulePath
1313

1414

@@ -25,7 +25,10 @@ def __call__(self, parser, namespace, values, option_string=None):
2525
def make_arg_parser():
2626
parser = argparse.ArgumentParser(
2727
description="vLLM OpenAI-Compatible RESTful API server.")
28-
parser.add_argument("--host", type=str, default=None, help="host name")
28+
parser.add_argument("--host",
29+
type=nullable_str,
30+
default=None,
31+
help="host name")
2932
parser.add_argument("--port", type=int, default=8000, help="port number")
3033
parser.add_argument(
3134
"--uvicorn-log-level",
@@ -49,13 +52,13 @@ def make_arg_parser():
4952
default=["*"],
5053
help="allowed headers")
5154
parser.add_argument("--api-key",
52-
type=str,
55+
type=nullable_str,
5356
default=None,
5457
help="If provided, the server will require this key "
5558
"to be presented in the header.")
5659
parser.add_argument("--served-model-name",
5760
nargs="+",
58-
type=str,
61+
type=nullable_str,
5962
default=None,
6063
help="The model name(s) used in the API. If multiple "
6164
"names are provided, the server will respond to any "
@@ -65,33 +68,33 @@ def make_arg_parser():
6568
"same as the `--model` argument.")
6669
parser.add_argument(
6770
"--lora-modules",
68-
type=str,
71+
type=nullable_str,
6972
default=None,
7073
nargs='+',
7174
action=LoRAParserAction,
7275
help="LoRA module configurations in the format name=path. "
7376
"Multiple modules can be specified.")
7477
parser.add_argument("--chat-template",
75-
type=str,
78+
type=nullable_str,
7679
default=None,
7780
help="The file path to the chat template, "
7881
"or the template in single-line form "
7982
"for the specified model")
8083
parser.add_argument("--response-role",
81-
type=str,
84+
type=nullable_str,
8285
default="assistant",
8386
help="The role name to return if "
8487
"`request.add_generation_prompt=true`.")
8588
parser.add_argument("--ssl-keyfile",
86-
type=str,
89+
type=nullable_str,
8790
default=None,
8891
help="The file path to the SSL key file")
8992
parser.add_argument("--ssl-certfile",
90-
type=str,
93+
type=nullable_str,
9194
default=None,
9295
help="The file path to the SSL cert file")
9396
parser.add_argument("--ssl-ca-certs",
94-
type=str,
97+
type=nullable_str,
9598
default=None,
9699
help="The CA certificates file")
97100
parser.add_argument(
@@ -102,12 +105,12 @@ def make_arg_parser():
102105
)
103106
parser.add_argument(
104107
"--root-path",
105-
type=str,
108+
type=nullable_str,
106109
default=None,
107110
help="FastAPI root_path when app is behind a path based routing proxy")
108111
parser.add_argument(
109112
"--middleware",
110-
type=str,
113+
type=nullable_str,
111114
action="append",
112115
default=[],
113116
help="Additional ASGI middleware to apply to the app. "

0 commit comments

Comments
 (0)