Skip to content

Commit 4b883e4

Browse files
committed
update engine args
1 parent d8e4ca3 commit 4b883e4

8 files changed

+49
-8
lines changed

catalog/deepseek-r1-distill-qwen-1-5b.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ spec:
2929
engine_args:
3030
tensor_parallel_size: 1
3131
max_model_len: 32768
32-
enforce_eager: true
32+
enforce_eager: false
3333
gpu_memory_utilization: 0.95
3434
enable_chunked_prefill: true
3535
served_model_name: deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B

catalog/deepseek-r1-distill-qwen-14b.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ spec:
2929
engine_args:
3030
tensor_parallel_size: 1
3131
max_model_len: 32768
32-
enforce_eager: true
32+
enforce_eager: false
3333
gpu_memory_utilization: 0.95
3434
enable_chunked_prefill: true
3535
served_model_name: deepseek-ai/DeepSeek-R1-Distill-Qwen-14B

catalog/qwen2-5-0-5b-instruct.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ spec:
2929
engine_args:
3030
tensor_parallel_size: 1
3131
max_model_len: 32768
32-
enforce_eager: true
32+
enforce_eager: false
3333
gpu_memory_utilization: 0.95
3434
enable_chunked_prefill: true
3535
tool_call_parser: hermes

catalog/qwen2-5-vl-3b-instruct.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ spec:
2929
engine_args:
3030
tensor_parallel_size: 1
3131
max_model_len: 32768
32-
enforce_eager: true
32+
enforce_eager: false
3333
gpu_memory_utilization: 0.95
3434
enable_chunked_prefill: true
3535
served_model_name: Qwen/Qwen2.5-VL-3B-Instruct
Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
apiVersion: v1
2+
kind: ModelCatalog
3+
metadata:
4+
name: qwen3-32b-fp8-non-thinking
5+
display_name: Qwen/Qwen3-32B-FP8-non-thinking
6+
labels:
7+
icon_url: 'https://cdn-thumbnails.huggingface.co/social-thumbnails/Qwen.png'
8+
hf_repo_url: 'https://huggingface.co/Qwen/Qwen3-32B-FP8'
9+
spec:
10+
model:
11+
registry: ''
12+
name: Qwen/Qwen3-32B-FP8
13+
file: model-00001-of-00007.safetensors
14+
version: latest
15+
task: text-generation
16+
engine:
17+
engine: vllm
18+
version: v1
19+
resources: {}
20+
replicas:
21+
num: 1
22+
deployment_options:
23+
scheduler:
24+
type: consistent_hash
25+
virtual_nodes: 150
26+
load_factor: 1.25
27+
variables:
28+
RAY_SCHEDULER_TYPE: consistent_hash
29+
engine_args:
30+
tensor_parallel_size: 1
31+
max_model_len: 32768
32+
enforce_eager: false
33+
gpu_memory_utilization: 0.95
34+
enable_chunked_prefill: true
35+
served_model_name: Qwen/Qwen3-32B-FP8
36+
tool_call_parser: hermes
37+
enable_reasoning: false
38+
chat_template: >-
39+
{%- if tools %}{{- '<|im_start|>system\n' }}{%- if messages[0].role == 'system' %}{{- messages[0].content + '\n\n' }}{%- endif %}{{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}{%- for tool in tools %}{{- "\n" }}{{- tool | tojson }}{%- endfor %}{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}{%- else %}{%- if messages[0].role == 'system' %}{{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}{%- endif %}{%- endif %}{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}{%- for message in messages[::-1] %}{%- set index = (messages|length - 1) - loop.index0 %}{%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}{%- set ns.multi_step_tool = false %}{%- set ns.last_query_index = index %}{%- endif %}{%- endfor %}{%- for message in messages %}{%- if message.content is string %}{%- set content = message.content %}{%- else %}{%- set content = '' %}{%- endif %}{%- if (message.role == "user") or (message.role == "system" and not loop.first) %}{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}{%- elif message.role == "assistant" %}{%- set reasoning_content = '' %}{%- if message.reasoning_content is string %}{%- set reasoning_content = message.reasoning_content %}{%- else %}{%- if '</think>' in content %}{%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}{%- set content = content.split('</think>')[-1].lstrip('\n') %}{%- endif %}{%- endif %}{%- if loop.index0 > ns.last_query_index %}{%- if loop.last or (not loop.last and reasoning_content) %}{{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}{%- else %}{{- '<|im_start|>' + message.role + '\n' + content }}{%- endif %}{%- else %}{{- '<|im_start|>' + message.role + '\n' + content }}{%- endif %}{%- if message.tool_calls %}{%- for tool_call in message.tool_calls %}{%- if (loop.first and content) or (not loop.first) %}{{- '\n' }}{%- endif %}{%- if tool_call.function %}{%- set tool_call = tool_call.function %}{%- endif %}{{- '<tool_call>\n{"name": "' }}{{- tool_call.name }}{{- '", "arguments": ' }}{%- if tool_call.arguments is string %}{{- tool_call.arguments }}{%- else %}{{- tool_call.arguments | tojson }}{%- endif %}{{- '}\n</tool_call>' }}{%- endfor %}{%- endif %}{{- '<|im_end|>\n' }}{%- elif message.role == "tool" %}{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}{{- '<|im_start|>user' }}{%- endif %}{{- '\n<tool_response>\n' }}{{- content }}{{- '\n</tool_response>' }}{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}{{- '<|im_end|>\n' }}{%- endif %}{%- endif %}{%- endfor %}{%- if add_generation_prompt %}{{- '<|im_start|>assistant\n<think>\n\n</think>\n\n' }}{%- endif %}

catalog/qwen3-32b-fp8.yaml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,10 @@ spec:
2929
engine_args:
3030
tensor_parallel_size: 1
3131
max_model_len: 32768
32-
enforce_eager: true
32+
enforce_eager: false
3333
gpu_memory_utilization: 0.95
3434
enable_chunked_prefill: true
3535
served_model_name: Qwen/Qwen3-32B-FP8
36-
tool_call_parser: hermes
36+
tool_call_parser: hermes
37+
enable_reasoning: true
38+
reasoning_parser: deepseek_r1

catalog/qwen3-4b-instruct.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ spec:
3131
engine_args:
3232
tensor_parallel_size: 1
3333
max_model_len: 4096
34-
enforce_eager: true
34+
enforce_eager: false
3535
gpu_memory_utilization: 0.95
3636
enable_chunked_prefill: true
3737
tool_call_parser: hermes

catalog/qwq-32b-preview.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ spec:
2929
engine_args:
3030
tensor_parallel_size: 1
3131
max_model_len: 32768
32-
enforce_eager: true
32+
enforce_eager: false
3333
gpu_memory_utilization: 0.95
3434
enable_chunked_prefill: true
3535
tool_call_parser: hermes

0 commit comments

Comments
 (0)