update server configs for large models (#36)

dhuangnm · web-flow · commit b0bd47d5eb54 · 2025-05-06T16:23:04.000-04:00
* update server configs for large models

* use 8 for tensor-parallel-size
diff --git a/RedHatAI/Llama-4-Scout-17B-16E-Instruct-FP8-dynamic/accuracy/server.yml b/RedHatAI/Llama-4-Scout-17B-16E-Instruct-FP8-dynamic/accuracy/server.yml
@@ -0,0 +1,3 @@
+trust-remote-code: true
+tensor-parallel-size: 2
+max-model-len: 16384
diff --git a/RedHatAI/Llama-4-Scout-17B-16E-Instruct-quantized.w4a16/accuracy/server.yml b/RedHatAI/Llama-4-Scout-17B-16E-Instruct-quantized.w4a16/accuracy/server.yml
@@ -0,0 +1,3 @@
+trust-remote-code: true
+tensor-parallel-size: 2
+max-model-len: 16384
diff --git a/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8/accuracy/server.yml b/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8/accuracy/server.yml
@@ -0,0 +1,3 @@
+trust-remote-code: true
+tensor-parallel-size: 8
+max-model-len: 16384
diff --git a/meta-llama/Llama-4-Maverick-17B-128E-Instruct/accuracy/server.yml b/meta-llama/Llama-4-Maverick-17B-128E-Instruct/accuracy/server.yml
@@ -1,5 +1,3 @@
-# server configs for https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct
-model: "meta-llama/Llama-4-Maverick-17B-128E-Instruct"
 trust-remote-code: true
 tensor-parallel-size: 8
 max-model-len: 16384
diff --git a/meta-llama/Llama-4-Scout-17B-16E-Instruct/accuracy/server.yml b/meta-llama/Llama-4-Scout-17B-16E-Instruct/accuracy/server.yml
@@ -0,0 +1,3 @@
+trust-remote-code: true
+tensor-parallel-size: 4
+max-model-len: 16384
