fix

Signed-off-by: yiliu30 <[email protected]>

Author: yiliu30

examples/pytorch/nlp/huggingface_models/language-modeling/quantization/auto_round/deepseek/run_evaluation.sh

@@ -107,7 +107,7 @@ fi
 
 # for fp8 kv cache
 if [[ "$KV_CACHE_DTYPE" == "fp8" ]]; then
-    export VLLM_FLASHINFER_DISABLE_Q_QUANTIZATION=0
+    export VLLM_FLASHINFER_DISABLE_Q_QUANTIZATION=1
     export VLLM_ATTENTION_BACKEND="FLASHINFER_MLA"
     echo "Using FP8 for KV cache"
 fi

examples/pytorch/nlp/huggingface_models/language-modeling/quantization/auto_round/deepseek/run_generate.sh

@@ -110,7 +110,7 @@ fi
 
 # for fp8 kv cache
 if [[ "$KV_CACHE_DTYPE" == "fp8" ]]; then
-    export VLLM_FLASHINFER_DISABLE_Q_QUANTIZATION=0
+    export VLLM_FLASHINFER_DISABLE_Q_QUANTIZATION=1
     export VLLM_ATTENTION_BACKEND="FLASHINFER_MLA"
     echo "Using FP8 for KV cache"
 fi

examples/pytorch/nlp/huggingface_models/language-modeling/quantization/auto_round/qwen/run_evaluation.sh

@@ -100,7 +100,7 @@ fi
 
 # for fp8 kv cache
 if [[ "$KV_CACHE_DTYPE" == "fp8" ]]; then
-    export VLLM_FLASHINFER_DISABLE_Q_QUANTIZATION=0
+    export VLLM_FLASHINFER_DISABLE_Q_QUANTIZATION=1
     export VLLM_ATTENTION_BACKEND="FLASHINFER"
     echo "Using FP8 for KV cache"
 fi

examples/pytorch/nlp/huggingface_models/language-modeling/quantization/auto_round/qwen/run_generate.sh

@@ -101,7 +101,7 @@ fi
 
 # for fp8 kv cache
 if [[ "$KV_CACHE_DTYPE" == "fp8" ]]; then
-    export VLLM_FLASHINFER_DISABLE_Q_QUANTIZATION=0
+    export VLLM_FLASHINFER_DISABLE_Q_QUANTIZATION=1
     export VLLM_ATTENTION_BACKEND="FLASHINFER"
     echo "Using FP8 for KV cache"
 fi