Disable AMP for Baichuan fp16 inference on single tile mode. (#4992) (#4996)

Author: cboss6

examples/gpu/llm/inference/run_benchmark.sh

@@ -162,7 +162,7 @@ Run_benchmark_baichuan2-13b-chat() {
     sub_model_name=baichuan2-13b
     dir=perf/${model}/beam${beam}_bs${bs}_input${input}_out${out}
     mkdir -p ${dir}
-    python -u run_generation.py --benchmark -m ${model} --sub-model-name ${sub_model_name} --use-static-cache --num-beams ${beam} --num-iter ${iter} --batch-size ${bs} --input-tokens ${input} --max-new-tokens ${out} --device xpu --ipex --dtype float16 --token-latency 2>&1 | tee log_e2e
+    python -u run_generation.py --benchmark -m ${model} --sub-model-name ${sub_model_name} --use-static-cache --num-beams ${beam} --num-iter ${iter} --batch-size ${bs} --input-tokens ${input} --max-new-tokens ${out} --device xpu --ipex --dtype float16 --token-latency --disable-auto-cast 2>&1 | tee log_e2e
     mv log_e2e ${dir}
     PROFILE=1 python -u run_generation.py --benchmark -m ${model} --sub-model-name ${sub_model_name} --use-static-cache --num-beams ${beam} --num-iter ${iter} --batch-size ${bs} --input-tokens ${input} --max-new-tokens ${out} --device xpu --ipex --dtype float16
     mv profile*pt ${dir}

examples/gpu/llm/inference/run_generation.py

@@ -115,6 +115,7 @@
 parser.add_argument("--acc-iter", default=-1, type=int)
 parser.add_argument("--use-static-cache", default=False, action="store_true", help="use static kv cache")
 parser.add_argument("--use-hf-code", default=True, action="store_false", help="use hf transformers code")
+parser.add_argument("--disable-auto-cast", default=False, action="store_true", help="whether to disable auto-mixed-precision feature")
 args = parser.parse_args()
 print(args)
 
@@ -144,7 +145,7 @@ def get_memory_usage(name, args):
 #    torch._C._jit_set_texpr_fuser_enabled(False)
 
 # dtype
-amp_enabled = True if args.dtype != "float32" else False
+amp_enabled = True if args.dtype != "float32" and not args.disable_auto_cast else False
 amp_dtype = getattr(torch, args.dtype)
 
 # load model