fix merge_lora_dtype (#842)

Author: Jintao-Huang

docs/source/LLM/Grok训练和推理.md

@@ -59,7 +59,7 @@ torchrun \
     --lora_rank 8 \
     --lora_alpha 32 \
     --lora_dropout_p 0.05 \
-    --lora_dtype bf16 \
+    --lora_dtype AUTO \
     --lora_target_modules DEFAULT \
     --gradient_checkpointing true \
     --batch_size 2 \

docs/source/LLM/命令行参数.md

@@ -51,7 +51,7 @@
 - `--lora_dropout_p`: 默认为`0.05`, 只有当`sft_type`指定为'lora'时才生效.
 - `--lora_bias_trainable`: 默认为`'none'`, 可以选择的值: 'none', 'all'. 如果你要将bias全都设置为可训练, 你可以设置为`'all'`.
 - `--lora_modules_to_save`: 默认为`[]`. 如果你想要训练embedding, lm_head, 或者layer_norm, 你可以设置此参数, 例如: `--lora_modules_to_save EMBEDDING LN lm_head`. 如果传入`'EMBEDDING'`, 则将Embedding层添加到`lora_modules_to_save`. 如果传入`'LN'`, 则将`RMSNorm`和`LayerNorm`添加到`lora_modules_to_save`.
-- `--lora_dtype`: 默认为`'fp32'`, 指定lora模块的dtype类型. 如果是`AUTO`则跟随原始模块的dtype类型. 你可以选择的值: 'fp16', 'bf16', 'fp32', 'AUTO'.
+- `--lora_dtype`: 默认为`'AUTO'`, 指定lora模块的dtype类型. 如果是`AUTO`则跟随原始模块的dtype类型. 你可以选择的值: 'fp16', 'bf16', 'fp32', 'AUTO'.
 - `--use_dora`: 默认为`False`, 是否使用`DoRA`.
 - `--use_rslora`: 默认为`False`, 是否使用`RS-LoRA`.
 - `--neftune_noise_alpha`: `NEFTune`添加的噪声系数, 可以提升模型在指令微调中的性能, 默认为`None`. 通常可以设置为5, 10, 15. 你可以查看[相关论文](https://arxiv.org/abs/2310.05914).

docs/source_en/LLM/Command-line-parameters.md

@@ -50,7 +50,7 @@
 - `--lora_dropout_p`: Default is `0.05`, only takes effect when `sft_type` is 'lora'.
 - `--lora_bias_trainable`: Default is `'none'`, options: 'none', 'all'. Set to `'all'` to make all biases trainable.
 - `--lora_modules_to_save`: Default is `[]`. If you want to train embedding, lm_head, or layer_norm, you can set this parameter, e.g. `--lora_modules_to_save EMBEDDING LN lm_head`. If passed `'EMBEDDING'`, Embedding layer will be added to `lora_modules_to_save`. If passed `'LN'`, `RMSNorm` and `LayerNorm` will be added to `lora_modules_to_save`.
-- `--lora_dtype`: Default is `'fp32'`, specifies dtype for lora modules. If `AUTO`, follow dtype of original module. Options: 'fp16', 'bf16', 'fp32', 'AUTO'.
+- `--lora_dtype`: Default is `'AUTO'`, specifies dtype for lora modules. If `AUTO`, follow dtype of original module. Options: 'fp16', 'bf16', 'fp32', 'AUTO'.
 - `--use_dora`: Default is `False`, whether to use `DoRA`.
 - `--use_rslora`: Default is `False`, whether to use `RS-LoRA`.
 - `--neftune_noise_alpha`: The noise coefficient added by `NEFTune` can improve performance of instruction fine-tuning, default is `None`. Usually can be set to 5, 10, 15. See [related paper](https://arxiv.org/abs/2310.05914).

docs/source_en/LLM/Grok-1-best-practice.md

@@ -57,7 +57,7 @@ torchrun \
     --lora_rank 8 \
     --lora_alpha 32 \
     --lora_dropout_p 0.05 \
-    --lora_dtype bf16 \
+    --lora_dtype AUTO \
     --lora_target_modules DEFAULT \
     --gradient_checkpointing true \
     --batch_size 2 \

examples/pytorch/llm/scripts/dbrx-instruct/lora_mp/sft.sh

@@ -19,7 +19,7 @@ swift sft \
     --lora_alpha 32 \
     --lora_dropout_p 0.05 \
     --lora_target_modules ALL \
-    --lora_dtype bf16 \
+    --lora_dtype AUTO \
     --gradient_checkpointing false \
     --batch_size 1 \
     --weight_decay 0.1 \

examples/pytorch/llm/scripts/grok-1/lora_ddp_ds/sft.sh

@@ -21,7 +21,7 @@ torchrun \
     --lora_rank 8 \
     --lora_alpha 32 \
     --lora_dropout_p 0.05 \
-    --lora_dtype bf16 \
+    --lora_dtype AUTO \
     --lora_target_modules DEFAULT \
     --gradient_checkpointing true \
     --batch_size 2 \

examples/pytorch/llm/scripts/llama2_70b_chat/qlora_fsdp/sft.sh

@@ -23,7 +23,7 @@ accelerate launch --config_file "./scripts/llama2_70b_chat/qlora_fsdp/fsdp_offlo
     --bnb_4bit_quant_storage bfloat16 \
     --lora_rank 8 \
     --lora_alpha 32 \
-    --lora_dtype bf16 \
+    --lora_dtype AUTO \
     --lora_dropout_p 0.05 \
     --lora_target_modules DEFAULT \
     --gradient_checkpointing true \

examples/pytorch/llm/scripts/xverse_moe_a4_2b/lora/sft.sh

@@ -11,7 +11,7 @@ swift sft \
     --num_train_epochs 1 \
     --max_length 1024 \
     --check_dataset_strategy warning \
-    --lora_dtype fp16 \
+    --lora_dtype AUTO \
     --lora_rank 8 \
     --lora_alpha 32 \
     --lora_dropout_p 0.05 \

swift/llm/infer.py

@@ -199,8 +199,7 @@ def prepare_model_template(
     if is_adapter(args.sft_type) and args.ckpt_dir is not None:
         model = Swift.from_pretrained(
             model, args.ckpt_dir, inference_mode=True)
-        if args.sft_type == 'adalora':
-            model = model.to(model.dtype)
+        model = model.to(model.dtype)
 
     if verbose:
         show_layers(model)

swift/llm/utils/argument.py

@@ -366,7 +366,7 @@ class SftArguments(ArgumentsBase):
     lora_bias_trainable: Literal['none', 'all'] = 'none'
     # e.g. ['wte', 'ln_1', 'ln_2', 'ln_f', 'lm_head']
     lora_modules_to_save: List[str] = field(default_factory=list)
-    lora_dtype: Literal['fp16', 'bf16', 'fp32', 'AUTO'] = 'fp32'
+    lora_dtype: Literal['fp16', 'bf16', 'fp32', 'AUTO'] = 'AUTO'
     lora_lr_ratio: float = None
     use_rslora: bool = False
     use_dora: bool = False