Fine-tuning only Whisper decoder

[bardenthenry]

I found that if I fine-tune Whisper in the PEFT LoRA, its distinctive style will become completely different from the original. Is it possible to use Lora to fine-tune only the decoder of Whisper?

[bardenthenry]

I want to use layers_to_transform in LoraConfig to specify that add lora layer only in decoder

decoder_id_ls = []
for id, (name, param) in enumerate(model.named_parameters()):
    if 'model.decoder' in name:
        decoder_id_ls.append(id)

target_modules = ["q_proj", "v_proj"]

config = LoraConfig(r=32, lora_alpha=64, target_modules=target_modules, lora_dropout=0.05, bias="none", layers_to_transform=decoder_id_ls)
model = get_peft_model(model, config)
model.print_trainable_parameters()

But I got error

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
/usr/src/app/fine_tune_decoder.ipynb Cell 5 line 3
      [1](vscode-notebook-cell://attached-container%2B7b22636f6e7461696e65724e616d65223a222f776869737065725f66696e6574756e655f7175616e74697a652d636f6e736f6c652d72756e2d643130386165336162393165222c22637764223a225c5c5c5c77736c2e6c6f63616c686f73745c5c5562756e74752d32322e30345c5c686f6d655c5c6772617469615c5c574849535045525f46494e4554554e455f5155414e54495a45227d/usr/src/app/fine_tune_decoder.ipynb#X12sdnNjb2RlLXJlbW90ZQ%3D%3D?line=0) print('Load LoRA Model')
      [2](vscode-notebook-cell://attached-container%2B7b22636f6e7461696e65724e616d65223a222f776869737065725f66696e6574756e655f7175616e74697a652d636f6e736f6c652d72756e2d643130386165336162393165222c22637764223a225c5c5c5c77736c2e6c6f63616c686f73745c5c5562756e74752d32322e30345c5c686f6d655c5c6772617469615c5c574849535045525f46494e4554554e455f5155414e54495a45227d/usr/src/app/fine_tune_decoder.ipynb#X12sdnNjb2RlLXJlbW90ZQ%3D%3D?line=1) config = LoraConfig(r=32, lora_alpha=64, target_modules=target_modules, lora_dropout=0.05, bias="none", layers_to_transform=decoder_id_ls)
----> [3](vscode-notebook-cell://attached-container%2B7b22636f6e7461696e65724e616d65223a222f776869737065725f66696e6574756e655f7175616e74697a652d636f6e736f6c652d72756e2d643130386165336162393165222c22637764223a225c5c5c5c77736c2e6c6f63616c686f73745c5c5562756e74752d32322e30345c5c686f6d655c5c6772617469615c5c574849535045525f46494e4554554e455f5155414e54495a45227d/usr/src/app/fine_tune_decoder.ipynb#X12sdnNjb2RlLXJlbW90ZQ%3D%3D?line=2) model = get_peft_model(model, config)
      [4](vscode-notebook-cell://attached-container%2B7b22636f6e7461696e65724e616d65223a222f776869737065725f66696e6574756e655f7175616e74697a652d636f6e736f6c652d72756e2d643130386165336162393165222c22637764223a225c5c5c5c77736c2e6c6f63616c686f73745c5c5562756e74752d32322e30345c5c686f6d655c5c6772617469615c5c574849535045525f46494e4554554e455f5155414e54495a45227d/usr/src/app/fine_tune_decoder.ipynb#X12sdnNjb2RlLXJlbW90ZQ%3D%3D?line=3) model.print_trainable_parameters()

File /usr/src/app/peft/mapping.py:103, in get_peft_model(model, peft_config, adapter_name)
    100 peft_config.base_model_name_or_path = model.__dict__.get("name_or_path", None)
    102 if peft_config.task_type not in MODEL_TYPE_TO_PEFT_MODEL_MAPPING.keys() and not peft_config.is_prompt_learning:
--> 103     return PeftModel(model, peft_config, adapter_name=adapter_name)
    104 if peft_config.is_prompt_learning:
    105     peft_config = _prepare_prompt_learning_config(peft_config, model_config)

File /usr/src/app/peft/peft_model.py:111, in PeftModel.__init__(self, model, peft_config, adapter_name)
    109 if not peft_config.is_prompt_learning:
    110     self.peft_config[adapter_name] = peft_config
--> 111     self.base_model = PEFT_TYPE_TO_MODEL_MAPPING[peft_config.peft_type](
    112         self.base_model, self.peft_config, adapter_name
    113     )
    114     self.set_additional_trainable_modules(peft_config, adapter_name)
    115 else:

File /usr/src/app/peft/tuners/lora.py:274, in LoraModel.__init__(self, model, config, adapter_name)
    273 def __init__(self, model, config, adapter_name) -> None:
--> 274     super().__init__(model, config, adapter_name)

File /usr/src/app/peft/tuners/tuners_utils.py:88, in BaseTuner.__init__(self, model, peft_config, adapter_name)
     85 if not hasattr(self, "config"):
     86     self.config = {"model_type": "custom"}
---> 88 self.inject_adapter(self.model, adapter_name)
     90 # Copy the peft_config in the injected model.
     91 self.model.peft_config = self.peft_config

File /usr/src/app/peft/tuners/tuners_utils.py:225, in BaseTuner.inject_adapter(self, model, adapter_name)
    222     self._create_and_replace(peft_config, adapter_name, target, target_name, parent, **optionnal_kwargs)
    224 if not is_target_modules_in_base_model:
--> 225     raise ValueError(
    226         f"Target modules {peft_config.target_modules} not found in the base model. "
    227         f"Please check the target modules and try again."
    228     )
    230 self._mark_only_adapters_as_trainable()
    232 if self.peft_config[adapter_name].inference_mode:

ValueError: Target modules ['q_proj', 'v_proj'] not found in the base model. Please check the target modules and try again.

[AlienKevin]

I think people might be misunderstanding the meaning of layers_to_transform. I still don't know what it means precisely but I found that you can directly feed LoraConfig the target_modules from the decoder:

from peft import LoraConfig, PeftModel, LoraModel, LoraConfig, get_peft_model

target_modules = []
for id, (name, param) in enumerate(model.named_modules()):
    if 'model.decoder' in name and ('q_proj' in name or 'v_proj' in name):
        target_modules.append(name)

config = LoraConfig(r=32, lora_alpha=64, target_modules=target_modules, lora_dropout=0.05, bias="none")

model = get_peft_model(model, config)
model.print_trainable_parameters()

Note the 'model.decoder' in name part is crucial for keeping only the decoder layers. This is the trainable parameter after filtering:

trainable params: 786,432 || all params: 73,380,352 || trainable%: 1.0717

Without the filtering, we got about 1.5x more trainable parameters:

trainable params: 1,179,648 || all params: 73,773,568 || trainable%: 1.5990

[DuyTa506]

just try to print the model architechture and map the name with the target module, may be in whisper repo, the name of layer is different from name on huggingface