removed lora parse args to simplify; tuned sweep config

Author: codinglabsong

src/bart_reddit_lora/model.py

@@ -13,15 +13,14 @@ def build_base_model(
 def build_peft_model(
     base_model: BartForConditionalGeneration,
     r: int = 8,
-    lora_alpha: int = 16,
     lora_dropout: float = 0.1,
     bias: str = "none",
     target_modules: list[str] = ("q_proj", "k_proj", "v_proj", "o_proj", "fc1", "fc2"),
     modules_to_save: list[str] = ("lm_head",),
 ) -> PeftModel:
     config = LoraConfig(
         r=r,
-        lora_alpha=lora_alpha,
+        lora_alpha=r * 2,
         target_modules=list(target_modules),
         lora_dropout=lora_dropout,
         bias=bias,

src/bart_reddit_lora/train.py

@@ -75,7 +75,6 @@ class CustomTrainingArgs(Seq2SeqTrainingArguments):
 
     # additional custom args
     peft_rank: int = field(default=32, metadata={"help": "LoRA adapter rank (r)."})
-    lora_alpha: int = 64
     hf_hub_repo_id: str | None = None
     run_test: bool = field(
         default=False,
@@ -171,11 +170,9 @@ def to_qa(ex):
     logger.info(
         f"Base model trainable params:\n{print_trainable_parameters(base_model)}"
     )
-    lora_model = build_peft_model(
-        base_model, training_args.peft_rank, training_args.lora_alpha
-    )
+    lora_model = build_peft_model(base_model, training_args.peft_rank)
     logger.info(
-        f"LoRA model (peft_rank={training_args.peft_rank}, lora_alpha={training_args.lora_alpha}) trainable params:\n{print_trainable_parameters(lora_model)}"
+        f"LoRA model (peft_rank={training_args.peft_rank}) trainable params:\n{print_trainable_parameters(lora_model)}"
     )
 
     # ---------- Train ----------

sweep.yaml

@@ -5,24 +5,24 @@ project: bart-base-reddit-lora
 entity:  codinglabsong-keio-jp
 
 method: bayes # {grid | random | bayes}
-run_cap: 10 # sweep run limit
+run_cap: 15 # sweep run limit
 
 metric: # what to optimise
   name: eval/loss # must match the key in evaluation.compute_metrics returns
-  goal: maximize
+  goal: minimize
 
 parameters:
   learning_rate:
-    min: 0.00001
-    max: 0.001
+    min: 0.00001 # 1e-5
+    max: 0.001 # 1e-3
     distribution: log_uniform_values
   num_train_epochs:
-    values: [2]
+    values: 4
   peft_rank:
-    values: [32]
+    values: [32, 64]
   train_sample:
-    values: [True]
+    values: True
 
-early_terminate:
-  type: hyperband
-  min_iter: 1
+# early_terminate:
+#   type: hyperband
+#   min_iter: 3