TST Add test suite for GraLoRA.

Author: HaohanTsao

src/peft/tuners/gralora/layer.py

@@ -224,8 +224,8 @@ def merge(self, safe_merge: bool = False, adapter_names: Optional[list[str]] = N
                 before merging the weights. This is useful if you want to check if the merge operation will produce
                 NaNs. Defaults to `False`.
             adapter_names (`list[str]`, *optional*):
-                The list of adapter names that should be merged. If None, all active adapters will be merged.
-                Defaults to `None`.
+                The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults
+                to `None`.
         """
         from peft.tuners.tuners_utils import check_adapters_to_merge
 
@@ -274,9 +274,8 @@ def get_delta_weight(self, adapter) -> torch.Tensor:
         """
         Compute the delta weight for GraLoRA adapter.
 
-        GraLoRA applies block-wise low-rank adaptation with information exchange.
-        This method computes the equivalent weight matrix that would be added to
-        the base weight during merge.
+        GraLoRA applies block-wise low-rank adaptation with information exchange. This method computes the equivalent
+        weight matrix that would be added to the base weight during merge.
 
         Args:
             adapter (str): The name of the adapter

src/peft/tuners/gralora/model.py

@@ -323,8 +323,8 @@ def merge_and_unload(
         self, progressbar: bool = False, safe_merge: bool = False, adapter_names: Optional[list[str]] = None
     ):
         r"""
-        This method merges the Gralora layers into the base model. This is needed if someone wants to use the base model
-        as a standalone model.
+        This method merges the Gralora layers into the base model. This is needed if someone wants to use the base
+        model as a standalone model.
 
         Args:
             progressbar (`bool`):

tests/test_config.py

@@ -27,6 +27,7 @@
     BoneConfig,
     C3AConfig,
     FourierFTConfig,
+    GraloraConfig,
     HRAConfig,
     IA3Config,
     LNTuningConfig,
@@ -64,6 +65,7 @@
     (BoneConfig, {}),
     (C3AConfig, {}),
     (FourierFTConfig, {}),
+    (GraloraConfig, {}),
     (HRAConfig, {}),
     (IA3Config, {}),
     (LNTuningConfig, {}),

tests/test_decoder_models.py

@@ -34,6 +34,7 @@
     CPTConfig,
     DeloraConfig,
     FourierFTConfig,
+    GraloraConfig,
     HRAConfig,
     IA3Config,
     LoraConfig,
@@ -137,6 +138,30 @@
             "target_modules": None,
         },
     ),
+    (
+        GraloraConfig,
+        {
+            "task_type": "CAUSAL_LM",
+            "r": 8,
+            "gralora_alpha": 16,
+            "target_modules": None,
+            "gralora_dropout": 0.05,
+            "gralora_k": 2,
+            "hybrid_r": 0,
+        },
+    ),
+    (
+        GraloraConfig,
+        {
+            "task_type": "CAUSAL_LM",
+            "r": 16,
+            "gralora_alpha": 32,
+            "target_modules": None,
+            "gralora_dropout": 0.05,
+            "gralora_k": 4,
+            "hybrid_r": 4,
+        },
+    ),
     (
         HRAConfig,
         {