feat: Add Gralora configuration and basic implementation

Author: yeonjoon-jung01

src/peft/__init__.py

@@ -64,6 +64,8 @@
     EvaConfig,
     FourierFTConfig,
     FourierFTModel,
+    GraloraConfig,
+    GraloraModel,
     HRAConfig,
     HRAModel,
     IA3Config,
@@ -163,6 +165,8 @@
     "EvaConfig",
     "FourierFTConfig",
     "FourierFTModel",
+    "GraloraConfig",
+    "GraloraModel",
     "HRAConfig",
     "HRAModel",
     "IA3Config",

src/peft/tuners/__init__.py

@@ -20,6 +20,7 @@
 from .cpt import CPTConfig, CPTEmbedding
 from .delora import DeloraConfig, DeloraModel
 from .fourierft import FourierFTConfig, FourierFTModel
+from .gralora import GraloraConfig, GraloraModel
 from .hra import HRAConfig, HRAModel
 from .ia3 import IA3Config, IA3Model
 from .ln_tuning import LNTuningConfig, LNTuningModel
@@ -74,6 +75,8 @@
     "EvaConfig",
     "FourierFTConfig",
     "FourierFTModel",
+    "GraloraConfig",
+    "GraloraModel",
     "HRAConfig",
     "HRAModel",
     "IA3Config",

src/peft/tuners/gralora/__init__.py

@@ -0,0 +1,20 @@
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .config import GraloraConfig
+from .layer import GraloraLayer
+from .model import GraloraModel
+
+
+__all__ = ["GraloraConfig", "GraloraLayer", "GraloraModel"]

src/peft/tuners/gralora/config.py

@@ -0,0 +1,82 @@
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from dataclasses import dataclass, field
+from typing import Optional, Union
+
+from peft.config import PeftConfig
+from peft.utils import PeftType
+
+
+@dataclass
+class GraloraConfig(PeftConfig):
+    r: int = field(default=8, metadata={"help": "gralora attention dimension"})
+    hybrid_r: int = field(
+        default=0, metadata={"help": "hybrid_r is the rank allocated to vanilla LoRA method when using Hybrid GraLoRA"}
+    )
+    target_modules: Optional[Union[list[str], str]] = field(
+        default=None,
+        metadata={
+            "help": (
+                "List of module names or regex expression of the module names to replace with gralora."
+                "For example, ['q', 'v'] or '.*decoder.*(SelfAttention|EncDecAttention).*(q|v)$'. "
+                "Only linear layers are supported."
+            )
+        },
+    )
+    gralora_alpha: int = field(default=8, metadata={"help": "gralora alpha"})
+    gralora_dropout: float = field(default=0.0, metadata={"help": "gralora dropout"})
+    gralora_k: int = field(default=2, metadata={"help": "gralora k"})
+    fan_in_fan_out: bool = field(
+        default=False,
+        metadata={"help": "Set this to True if the layer to replace stores weight like (fan_in, fan_out)"},
+    )
+    bias: str = field(
+        default="none", metadata={"help": "Bias type for gralora. Can be 'none', 'all' or 'gralora_only'"}
+    )
+    modules_to_save: Optional[list[str]] = field(
+        default=None,
+        metadata={
+            "help": (
+                "List of modules apart from gralora layers to be set as trainable and saved in the final checkpoint. For"
+                " example, in Sequence Classification or Token Classification tasks, the final layer"
+                " `classifier/score` are randomly initialized and as such need to be trainable and saved."
+            )
+        },
+    )
+    layers_to_transform: Optional[Union[list[int], int]] = field(
+        default=None,
+        metadata={
+            "help": (
+                "The layer indexes to transform, is this argument is specified, PEFT will transform only the layers"
+                " indexes that are specified inside this list. If a single integer is passed, PEFT will transform only"
+                " the layer at this index."
+            )
+        },
+    )
+    layers_pattern: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": (
+                "The layer pattern name, used only if `layers_to_transform` is different to None and if the layer"
+                " pattern is not in the common layers pattern."
+            )
+        },
+    )
+
+    def __post_init__(self):
+        self.peft_type = PeftType.GRALORA
+        self.target_modules = (
+            set(self.target_modules) if isinstance(self.target_modules, list) else self.target_modules
+        )