FEAT Add RandLoRA to PEFT (#2464)

Implements "RandLoRA: Full-rank parameter-efficient fine-tuning of large
models", https://arxiv.org/abs/2502.00987.

Author: PaulAlbert31

src/peft/__init__.py

@@ -87,6 +87,8 @@
     PromptEncoderReparameterizationType,
     PromptTuningConfig,
     PromptTuningInit,
+    RandLoraConfig,
+    RandLoraModel,
     TrainableTokensConfig,
     TrainableTokensModel,
     VBLoRAConfig,
@@ -178,6 +180,8 @@
     "PromptLearningConfig",
     "PromptTuningConfig",
     "PromptTuningInit",
+    "RandLoraConfig",
+    "RandLoraModel",
     "TaskType",
     "TrainableTokensConfig",
     "TrainableTokensModel",

src/peft/tuners/__init__.py

@@ -39,6 +39,7 @@
 from .poly import PolyConfig, PolyModel
 from .prefix_tuning import PrefixEncoder, PrefixTuningConfig
 from .prompt_tuning import PromptEmbedding, PromptTuningConfig, PromptTuningInit
+from .randlora import RandLoraConfig, RandLoraModel
 from .trainable_tokens import TrainableTokensConfig, TrainableTokensModel
 from .vblora import VBLoRAConfig, VBLoRAModel
 from .vera import VeraConfig, VeraModel
@@ -89,6 +90,8 @@
     "PromptEncoderReparameterizationType",
     "PromptTuningConfig",
     "PromptTuningInit",
+    "RandLoraConfig",
+    "RandLoraModel",
     "TrainableTokensConfig",
     "TrainableTokensModel",
     "VBLoRAConfig",

src/peft/tuners/randlora/__init__.py

@@ -0,0 +1,40 @@
+# Copyright 2025-present the HuggingFace Inc. team.
+
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from peft.import_utils import is_bnb_4bit_available, is_bnb_available
+from peft.utils import register_peft_method
+
+from .config import RandLoraConfig
+from .layer import Linear, RandLoraLayer
+from .model import RandLoraModel
+
+
+__all__ = ["Linear", "RandLoraConfig", "RandLoraLayer", "RandLoraModel"]
+
+register_peft_method(name="randlora", config_cls=RandLoraConfig, model_cls=RandLoraModel, prefix="randlora_")
+
+
+def __getattr__(name):
+    if (name == "Linear8bitLt") and is_bnb_available():
+        from .bnb import Linear8bitLt
+
+        return Linear8bitLt
+
+    if (name == "Linear4bit") and is_bnb_4bit_available():
+        from .bnb import Linear4bit
+
+        return Linear4bit
+
+    raise AttributeError(f"module {__name__} has no attribute {name}")