feat: alora migration documentation and nit fixes

Signed-off-by: yashasvi <yashasvi@ibm.com>

Author: YashasviChaurasia

docs/tuning-techniques.md

@@ -218,13 +218,13 @@ Activated LoRA (aLoRA) is a new low rank adapter architecture that allows for re
 
 **Note** Often (not always) aLoRA requires higher rank (r) than LoRA. r=32 can be a good starting point for challenging tasks.
 
-**Installation** The Activated LoRA requirements are an optional install in pyproject.toml (activated-lora)
+**Installation** Native aLoRA support requires PEFT release with `alora_invocation_tokens` support at [PR#2609](https://github.com/huggingface/peft/pull/2609)
 
 Set `peft_method` to `"alora"`. 
 
 You *must* pass in an invocation_string argument. This invocation_string *must be present* in both training data inputs and the input at test time. A good solution is to set invocation_string = response_template, this will ensure that every training input will have the invocation_string present. We keep these separate arguments for flexibility. It is most robust if the invocation_string begins and ends with special tokens.
 
-You can additionally pass any arguments from [aLoraConfig](https://github.com/IBM/activated-lora/blob/fms-hf-tuning/alora/config.py#L35), see the LoRA section for examples.
+You can additionally pass any arguments from `LoraConfig`, see the LoRA section for examples.
 
 Example command to run, here using the ([Granite Instruct response template](https://huggingface.co/ibm-granite/granite-3.0-8b-instruct/blob/main/tokenizer_config.json#L188)) as the invocation sequence:
 
@@ -306,7 +306,7 @@ class SaveBestModelCallback(TrainerCallback):
 Example inference:
 ```py
 # Load the model
-loaded_model = TunedCausalLM.load(ALORA_MODEL, BASE_MODEL_NAME, use_alora=True)
+loaded_model = TunedCausalLM.load(ALORA_MODEL, BASE_MODEL_NAME)
 
 # Retrieve the invocation string from the model config
 invocation_string = loaded_model.peft_model.peft_config[

tests/test_sft_trainer.py

@@ -97,7 +97,7 @@
     load_and_validate_data_config,
 )
 from tuning.data.data_handlers import DataHandler, DataHandlerType
-from tuning.utils.import_utils import is_alora_available, is_fms_accelerate_available
+from tuning.utils.import_utils import is_fms_accelerate_available
 
 MODEL_NAME = MAYKEYE_TINY_LLAMA_CACHED
 
@@ -153,7 +153,6 @@
 
 if hasattr(HFLoraConfig, "alora_invocation_tokens"):
     PEFT_ALORA_ARGS = peft_config.LoraConfig(r=8, lora_alpha=32, lora_dropout=0.05)
-    PEFT_ALORA_ARGS.alora_invocation_tokens = [42]
 else:
     PEFT_ALORA_ARGS = None
 
@@ -745,10 +744,6 @@ def test_run_causallm_lora_and_inference(request, target_modules, expected):
         assert "Simply put, the theory of relativity states that" in output_inference
 
 
-@pytest.mark.skipif(
-    not is_alora_available(),
-    reason="Only runs if alora is installed",
-)
 @pytest.mark.parametrize(
     "target_modules,expected",
     target_modules_val_map,

tuning/sft_trainer.py

@@ -676,8 +676,6 @@ def parse_arguments(parser, json_config=None):
         invocation_string = additional.invocation_string
 
     if peft_method == peft_config.PEFT_METHOD.ALORA.value:
-        if invocation_string is None:
-            raise ValueError("invocation_string is required for aLoRA usage")
         tune_config = lora_config
         setattr(tune_config, "alora_invocation_string", invocation_string)
     elif peft_method == peft_config.PEFT_METHOD.LORA.value:

tuning/utils/config_utils.py

@@ -61,7 +61,10 @@ def create_tuning_config(peft_method, **kwargs):
         "pt",
         "None",
     ], f"peft config {peft_method} not defined in peft.py"
-    if peft_method in ("alora", "lora"):
+    if peft_method in (
+        peft_config.PEFT_METHOD.ALORA.value,
+        peft_config.PEFT_METHOD.LORA.value,
+    ):
         tune_config = peft_config.LoraConfig()
         update_config(tune_config, **kwargs)
     elif peft_method == "pt":

tuning/utils/import_utils.py

@@ -32,14 +32,3 @@ def is_fms_accelerate_available(
         if not _is_package_available(n):
             return False
     return True
-
-
-def is_alora_available() -> bool:
-    try:
-        # Third Party
-        from peft import LoraConfig  # pylint: disable=import-outside-toplevel
-
-        # Check if LoraConfig has the new Activated LoRA field
-        return hasattr(LoraConfig, "alora_invocation_tokens")
-    except ImportError:
-        return False