From 7a7fb877241b0f0637ac36d46d36526476bf0d03 Mon Sep 17 00:00:00 2001 From: "Liu, Kaixuan" Date: Thu, 23 Oct 2025 07:15:20 +0000 Subject: [PATCH 1/4] use `max_length` to replace `max_seq_length`; correct README for delora_finetuning Signed-off-by: Liu, Kaixuan --- examples/alora_finetuning/README.md | 2 +- examples/bone_finetuning/README.md | 2 +- examples/corda_finetuning/README.md | 2 +- examples/delora_finetuning/README.md | 6 ++-- examples/dora_finetuning/README.md | 2 +- examples/lorafa_finetune/README.md | 2 +- examples/miss_finetuning/README.md | 2 +- examples/olora_finetuning/README.md | 2 +- examples/pissa_finetuning/README.md | 2 +- examples/randlora_finetuning/README.md | 2 +- examples/road_finetuning/README.md | 2 +- examples/sft/train.py | 2 +- examples/sft/utils.py | 4 +-- examples/shira_finetuning/README.md | 2 +- .../peft_lora_token_cls.ipynb | 30 +++++++++---------- examples/waveft_finetuning/README.md | 2 +- method_comparison/MetaMathQA/README.md | 2 +- .../MetaMathQA/default_training_params.json | 2 +- .../results/adalora--llama-3.2-3B-rank32.json | 2 +- ...daptionprompt--llama-3.2-3B-lr_0.0005.json | 2 +- .../results/boft--llama-3.2-3B-default.json | 2 +- .../results/bone--llama-3.2-3B-bat.json | 2 +- .../results/bone--llama-3.2-3B-default.json | 2 +- .../results/c3a--llama-3.2-3B-default.json | 2 +- .../fourierft--llama-3.2-3B-default.json | 2 +- ...rierft--llama-3.2-3B-n_frequency-5000.json | 2 +- ...l-finetuning--llama-3.2-3B-lr_0.00001.json | 2 +- .../results/ia3--llama-3.2-3B-default.json | 2 +- .../results/ia3--llama-3.2-3B-lr_0.001.json | 2 +- .../ln_tuning--llama-3.2-3B-default.json | 2 +- .../results/loha--llama-3.2-3B-rank32.json | 2 +- .../results/lokr--llama-3.2-3B-rank32.json | 2 +- .../lora--llama-3.2-3B-rank32-dora.json | 2 +- .../lora--llama-3.2-3B-rank32-lorafa.json | 2 +- .../results/lora--llama-3.2-3B-rank32.json | 2 +- .../lora--llama-3.2-3B-rank64-rslora.json | 2 +- .../results/lora--llama-3.2-3B-rank64.json | 2 +- .../results/miss--llama-3.2-3B-bat.json | 2 +- .../results/miss--llama-3.2-3B-default.json | 2 +- .../results/miss--llama-3.2-3B-mini.json | 2 +- .../results/oft--llama-3.2-3B-rank32.json | 2 +- .../prefixtuning--llama-3.2-3B-lr_0.001.json | 2 +- .../prompt_tuning--llama-3.2-3B-default.json | 2 +- .../prompt_tuning--llama-3.2-3B-lr_0.001.json | 2 +- .../ptuning--llama-3.2-3B-default.json | 2 +- .../randlora--llama-3.2-3B-default.json | 2 +- ...llama-3.2-3B-lr_0.0003-random_seed_42.json | 2 +- ...rainable_tokens--llama-3.2-3B-sos+eos.json | 2 +- .../results/vblora--llama-3.2-3B-default.json | 2 +- .../results/vera--llama-3.2-3B-default.json | 2 +- method_comparison/MetaMathQA/run.py | 2 +- method_comparison/MetaMathQA/utils.py | 12 ++++---- scripts/train_memory.py | 12 ++++---- 53 files changed, 80 insertions(+), 80 deletions(-) diff --git a/examples/alora_finetuning/README.md b/examples/alora_finetuning/README.md index e6b8da0bcd..da0f1133a7 100644 --- a/examples/alora_finetuning/README.md +++ b/examples/alora_finetuning/README.md @@ -32,7 +32,7 @@ trainer = Trainer( model=peft_model, train_dataset=dataset, dataset_text_field="text", - max_seq_length=2048, + max_length=2048, tokenizer=tokenizer, data_collator=data_collator, ) diff --git a/examples/bone_finetuning/README.md b/examples/bone_finetuning/README.md index 42b604f92d..bfc25f780b 100644 --- a/examples/bone_finetuning/README.md +++ b/examples/bone_finetuning/README.md @@ -28,7 +28,7 @@ peft_model.print_trainable_parameters() dataset = load_dataset("imdb", split="train[:1%]") -training_args = SFTConfig(dataset_text_field="text", max_seq_length=128) +training_args = SFTConfig(dataset_text_field="text", max_length=128) trainer = SFTTrainer( model=peft_model, args=training_args, diff --git a/examples/corda_finetuning/README.md b/examples/corda_finetuning/README.md index 9d72fc2f11..d86bcdcd7c 100644 --- a/examples/corda_finetuning/README.md +++ b/examples/corda_finetuning/README.md @@ -109,7 +109,7 @@ preprocess_corda(model, lora_config, run_model=run_model) peft_model = get_peft_model(model, lora_config) peft_model.print_trainable_parameters() -training_args = SFTConfig(dataset_text_field="text", max_seq_length=128) +training_args = SFTConfig(dataset_text_field="text", max_length=128) trainer = SFTTrainer( model=peft_model, args=training_args, diff --git a/examples/delora_finetuning/README.md b/examples/delora_finetuning/README.md index 20fc858bec..20c18ae059 100644 --- a/examples/delora_finetuning/README.md +++ b/examples/delora_finetuning/README.md @@ -26,7 +26,7 @@ peft_model.print_trainable_parameters() dataset = load_dataset("imdb", split="train[:1%]") -training_args = SFTConfig(dataset_text_field="text", max_seq_length=128) +training_args = SFTConfig(dataset_text_field="text", max_length=128) trainer = SFTTrainer( model=peft_model, args=training_args, @@ -52,7 +52,7 @@ peft_model = PeftModel.from_pretrained(model, "delora-llama-3-8b") ## Advanced Usage In this script the default DeLoRA layers are the query and value layers of the Llama model. Adding adapters on more layers will increase memory usage. If you wish to choose a different set of layers for DeLoRA to be applied on, you can simply define it using: ```bash -python examples/delora_finetuning/delora_finetuning.py --base_model meta-llama/Meta-Llama-3-8B --delora_target_modules "q_proj,k_proj,v_proj,o_proj" +python examples/delora_finetuning/delora_finetuning.py --base_model meta-llama/Meta-Llama-3-8B --target_modules "q_proj,k_proj,v_proj,o_proj" ``` Using different lambdas for different layers is also possible by setting `lambda_pattern`. @@ -74,7 +74,7 @@ python delora_finetuning.py \ --rank 32 \ --delora_lambda 15 \ --module_dropout 0.1 \ - --delora_target_modules "q_proj,v_proj" \ + --target_modules "q_proj,v_proj" \ --hub_model_id "YOUR_HF_REPO" \ --push_to_hub ``` diff --git a/examples/dora_finetuning/README.md b/examples/dora_finetuning/README.md index 91ffb2a529..20250476e4 100644 --- a/examples/dora_finetuning/README.md +++ b/examples/dora_finetuning/README.md @@ -24,7 +24,7 @@ trainer = transformers.Trainer( model=peft_model, train_dataset=dataset, dataset_text_field="text", - max_seq_length=2048, + max_length=2048, tokenizer=tokenizer, ) trainer.train() diff --git a/examples/lorafa_finetune/README.md b/examples/lorafa_finetune/README.md index 432c93ad83..1c1314dbd6 100644 --- a/examples/lorafa_finetune/README.md +++ b/examples/lorafa_finetune/README.md @@ -40,7 +40,7 @@ trainer = transformers.Trainer( model=peft_model, train_dataset=dataset, dataset_text_field="text", - max_seq_length=2048, + max_length=2048, processing_class=tokenizer, optimizers=(optimizer, None), ) diff --git a/examples/miss_finetuning/README.md b/examples/miss_finetuning/README.md index bcc1bb33d8..1e11064d8d 100644 --- a/examples/miss_finetuning/README.md +++ b/examples/miss_finetuning/README.md @@ -36,7 +36,7 @@ peft_model.print_trainable_parameters() dataset = load_dataset("imdb", split="train[:1%]") -training_args = SFTConfig(dataset_text_field="text", max_seq_length=128) +training_args = SFTConfig(dataset_text_field="text", max_length=128) trainer = SFTTrainer( model=peft_model, args=training_args, diff --git a/examples/olora_finetuning/README.md b/examples/olora_finetuning/README.md index bb548b77b2..3b9bbf68ee 100644 --- a/examples/olora_finetuning/README.md +++ b/examples/olora_finetuning/README.md @@ -18,7 +18,7 @@ lora_config = LoraConfig( init_lora_weights="olora" ) peft_model = get_peft_model(model, lora_config) -training_args = SFTConfig(dataset_text_field="text", max_seq_length=128) +training_args = SFTConfig(dataset_text_field="text", max_length=128) trainer = SFTTrainer( model=peft_model, train_dataset=dataset, diff --git a/examples/pissa_finetuning/README.md b/examples/pissa_finetuning/README.md index 4c0734bdcf..6ab1515679 100644 --- a/examples/pissa_finetuning/README.md +++ b/examples/pissa_finetuning/README.md @@ -23,7 +23,7 @@ peft_model.print_trainable_parameters() dataset = load_dataset("imdb", split="train[:1%]") -training_args = SFTConfig(dataset_text_field="text", max_seq_length=128) +training_args = SFTConfig(dataset_text_field="text", max_length=128) trainer = SFTTrainer( model=peft_model, args=training_args, diff --git a/examples/randlora_finetuning/README.md b/examples/randlora_finetuning/README.md index fa9d2d61de..f8eb7d77d0 100644 --- a/examples/randlora_finetuning/README.md +++ b/examples/randlora_finetuning/README.md @@ -20,7 +20,7 @@ trainer = transformers.Trainer( model=peft_model, train_dataset=dataset, dataset_text_field="text", - max_seq_length=2048, + max_length=2048, processing_class=tokenizer, ) trainer.train() diff --git a/examples/road_finetuning/README.md b/examples/road_finetuning/README.md index b9ce14017c..00a043936f 100644 --- a/examples/road_finetuning/README.md +++ b/examples/road_finetuning/README.md @@ -26,7 +26,7 @@ trainer = transformers.Trainer( model=peft_model, train_dataset=dataset, dataset_text_field="text", - max_seq_length=2048, + max_length=2048, tokenizer=tokenizer, ) trainer.train() diff --git a/examples/sft/train.py b/examples/sft/train.py index 5a34f69357..77efac3609 100644 --- a/examples/sft/train.py +++ b/examples/sft/train.py @@ -18,7 +18,7 @@ class ModelArguments: model_name_or_path: str = field( metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"} ) - max_seq_length: Optional[int] = field( + max_length: Optional[int] = field( default=512, metadata={"help": "The maximum total input sequence length after tokenization."}, ) diff --git a/examples/sft/utils.py b/examples/sft/utils.py index 0e24796de4..96d49e7a2f 100644 --- a/examples/sft/utils.py +++ b/examples/sft/utils.py @@ -124,7 +124,7 @@ def create_and_prepare_model(args, data_args, training_args): # Load model model, _ = FastLanguageModel.from_pretrained( model_name=args.model_name_or_path, - max_seq_length=training_args.max_seq_length, + max_length=training_args.max_length, dtype=None, load_in_4bit=args.use_4bit_quantization, ) @@ -211,7 +211,7 @@ def create_and_prepare_model(args, data_args, training_args): else args.lora_target_modules, use_gradient_checkpointing=training_args.gradient_checkpointing, random_state=training_args.seed, - max_seq_length=training_args.max_seq_length, + max_length=training_args.max_length, ) return model, peft_config, tokenizer diff --git a/examples/shira_finetuning/README.md b/examples/shira_finetuning/README.md index 45b1e99be6..76d062e2c7 100644 --- a/examples/shira_finetuning/README.md +++ b/examples/shira_finetuning/README.md @@ -18,7 +18,7 @@ shira_config = ShiraConfig( r=32, ) peft_model = get_peft_model(model, shira_config) -training_args = SFTConfig(dataset_text_field="text", max_seq_length=128) +training_args = SFTConfig(dataset_text_field="text", max_length=128) trainer = SFTTrainer( model=peft_model, train_dataset=dataset, diff --git a/examples/token_classification/peft_lora_token_cls.ipynb b/examples/token_classification/peft_lora_token_cls.ipynb index 0c6c686ae5..9e5ba813cd 100644 --- a/examples/token_classification/peft_lora_token_cls.ipynb +++ b/examples/token_classification/peft_lora_token_cls.ipynb @@ -401,7 +401,7 @@ " \"cached_{}_{}_{}\".format(\n", " mode,\n", " list(filter(None, args.model_name_or_path.split(\"/\"))).pop(),\n", - " str(args.max_seq_length),\n", + " str(args.max_length),\n", " ),\n", " )\n", " if os.path.exists(cached_features_file) and not args.overwrite_cache:\n", @@ -413,7 +413,7 @@ " features = convert_examples_to_features(\n", " examples,\n", " labels,\n", - " args.max_seq_length,\n", + " args.max_length,\n", " tokenizer,\n", " cls_token_at_end=bool(args.model_type in [\"xlnet\"]),\n", " # xlnet has a cls token at the end\n", @@ -579,7 +579,7 @@ "def convert_examples_to_features(\n", " examples,\n", " label_list,\n", - " max_seq_length,\n", + " max_length,\n", " tokenizer,\n", " cls_token_at_end=False,\n", " cls_token=\"[CLS]\",\n", @@ -627,11 +627,11 @@ "\n", " # Account for [CLS] and [SEP] with \"- 2\" and with \"- 3\" for RoBERTa.\n", " special_tokens_count = 3 if sep_token_extra else 2\n", - " if len(tokens) > max_seq_length - special_tokens_count:\n", - " tokens = tokens[: (max_seq_length - special_tokens_count)]\n", - " token_boxes = token_boxes[: (max_seq_length - special_tokens_count)]\n", - " actual_bboxes = actual_bboxes[: (max_seq_length - special_tokens_count)]\n", - " label_ids = label_ids[: (max_seq_length - special_tokens_count)]\n", + " if len(tokens) > max_length - special_tokens_count:\n", + " tokens = tokens[: (max_length - special_tokens_count)]\n", + " token_boxes = token_boxes[: (max_length - special_tokens_count)]\n", + " actual_bboxes = actual_bboxes[: (max_length - special_tokens_count)]\n", + " label_ids = label_ids[: (max_length - special_tokens_count)]\n", "\n", " # The convention in BERT is:\n", " # (a) For sequence pairs:\n", @@ -683,7 +683,7 @@ " input_mask = [1 if mask_padding_with_zero else 0] * len(input_ids)\n", "\n", " # Zero-pad up to the sequence length.\n", - " padding_length = max_seq_length - len(input_ids)\n", + " padding_length = max_length - len(input_ids)\n", " if pad_on_left:\n", " input_ids = ([pad_token] * padding_length) + input_ids\n", " input_mask = ([0 if mask_padding_with_zero else 1] * padding_length) + input_mask\n", @@ -697,11 +697,11 @@ " label_ids += [pad_token_label_id] * padding_length\n", " token_boxes += [pad_token_box] * padding_length\n", "\n", - " assert len(input_ids) == max_seq_length\n", - " assert len(input_mask) == max_seq_length\n", - " assert len(segment_ids) == max_seq_length\n", - " assert len(label_ids) == max_seq_length\n", - " assert len(token_boxes) == max_seq_length\n", + " assert len(input_ids) == max_length\n", + " assert len(input_mask) == max_length\n", + " assert len(segment_ids) == max_length\n", + " assert len(label_ids) == max_length\n", + " assert len(token_boxes) == max_length\n", "\n", " if ex_index < 5:\n", " logger.info(\"*** Example ***\")\n", @@ -748,7 +748,7 @@ " \"overwrite_cache\": True,\n", " \"data_dir\": \"data/\",\n", " \"model_name_or_path\": \"microsoft/layoutlm-base-uncased\",\n", - " \"max_seq_length\": 512,\n", + " \"max_length\": 512,\n", " \"model_type\": \"layoutlm\",\n", "}\n", "\n", diff --git a/examples/waveft_finetuning/README.md b/examples/waveft_finetuning/README.md index ad2d231698..820ead27aa 100644 --- a/examples/waveft_finetuning/README.md +++ b/examples/waveft_finetuning/README.md @@ -20,7 +20,7 @@ waveft_config = WaveFTConfig( n_frequency=2592, ) peft_model = get_peft_model(model, waveft_config) -training_args = SFTConfig(dataset_text_field="text", max_seq_length=128) +training_args = SFTConfig(dataset_text_field="text", max_length=128) trainer = SFTTrainer( model=peft_model, train_dataset=dataset, diff --git a/method_comparison/MetaMathQA/README.md b/method_comparison/MetaMathQA/README.md index 4c76240661..130054eafe 100644 --- a/method_comparison/MetaMathQA/README.md +++ b/method_comparison/MetaMathQA/README.md @@ -118,7 +118,7 @@ Results are stored in one of the result directories. An example output could loo "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 51, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/default_training_params.json b/method_comparison/MetaMathQA/default_training_params.json index a200a41ed9..819d40abe4 100644 --- a/method_comparison/MetaMathQA/default_training_params.json +++ b/method_comparison/MetaMathQA/default_training_params.json @@ -1,7 +1,7 @@ { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/adalora--llama-3.2-3B-rank32.json b/method_comparison/MetaMathQA/results/adalora--llama-3.2-3B-rank32.json index 8b38ea15e8..2c799cd731 100644 --- a/method_comparison/MetaMathQA/results/adalora--llama-3.2-3B-rank32.json +++ b/method_comparison/MetaMathQA/results/adalora--llama-3.2-3B-rank32.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/adaptionprompt--llama-3.2-3B-lr_0.0005.json b/method_comparison/MetaMathQA/results/adaptionprompt--llama-3.2-3B-lr_0.0005.json index c35ccc865b..817187822b 100644 --- a/method_comparison/MetaMathQA/results/adaptionprompt--llama-3.2-3B-lr_0.0005.json +++ b/method_comparison/MetaMathQA/results/adaptionprompt--llama-3.2-3B-lr_0.0005.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/boft--llama-3.2-3B-default.json b/method_comparison/MetaMathQA/results/boft--llama-3.2-3B-default.json index 83ddbc7c63..052c5b1d03 100644 --- a/method_comparison/MetaMathQA/results/boft--llama-3.2-3B-default.json +++ b/method_comparison/MetaMathQA/results/boft--llama-3.2-3B-default.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/bone--llama-3.2-3B-bat.json b/method_comparison/MetaMathQA/results/bone--llama-3.2-3B-bat.json index 069bbfe107..0eb718c4f9 100644 --- a/method_comparison/MetaMathQA/results/bone--llama-3.2-3B-bat.json +++ b/method_comparison/MetaMathQA/results/bone--llama-3.2-3B-bat.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/bone--llama-3.2-3B-default.json b/method_comparison/MetaMathQA/results/bone--llama-3.2-3B-default.json index a473c5827e..009bf52a1e 100644 --- a/method_comparison/MetaMathQA/results/bone--llama-3.2-3B-default.json +++ b/method_comparison/MetaMathQA/results/bone--llama-3.2-3B-default.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/c3a--llama-3.2-3B-default.json b/method_comparison/MetaMathQA/results/c3a--llama-3.2-3B-default.json index dedfb5f288..cae9c233d1 100644 --- a/method_comparison/MetaMathQA/results/c3a--llama-3.2-3B-default.json +++ b/method_comparison/MetaMathQA/results/c3a--llama-3.2-3B-default.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/fourierft--llama-3.2-3B-default.json b/method_comparison/MetaMathQA/results/fourierft--llama-3.2-3B-default.json index f3bc26876c..28dafa6514 100644 --- a/method_comparison/MetaMathQA/results/fourierft--llama-3.2-3B-default.json +++ b/method_comparison/MetaMathQA/results/fourierft--llama-3.2-3B-default.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/fourierft--llama-3.2-3B-n_frequency-5000.json b/method_comparison/MetaMathQA/results/fourierft--llama-3.2-3B-n_frequency-5000.json index 3c7241b5f0..26c8726cc4 100644 --- a/method_comparison/MetaMathQA/results/fourierft--llama-3.2-3B-n_frequency-5000.json +++ b/method_comparison/MetaMathQA/results/fourierft--llama-3.2-3B-n_frequency-5000.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/full-finetuning--llama-3.2-3B-lr_0.00001.json b/method_comparison/MetaMathQA/results/full-finetuning--llama-3.2-3B-lr_0.00001.json index 4f15dc9eb2..e934287464 100644 --- a/method_comparison/MetaMathQA/results/full-finetuning--llama-3.2-3B-lr_0.00001.json +++ b/method_comparison/MetaMathQA/results/full-finetuning--llama-3.2-3B-lr_0.00001.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/ia3--llama-3.2-3B-default.json b/method_comparison/MetaMathQA/results/ia3--llama-3.2-3B-default.json index 50d2efa1f9..03b530d7c6 100644 --- a/method_comparison/MetaMathQA/results/ia3--llama-3.2-3B-default.json +++ b/method_comparison/MetaMathQA/results/ia3--llama-3.2-3B-default.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/ia3--llama-3.2-3B-lr_0.001.json b/method_comparison/MetaMathQA/results/ia3--llama-3.2-3B-lr_0.001.json index f1982e8ef3..45d060600f 100644 --- a/method_comparison/MetaMathQA/results/ia3--llama-3.2-3B-lr_0.001.json +++ b/method_comparison/MetaMathQA/results/ia3--llama-3.2-3B-lr_0.001.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/ln_tuning--llama-3.2-3B-default.json b/method_comparison/MetaMathQA/results/ln_tuning--llama-3.2-3B-default.json index 2e8dabe44e..abf9fd8208 100644 --- a/method_comparison/MetaMathQA/results/ln_tuning--llama-3.2-3B-default.json +++ b/method_comparison/MetaMathQA/results/ln_tuning--llama-3.2-3B-default.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/loha--llama-3.2-3B-rank32.json b/method_comparison/MetaMathQA/results/loha--llama-3.2-3B-rank32.json index 7baa8eb00f..3f18f1888d 100644 --- a/method_comparison/MetaMathQA/results/loha--llama-3.2-3B-rank32.json +++ b/method_comparison/MetaMathQA/results/loha--llama-3.2-3B-rank32.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/lokr--llama-3.2-3B-rank32.json b/method_comparison/MetaMathQA/results/lokr--llama-3.2-3B-rank32.json index 07ae5b1b5a..146a07fc66 100644 --- a/method_comparison/MetaMathQA/results/lokr--llama-3.2-3B-rank32.json +++ b/method_comparison/MetaMathQA/results/lokr--llama-3.2-3B-rank32.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank32-dora.json b/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank32-dora.json index a2b645db9b..2ff59efd89 100644 --- a/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank32-dora.json +++ b/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank32-dora.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank32-lorafa.json b/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank32-lorafa.json index 35c7a9981d..18b887cbdf 100644 --- a/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank32-lorafa.json +++ b/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank32-lorafa.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank32.json b/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank32.json index f3d348c07e..f6e1a4da5c 100644 --- a/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank32.json +++ b/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank32.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank64-rslora.json b/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank64-rslora.json index dbeb788d4e..6820aea352 100644 --- a/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank64-rslora.json +++ b/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank64-rslora.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank64.json b/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank64.json index 5dd3d50954..1b8c5d827d 100644 --- a/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank64.json +++ b/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank64.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/miss--llama-3.2-3B-bat.json b/method_comparison/MetaMathQA/results/miss--llama-3.2-3B-bat.json index fc788b3e62..44419fdc75 100644 --- a/method_comparison/MetaMathQA/results/miss--llama-3.2-3B-bat.json +++ b/method_comparison/MetaMathQA/results/miss--llama-3.2-3B-bat.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/miss--llama-3.2-3B-default.json b/method_comparison/MetaMathQA/results/miss--llama-3.2-3B-default.json index 89af459238..2577d969e7 100644 --- a/method_comparison/MetaMathQA/results/miss--llama-3.2-3B-default.json +++ b/method_comparison/MetaMathQA/results/miss--llama-3.2-3B-default.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/miss--llama-3.2-3B-mini.json b/method_comparison/MetaMathQA/results/miss--llama-3.2-3B-mini.json index 66e5f97592..561999b293 100644 --- a/method_comparison/MetaMathQA/results/miss--llama-3.2-3B-mini.json +++ b/method_comparison/MetaMathQA/results/miss--llama-3.2-3B-mini.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/oft--llama-3.2-3B-rank32.json b/method_comparison/MetaMathQA/results/oft--llama-3.2-3B-rank32.json index b57f300fa3..6fb06abad6 100644 --- a/method_comparison/MetaMathQA/results/oft--llama-3.2-3B-rank32.json +++ b/method_comparison/MetaMathQA/results/oft--llama-3.2-3B-rank32.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/prefixtuning--llama-3.2-3B-lr_0.001.json b/method_comparison/MetaMathQA/results/prefixtuning--llama-3.2-3B-lr_0.001.json index 9c1717d39a..4b89e7f59c 100644 --- a/method_comparison/MetaMathQA/results/prefixtuning--llama-3.2-3B-lr_0.001.json +++ b/method_comparison/MetaMathQA/results/prefixtuning--llama-3.2-3B-lr_0.001.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/prompt_tuning--llama-3.2-3B-default.json b/method_comparison/MetaMathQA/results/prompt_tuning--llama-3.2-3B-default.json index 013c9ebf41..b96841e610 100644 --- a/method_comparison/MetaMathQA/results/prompt_tuning--llama-3.2-3B-default.json +++ b/method_comparison/MetaMathQA/results/prompt_tuning--llama-3.2-3B-default.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/prompt_tuning--llama-3.2-3B-lr_0.001.json b/method_comparison/MetaMathQA/results/prompt_tuning--llama-3.2-3B-lr_0.001.json index 2ce456649c..7ca4f4f9ea 100644 --- a/method_comparison/MetaMathQA/results/prompt_tuning--llama-3.2-3B-lr_0.001.json +++ b/method_comparison/MetaMathQA/results/prompt_tuning--llama-3.2-3B-lr_0.001.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/ptuning--llama-3.2-3B-default.json b/method_comparison/MetaMathQA/results/ptuning--llama-3.2-3B-default.json index 5ad6db2181..2bab7e708e 100644 --- a/method_comparison/MetaMathQA/results/ptuning--llama-3.2-3B-default.json +++ b/method_comparison/MetaMathQA/results/ptuning--llama-3.2-3B-default.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/randlora--llama-3.2-3B-default.json b/method_comparison/MetaMathQA/results/randlora--llama-3.2-3B-default.json index 1025019a88..807694cc76 100644 --- a/method_comparison/MetaMathQA/results/randlora--llama-3.2-3B-default.json +++ b/method_comparison/MetaMathQA/results/randlora--llama-3.2-3B-default.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/shira--llama-3.2-3B-lr_0.0003-random_seed_42.json b/method_comparison/MetaMathQA/results/shira--llama-3.2-3B-lr_0.0003-random_seed_42.json index 1263cd479f..4cbf791f03 100644 --- a/method_comparison/MetaMathQA/results/shira--llama-3.2-3B-lr_0.0003-random_seed_42.json +++ b/method_comparison/MetaMathQA/results/shira--llama-3.2-3B-lr_0.0003-random_seed_42.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/trainable_tokens--llama-3.2-3B-sos+eos.json b/method_comparison/MetaMathQA/results/trainable_tokens--llama-3.2-3B-sos+eos.json index eff36db4db..5e46377eb4 100644 --- a/method_comparison/MetaMathQA/results/trainable_tokens--llama-3.2-3B-sos+eos.json +++ b/method_comparison/MetaMathQA/results/trainable_tokens--llama-3.2-3B-sos+eos.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/vblora--llama-3.2-3B-default.json b/method_comparison/MetaMathQA/results/vblora--llama-3.2-3B-default.json index ccf041765e..f9180766da 100644 --- a/method_comparison/MetaMathQA/results/vblora--llama-3.2-3B-default.json +++ b/method_comparison/MetaMathQA/results/vblora--llama-3.2-3B-default.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/vera--llama-3.2-3B-default.json b/method_comparison/MetaMathQA/results/vera--llama-3.2-3B-default.json index 690c350729..44f0965d73 100644 --- a/method_comparison/MetaMathQA/results/vera--llama-3.2-3B-default.json +++ b/method_comparison/MetaMathQA/results/vera--llama-3.2-3B-default.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_seq_length": 768, + "max_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/run.py b/method_comparison/MetaMathQA/run.py index c03c801f5c..dc9ce7d49b 100644 --- a/method_comparison/MetaMathQA/run.py +++ b/method_comparison/MetaMathQA/run.py @@ -381,7 +381,7 @@ def main(*, path_experiment: str, experiment_name: str, clean: bool) -> None: # initialize objects accelerator_memory_init = init_accelerator() - tokenizer = get_tokenizer(model_id=train_config.model_id, max_seq_length=train_config.max_seq_length) + tokenizer = get_tokenizer(model_id=train_config.model_id, max_length=train_config.max_length) model_info = get_base_model_info(train_config.model_id) metamath_info = get_dataset_info("meta-math/MetaMathQA") diff --git a/method_comparison/MetaMathQA/utils.py b/method_comparison/MetaMathQA/utils.py index 531554a6a0..b88ae99a2b 100644 --- a/method_comparison/MetaMathQA/utils.py +++ b/method_comparison/MetaMathQA/utils.py @@ -72,7 +72,7 @@ class TrainConfig: Args: model_id: The model identifier dtype: The data type to use for the model - max_seq_length: The maximum sequence length + max_length: The maximum sequence length batch_size: The batch size for training batch_size_eval: The batch size for eval/test, can be much higher than for training max_steps: The maximum number of steps to train for @@ -92,7 +92,7 @@ class TrainConfig: model_id: str dtype: Literal["float32", "float16", "bfloat16", "int8", "int4"] - max_seq_length: int + max_length: int batch_size: int batch_size_eval: int max_steps: int @@ -114,8 +114,8 @@ def __post_init__(self) -> None: raise ValueError(f"Invalid model_id: {self.model_id}") if self.dtype not in ["float32", "float16", "bfloat16", "int8", "int4"]: raise ValueError(f"Invalid dtype: {self.dtype}") - if self.max_seq_length < 0: - raise ValueError(f"Invalid max_seq_length: {self.max_seq_length}") + if self.max_length < 0: + raise ValueError(f"Invalid max_length: {self.max_length}") if self.batch_size <= 0: raise ValueError(f"Invalid batch_size: {self.batch_size}") if self.batch_size_eval <= 0: @@ -194,9 +194,9 @@ def init_accelerator() -> int: return accelerator_memory_init -def get_tokenizer(*, model_id: str, max_seq_length: int): +def get_tokenizer(*, model_id: str, max_length: int): tokenizer = AutoTokenizer.from_pretrained(model_id) - tokenizer.model_max_length = max_seq_length + tokenizer.model_max_length = max_length if not tokenizer.pad_token: tokenizer.pad_token = tokenizer.eos_token return tokenizer diff --git a/scripts/train_memory.py b/scripts/train_memory.py index c764fe84ae..1d29bc5a2e 100644 --- a/scripts/train_memory.py +++ b/scripts/train_memory.py @@ -26,7 +26,7 @@ Train the google/gemma-2-2b model with a LoRA config json at the indicated location. ```bash -python train_memory.py "google/gemma-2-2b" --max_seq_length 256 --batch_size 1 --rank 32 --dtype bfloat16 --path_config +python train_memory.py "google/gemma-2-2b" --max_length 256 --batch_size 1 --rank 32 --dtype bfloat16 --path_config ``` Fully fine-tune the model (i.e. without LoRA) by setting the rank to 0: @@ -38,7 +38,7 @@ Get an estimate of the size of the hidden states by passing `--monitor_tensors`. This trains just for a single epoch. For realistic estimates, the batch size for this: ```bash -python train_memory.py "google/gemma-2-2b" --max_seq_length 256 --batch_size 32 --rank 32 --dtype bfloat16 --path_config configs/lora_rank-32_embedding-lora/ --monitor_tensors +python train_memory.py "google/gemma-2-2b" --max_length 256 --batch_size 32 --rank 32 --dtype bfloat16 --path_config configs/lora_rank-32_embedding-lora/ --monitor_tensors ``` """ @@ -106,14 +106,14 @@ def tokenize(samples): return data -def train(model_id, rank, dtype, monitor_tensors, max_seq_length, batch_size, max_steps, path_config): +def train(model_id, rank, dtype, monitor_tensors, max_length, batch_size, max_steps, path_config): init_accelerator() device_module = getattr(torch, device, torch.cuda) accelerator_memory_init = device_module.max_memory_allocated() accelerator_memory_log = [] tokenizer = AutoTokenizer.from_pretrained(model_id) - tokenizer.model_max_length = max_seq_length + tokenizer.model_max_length = max_length if not tokenizer.pad_token: tokenizer.pad_token = tokenizer.eos_token data = get_data(tokenizer) @@ -259,7 +259,7 @@ def unpack(x): action="store_true", help="Monitor tensor sizes during training for a single training step, off by default", ) - parser.add_argument("--max_seq_length", type=int, default=128, help="Maximum sequence length, default 128") + parser.add_argument("--max_length", type=int, default=128, help="Maximum sequence length, default 128") parser.add_argument("--batch_size", type=int, default=1, help="Batch size, default 1") parser.add_argument("--max_steps", type=int, default=50, help="Maximum number of training steps, default 50") parser.add_argument("--path_config", type=str, default=None, help="Path to LoRA config") @@ -269,7 +269,7 @@ def unpack(x): rank=args.rank, dtype=args.dtype, monitor_tensors=args.monitor_tensors, - max_seq_length=args.max_seq_length, + max_length=args.max_length, batch_size=args.batch_size, max_steps=args.max_steps, path_config=args.path_config, From 2e5e99a7e6cd5c033b60f337ce7c3c49dc999b90 Mon Sep 17 00:00:00 2001 From: "Liu, Kaixuan" Date: Thu, 23 Oct 2025 07:43:44 +0000 Subject: [PATCH 2/4] Revert "use `max_length` to replace `max_seq_length`; correct README for" This reverts commit 7a7fb877241b0f0637ac36d46d36526476bf0d03. --- examples/alora_finetuning/README.md | 2 +- examples/bone_finetuning/README.md | 2 +- examples/corda_finetuning/README.md | 2 +- examples/delora_finetuning/README.md | 6 ++-- examples/dora_finetuning/README.md | 2 +- examples/lorafa_finetune/README.md | 2 +- examples/miss_finetuning/README.md | 2 +- examples/olora_finetuning/README.md | 2 +- examples/pissa_finetuning/README.md | 2 +- examples/randlora_finetuning/README.md | 2 +- examples/road_finetuning/README.md | 2 +- examples/sft/train.py | 2 +- examples/sft/utils.py | 4 +-- examples/shira_finetuning/README.md | 2 +- .../peft_lora_token_cls.ipynb | 30 +++++++++---------- examples/waveft_finetuning/README.md | 2 +- method_comparison/MetaMathQA/README.md | 2 +- .../MetaMathQA/default_training_params.json | 2 +- .../results/adalora--llama-3.2-3B-rank32.json | 2 +- ...daptionprompt--llama-3.2-3B-lr_0.0005.json | 2 +- .../results/boft--llama-3.2-3B-default.json | 2 +- .../results/bone--llama-3.2-3B-bat.json | 2 +- .../results/bone--llama-3.2-3B-default.json | 2 +- .../results/c3a--llama-3.2-3B-default.json | 2 +- .../fourierft--llama-3.2-3B-default.json | 2 +- ...rierft--llama-3.2-3B-n_frequency-5000.json | 2 +- ...l-finetuning--llama-3.2-3B-lr_0.00001.json | 2 +- .../results/ia3--llama-3.2-3B-default.json | 2 +- .../results/ia3--llama-3.2-3B-lr_0.001.json | 2 +- .../ln_tuning--llama-3.2-3B-default.json | 2 +- .../results/loha--llama-3.2-3B-rank32.json | 2 +- .../results/lokr--llama-3.2-3B-rank32.json | 2 +- .../lora--llama-3.2-3B-rank32-dora.json | 2 +- .../lora--llama-3.2-3B-rank32-lorafa.json | 2 +- .../results/lora--llama-3.2-3B-rank32.json | 2 +- .../lora--llama-3.2-3B-rank64-rslora.json | 2 +- .../results/lora--llama-3.2-3B-rank64.json | 2 +- .../results/miss--llama-3.2-3B-bat.json | 2 +- .../results/miss--llama-3.2-3B-default.json | 2 +- .../results/miss--llama-3.2-3B-mini.json | 2 +- .../results/oft--llama-3.2-3B-rank32.json | 2 +- .../prefixtuning--llama-3.2-3B-lr_0.001.json | 2 +- .../prompt_tuning--llama-3.2-3B-default.json | 2 +- .../prompt_tuning--llama-3.2-3B-lr_0.001.json | 2 +- .../ptuning--llama-3.2-3B-default.json | 2 +- .../randlora--llama-3.2-3B-default.json | 2 +- ...llama-3.2-3B-lr_0.0003-random_seed_42.json | 2 +- ...rainable_tokens--llama-3.2-3B-sos+eos.json | 2 +- .../results/vblora--llama-3.2-3B-default.json | 2 +- .../results/vera--llama-3.2-3B-default.json | 2 +- method_comparison/MetaMathQA/run.py | 2 +- method_comparison/MetaMathQA/utils.py | 12 ++++---- scripts/train_memory.py | 12 ++++---- 53 files changed, 80 insertions(+), 80 deletions(-) diff --git a/examples/alora_finetuning/README.md b/examples/alora_finetuning/README.md index da0f1133a7..e6b8da0bcd 100644 --- a/examples/alora_finetuning/README.md +++ b/examples/alora_finetuning/README.md @@ -32,7 +32,7 @@ trainer = Trainer( model=peft_model, train_dataset=dataset, dataset_text_field="text", - max_length=2048, + max_seq_length=2048, tokenizer=tokenizer, data_collator=data_collator, ) diff --git a/examples/bone_finetuning/README.md b/examples/bone_finetuning/README.md index bfc25f780b..42b604f92d 100644 --- a/examples/bone_finetuning/README.md +++ b/examples/bone_finetuning/README.md @@ -28,7 +28,7 @@ peft_model.print_trainable_parameters() dataset = load_dataset("imdb", split="train[:1%]") -training_args = SFTConfig(dataset_text_field="text", max_length=128) +training_args = SFTConfig(dataset_text_field="text", max_seq_length=128) trainer = SFTTrainer( model=peft_model, args=training_args, diff --git a/examples/corda_finetuning/README.md b/examples/corda_finetuning/README.md index d86bcdcd7c..9d72fc2f11 100644 --- a/examples/corda_finetuning/README.md +++ b/examples/corda_finetuning/README.md @@ -109,7 +109,7 @@ preprocess_corda(model, lora_config, run_model=run_model) peft_model = get_peft_model(model, lora_config) peft_model.print_trainable_parameters() -training_args = SFTConfig(dataset_text_field="text", max_length=128) +training_args = SFTConfig(dataset_text_field="text", max_seq_length=128) trainer = SFTTrainer( model=peft_model, args=training_args, diff --git a/examples/delora_finetuning/README.md b/examples/delora_finetuning/README.md index 20c18ae059..20fc858bec 100644 --- a/examples/delora_finetuning/README.md +++ b/examples/delora_finetuning/README.md @@ -26,7 +26,7 @@ peft_model.print_trainable_parameters() dataset = load_dataset("imdb", split="train[:1%]") -training_args = SFTConfig(dataset_text_field="text", max_length=128) +training_args = SFTConfig(dataset_text_field="text", max_seq_length=128) trainer = SFTTrainer( model=peft_model, args=training_args, @@ -52,7 +52,7 @@ peft_model = PeftModel.from_pretrained(model, "delora-llama-3-8b") ## Advanced Usage In this script the default DeLoRA layers are the query and value layers of the Llama model. Adding adapters on more layers will increase memory usage. If you wish to choose a different set of layers for DeLoRA to be applied on, you can simply define it using: ```bash -python examples/delora_finetuning/delora_finetuning.py --base_model meta-llama/Meta-Llama-3-8B --target_modules "q_proj,k_proj,v_proj,o_proj" +python examples/delora_finetuning/delora_finetuning.py --base_model meta-llama/Meta-Llama-3-8B --delora_target_modules "q_proj,k_proj,v_proj,o_proj" ``` Using different lambdas for different layers is also possible by setting `lambda_pattern`. @@ -74,7 +74,7 @@ python delora_finetuning.py \ --rank 32 \ --delora_lambda 15 \ --module_dropout 0.1 \ - --target_modules "q_proj,v_proj" \ + --delora_target_modules "q_proj,v_proj" \ --hub_model_id "YOUR_HF_REPO" \ --push_to_hub ``` diff --git a/examples/dora_finetuning/README.md b/examples/dora_finetuning/README.md index 20250476e4..91ffb2a529 100644 --- a/examples/dora_finetuning/README.md +++ b/examples/dora_finetuning/README.md @@ -24,7 +24,7 @@ trainer = transformers.Trainer( model=peft_model, train_dataset=dataset, dataset_text_field="text", - max_length=2048, + max_seq_length=2048, tokenizer=tokenizer, ) trainer.train() diff --git a/examples/lorafa_finetune/README.md b/examples/lorafa_finetune/README.md index 1c1314dbd6..432c93ad83 100644 --- a/examples/lorafa_finetune/README.md +++ b/examples/lorafa_finetune/README.md @@ -40,7 +40,7 @@ trainer = transformers.Trainer( model=peft_model, train_dataset=dataset, dataset_text_field="text", - max_length=2048, + max_seq_length=2048, processing_class=tokenizer, optimizers=(optimizer, None), ) diff --git a/examples/miss_finetuning/README.md b/examples/miss_finetuning/README.md index 1e11064d8d..bcc1bb33d8 100644 --- a/examples/miss_finetuning/README.md +++ b/examples/miss_finetuning/README.md @@ -36,7 +36,7 @@ peft_model.print_trainable_parameters() dataset = load_dataset("imdb", split="train[:1%]") -training_args = SFTConfig(dataset_text_field="text", max_length=128) +training_args = SFTConfig(dataset_text_field="text", max_seq_length=128) trainer = SFTTrainer( model=peft_model, args=training_args, diff --git a/examples/olora_finetuning/README.md b/examples/olora_finetuning/README.md index 3b9bbf68ee..bb548b77b2 100644 --- a/examples/olora_finetuning/README.md +++ b/examples/olora_finetuning/README.md @@ -18,7 +18,7 @@ lora_config = LoraConfig( init_lora_weights="olora" ) peft_model = get_peft_model(model, lora_config) -training_args = SFTConfig(dataset_text_field="text", max_length=128) +training_args = SFTConfig(dataset_text_field="text", max_seq_length=128) trainer = SFTTrainer( model=peft_model, train_dataset=dataset, diff --git a/examples/pissa_finetuning/README.md b/examples/pissa_finetuning/README.md index 6ab1515679..4c0734bdcf 100644 --- a/examples/pissa_finetuning/README.md +++ b/examples/pissa_finetuning/README.md @@ -23,7 +23,7 @@ peft_model.print_trainable_parameters() dataset = load_dataset("imdb", split="train[:1%]") -training_args = SFTConfig(dataset_text_field="text", max_length=128) +training_args = SFTConfig(dataset_text_field="text", max_seq_length=128) trainer = SFTTrainer( model=peft_model, args=training_args, diff --git a/examples/randlora_finetuning/README.md b/examples/randlora_finetuning/README.md index f8eb7d77d0..fa9d2d61de 100644 --- a/examples/randlora_finetuning/README.md +++ b/examples/randlora_finetuning/README.md @@ -20,7 +20,7 @@ trainer = transformers.Trainer( model=peft_model, train_dataset=dataset, dataset_text_field="text", - max_length=2048, + max_seq_length=2048, processing_class=tokenizer, ) trainer.train() diff --git a/examples/road_finetuning/README.md b/examples/road_finetuning/README.md index 00a043936f..b9ce14017c 100644 --- a/examples/road_finetuning/README.md +++ b/examples/road_finetuning/README.md @@ -26,7 +26,7 @@ trainer = transformers.Trainer( model=peft_model, train_dataset=dataset, dataset_text_field="text", - max_length=2048, + max_seq_length=2048, tokenizer=tokenizer, ) trainer.train() diff --git a/examples/sft/train.py b/examples/sft/train.py index 77efac3609..5a34f69357 100644 --- a/examples/sft/train.py +++ b/examples/sft/train.py @@ -18,7 +18,7 @@ class ModelArguments: model_name_or_path: str = field( metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"} ) - max_length: Optional[int] = field( + max_seq_length: Optional[int] = field( default=512, metadata={"help": "The maximum total input sequence length after tokenization."}, ) diff --git a/examples/sft/utils.py b/examples/sft/utils.py index 96d49e7a2f..0e24796de4 100644 --- a/examples/sft/utils.py +++ b/examples/sft/utils.py @@ -124,7 +124,7 @@ def create_and_prepare_model(args, data_args, training_args): # Load model model, _ = FastLanguageModel.from_pretrained( model_name=args.model_name_or_path, - max_length=training_args.max_length, + max_seq_length=training_args.max_seq_length, dtype=None, load_in_4bit=args.use_4bit_quantization, ) @@ -211,7 +211,7 @@ def create_and_prepare_model(args, data_args, training_args): else args.lora_target_modules, use_gradient_checkpointing=training_args.gradient_checkpointing, random_state=training_args.seed, - max_length=training_args.max_length, + max_seq_length=training_args.max_seq_length, ) return model, peft_config, tokenizer diff --git a/examples/shira_finetuning/README.md b/examples/shira_finetuning/README.md index 76d062e2c7..45b1e99be6 100644 --- a/examples/shira_finetuning/README.md +++ b/examples/shira_finetuning/README.md @@ -18,7 +18,7 @@ shira_config = ShiraConfig( r=32, ) peft_model = get_peft_model(model, shira_config) -training_args = SFTConfig(dataset_text_field="text", max_length=128) +training_args = SFTConfig(dataset_text_field="text", max_seq_length=128) trainer = SFTTrainer( model=peft_model, train_dataset=dataset, diff --git a/examples/token_classification/peft_lora_token_cls.ipynb b/examples/token_classification/peft_lora_token_cls.ipynb index 9e5ba813cd..0c6c686ae5 100644 --- a/examples/token_classification/peft_lora_token_cls.ipynb +++ b/examples/token_classification/peft_lora_token_cls.ipynb @@ -401,7 +401,7 @@ " \"cached_{}_{}_{}\".format(\n", " mode,\n", " list(filter(None, args.model_name_or_path.split(\"/\"))).pop(),\n", - " str(args.max_length),\n", + " str(args.max_seq_length),\n", " ),\n", " )\n", " if os.path.exists(cached_features_file) and not args.overwrite_cache:\n", @@ -413,7 +413,7 @@ " features = convert_examples_to_features(\n", " examples,\n", " labels,\n", - " args.max_length,\n", + " args.max_seq_length,\n", " tokenizer,\n", " cls_token_at_end=bool(args.model_type in [\"xlnet\"]),\n", " # xlnet has a cls token at the end\n", @@ -579,7 +579,7 @@ "def convert_examples_to_features(\n", " examples,\n", " label_list,\n", - " max_length,\n", + " max_seq_length,\n", " tokenizer,\n", " cls_token_at_end=False,\n", " cls_token=\"[CLS]\",\n", @@ -627,11 +627,11 @@ "\n", " # Account for [CLS] and [SEP] with \"- 2\" and with \"- 3\" for RoBERTa.\n", " special_tokens_count = 3 if sep_token_extra else 2\n", - " if len(tokens) > max_length - special_tokens_count:\n", - " tokens = tokens[: (max_length - special_tokens_count)]\n", - " token_boxes = token_boxes[: (max_length - special_tokens_count)]\n", - " actual_bboxes = actual_bboxes[: (max_length - special_tokens_count)]\n", - " label_ids = label_ids[: (max_length - special_tokens_count)]\n", + " if len(tokens) > max_seq_length - special_tokens_count:\n", + " tokens = tokens[: (max_seq_length - special_tokens_count)]\n", + " token_boxes = token_boxes[: (max_seq_length - special_tokens_count)]\n", + " actual_bboxes = actual_bboxes[: (max_seq_length - special_tokens_count)]\n", + " label_ids = label_ids[: (max_seq_length - special_tokens_count)]\n", "\n", " # The convention in BERT is:\n", " # (a) For sequence pairs:\n", @@ -683,7 +683,7 @@ " input_mask = [1 if mask_padding_with_zero else 0] * len(input_ids)\n", "\n", " # Zero-pad up to the sequence length.\n", - " padding_length = max_length - len(input_ids)\n", + " padding_length = max_seq_length - len(input_ids)\n", " if pad_on_left:\n", " input_ids = ([pad_token] * padding_length) + input_ids\n", " input_mask = ([0 if mask_padding_with_zero else 1] * padding_length) + input_mask\n", @@ -697,11 +697,11 @@ " label_ids += [pad_token_label_id] * padding_length\n", " token_boxes += [pad_token_box] * padding_length\n", "\n", - " assert len(input_ids) == max_length\n", - " assert len(input_mask) == max_length\n", - " assert len(segment_ids) == max_length\n", - " assert len(label_ids) == max_length\n", - " assert len(token_boxes) == max_length\n", + " assert len(input_ids) == max_seq_length\n", + " assert len(input_mask) == max_seq_length\n", + " assert len(segment_ids) == max_seq_length\n", + " assert len(label_ids) == max_seq_length\n", + " assert len(token_boxes) == max_seq_length\n", "\n", " if ex_index < 5:\n", " logger.info(\"*** Example ***\")\n", @@ -748,7 +748,7 @@ " \"overwrite_cache\": True,\n", " \"data_dir\": \"data/\",\n", " \"model_name_or_path\": \"microsoft/layoutlm-base-uncased\",\n", - " \"max_length\": 512,\n", + " \"max_seq_length\": 512,\n", " \"model_type\": \"layoutlm\",\n", "}\n", "\n", diff --git a/examples/waveft_finetuning/README.md b/examples/waveft_finetuning/README.md index 820ead27aa..ad2d231698 100644 --- a/examples/waveft_finetuning/README.md +++ b/examples/waveft_finetuning/README.md @@ -20,7 +20,7 @@ waveft_config = WaveFTConfig( n_frequency=2592, ) peft_model = get_peft_model(model, waveft_config) -training_args = SFTConfig(dataset_text_field="text", max_length=128) +training_args = SFTConfig(dataset_text_field="text", max_seq_length=128) trainer = SFTTrainer( model=peft_model, train_dataset=dataset, diff --git a/method_comparison/MetaMathQA/README.md b/method_comparison/MetaMathQA/README.md index 130054eafe..4c76240661 100644 --- a/method_comparison/MetaMathQA/README.md +++ b/method_comparison/MetaMathQA/README.md @@ -118,7 +118,7 @@ Results are stored in one of the result directories. An example output could loo "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 51, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/default_training_params.json b/method_comparison/MetaMathQA/default_training_params.json index 819d40abe4..a200a41ed9 100644 --- a/method_comparison/MetaMathQA/default_training_params.json +++ b/method_comparison/MetaMathQA/default_training_params.json @@ -1,7 +1,7 @@ { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/adalora--llama-3.2-3B-rank32.json b/method_comparison/MetaMathQA/results/adalora--llama-3.2-3B-rank32.json index 2c799cd731..8b38ea15e8 100644 --- a/method_comparison/MetaMathQA/results/adalora--llama-3.2-3B-rank32.json +++ b/method_comparison/MetaMathQA/results/adalora--llama-3.2-3B-rank32.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/adaptionprompt--llama-3.2-3B-lr_0.0005.json b/method_comparison/MetaMathQA/results/adaptionprompt--llama-3.2-3B-lr_0.0005.json index 817187822b..c35ccc865b 100644 --- a/method_comparison/MetaMathQA/results/adaptionprompt--llama-3.2-3B-lr_0.0005.json +++ b/method_comparison/MetaMathQA/results/adaptionprompt--llama-3.2-3B-lr_0.0005.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/boft--llama-3.2-3B-default.json b/method_comparison/MetaMathQA/results/boft--llama-3.2-3B-default.json index 052c5b1d03..83ddbc7c63 100644 --- a/method_comparison/MetaMathQA/results/boft--llama-3.2-3B-default.json +++ b/method_comparison/MetaMathQA/results/boft--llama-3.2-3B-default.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/bone--llama-3.2-3B-bat.json b/method_comparison/MetaMathQA/results/bone--llama-3.2-3B-bat.json index 0eb718c4f9..069bbfe107 100644 --- a/method_comparison/MetaMathQA/results/bone--llama-3.2-3B-bat.json +++ b/method_comparison/MetaMathQA/results/bone--llama-3.2-3B-bat.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/bone--llama-3.2-3B-default.json b/method_comparison/MetaMathQA/results/bone--llama-3.2-3B-default.json index 009bf52a1e..a473c5827e 100644 --- a/method_comparison/MetaMathQA/results/bone--llama-3.2-3B-default.json +++ b/method_comparison/MetaMathQA/results/bone--llama-3.2-3B-default.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/c3a--llama-3.2-3B-default.json b/method_comparison/MetaMathQA/results/c3a--llama-3.2-3B-default.json index cae9c233d1..dedfb5f288 100644 --- a/method_comparison/MetaMathQA/results/c3a--llama-3.2-3B-default.json +++ b/method_comparison/MetaMathQA/results/c3a--llama-3.2-3B-default.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/fourierft--llama-3.2-3B-default.json b/method_comparison/MetaMathQA/results/fourierft--llama-3.2-3B-default.json index 28dafa6514..f3bc26876c 100644 --- a/method_comparison/MetaMathQA/results/fourierft--llama-3.2-3B-default.json +++ b/method_comparison/MetaMathQA/results/fourierft--llama-3.2-3B-default.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/fourierft--llama-3.2-3B-n_frequency-5000.json b/method_comparison/MetaMathQA/results/fourierft--llama-3.2-3B-n_frequency-5000.json index 26c8726cc4..3c7241b5f0 100644 --- a/method_comparison/MetaMathQA/results/fourierft--llama-3.2-3B-n_frequency-5000.json +++ b/method_comparison/MetaMathQA/results/fourierft--llama-3.2-3B-n_frequency-5000.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/full-finetuning--llama-3.2-3B-lr_0.00001.json b/method_comparison/MetaMathQA/results/full-finetuning--llama-3.2-3B-lr_0.00001.json index e934287464..4f15dc9eb2 100644 --- a/method_comparison/MetaMathQA/results/full-finetuning--llama-3.2-3B-lr_0.00001.json +++ b/method_comparison/MetaMathQA/results/full-finetuning--llama-3.2-3B-lr_0.00001.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/ia3--llama-3.2-3B-default.json b/method_comparison/MetaMathQA/results/ia3--llama-3.2-3B-default.json index 03b530d7c6..50d2efa1f9 100644 --- a/method_comparison/MetaMathQA/results/ia3--llama-3.2-3B-default.json +++ b/method_comparison/MetaMathQA/results/ia3--llama-3.2-3B-default.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/ia3--llama-3.2-3B-lr_0.001.json b/method_comparison/MetaMathQA/results/ia3--llama-3.2-3B-lr_0.001.json index 45d060600f..f1982e8ef3 100644 --- a/method_comparison/MetaMathQA/results/ia3--llama-3.2-3B-lr_0.001.json +++ b/method_comparison/MetaMathQA/results/ia3--llama-3.2-3B-lr_0.001.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/ln_tuning--llama-3.2-3B-default.json b/method_comparison/MetaMathQA/results/ln_tuning--llama-3.2-3B-default.json index abf9fd8208..2e8dabe44e 100644 --- a/method_comparison/MetaMathQA/results/ln_tuning--llama-3.2-3B-default.json +++ b/method_comparison/MetaMathQA/results/ln_tuning--llama-3.2-3B-default.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/loha--llama-3.2-3B-rank32.json b/method_comparison/MetaMathQA/results/loha--llama-3.2-3B-rank32.json index 3f18f1888d..7baa8eb00f 100644 --- a/method_comparison/MetaMathQA/results/loha--llama-3.2-3B-rank32.json +++ b/method_comparison/MetaMathQA/results/loha--llama-3.2-3B-rank32.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/lokr--llama-3.2-3B-rank32.json b/method_comparison/MetaMathQA/results/lokr--llama-3.2-3B-rank32.json index 146a07fc66..07ae5b1b5a 100644 --- a/method_comparison/MetaMathQA/results/lokr--llama-3.2-3B-rank32.json +++ b/method_comparison/MetaMathQA/results/lokr--llama-3.2-3B-rank32.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank32-dora.json b/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank32-dora.json index 2ff59efd89..a2b645db9b 100644 --- a/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank32-dora.json +++ b/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank32-dora.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank32-lorafa.json b/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank32-lorafa.json index 18b887cbdf..35c7a9981d 100644 --- a/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank32-lorafa.json +++ b/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank32-lorafa.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank32.json b/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank32.json index f6e1a4da5c..f3d348c07e 100644 --- a/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank32.json +++ b/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank32.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank64-rslora.json b/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank64-rslora.json index 6820aea352..dbeb788d4e 100644 --- a/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank64-rslora.json +++ b/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank64-rslora.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank64.json b/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank64.json index 1b8c5d827d..5dd3d50954 100644 --- a/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank64.json +++ b/method_comparison/MetaMathQA/results/lora--llama-3.2-3B-rank64.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/miss--llama-3.2-3B-bat.json b/method_comparison/MetaMathQA/results/miss--llama-3.2-3B-bat.json index 44419fdc75..fc788b3e62 100644 --- a/method_comparison/MetaMathQA/results/miss--llama-3.2-3B-bat.json +++ b/method_comparison/MetaMathQA/results/miss--llama-3.2-3B-bat.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/miss--llama-3.2-3B-default.json b/method_comparison/MetaMathQA/results/miss--llama-3.2-3B-default.json index 2577d969e7..89af459238 100644 --- a/method_comparison/MetaMathQA/results/miss--llama-3.2-3B-default.json +++ b/method_comparison/MetaMathQA/results/miss--llama-3.2-3B-default.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/miss--llama-3.2-3B-mini.json b/method_comparison/MetaMathQA/results/miss--llama-3.2-3B-mini.json index 561999b293..66e5f97592 100644 --- a/method_comparison/MetaMathQA/results/miss--llama-3.2-3B-mini.json +++ b/method_comparison/MetaMathQA/results/miss--llama-3.2-3B-mini.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/oft--llama-3.2-3B-rank32.json b/method_comparison/MetaMathQA/results/oft--llama-3.2-3B-rank32.json index 6fb06abad6..b57f300fa3 100644 --- a/method_comparison/MetaMathQA/results/oft--llama-3.2-3B-rank32.json +++ b/method_comparison/MetaMathQA/results/oft--llama-3.2-3B-rank32.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/prefixtuning--llama-3.2-3B-lr_0.001.json b/method_comparison/MetaMathQA/results/prefixtuning--llama-3.2-3B-lr_0.001.json index 4b89e7f59c..9c1717d39a 100644 --- a/method_comparison/MetaMathQA/results/prefixtuning--llama-3.2-3B-lr_0.001.json +++ b/method_comparison/MetaMathQA/results/prefixtuning--llama-3.2-3B-lr_0.001.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/prompt_tuning--llama-3.2-3B-default.json b/method_comparison/MetaMathQA/results/prompt_tuning--llama-3.2-3B-default.json index b96841e610..013c9ebf41 100644 --- a/method_comparison/MetaMathQA/results/prompt_tuning--llama-3.2-3B-default.json +++ b/method_comparison/MetaMathQA/results/prompt_tuning--llama-3.2-3B-default.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/prompt_tuning--llama-3.2-3B-lr_0.001.json b/method_comparison/MetaMathQA/results/prompt_tuning--llama-3.2-3B-lr_0.001.json index 7ca4f4f9ea..2ce456649c 100644 --- a/method_comparison/MetaMathQA/results/prompt_tuning--llama-3.2-3B-lr_0.001.json +++ b/method_comparison/MetaMathQA/results/prompt_tuning--llama-3.2-3B-lr_0.001.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/ptuning--llama-3.2-3B-default.json b/method_comparison/MetaMathQA/results/ptuning--llama-3.2-3B-default.json index 2bab7e708e..5ad6db2181 100644 --- a/method_comparison/MetaMathQA/results/ptuning--llama-3.2-3B-default.json +++ b/method_comparison/MetaMathQA/results/ptuning--llama-3.2-3B-default.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/randlora--llama-3.2-3B-default.json b/method_comparison/MetaMathQA/results/randlora--llama-3.2-3B-default.json index 807694cc76..1025019a88 100644 --- a/method_comparison/MetaMathQA/results/randlora--llama-3.2-3B-default.json +++ b/method_comparison/MetaMathQA/results/randlora--llama-3.2-3B-default.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/shira--llama-3.2-3B-lr_0.0003-random_seed_42.json b/method_comparison/MetaMathQA/results/shira--llama-3.2-3B-lr_0.0003-random_seed_42.json index 4cbf791f03..1263cd479f 100644 --- a/method_comparison/MetaMathQA/results/shira--llama-3.2-3B-lr_0.0003-random_seed_42.json +++ b/method_comparison/MetaMathQA/results/shira--llama-3.2-3B-lr_0.0003-random_seed_42.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/trainable_tokens--llama-3.2-3B-sos+eos.json b/method_comparison/MetaMathQA/results/trainable_tokens--llama-3.2-3B-sos+eos.json index 5e46377eb4..eff36db4db 100644 --- a/method_comparison/MetaMathQA/results/trainable_tokens--llama-3.2-3B-sos+eos.json +++ b/method_comparison/MetaMathQA/results/trainable_tokens--llama-3.2-3B-sos+eos.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/vblora--llama-3.2-3B-default.json b/method_comparison/MetaMathQA/results/vblora--llama-3.2-3B-default.json index f9180766da..ccf041765e 100644 --- a/method_comparison/MetaMathQA/results/vblora--llama-3.2-3B-default.json +++ b/method_comparison/MetaMathQA/results/vblora--llama-3.2-3B-default.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/results/vera--llama-3.2-3B-default.json b/method_comparison/MetaMathQA/results/vera--llama-3.2-3B-default.json index 44f0965d73..690c350729 100644 --- a/method_comparison/MetaMathQA/results/vera--llama-3.2-3B-default.json +++ b/method_comparison/MetaMathQA/results/vera--llama-3.2-3B-default.json @@ -7,7 +7,7 @@ "train_config": { "model_id": "meta-llama/Llama-3.2-3B", "dtype": "bfloat16", - "max_length": 768, + "max_seq_length": 768, "batch_size": 4, "batch_size_eval": 50, "max_steps": 5000, diff --git a/method_comparison/MetaMathQA/run.py b/method_comparison/MetaMathQA/run.py index dc9ce7d49b..c03c801f5c 100644 --- a/method_comparison/MetaMathQA/run.py +++ b/method_comparison/MetaMathQA/run.py @@ -381,7 +381,7 @@ def main(*, path_experiment: str, experiment_name: str, clean: bool) -> None: # initialize objects accelerator_memory_init = init_accelerator() - tokenizer = get_tokenizer(model_id=train_config.model_id, max_length=train_config.max_length) + tokenizer = get_tokenizer(model_id=train_config.model_id, max_seq_length=train_config.max_seq_length) model_info = get_base_model_info(train_config.model_id) metamath_info = get_dataset_info("meta-math/MetaMathQA") diff --git a/method_comparison/MetaMathQA/utils.py b/method_comparison/MetaMathQA/utils.py index b88ae99a2b..531554a6a0 100644 --- a/method_comparison/MetaMathQA/utils.py +++ b/method_comparison/MetaMathQA/utils.py @@ -72,7 +72,7 @@ class TrainConfig: Args: model_id: The model identifier dtype: The data type to use for the model - max_length: The maximum sequence length + max_seq_length: The maximum sequence length batch_size: The batch size for training batch_size_eval: The batch size for eval/test, can be much higher than for training max_steps: The maximum number of steps to train for @@ -92,7 +92,7 @@ class TrainConfig: model_id: str dtype: Literal["float32", "float16", "bfloat16", "int8", "int4"] - max_length: int + max_seq_length: int batch_size: int batch_size_eval: int max_steps: int @@ -114,8 +114,8 @@ def __post_init__(self) -> None: raise ValueError(f"Invalid model_id: {self.model_id}") if self.dtype not in ["float32", "float16", "bfloat16", "int8", "int4"]: raise ValueError(f"Invalid dtype: {self.dtype}") - if self.max_length < 0: - raise ValueError(f"Invalid max_length: {self.max_length}") + if self.max_seq_length < 0: + raise ValueError(f"Invalid max_seq_length: {self.max_seq_length}") if self.batch_size <= 0: raise ValueError(f"Invalid batch_size: {self.batch_size}") if self.batch_size_eval <= 0: @@ -194,9 +194,9 @@ def init_accelerator() -> int: return accelerator_memory_init -def get_tokenizer(*, model_id: str, max_length: int): +def get_tokenizer(*, model_id: str, max_seq_length: int): tokenizer = AutoTokenizer.from_pretrained(model_id) - tokenizer.model_max_length = max_length + tokenizer.model_max_length = max_seq_length if not tokenizer.pad_token: tokenizer.pad_token = tokenizer.eos_token return tokenizer diff --git a/scripts/train_memory.py b/scripts/train_memory.py index 1d29bc5a2e..c764fe84ae 100644 --- a/scripts/train_memory.py +++ b/scripts/train_memory.py @@ -26,7 +26,7 @@ Train the google/gemma-2-2b model with a LoRA config json at the indicated location. ```bash -python train_memory.py "google/gemma-2-2b" --max_length 256 --batch_size 1 --rank 32 --dtype bfloat16 --path_config +python train_memory.py "google/gemma-2-2b" --max_seq_length 256 --batch_size 1 --rank 32 --dtype bfloat16 --path_config ``` Fully fine-tune the model (i.e. without LoRA) by setting the rank to 0: @@ -38,7 +38,7 @@ Get an estimate of the size of the hidden states by passing `--monitor_tensors`. This trains just for a single epoch. For realistic estimates, the batch size for this: ```bash -python train_memory.py "google/gemma-2-2b" --max_length 256 --batch_size 32 --rank 32 --dtype bfloat16 --path_config configs/lora_rank-32_embedding-lora/ --monitor_tensors +python train_memory.py "google/gemma-2-2b" --max_seq_length 256 --batch_size 32 --rank 32 --dtype bfloat16 --path_config configs/lora_rank-32_embedding-lora/ --monitor_tensors ``` """ @@ -106,14 +106,14 @@ def tokenize(samples): return data -def train(model_id, rank, dtype, monitor_tensors, max_length, batch_size, max_steps, path_config): +def train(model_id, rank, dtype, monitor_tensors, max_seq_length, batch_size, max_steps, path_config): init_accelerator() device_module = getattr(torch, device, torch.cuda) accelerator_memory_init = device_module.max_memory_allocated() accelerator_memory_log = [] tokenizer = AutoTokenizer.from_pretrained(model_id) - tokenizer.model_max_length = max_length + tokenizer.model_max_length = max_seq_length if not tokenizer.pad_token: tokenizer.pad_token = tokenizer.eos_token data = get_data(tokenizer) @@ -259,7 +259,7 @@ def unpack(x): action="store_true", help="Monitor tensor sizes during training for a single training step, off by default", ) - parser.add_argument("--max_length", type=int, default=128, help="Maximum sequence length, default 128") + parser.add_argument("--max_seq_length", type=int, default=128, help="Maximum sequence length, default 128") parser.add_argument("--batch_size", type=int, default=1, help="Batch size, default 1") parser.add_argument("--max_steps", type=int, default=50, help="Maximum number of training steps, default 50") parser.add_argument("--path_config", type=str, default=None, help="Path to LoRA config") @@ -269,7 +269,7 @@ def unpack(x): rank=args.rank, dtype=args.dtype, monitor_tensors=args.monitor_tensors, - max_length=args.max_length, + max_seq_length=args.max_seq_length, batch_size=args.batch_size, max_steps=args.max_steps, path_config=args.path_config, From b87e0928c60c423e9d46a8274456f8e6e5e48235 Mon Sep 17 00:00:00 2001 From: "Liu, Kaixuan" Date: Thu, 23 Oct 2025 08:06:35 +0000 Subject: [PATCH 3/4] use `max_length` for examples Signed-off-by: Liu, Kaixuan --- examples/alora_finetuning/README.md | 2 +- examples/bone_finetuning/README.md | 2 +- examples/corda_finetuning/README.md | 2 +- examples/delora_finetuning/README.md | 6 +++--- examples/dora_finetuning/README.md | 2 +- examples/lorafa_finetune/README.md | 2 +- examples/miss_finetuning/README.md | 2 +- examples/olora_finetuning/README.md | 2 +- examples/pissa_finetuning/README.md | 2 +- examples/randlora_finetuning/README.md | 2 +- examples/road_finetuning/README.md | 2 +- examples/sft/train.py | 2 +- examples/shira_finetuning/README.md | 2 +- examples/waveft_finetuning/README.md | 2 +- 14 files changed, 16 insertions(+), 16 deletions(-) diff --git a/examples/alora_finetuning/README.md b/examples/alora_finetuning/README.md index e6b8da0bcd..da0f1133a7 100644 --- a/examples/alora_finetuning/README.md +++ b/examples/alora_finetuning/README.md @@ -32,7 +32,7 @@ trainer = Trainer( model=peft_model, train_dataset=dataset, dataset_text_field="text", - max_seq_length=2048, + max_length=2048, tokenizer=tokenizer, data_collator=data_collator, ) diff --git a/examples/bone_finetuning/README.md b/examples/bone_finetuning/README.md index 42b604f92d..bfc25f780b 100644 --- a/examples/bone_finetuning/README.md +++ b/examples/bone_finetuning/README.md @@ -28,7 +28,7 @@ peft_model.print_trainable_parameters() dataset = load_dataset("imdb", split="train[:1%]") -training_args = SFTConfig(dataset_text_field="text", max_seq_length=128) +training_args = SFTConfig(dataset_text_field="text", max_length=128) trainer = SFTTrainer( model=peft_model, args=training_args, diff --git a/examples/corda_finetuning/README.md b/examples/corda_finetuning/README.md index 9d72fc2f11..d86bcdcd7c 100644 --- a/examples/corda_finetuning/README.md +++ b/examples/corda_finetuning/README.md @@ -109,7 +109,7 @@ preprocess_corda(model, lora_config, run_model=run_model) peft_model = get_peft_model(model, lora_config) peft_model.print_trainable_parameters() -training_args = SFTConfig(dataset_text_field="text", max_seq_length=128) +training_args = SFTConfig(dataset_text_field="text", max_length=128) trainer = SFTTrainer( model=peft_model, args=training_args, diff --git a/examples/delora_finetuning/README.md b/examples/delora_finetuning/README.md index 20fc858bec..20c18ae059 100644 --- a/examples/delora_finetuning/README.md +++ b/examples/delora_finetuning/README.md @@ -26,7 +26,7 @@ peft_model.print_trainable_parameters() dataset = load_dataset("imdb", split="train[:1%]") -training_args = SFTConfig(dataset_text_field="text", max_seq_length=128) +training_args = SFTConfig(dataset_text_field="text", max_length=128) trainer = SFTTrainer( model=peft_model, args=training_args, @@ -52,7 +52,7 @@ peft_model = PeftModel.from_pretrained(model, "delora-llama-3-8b") ## Advanced Usage In this script the default DeLoRA layers are the query and value layers of the Llama model. Adding adapters on more layers will increase memory usage. If you wish to choose a different set of layers for DeLoRA to be applied on, you can simply define it using: ```bash -python examples/delora_finetuning/delora_finetuning.py --base_model meta-llama/Meta-Llama-3-8B --delora_target_modules "q_proj,k_proj,v_proj,o_proj" +python examples/delora_finetuning/delora_finetuning.py --base_model meta-llama/Meta-Llama-3-8B --target_modules "q_proj,k_proj,v_proj,o_proj" ``` Using different lambdas for different layers is also possible by setting `lambda_pattern`. @@ -74,7 +74,7 @@ python delora_finetuning.py \ --rank 32 \ --delora_lambda 15 \ --module_dropout 0.1 \ - --delora_target_modules "q_proj,v_proj" \ + --target_modules "q_proj,v_proj" \ --hub_model_id "YOUR_HF_REPO" \ --push_to_hub ``` diff --git a/examples/dora_finetuning/README.md b/examples/dora_finetuning/README.md index 91ffb2a529..20250476e4 100644 --- a/examples/dora_finetuning/README.md +++ b/examples/dora_finetuning/README.md @@ -24,7 +24,7 @@ trainer = transformers.Trainer( model=peft_model, train_dataset=dataset, dataset_text_field="text", - max_seq_length=2048, + max_length=2048, tokenizer=tokenizer, ) trainer.train() diff --git a/examples/lorafa_finetune/README.md b/examples/lorafa_finetune/README.md index 432c93ad83..1c1314dbd6 100644 --- a/examples/lorafa_finetune/README.md +++ b/examples/lorafa_finetune/README.md @@ -40,7 +40,7 @@ trainer = transformers.Trainer( model=peft_model, train_dataset=dataset, dataset_text_field="text", - max_seq_length=2048, + max_length=2048, processing_class=tokenizer, optimizers=(optimizer, None), ) diff --git a/examples/miss_finetuning/README.md b/examples/miss_finetuning/README.md index bcc1bb33d8..1e11064d8d 100644 --- a/examples/miss_finetuning/README.md +++ b/examples/miss_finetuning/README.md @@ -36,7 +36,7 @@ peft_model.print_trainable_parameters() dataset = load_dataset("imdb", split="train[:1%]") -training_args = SFTConfig(dataset_text_field="text", max_seq_length=128) +training_args = SFTConfig(dataset_text_field="text", max_length=128) trainer = SFTTrainer( model=peft_model, args=training_args, diff --git a/examples/olora_finetuning/README.md b/examples/olora_finetuning/README.md index bb548b77b2..3b9bbf68ee 100644 --- a/examples/olora_finetuning/README.md +++ b/examples/olora_finetuning/README.md @@ -18,7 +18,7 @@ lora_config = LoraConfig( init_lora_weights="olora" ) peft_model = get_peft_model(model, lora_config) -training_args = SFTConfig(dataset_text_field="text", max_seq_length=128) +training_args = SFTConfig(dataset_text_field="text", max_length=128) trainer = SFTTrainer( model=peft_model, train_dataset=dataset, diff --git a/examples/pissa_finetuning/README.md b/examples/pissa_finetuning/README.md index 4c0734bdcf..6ab1515679 100644 --- a/examples/pissa_finetuning/README.md +++ b/examples/pissa_finetuning/README.md @@ -23,7 +23,7 @@ peft_model.print_trainable_parameters() dataset = load_dataset("imdb", split="train[:1%]") -training_args = SFTConfig(dataset_text_field="text", max_seq_length=128) +training_args = SFTConfig(dataset_text_field="text", max_length=128) trainer = SFTTrainer( model=peft_model, args=training_args, diff --git a/examples/randlora_finetuning/README.md b/examples/randlora_finetuning/README.md index fa9d2d61de..f8eb7d77d0 100644 --- a/examples/randlora_finetuning/README.md +++ b/examples/randlora_finetuning/README.md @@ -20,7 +20,7 @@ trainer = transformers.Trainer( model=peft_model, train_dataset=dataset, dataset_text_field="text", - max_seq_length=2048, + max_length=2048, processing_class=tokenizer, ) trainer.train() diff --git a/examples/road_finetuning/README.md b/examples/road_finetuning/README.md index b9ce14017c..00a043936f 100644 --- a/examples/road_finetuning/README.md +++ b/examples/road_finetuning/README.md @@ -26,7 +26,7 @@ trainer = transformers.Trainer( model=peft_model, train_dataset=dataset, dataset_text_field="text", - max_seq_length=2048, + max_length=2048, tokenizer=tokenizer, ) trainer.train() diff --git a/examples/sft/train.py b/examples/sft/train.py index 5a34f69357..77efac3609 100644 --- a/examples/sft/train.py +++ b/examples/sft/train.py @@ -18,7 +18,7 @@ class ModelArguments: model_name_or_path: str = field( metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"} ) - max_seq_length: Optional[int] = field( + max_length: Optional[int] = field( default=512, metadata={"help": "The maximum total input sequence length after tokenization."}, ) diff --git a/examples/shira_finetuning/README.md b/examples/shira_finetuning/README.md index 45b1e99be6..76d062e2c7 100644 --- a/examples/shira_finetuning/README.md +++ b/examples/shira_finetuning/README.md @@ -18,7 +18,7 @@ shira_config = ShiraConfig( r=32, ) peft_model = get_peft_model(model, shira_config) -training_args = SFTConfig(dataset_text_field="text", max_seq_length=128) +training_args = SFTConfig(dataset_text_field="text", max_length=128) trainer = SFTTrainer( model=peft_model, train_dataset=dataset, diff --git a/examples/waveft_finetuning/README.md b/examples/waveft_finetuning/README.md index ad2d231698..820ead27aa 100644 --- a/examples/waveft_finetuning/README.md +++ b/examples/waveft_finetuning/README.md @@ -20,7 +20,7 @@ waveft_config = WaveFTConfig( n_frequency=2592, ) peft_model = get_peft_model(model, waveft_config) -training_args = SFTConfig(dataset_text_field="text", max_seq_length=128) +training_args = SFTConfig(dataset_text_field="text", max_length=128) trainer = SFTTrainer( model=peft_model, train_dataset=dataset, From 6a3909a00d26f850bbe08fed8e72e4de20b7bffb Mon Sep 17 00:00:00 2001 From: "Liu, Kaixuan" Date: Thu, 23 Oct 2025 13:30:04 +0000 Subject: [PATCH 4/4] revert the change in sft Signed-off-by: Liu, Kaixuan --- examples/sft/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/sft/train.py b/examples/sft/train.py index 77efac3609..5a34f69357 100644 --- a/examples/sft/train.py +++ b/examples/sft/train.py @@ -18,7 +18,7 @@ class ModelArguments: model_name_or_path: str = field( metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"} ) - max_length: Optional[int] = field( + max_seq_length: Optional[int] = field( default=512, metadata={"help": "The maximum total input sequence length after tokenization."}, )