diff --git a/examples/cpt_finetuning/cpt_train_and_inference.ipynb b/examples/cpt_finetuning/cpt_train_and_inference.ipynb index 0f3e13c090..789a9b40c8 100644 --- a/examples/cpt_finetuning/cpt_train_and_inference.ipynb +++ b/examples/cpt_finetuning/cpt_train_and_inference.ipynb @@ -159,7 +159,7 @@ " TrainingArguments,\n", ")\n", "\n", - "from peft import CPTConfig, get_peft_model\n", + "from peft import CPTConfig, TaskType, get_peft_model\n", "\n", "\n", "MAX_INPUT_LENGTH = 1024\n", @@ -559,6 +559,7 @@ "\n", "# Initialize the CPT configuration\n", "config = CPTConfig(\n", + " task_type=TaskType.CAUSAL_LM,\n", " cpt_token_ids=context_ids,\n", " cpt_mask=context_attention_mask,\n", " cpt_tokens_type_mask=context_input_type_mask,\n", @@ -761,7 +762,6 @@ " num_train_epochs=5,\n", " fp16=True,\n", " save_strategy='no',\n", - " logging_dir=\"logs\",\n", " report_to=\"none\"\n", ")\n", "\n", diff --git a/setup.py b/setup.py index a677afe011..665e40d3db 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ from setuptools import find_packages, setup -VERSION = "0.17.2.dev0" +VERSION = "0.18.0.rc0" extras = {} extras["quality"] = [ diff --git a/src/peft/__init__.py b/src/peft/__init__.py index af26f8309b..e8009bc21e 100644 --- a/src/peft/__init__.py +++ b/src/peft/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "0.17.2.dev0" +__version__ = "0.18.0.rc0" from .auto import ( MODEL_TYPE_TO_PEFT_MODEL_MAPPING, diff --git a/src/peft/tuners/cpt/config.py b/src/peft/tuners/cpt/config.py index 324c22995d..d23b0acc65 100644 --- a/src/peft/tuners/cpt/config.py +++ b/src/peft/tuners/cpt/config.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import warnings from dataclasses import dataclass, field from typing import Literal, Optional @@ -81,13 +80,7 @@ def __post_init__(self): self.num_transformer_submodules = 1 # Number of transformer submodules used. self.peft_type = PeftType.CPT # Specifies that the PEFT type is CPT. if self.task_type != TaskType.CAUSAL_LM: - # TODO: adjust this to raise an error with PEFT v0.18.0 - warnings.warn( - f"{self.__class__.__name__} only supports task_type = {TaskType.CAUSAL_LM.value}, " - "setting it automatically. This will raise an error starting from PEFT v0.18.0.", - FutureWarning, - ) - self.task_type = TaskType.CAUSAL_LM # Ensures task type is causal language modeling. + raise ValueError(f"{self.__class__.__name__} only supports task_type = {TaskType.CAUSAL_LM.value}.") if self.cpt_token_ids is None: self.cpt_token_ids = [0] diff --git a/tests/test_config.py b/tests/test_config.py index 4a6d8cffbd..4722f7be01 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -26,6 +26,7 @@ BOFTConfig, BoneConfig, C3AConfig, + CPTConfig, FourierFTConfig, HRAConfig, IA3Config, @@ -134,6 +135,8 @@ def test_from_peft_type(self): if expected_cls == AdaLoraConfig: mandatory_config_kwargs = {"total_step": 1} + elif expected_cls == CPTConfig: + mandatory_config_kwargs = {"task_type": TaskType.CAUSAL_LM} config = PeftConfig.from_peft_type(peft_type=peft_type, **mandatory_config_kwargs) assert type(config) is expected_cls diff --git a/tests/test_cpt.py b/tests/test_cpt.py index 6b747f8f41..64a518e4ed 100644 --- a/tests/test_cpt.py +++ b/tests/test_cpt.py @@ -55,6 +55,7 @@ def config_text(): opt_projection_epsilon=0.2, opt_projection_format_epsilon=0.1, tokenizer_name_or_path=MODEL_NAME, + task_type=TaskType.CAUSAL_LM, ) return config @@ -68,6 +69,7 @@ def config_random(): opt_projection_epsilon=0.2, opt_projection_format_epsilon=0.1, tokenizer_name_or_path=MODEL_NAME, + task_type=TaskType.CAUSAL_LM, ) return config @@ -227,12 +229,14 @@ def test_model_initialization_random(global_tokenizer, config_random): assert model is not None, "PEFT model initialization failed" -def test_model_initialization_wrong_task_type_warns(): - # TODO: adjust this test to check for an error with PEFT v0.18.0 - msg = "CPTConfig only supports task_type = CAUSAL_LM, setting it automatically" - with pytest.warns(FutureWarning, match=msg): - config = CPTConfig(task_type=TaskType.SEQ_CLS) - assert config.task_type == TaskType.CAUSAL_LM +def test_model_initialization_wrong_task_type_raises(): + msg = "CPTConfig only supports task_type = CAUSAL_LM." + with pytest.raises(ValueError, match=msg): + CPTConfig(task_type=TaskType.SEQ_CLS) + + msg = "CPTConfig only supports task_type = CAUSAL_LM." + with pytest.raises(ValueError, match=msg): + CPTConfig() def test_model_training_random(sst_data, global_tokenizer, collator, config_random): diff --git a/tests/test_decoder_models.py b/tests/test_decoder_models.py index ed7ff2b9e1..8d10e81c5f 100644 --- a/tests/test_decoder_models.py +++ b/tests/test_decoder_models.py @@ -708,6 +708,7 @@ def process(samples): ( CPTConfig, { + "task_type": "CAUSAL_LM", "cpt_token_ids": [0, 1, 2, 3, 4, 5, 6, 7], # Example token IDs for testing "cpt_mask": [1, 1, 1, 1, 1, 1, 1, 1], "cpt_tokens_type_mask": [1, 2, 2, 2, 3, 3, 4, 4], diff --git a/tests/test_vision_models.py b/tests/test_vision_models.py index 74e5d654de..0a9aec9660 100644 --- a/tests/test_vision_models.py +++ b/tests/test_vision_models.py @@ -29,6 +29,7 @@ ) from peft import ( + BOFTConfig, HRAConfig, LoHaConfig, LoKrConfig, @@ -50,10 +51,12 @@ r=1, oft_block_size=0, target_modules=["convolution"], modules_to_save=["classifier", "normalization"] ), "hra": HRAConfig(target_modules=["convolution"], modules_to_save=["classifier", "normalization"]), - # TODO: cannot use BOFT because some convolutional kernel dimensions are even (64) and others odd (147). There is no - # common denominator for the boft_block_size except 1, but using 1 results in an error in the fbd_cuda kernel: + # Cannot target multiple layers with BOFT because some convolutional kernel dimensions vary and there is no common + # denominator for the boft_block_size except 1, but using 1 results in an error in the fbd_cuda kernel: # > Error in forward_fast_block_diag_cuda_kernel: an illegal memory access was encountered - # "boft": BOFTConfig(target_modules=["convolution"], modules_to_save=["classifier", "normalization"], boft_block_size=2), + "boft": BOFTConfig( + target_modules=["0.layer.0.convolution"], modules_to_save=["classifier", "normalization"], boft_block_size=2 + ), }