make sure to validate the config before normalizing so defaults get set (axolotl-ai-cloud#2554)

winglian · web-flow · commit 1447beb1328f · 2025-04-24T13:01:43.000-04:00
* make sure to validate the config before normalizing so defaults get set

* validation not needed for particular test

* remove duplicate validations

* set qlora correctly
diff --git a/tests/e2e/integrations/test_cut_cross_entropy.py b/tests/e2e/integrations/test_cut_cross_entropy.py
@@ -8,7 +8,7 @@
 from axolotl.common.datasets import load_datasets
 from axolotl.train import train
 from axolotl.utils import get_pytorch_version
-from axolotl.utils.config import normalize_config, prepare_plugins
+from axolotl.utils.config import normalize_config, prepare_plugins, validate_config
 from axolotl.utils.dict import DictDefault
 
 from ..utils import check_model_output_exists
@@ -56,6 +56,7 @@ class TestCutCrossEntropyIntegration:
     # pylint: disable=redefined-outer-name
     def test_llama_w_cce(self, min_cfg, temp_dir):
         cfg = DictDefault(min_cfg)
+        cfg = validate_config(cfg)
         prepare_plugins(cfg)
         normalize_config(cfg)
         cli_args = TrainerCliArgs()
@@ -101,6 +102,7 @@ def test_qwen2_w_cce(self, temp_dir):
                 "bf16": "auto",
             }
         )
+        cfg = validate_config(cfg)
         prepare_plugins(cfg)
         normalize_config(cfg)
         cli_args = TrainerCliArgs()
@@ -129,6 +131,7 @@ def test_llama_w_cce_and_attention(self, min_cfg, temp_dir, attention_type):
                 attention_type: True,
             }
         )
+        cfg = validate_config(cfg)
         prepare_plugins(cfg)
         normalize_config(cfg)
         cli_args = TrainerCliArgs()
diff --git a/tests/e2e/integrations/test_liger.py b/tests/e2e/integrations/test_liger.py
@@ -5,7 +5,7 @@
 from axolotl.cli.args import TrainerCliArgs
 from axolotl.common.datasets import load_datasets
 from axolotl.train import train
-from axolotl.utils.config import normalize_config, prepare_plugins
+from axolotl.utils.config import normalize_config, prepare_plugins, validate_config
 from axolotl.utils.dict import DictDefault
 
 from tests.e2e.utils import check_model_output_exists, require_torch_2_4_1
@@ -54,6 +54,7 @@ def test_llama_wo_flce(self, temp_dir):
             }
         )
         # pylint: disable=duplicate-code
+        cfg = validate_config(cfg)
         prepare_plugins(cfg)
         normalize_config(cfg)
         cli_args = TrainerCliArgs()
@@ -100,6 +101,7 @@ def test_llama_w_flce(self, temp_dir):
             }
         )
         # pylint: disable=duplicate-code
+        cfg = validate_config(cfg)
         prepare_plugins(cfg)
         normalize_config(cfg)
         cli_args = TrainerCliArgs()
diff --git a/tests/e2e/patched/test_4d_multipack_llama.py b/tests/e2e/patched/test_4d_multipack_llama.py
@@ -9,7 +9,7 @@
 from axolotl.cli.args import TrainerCliArgs
 from axolotl.common.datasets import load_datasets
 from axolotl.train import train
-from axolotl.utils.config import normalize_config
+from axolotl.utils.config import normalize_config, validate_config
 from axolotl.utils.dict import DictDefault
 
 from ..utils import check_model_output_exists, with_temp_dir
@@ -60,6 +60,7 @@ def test_sdp_lora_packing(self, temp_dir):
                 "fp16": True,
             }
         )
+        cfg = validate_config(cfg)
         normalize_config(cfg)
         cli_args = TrainerCliArgs()
         dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
@@ -104,6 +105,7 @@ def test_torch_lora_packing(self, temp_dir):
                 "fp16": True,
             }
         )
+        cfg = validate_config(cfg)
         normalize_config(cfg)
         cli_args = TrainerCliArgs()
         dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
diff --git a/tests/e2e/patched/test_falcon_samplepack.py b/tests/e2e/patched/test_falcon_samplepack.py
@@ -9,7 +9,7 @@
 from axolotl.cli.args import TrainerCliArgs
 from axolotl.common.datasets import load_datasets
 from axolotl.train import train
-from axolotl.utils.config import normalize_config
+from axolotl.utils.config import normalize_config, validate_config
 from axolotl.utils.dict import DictDefault
 
 from ..utils import check_model_output_exists, with_temp_dir
@@ -63,6 +63,7 @@ def test_qlora(self, temp_dir):
                 "bf16": "auto",
             }
         )
+        cfg = validate_config(cfg)
         normalize_config(cfg)
         cli_args = TrainerCliArgs()
         dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
@@ -103,6 +104,7 @@ def test_ft(self, temp_dir):
                 "bf16": "auto",
             }
         )
+        cfg = validate_config(cfg)
         normalize_config(cfg)
         cli_args = TrainerCliArgs()
         dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
diff --git a/tests/e2e/patched/test_fused_llama.py b/tests/e2e/patched/test_fused_llama.py
@@ -12,7 +12,7 @@
 from axolotl.cli.args import TrainerCliArgs
 from axolotl.common.datasets import load_datasets
 from axolotl.train import train
-from axolotl.utils.config import normalize_config
+from axolotl.utils.config import normalize_config, validate_config
 from axolotl.utils.dict import DictDefault
 
 from ..utils import check_model_output_exists, with_temp_dir
@@ -67,6 +67,7 @@ def test_fft_packing(self, temp_dir):
             cfg.bf16 = True
         else:
             cfg.fp16 = True
+        cfg = validate_config(cfg)
         normalize_config(cfg)
         cli_args = TrainerCliArgs()
         dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
diff --git a/tests/e2e/patched/test_llama_s2_attention.py b/tests/e2e/patched/test_llama_s2_attention.py
@@ -11,7 +11,7 @@
 from axolotl.cli.args import TrainerCliArgs
 from axolotl.common.datasets import load_datasets
 from axolotl.train import train
-from axolotl.utils.config import normalize_config
+from axolotl.utils.config import normalize_config, validate_config
 from axolotl.utils.dict import DictDefault
 
 from ..utils import check_model_output_exists, with_temp_dir
@@ -65,6 +65,7 @@ def test_lora_s2_attn(self, temp_dir):
             }
         )
 
+        cfg = validate_config(cfg)
         normalize_config(cfg)
         cli_args = TrainerCliArgs()
         dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
@@ -105,6 +106,7 @@ def test_fft_s2_attn(self, temp_dir):
             }
         )
 
+        cfg = validate_config(cfg)
         normalize_config(cfg)
         cli_args = TrainerCliArgs()
         dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
diff --git a/tests/e2e/patched/test_lora_llama_multipack.py b/tests/e2e/patched/test_lora_llama_multipack.py
@@ -12,7 +12,7 @@
 from axolotl.cli.args import TrainerCliArgs
 from axolotl.common.datasets import load_datasets
 from axolotl.train import train
-from axolotl.utils.config import normalize_config
+from axolotl.utils.config import normalize_config, validate_config
 from axolotl.utils.dict import DictDefault
 
 from ..utils import check_model_output_exists, with_temp_dir
@@ -70,6 +70,7 @@ def test_lora_packing(self, temp_dir):
         else:
             cfg.fp16 = True
 
+        cfg = validate_config(cfg)
         normalize_config(cfg)
         cli_args = TrainerCliArgs()
         dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
@@ -120,6 +121,7 @@ def test_lora_gptq_packed(self, temp_dir):
                 "lr_scheduler": "cosine",
             }
         )
+        cfg = validate_config(cfg)
         normalize_config(cfg)
         cli_args = TrainerCliArgs()
         dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
diff --git a/tests/e2e/patched/test_mistral_samplepack.py b/tests/e2e/patched/test_mistral_samplepack.py
@@ -9,7 +9,7 @@
 from axolotl.cli.args import TrainerCliArgs
 from axolotl.common.datasets import load_datasets
 from axolotl.train import train
-from axolotl.utils.config import normalize_config
+from axolotl.utils.config import normalize_config, validate_config
 from axolotl.utils.dict import DictDefault
 
 from ..utils import check_model_output_exists, with_temp_dir
@@ -63,6 +63,7 @@ def test_lora_packing(self, temp_dir):
                 "bf16": "auto",
             }
         )
+        cfg = validate_config(cfg)
         normalize_config(cfg)
         cli_args = TrainerCliArgs()
         dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
@@ -104,6 +105,7 @@ def test_ft_packing(self, temp_dir):
                 "bf16": "auto",
             }
         )
+        cfg = validate_config(cfg)
         normalize_config(cfg)
         cli_args = TrainerCliArgs()
         dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
diff --git a/tests/e2e/patched/test_mixtral_samplepack.py b/tests/e2e/patched/test_mixtral_samplepack.py
@@ -9,7 +9,7 @@
 from axolotl.cli.args import TrainerCliArgs
 from axolotl.common.datasets import load_datasets
 from axolotl.train import train
-from axolotl.utils.config import normalize_config
+from axolotl.utils.config import normalize_config, validate_config
 from axolotl.utils.dict import DictDefault
 
 from ..utils import check_model_output_exists, with_temp_dir
@@ -60,6 +60,7 @@ def test_qlora(self, temp_dir):
                 "bf16": "auto",
             }
         )
+        cfg = validate_config(cfg)
         normalize_config(cfg)
         cli_args = TrainerCliArgs()
         dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
diff --git a/tests/e2e/patched/test_model_patches.py b/tests/e2e/patched/test_model_patches.py
@@ -6,7 +6,7 @@
 
 import transformers
 
-from axolotl.utils.config import normalize_config
+from axolotl.utils.config import normalize_config, validate_config
 from axolotl.utils.dict import DictDefault
 from axolotl.utils.models import load_model, load_tokenizer
 
@@ -47,6 +47,7 @@ def test_mixtral_multipack(self, temp_dir):
                 "eval_steps": 10,
             }
         )
+        cfg = validate_config(cfg)
         normalize_config(cfg)
         tokenizer = load_tokenizer(cfg)
         load_model(cfg, tokenizer, inference=False)
@@ -79,6 +80,7 @@ def test_mistral_multipack(self, temp_dir):
                 "eval_steps": 10,
             }
         )
+        cfg = validate_config(cfg)
         normalize_config(cfg)
         tokenizer = load_tokenizer(cfg)
         load_model(cfg, tokenizer, inference=False)
diff --git a/tests/e2e/patched/test_phi_multipack.py b/tests/e2e/patched/test_phi_multipack.py
@@ -9,7 +9,7 @@
 from axolotl.cli.args import TrainerCliArgs
 from axolotl.common.datasets import load_datasets
 from axolotl.train import train
-from axolotl.utils.config import normalize_config
+from axolotl.utils.config import normalize_config, validate_config
 from axolotl.utils.dict import DictDefault
 
 from ..utils import check_model_output_exists, with_temp_dir
@@ -63,6 +63,7 @@ def test_ft_packed(self, temp_dir):
             }
         )
 
+        cfg = validate_config(cfg)
         normalize_config(cfg)
         cli_args = TrainerCliArgs()
         dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
@@ -82,7 +83,7 @@ def test_qlora_packed(self, temp_dir):
                 "sample_packing": True,
                 "flash_attention": True,
                 "pad_to_sequence_len": True,
-                "load_in_8bit": False,
+                "load_in_4bit": True,
                 "adapter": "qlora",
                 "lora_r": 64,
                 "lora_alpha": 32,
@@ -114,6 +115,7 @@ def test_qlora_packed(self, temp_dir):
             }
         )
 
+        cfg = validate_config(cfg)
         normalize_config(cfg)
         cli_args = TrainerCliArgs()
         dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
diff --git a/tests/e2e/patched/test_resume.py b/tests/e2e/patched/test_resume.py
@@ -12,7 +12,7 @@
 from axolotl.cli.args import TrainerCliArgs
 from axolotl.common.datasets import load_datasets
 from axolotl.train import train
-from axolotl.utils.config import normalize_config
+from axolotl.utils.config import normalize_config, validate_config
 from axolotl.utils.dict import DictDefault
 
 from ..utils import check_model_output_exists, most_recent_subdir
@@ -68,6 +68,7 @@ def test_resume_lora_packed(self, temp_dir):
             cfg.bf16 = True
         else:
             cfg.fp16 = True
+        cfg = validate_config(cfg)
         normalize_config(cfg)
         cli_args = TrainerCliArgs()
         dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
diff --git a/tests/e2e/patched/test_unsloth_qlora.py b/tests/e2e/patched/test_unsloth_qlora.py
@@ -10,7 +10,7 @@
 from axolotl.cli.args import TrainerCliArgs
 from axolotl.common.datasets import load_datasets
 from axolotl.train import train
-from axolotl.utils.config import normalize_config
+from axolotl.utils.config import normalize_config, validate_config
 from axolotl.utils.dict import DictDefault
 
 from ..utils import check_model_output_exists, check_tensorboard
@@ -72,6 +72,7 @@ def test_unsloth_llama_qlora_fa2(self, temp_dir, sample_packing):
             }
         )
 
+        cfg = validate_config(cfg)
         normalize_config(cfg)
         cli_args = TrainerCliArgs()
         dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
@@ -122,6 +123,7 @@ def test_unsloth_llama_qlora_unpacked(self, temp_dir):
             }
         )
 
+        cfg = validate_config(cfg)
         normalize_config(cfg)
         cli_args = TrainerCliArgs()
         dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
@@ -177,6 +179,7 @@ def test_unsloth_llama_qlora_unpacked_no_fa2_fp16(self, temp_dir, sdp_attention)
             }
         )
 
+        cfg = validate_config(cfg)
         normalize_config(cfg)
         cli_args = TrainerCliArgs()
         dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
diff --git a/tests/e2e/test_embeddings_lr.py b/tests/e2e/test_embeddings_lr.py
@@ -102,6 +102,7 @@ def test_train_w_embedding_lr(self, temp_dir):
                 "use_tensorboard": True,
             }
         )
+        cfg = validate_config(cfg)
         normalize_config(cfg)
         cli_args = TrainerCliArgs()
         dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
diff --git a/tests/e2e/test_llama_vision.py b/tests/e2e/test_llama_vision.py
@@ -109,6 +109,7 @@ def test_lora_llama_vision_multimodal_dataset(self, temp_dir):
                 "bf16": True,
             }
         )
+        cfg = validate_config(cfg)
         normalize_config(cfg)
         cli_args = TrainerCliArgs()
         dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
diff --git a/tests/e2e/test_phi.py b/tests/e2e/test_phi.py
@@ -79,7 +79,7 @@ def test_phi_qlora(self, temp_dir):
                 "tokenizer_type": "AutoTokenizer",
                 "sequence_len": 2048,
                 "sample_packing": False,
-                "load_in_8bit": False,
+                "load_in_4bit": True,
                 "adapter": "qlora",
                 "lora_r": 64,
                 "lora_alpha": 32,
@@ -111,6 +111,7 @@ def test_phi_qlora(self, temp_dir):
                 "bf16": "auto",
             }
         )
+        cfg = validate_config(cfg)
         normalize_config(cfg)
         cli_args = TrainerCliArgs()
         dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
diff --git a/tests/e2e/test_process_reward_model_smollm2.py b/tests/e2e/test_process_reward_model_smollm2.py
@@ -9,7 +9,7 @@
 from axolotl.cli.args import TrainerCliArgs
 from axolotl.common.datasets import load_datasets
 from axolotl.train import train
-from axolotl.utils.config import normalize_config
+from axolotl.utils.config import normalize_config, validate_config
 from axolotl.utils.dict import DictDefault
 
 from .utils import check_model_output_exists, check_tensorboard, with_temp_dir
@@ -57,6 +57,7 @@ def test_prm(self, temp_dir):
                 "seed": 42,
             }
         )
+        cfg = validate_config(cfg)
         normalize_config(cfg)
         cli_args = TrainerCliArgs()
         dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
diff --git a/tests/test_exact_deduplication.py b/tests/test_exact_deduplication.py
@@ -11,7 +11,7 @@
 import pytest
 from datasets import Dataset
 
-from axolotl.utils.config import normalize_config
+from axolotl.utils.config import normalize_config, validate_config
 from axolotl.utils.data import prepare_dataset
 from axolotl.utils.data.rl import load_prepare_preference_datasets
 from axolotl.utils.data.utils import deduplicate_and_log_datasets
@@ -319,6 +319,7 @@ def setUp(self) -> None:
                 "num_epochs": 1,
             }
         )
+        self.cfg_1 = validate_config(self.cfg_1)
         normalize_config(self.cfg_1)
 
     @pytest.mark.skip(reason="TODO: fix hf hub offline to work with HF rate limits")

Original file line number	Diff line number	Diff line change
`@@ -5,7 +5,7 @@`
`5`	`5`	`from axolotl.cli.args import TrainerCliArgs`
`6`	`6`	`from axolotl.common.datasets import load_datasets`
`7`	`7`	`from axolotl.train import train`
`8`		`-from axolotl.utils.config import normalize_config, prepare_plugins`
	`8`	`+from axolotl.utils.config import normalize_config, prepare_plugins, validate_config`
`9`	`9`	`from axolotl.utils.dict import DictDefault`
`10`	`10`
`11`	`11`	`from tests.e2e.utils import check_model_output_exists, require_torch_2_4_1`
`@@ -54,6 +54,7 @@ def test_llama_wo_flce(self, temp_dir):`
`54`	`54`	`}`
`55`	`55`	`)`
`56`	`56`	`# pylint: disable=duplicate-code`
	`57`	`+ cfg = validate_config(cfg)`
`57`	`58`	`prepare_plugins(cfg)`
`58`	`59`	`normalize_config(cfg)`
`59`	`60`	`cli_args = TrainerCliArgs()`
`@@ -100,6 +101,7 @@ def test_llama_w_flce(self, temp_dir):`
`100`	`101`	`}`
`101`	`102`	`)`
`102`	`103`	`# pylint: disable=duplicate-code`
	`104`	`+ cfg = validate_config(cfg)`
`103`	`105`	`prepare_plugins(cfg)`
`104`	`106`	`normalize_config(cfg)`
`105`	`107`	`cli_args = TrainerCliArgs()`
Original file line number	Diff line number	Diff line change
`@@ -9,7 +9,7 @@`
`9`	`9`	`from axolotl.cli.args import TrainerCliArgs`
`10`	`10`	`from axolotl.common.datasets import load_datasets`
`11`	`11`	`from axolotl.train import train`
`12`		`-from axolotl.utils.config import normalize_config`
	`12`	`+from axolotl.utils.config import normalize_config, validate_config`
`13`	`13`	`from axolotl.utils.dict import DictDefault`
`14`	`14`
`15`	`15`	`from ..utils import check_model_output_exists, with_temp_dir`
`@@ -60,6 +60,7 @@ def test_sdp_lora_packing(self, temp_dir):`
`60`	`60`	`"fp16": True,`
`61`	`61`	`}`
`62`	`62`	`)`
	`63`	`+ cfg = validate_config(cfg)`
`63`	`64`	`normalize_config(cfg)`
`64`	`65`	`cli_args = TrainerCliArgs()`
`65`	`66`	`dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)`
`@@ -104,6 +105,7 @@ def test_torch_lora_packing(self, temp_dir):`
`104`	`105`	`"fp16": True,`
`105`	`106`	`}`
`106`	`107`	`)`
	`108`	`+ cfg = validate_config(cfg)`
`107`	`109`	`normalize_config(cfg)`
`108`	`110`	`cli_args = TrainerCliArgs()`
`109`	`111`	`dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)`
Original file line number	Diff line number	Diff line change
`@@ -11,7 +11,7 @@`
`11`	`11`	`from axolotl.cli.args import TrainerCliArgs`
`12`	`12`	`from axolotl.common.datasets import load_datasets`
`13`	`13`	`from axolotl.train import train`
`14`		`-from axolotl.utils.config import normalize_config`
	`14`	`+from axolotl.utils.config import normalize_config, validate_config`
`15`	`15`	`from axolotl.utils.dict import DictDefault`
`16`	`16`
`17`	`17`	`from ..utils import check_model_output_exists, with_temp_dir`
`@@ -65,6 +65,7 @@ def test_lora_s2_attn(self, temp_dir):`
`65`	`65`	`}`
`66`	`66`	`)`
`67`	`67`
	`68`	`+ cfg = validate_config(cfg)`
`68`	`69`	`normalize_config(cfg)`
`69`	`70`	`cli_args = TrainerCliArgs()`
`70`	`71`	`dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)`
`@@ -105,6 +106,7 @@ def test_fft_s2_attn(self, temp_dir):`
`105`	`106`	`}`
`106`	`107`	`)`
`107`	`108`
	`109`	`+ cfg = validate_config(cfg)`
`108`	`110`	`normalize_config(cfg)`
`109`	`111`	`cli_args = TrainerCliArgs()`
`110`	`112`	`dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)`
Original file line number	Diff line number	Diff line change
`@@ -6,7 +6,7 @@`
`6`	`6`
`7`	`7`	`import transformers`
`8`	`8`
`9`		`-from axolotl.utils.config import normalize_config`
	`9`	`+from axolotl.utils.config import normalize_config, validate_config`
`10`	`10`	`from axolotl.utils.dict import DictDefault`
`11`	`11`	`from axolotl.utils.models import load_model, load_tokenizer`
`12`	`12`
`@@ -47,6 +47,7 @@ def test_mixtral_multipack(self, temp_dir):`
`47`	`47`	`"eval_steps": 10,`
`48`	`48`	`}`
`49`	`49`	`)`
	`50`	`+ cfg = validate_config(cfg)`
`50`	`51`	`normalize_config(cfg)`
`51`	`52`	`tokenizer = load_tokenizer(cfg)`
`52`	`53`	`load_model(cfg, tokenizer, inference=False)`
`@@ -79,6 +80,7 @@ def test_mistral_multipack(self, temp_dir):`
`79`	`80`	`"eval_steps": 10,`
`80`	`81`	`}`
`81`	`82`	`)`
	`83`	`+ cfg = validate_config(cfg)`
`82`	`84`	`normalize_config(cfg)`
`83`	`85`	`tokenizer = load_tokenizer(cfg)`
`84`	`86`	`load_model(cfg, tokenizer, inference=False)`