[bugfix] fix streaming & packing (#5403)

Author: Jintao-Huang

requirements/install_all.sh

@@ -7,7 +7,7 @@ pip install autoawq -U --no-deps
 pip install auto_gptq optimum bitsandbytes "gradio<5.33" -U
 pip install git+https://github.com/modelscope/ms-swift.git
 pip install timm -U
-pip install "deepspeed<0.17" -U
+pip install "deepspeed" -U
 pip install qwen_vl_utils qwen_omni_utils decord librosa icecream soundfile -U
 pip install liger_kernel nvitop pre-commit math_verify py-spy -U
 # flash-attn: https://github.com/Dao-AILab/flash-attention/releases

swift/llm/train/sft.py

@@ -127,14 +127,7 @@ def _prepare_dataset(self):
             if i == 1 and predict_with_generate:
                 # val_dataset
                 continue
-            if args.streaming:
-                preprocessor = EncodePreprocessor(template=template)
-                dataset = preprocessor(
-                    dataset,
-                    num_proc=args.dataset_num_proc,
-                    load_from_cache_file=args.load_from_cache_file,
-                    strict=args.strict)
-            elif (args.model_meta.is_multimodal or args.lazy_tokenize):
+            if (args.model_meta.is_multimodal or args.lazy_tokenize) and not args.streaming:
                 dataset = LazyLLMDataset(dataset, template.encode, strict=args.strict, random_state=args.data_seed)
             if args.packing:
                 packing_dataset_cls = IterablePackingDataset if args.streaming else PackingDataset
@@ -144,6 +137,13 @@ def _prepare_dataset(self):
                     num_proc=args.dataset_num_proc,
                     strict=args.strict,
                     load_from_cache_file=args.load_from_cache_file)
+            elif args.streaming:
+                preprocessor = EncodePreprocessor(template=template)
+                dataset = preprocessor(
+                    dataset,
+                    num_proc=args.dataset_num_proc,
+                    load_from_cache_file=args.load_from_cache_file,
+                    strict=args.strict)
             datasets[i] = dataset
         self._show_dataset(*datasets)
         return datasets

swift/megatron/argument/megatron_args.py

@@ -378,6 +378,9 @@ def __post_init__(self):
         self.tensorboard_dir = to_abspath(self.tensorboard_dir)
         self.extra_megatron_kwargs = json_parse_to_dict(self.extra_megatron_kwargs)
         self._init_no_rope_fusion()
+        if self.load is None and self.no_initialization:
+            raise ValueError('You did not pass `--load`, so you need to set `--no_initialization false` '
+                             'to allow the model to initialize weights properly.')
 
     def _init_no_rope_fusion(self):
         if self.no_rope_fusion is not None: