remove old dataset. (#2561)

Author: Jonathans575

examples/run_finetune.py

@@ -148,7 +148,7 @@ def main():
         model_config.fuse_attention_ffn = model_args.fuse_attention_ffn
     model_config.pp_seg_method = training_args.pp_seg_method
     model_config.seq_length = data_args.max_length
-    model_config.max_sequence_length = training_args.max_seq_length
+    model_config.max_sequence_length = training_args.max_seq_len
     model_config.num_nextn_predict_layers = model_args.num_nextn_predict_layers
     logger.info(f"Final model config: {model_config}")
     logger.info("Creating model")
@@ -213,11 +213,11 @@ def neft_post_hook(module, input, output):
 
     dataset_config = {
         "tokenizer": tokenizer,
-        "max_seq_len": training_args.max_seq_length,
+        "max_seq_len": training_args.max_seq_len,
         "random_seed": training_args.seed,
-        "num_replicas": 1,
-        "rank": 0,
-        "num_samples_each_epoch": 6000000,
+        "num_replicas": training_args.dataset_world_size,
+        "rank": training_args.dataset_rank,
+        "num_samples_each_epoch": data_args.num_samples_each_epoch,
         "random_shuffle": data_args.random_shuffle,
         "greedy_intokens": data_args.greedy_intokens,
         "packing": data_args.packing,
@@ -251,7 +251,7 @@ def neft_post_hook(module, input, output):
         collate_fn,
         tokenizer=tokenizer,
         model_args=model_args,
-        max_seq_len=training_args.max_seq_length + model_config.num_nextn_predict_layers,
+        max_seq_len=training_args.max_seq_len + model_config.num_nextn_predict_layers,
     )
     trainer = SFTTrainer(
         model=model,

paddleformers/data/__init__.py

@@ -20,22 +20,6 @@
 
 import_structure = {
     "sampler": ["SamplerHelper"],
-    "causal_dataset": [
-        "check_data_split",
-        "get_train_valid_test_split_",
-        "get_datasets_weights_and_num_samples",
-        "print_rank_0",
-        "build_train_valid_test_datasets",
-        "_build_train_valid_test_datasets",
-        "get_indexed_dataset_",
-        "GPTDataset",
-        "_build_index_mappings",
-        "_num_tokens",
-        "_num_epochs",
-        "_build_doc_idx",
-        "_build_sample_idx",
-        "_build_shuffle_idx",
-    ],
     "data_collator": [
         "DataCollatorForSeq2Seq",
         "default_data_collator",
@@ -54,7 +38,6 @@
         "DataCollatorForLanguageModeling",
     ],
     "dist_dataloader": ["DummyDataset", "IterableDummyDataset", "DistDataLoader", "init_dataloader_comm_group"],
-    "blendable_dataset": ["print_rank_0", "BlendableDataset"],
     "collate": ["Dict", "Pad", "Stack", "Tuple"],
     "vocab": ["Vocab"],
     "tokenizer": ["BaseTokenizer"],
@@ -91,8 +74,6 @@
 
 
 if TYPE_CHECKING:
-    from .blendable_dataset import *
-    from .causal_dataset import *
     from .collate import *
     from .data_collator import *
     from .dist_dataloader import *