remove helper.so and add a new way to compile it when start training

Author: rankcoder

mft_peft_hf/src/data/gpt2_multi_task_dataset.py

@@ -353,3 +353,16 @@ def load_dataset_from_jsonl(args, shard_data=False, world_size=1, global_rank=0,
         blending_valid_dataset = GPT2BlendableDataset(all_valid_datasets, valid_sample_weights, global_valid_num, local_valid_num)
 
     return blending_train_dataset, blending_valid_dataset
+
+
+def compile_helper():
+    """Compile helper function at runtime. Make sure this
+    is invoked on a single process."""
+    import os
+    import subprocess
+    path = os.path.abspath(os.path.dirname(__file__))
+    ret = subprocess.run(["make", "-C", path])
+    if ret.returncode != 0:
+        print("Making C++ dataset helpers module failed, exiting.")
+        import sys
+        sys.exit(1)

mft_peft_hf/src/data/helpers.cpython-38-x86_64-linux-gnu.so

@@ -44,7 +44,7 @@
 from accelerate.logging import get_logger
 
 sys.path.append("..")
-from data.gpt2_multi_task_dataset import load_dataset_from_jsonl
+from data.gpt2_multi_task_dataset import load_dataset_from_jsonl, compile_helper
 from utils.common_utils import generate_task_id, TASK2ID, ID2TASK
 from train_utils import accelerate_train
 from model_mapping import MODEL_TYPES, QLORA_TARGETING_MODULES, MODEL_SPECIAL_TOKENS
@@ -226,9 +226,12 @@ def main():
     if accelerator.is_local_main_process:
         datasets.utils.logging.set_verbosity_warning()
         transformers.utils.logging.set_verbosity_info()
+        compile_helper()
+        time.sleep(10)
     else:
         datasets.utils.logging.set_verbosity_error()
         transformers.utils.logging.set_verbosity_error()
+        time.sleep(10)
 
     if args.seed is not None:
         set_seed(args.seed)