Merge pull request #49 from fdcp/main

Author: anxiangsir

examples/llava_ov_1_5/sample_packing/1_s1_get_tokenlens_v3-sft.py

@@ -207,7 +207,8 @@ def merge_files_by_token(input_files, output_file, max_token=MAX_TOKEN_LEN):
         return (None, 0)
 
     def sort_key(line):
-        _, token_str = line.strip().split(':', 1)
+        # _, token_str = line.strip().split(':', 1)
+        token_str = line.strip().split(':')[-1]
         return int(token_str)
 
     try:

examples_offline_packing/bmr_packing/s1_get_tokenlens_v4-sft.py

@@ -263,7 +263,7 @@ def merge_files_by_token(input_files, output_file, max_token=MAX_TOKEN_LEN):
 
     # 定义排序键（按token_len整数排序）
     def sort_key(line):
-        _, token_str = line.strip().split(':', 1)
+        token_str = line.strip().split(':')[-1]
         return int(token_str)
 
     try:

examples_offline_packing/captions-packing/s1_get_tokenlens_v2.py

@@ -218,7 +218,7 @@ def merge_files_by_token(input_files, output_file, max_token=MAX_TOKEN_LEN):
 
     # 定义排序键（按token_len整数排序）
     def sort_key(line):
-        _, token_str = line.strip().split(':', 1)
+        token_str = line.strip().split(':')[-1]
         return int(token_str)
 
     try:

tools/data_preprocess/offline_packing/s1_get_tokenlens_v2.py

@@ -311,7 +311,8 @@ def merge_files_by_token(input_files, output_file, max_token=MAX_TOKEN_LEN):
 
     # 定义排序键（按token_len整数排序）
     def sort_key(line):
-        _, token_str = line.strip().split(':', 1)
+        # _, token_str = line.strip().split(':', 1)
+        token_str = line.strip().split(':')[-1]
         return int(token_str)
 
     try:

tools/data_preprocess/offline_packing/s1_get_tokenlens_v3-sft.py

@@ -321,7 +321,8 @@ def merge_files_by_token(input_files, output_file, max_token=MAX_TOKEN_LEN):
 
     # 定义排序键（按token_len整数排序）
     def sort_key(line):
-        _, token_str = line.strip().split(':', 1)
+        # _, token_str = line.strip().split(':', 1)
+        token_str = line.strip().split(':')[-1]
         return int(token_str)
 
     try: