bug fix (examples_offline_packing) : processing samples whose names contain some ':'

Author: fdcp

examples_offline_packing/bmr_packing/s1_get_tokenlens_v4-sft.py

@@ -263,7 +263,7 @@ def merge_files_by_token(input_files, output_file, max_token=MAX_TOKEN_LEN):
 
     # 定义排序键（按token_len整数排序）
     def sort_key(line):
-        _, token_str = line.strip().split(':', 1)
+        token_str = line.strip().split(':')[-1]
         return int(token_str)
 
     try:

examples_offline_packing/captions-packing/s1_get_tokenlens_v2.py

@@ -218,7 +218,7 @@ def merge_files_by_token(input_files, output_file, max_token=MAX_TOKEN_LEN):
 
     # 定义排序键（按token_len整数排序）
     def sort_key(line):
-        _, token_str = line.strip().split(':', 1)
+        token_str = line.strip().split(':')[-1]
         return int(token_str)
 
     try: