feat: add ChatMLConvertor for chatml-jsonl format and enhance Executor

Author: zihan-huang

.gitignore

@@ -0,0 +1,2 @@
+__pycache__/
+*.egg-info/

dingo/data/converter/base.py

@@ -45,6 +45,43 @@ def find_levels_image(cls, data: json, levels: str) -> List:
         res = reduce(lambda x, y: x[y], levels.split('.'), data)
         return res if isinstance(res, List) else [res]
 
+@BaseConverter.register("chatml-jsonl")
+class ChatMLConvertor(BaseConverter):
+    """
+    ddm chatml file converter.
+    """
+
+    def __init__(self):
+        super().__init__()
+
+    @classmethod
+    def convertor(cls, input_args: InputArgs) -> Callable:
+        def _convert(raw: Union[str, Dict]):
+            j = raw
+            if isinstance(raw, str):
+                j = json.loads(raw)
+
+            dialogs: list = j["dialogs"]
+            prompt = ""
+            content = ""
+
+            for i in dialogs[:-1]:
+                prompt += f"{i['role']:}\n\n"
+                prompt += f"{i['content']}\n\n"
+
+            if len(dialogs) > 1:
+                prompt += dialogs[-1]["role"]
+                content += dialogs[-1]["content"]
+
+            return MetaData(**{
+                'data_id': j["_id"],
+                'prompt': prompt,
+                'content': content,
+                'raw_data': j
+            })
+
+        return _convert
+
 
 @BaseConverter.register('json')
 class JsonConverter(BaseConverter):

dingo/exec/local.py

@@ -94,7 +94,7 @@ def evaluate(self):
             group (Any): _description_
             group_type (str): _description_
         """
-        with concurrent.futures.ThreadPoolExecutor(max_workers=self.input_args.max_workers) as executor:
+        with concurrent.futures.ProcessPoolExecutor(max_workers=self.input_args.max_workers) as executor:
             data_iter = self.load_data()
             data_iter = itertools.islice(data_iter, self.input_args.start_index, None)