Merge pull request #305 from shijinpjlab/dev_1218

feat: support excel

Author: shijinpjlab

dingo/config/__init__.py

@@ -1,2 +1,2 @@
-from dingo.config.input_args import (DatasetArgs, DatasetFieldArgs, DatasetHFConfigArgs, DatasetS3ConfigArgs, DatasetSqlArgs, EvalPipline, EvalPiplineConfig, EvaluatorLLMArgs,  # noqa E402.
-                                     EvaluatorRuleArgs, ExecutorArgs, ExecutorResultSaveArgs, InputArgs)
+from dingo.config.input_args import (DatasetArgs, DatasetExcelArgs, DatasetFieldArgs, DatasetHFConfigArgs, DatasetS3ConfigArgs, DatasetSqlArgs, EvalPipline, EvalPiplineConfig,  # noqa E402.
+                                     EvaluatorLLMArgs, EvaluatorRuleArgs, ExecutorArgs, ExecutorResultSaveArgs, InputArgs)

dingo/config/input_args.py

@@ -27,6 +27,11 @@ class DatasetSqlArgs(BaseModel):
     connect_args: str = ''  # 连接参数，如 ?charset=utf8mb4
 
 
+class DatasetExcelArgs(BaseModel):
+    sheet_name: str | int = 0  # 默认读取第一个工作表
+    has_header: bool = True  # 第一行是否为列名，False 则使用列序号作为列名
+
+
 class DatasetFieldArgs(BaseModel):
     id: str = ''
     prompt: str = ''
@@ -43,6 +48,7 @@ class DatasetArgs(BaseModel):
     hf_config: DatasetHFConfigArgs = DatasetHFConfigArgs()
     s3_config: DatasetS3ConfigArgs = DatasetS3ConfigArgs()
     sql_config: DatasetSqlArgs = DatasetSqlArgs()
+    excel_config: DatasetExcelArgs = DatasetExcelArgs()
 
 
 class ExecutorResultSaveArgs(BaseModel):

dingo/data/converter/base.py

@@ -261,6 +261,25 @@ def _convert(raw: Union[str, Dict]):
         return _convert
 
 
+@BaseConverter.register("excel")
+class ExcelConverter(BaseConverter):
+    """Excel file converter."""
+
+    def __init__(self):
+        super().__init__()
+
+    @classmethod
+    def convertor(cls, input_args: InputArgs) -> Callable:
+        def _convert(raw: Union[str, Dict]):
+            j = raw
+            if isinstance(raw, str):
+                j = json.loads(raw)
+            data_dict = j
+            return Data(**data_dict)
+
+        return _convert
+
+
 @BaseConverter.register("listjson")
 class ListJsonConverter(BaseConverter):
     """List json file converter."""