dataeval_dingo/examples/multi_turn_dialogues/sdk_mtbench_rule_all.py at dev · shijinpjlab/dataeval_dingo · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
from dingo.config import InputArgs
from dingo.exec import Executor

if __name__ == '__main__':
    input_data = {
        "input_path": "lmsys/mt_bench_human_judgments",
        "dataset": {
            "source": "hugging_face",
            "format": "multi_turn_dialog",
            "hf_config": {
                "huggingface_split": "human"
            }
        },
        "executor": {
            "result_save": {
                "bad": True,
                "good": True
            },
            "end_index": 5,
            "multi_turn_mode": "all"
        },
        "evaluator": [
            {
                "fields": {"content": "conversation_a"},
                "evals": [
                    {"name": "RuleEnterAndSpace"},
                    {"name": "RuleAbnormalChar"}
                ],
            }
        ]
    }
    input_args = InputArgs(**input_data)
    executor = Executor.exec_map["local"](input_args)
    result = executor.execute()
    print(result)