[Dataset] add dataset LCB_pro

opencompass/configs/datasets/livecodebench/livecodebench_pro_gen.py → opencompass/configs/datasets/livecodebench_pro/livecodebench_pro_gen.py

@@ -3,6 +3,7 @@
 from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.datasets import (
     LCBProDataset,
+    LCBProEvaluator,
 )
 
 lcb_pro_reader_cfg = dict(
@@ -29,7 +30,14 @@
 )
 
 lcb_pro_eval_cfg = dict(
-    evaluator=dict()
+    evaluator=dict(
+        type=LCBProEvaluator,
+        submit_url='http://lightcpverifier.ailab.ailab.ai/submit',
+        result_url='http://lightcpverifier.ailab.ailab.ai/result/{submission_id}',
+        timeout=10,
+        poll_interval=10,
+        max_retries=3,
+    )
 )
 
 lcb_pro_datasets = [

opencompass/datasets/__init__.py

@@ -93,6 +93,7 @@
 from .lcsts import *  # noqa: F401, F403
 from .leval import *  # noqa: F401, F403
 from .livecodebench import *  # noqa: F401, F403
+from .livecodebench_pro import *  # noqa: F401, F403
 from .livemathbench import *  # noqa: F401, F403
 from .livereasonbench import *  # noqa: F401, F403
 from .livestembench import *  # noqa: F401, F403

opencompass/datasets/livecodebench_pro/__init__.py

@@ -0,0 +1,4 @@
+from .livecodebench_pro import LCBProDataset  # noqa: F401, F403
+from .livecodebench_pro_evaluator import LCBProEvaluator  # noqa: F401, F403
+
+__all__ = ['LCBProDataset', 'LCBProEvaluator']

opencompass/datasets/livecodebench_pro/livecodebench_pro.py

@@ -0,0 +1,26 @@
+import json
+
+from datasets import Dataset
+
+from opencompass.utils import get_data_path  # noqa: F401, F403
+
+from ..base import BaseDataset
+
+
+class LCBProDataset(BaseDataset):
+
+    @staticmethod
+    def load(path, **kwargs):
+        path = get_data_path(path)
+        dataset_list = []
+        li = 0
+        with open(path, 'r', encoding='utf-8') as f:
+            for line in f:
+                line = line.strip()
+                data = json.loads(line)
+                dataset_list.append({
+                    'id_ddm': data['id_ddm'],
+                    'problem': data['dialogs'][0]['content']
+                })
+                li += 1
+        return Dataset.from_list(dataset_list)