[Update] Update LiveMathBench Hard Configs (open-compass#1826)

* support G-Pass@k and livemathbench

* fix bugs

* fix comments of GPassKEvaluator

* update saved details of GPassKEvaluator

* update saved details of GPassKEvaluator

* fix eval api configs & update openai_api for ease of debugging

* update huggingface path

* fix method name of G-Pass@k

* fix default value of eval_model_name

* refactor G-Pass@k evaluator

* log generation params for each backend

* fix evaluation resume

* add notimplementerror

* update livemathbench-hard configs

* remove max_out_len from livemathbench_hard_greedy_gen_9befbf.py

* remove max_out_len from livemathbench_hard_gen_9befbf.py

* rename livemathbench_hard_gen_9befbf.py to livemathbench_hard_gen_353ae7.py

* rename livemathbench_hard_greedy_gen_9befbf.py to livemathbench_hard_greedy_gen_353ae7.py

* update livemathbench_gen_9befbf.py

* remove whitespace

* upload livemathbench hard configs

Author: jnanliu

opencompass/configs/datasets/livemathbench/livemathbench_gen_9befbf.py

@@ -48,4 +48,4 @@
         )
     )
 )
-livemathbench_datasets = [livemathbench_dataset]
+livemathbench_datasets = [livemathbench_dataset]

opencompass/configs/datasets/livemathbench/livemathbench_gen_f1c095.py

@@ -1,4 +1,4 @@
 from mmengine.config import read_base
 
 with read_base():
-    from .livemathbench_greedy_gen_efb20d import livemathbench_datasets  # noqa: F401, F403
+    from .livemathbench_greedy_gen_9befbf import livemathbench_datasets  # noqa: F401, F403

opencompass/configs/datasets/livemathbench/livemathbench_greedy_gen.py

@@ -0,0 +1,50 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+
+from opencompass.datasets.livemathbench import LiveMathBenchDataset, LiveMathBenchEvaluator
+
+
+livemathbench_dataset = dict(
+    type=LiveMathBenchDataset,
+    path='',
+    k=16,
+    replication=3,
+    dataset_splits=['hard'],
+    dataset_languages=['cn', 'en'],
+    cot=True,
+    version='202412',
+    abbr='LiveMathBench-v202412-Hard',
+    reader_cfg=dict(
+        input_columns=['prompt'], 
+        output_column='answer'
+    ),
+    infer_cfg=dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(
+                round=[
+                    dict(role='HUMAN', prompt='{prompt}'),
+                ]
+            )
+        ),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(
+            type=GenInferencer
+        ),
+    ),
+    eval_cfg=dict(
+        evaluator=dict(
+            type=LiveMathBenchEvaluator,
+            model_name='',
+            url=[],
+            use_extract_model=False,
+            extract_url=[],
+            extract_model_name='',
+            k=[4, 8, 16],
+            replication=3,
+            thresholds=[0.0, 0.25, 0.5, 0.75, 1.0]
+        )
+    )
+)
+livemathbench_datasets = [livemathbench_dataset]

opencompass/configs/datasets/livemathbench/livemathbench_hard_gen_353ae7.py

@@ -0,0 +1,50 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+
+from opencompass.datasets.livemathbench import LiveMathBenchDataset, LiveMathBenchEvaluator
+
+
+livemathbench_dataset = dict(
+    type=LiveMathBenchDataset,
+    path='',
+    k=1,
+    replication=1,
+    dataset_splits=['hard'],
+    dataset_languages=['cn', 'en'],
+    cot=True,
+    version='202412',
+    abbr='LiveMathBench-v202412-Hard',
+    reader_cfg=dict(
+        input_columns=['prompt'], 
+        output_column='answer'
+    ),
+    infer_cfg=dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(
+                round=[
+                    dict(role='HUMAN', prompt='{prompt}'),
+                ]
+            )
+        ),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(
+            type=GenInferencer
+        ),
+    ),
+    eval_cfg=dict(
+        evaluator=dict(
+            type=LiveMathBenchEvaluator,
+            model_name='',
+            url=[],
+            use_extract_model=False,
+            extract_url=[],
+            extract_model_name='',
+            k=[1],
+            replication=1,
+            thresholds=[0.0]
+        )
+    )
+)
+livemathbench_datasets = [livemathbench_dataset]

opencompass/configs/datasets/livemathbench/livemathbench_hard_greedy_gen_353ae7.py

@@ -48,6 +48,7 @@ def load(path: str,
         if path != '':
             path = get_data_path(path)
             path = os.path.join(path, version)
+
         for split, language in product(dataset_splits, dataset_languages):
             dataset_info[f'{split}_{language}'] = {
                 'single-choice': 0,
@@ -64,7 +65,6 @@ def load(path: str,
 
             if path != '':
                 file_path = os.path.join(path, f'{split}_{language}.jsonl')
-
                 if not os.path.exists(file_path):
                     raise FileNotFoundError(
                         f'File {file_path} does not exist, please check the '

opencompass/datasets/livemathbench/livemathbench.py

@@ -164,8 +164,6 @@ def generate(self,
         self.logger.info('Generation Config of LMdeploy: ')
         self.logger.info(gen_config)
 
-
-
         results = []
         outputs = self.pipe(messages, gen_config=gen_config, do_preprocess=False)
         for output in outputs: