Merge pull request #14 from TheRoadQaQ/main

Reasoner process, Answer method

Author: scuuy

configs/process/text_process_reasoner_ansfilter.yaml

@@ -1,6 +1,6 @@
 model_cache_path: '../ckpt' # Path to cache models
 dependencies: [text]
-save_path: "./processed.jsonl"
+save_path: "../dataflow-develop/processed.jsonl"
 
 data:
   text:
@@ -9,21 +9,20 @@ data:
     dataset_split: 'train'  
     name: 'default' 
     revision: null
-    data_path: 'demos/text_process/reasoners/math_5_samples.json'  # Local data path, supports json, jsonl, parquet formats
+    data_path: './demos/text_process/reasoners/math_5_samples.json'  # Local data path, supports json, jsonl, parquet formats
     formatter: "TextFormatter" # Data loader type
     keys: 'answer' # Key name to be processed, for sft data, it can be specified as ['instruction','input','output']
 
 processors:
-  AnswerFormatterFilter:
-    type: "default"
+  AnswerFormatterFilter: {}
   AnswerNgramFilter:
-    min_score: 0.5
+    min_score: 0.1
     max_score: 1.0
     ngrams: 5
   AnswerGroundTruthFilter: 
-    compare_method: exact # exact/math_verify/xverify
+    compare_method: math_verify # exact or math_verify
   AnswerTokenLengthFilter:
-    max_answer_token_length: 1024
+    max_answer_token_length: 512
     tokenizer_dir: '../Qwen2.5-0.5B-Instruct'
 
 

dataflow/core/process/reasoner.py

@@ -19,10 +19,11 @@ def __init__(self, args=None):
         self.filter_name = "ReasonerFilter"
         self.args = args
 
-        api_args = args['api_args']
-        self.model_name = api_args['model_name']
-        self.api_url = api_args['api_url']
-        self.mode_test = api_args['mode_test']
+        api_args = args.get('api_args', None)
+        if api_args is not None:
+            self.model_name = api_args['model_name']
+            self.api_url = api_args['api_url']
+            self.mode_test = api_args['mode_test']
     def filter_func(self, dataset):
         pass
 

dataflow/process/text/reasoners/answer_formatter_filter.py

@@ -1,25 +1,15 @@
-from dataflow.core import TextFilter
+from dataflow.core import TextFilter, ReasonerFilter
 import numpy as np
 from dataflow.utils.registry import PROCESSOR_REGISTRY
 import re
 
 @PROCESSOR_REGISTRY.register()
-class AnswerFormatterFilter(TextFilter):
+class AnswerFormatterFilter(ReasonerFilter):
     def __init__(self, args_dict: dict):
         super().__init__(args_dict)
         self.filter_name = 'AnswerFormatterFilter'
 
     def is_valid_answer(answer: str) -> bool:
-        # start with "Solution:"
-        if not answer.startswith("Solution:"):
-            return False
-        
-        # check that every step start with "→" or not
-        #steps = answer.split("\n")
-        #for step in steps:
-        #    if step.strip() and not step.strip().startswith("→"):
-        #        return False
-        
         # check final answer in \boxed{} or not 
         if not re.search(r'\\boxed{.*}', answer):
             return False

dataflow/process/text/reasoners/answer_ground_truth_filter.py

@@ -1,4 +1,5 @@
-from dataflow.core import TextFilter
+from dataflow.core import ReasonerFilter
+from math_verify import parse, verify, LatexExtractionConfig
 import numpy as np
 from dataflow.utils.registry import PROCESSOR_REGISTRY
 #from math_verify import parse, verify, LatexExtractionConfig
@@ -214,18 +215,35 @@ def _get_last_number_answer(self, pred_str, use_last_number):
 
 
 @PROCESSOR_REGISTRY.register()
-class AnswerGroundTruthFilter(TextFilter):
+class AnswerGroundTruthFilter(ReasonerFilter):
     def __init__(self, args_dict: dict):
         super().__init__(args_dict)
         self.filter_name = 'AnswerGroundTruthFilter'
         unit_manager = UnitTextManager()
         string_cleaner = StringCleaner(unit_manager)
         self.answer_extractor = AnswerExtractor(string_cleaner)
 
+        name2compare = {
+            'exact': self.exact_compare,
+            'math_verify': self.math_verify_compare
+        }
+
+        self.compare = name2compare[args_dict.get('compare_method', 'exact')]
+
+    def exact_compare(self, answer, ground_truth):
+        return answer == ground_truth
+    
+    def math_verify_compare(self, answer, ground_truth):
+        try:
+            return verify(parse(ground_truth), parse(answer))
+        except:
+            return False
+
     def filter_func(self, dataset):
         indexes = np.zeros(len(dataset)).astype(int)
         for i in range(len(dataset)):
             final_answer =  self.answer_extractor.extract_answer(dataset[i]['answer'], dataset[i].get('data_name', None))
-            if 'ground_truth_answer'in dataset[i] and final_answer == dataset[i]['ground_truth_answer']:
-                indexes[i] = 1
+            if 'ground_truth_answer' in dataset[i]:
+                if self.compare(final_answer, dataset[i]['ground_truth_answer']):
+                    indexes[i] = 1
         return indexes

dataflow/process/text/reasoners/answer_ngram_filter.py

@@ -1,11 +1,11 @@
-from dataflow.core import TextFilter
+from dataflow.core import ReasonerFilter
 import numpy as np
 import re
 from dataflow.utils.registry import PROCESSOR_REGISTRY
 from dataflow.Eval.Text import NgramScorer
 
 @PROCESSOR_REGISTRY.register()
-class AnswerNgramFilter(TextFilter):
+class AnswerNgramFilter(ReasonerFilter):
     def __init__(self, args_dict: dict):
         super().__init__(args_dict)
         self.filter_name = 'AnswerNgramFilter'

dataflow/process/text/reasoners/answer_token_length_filter.py

@@ -1,10 +1,10 @@
-from dataflow.core import TextFilter
+from dataflow.core import ReasonerFilter
 import numpy as np
 from dataflow.utils.registry import PROCESSOR_REGISTRY
 from transformers import AutoTokenizer
 
 @PROCESSOR_REGISTRY.register()
-class AnswerTokenLengthFilter(TextFilter):
+class AnswerTokenLengthFilter(ReasonerFilter):
     def __init__(self, args_dict: dict):
         super().__init__(args_dict)
         self.filter_name = 'AnswerTokenLengthFilter'
@@ -13,7 +13,6 @@ def __init__(self, args_dict: dict):
 
     def filter_func(self, dataset):
         def get_token_count(input_string):
-            # 使用 tokenizer 对字符串进行编码，并获取 token 数目
             tokens = self.tokenizer.encode(input_string, add_special_tokens=False)
             return len(tokens)
 

demos/text_process/reasoners/math_5_samples.json

@@ -7,9 +7,9 @@
     },
     {
         "answer": 
-         "1. Find critical points:\n        → f'(x) = 3x² - 6x\n        → Set derivative to zero: 3x(x-2) = 0 ⇒ x=0, x=2\n    \n        2. Evaluate function at critical points and endpoints:\n        → f(-1) = (-1)^3 - 3(-1)^2 + 4 = -1 -3 +4 = 0.0000\n        → f(0) = 0³ - 3(0)² +4 = 4.0000\n        → f(2) = 8 - 12 +4 = 0.0000\n        → f(3) = 27 - 27 +4 = 4.0000\n    \n        3. Compare values:\n        → Minimum occurs at x=-1 and x=2\n    \n        Verification:\n        → Second derivative test: f''(x) = 6x-6\n        → f''(-1) = -12 < 0 (local max)\n        → f''(2) = 6 > 0 (local min)\n    \n        \\boxed{0}"
+         "Solution:\n        1. Find critical points:\n        → f'(x) = 3x² - 6x\n        → Set derivative to zero: 3x(x-2) = 0 ⇒ x=0, x=2\n    \n        2. Evaluate function at critical points and endpoints:\n        → f(-1) = (-1)^3 - 3(-1)^2 + 4 = -1 -3 +4 = 0.0000\n        → f(0) = 0³ - 3(0)² +4 = 4.0000\n        → f(2) = 8 - 12 +4 = 0.0000\n        → f(3) = 27 - 27 +4 = 4.0000\n    \n        3. Compare values:\n        → Minimum occurs at x=-1 and x=2\n    \n        Verification:\n        → Second derivative test: f''(x) = 6x-6\n        → f''(-1) = -12 < 0 (local max)\n        → f''(2) = 6 > 0 (local min)\n    \n        \\boxed{0.5}"
         ,
-        "ground_truth_answer": "0"
+        "ground_truth_answer": "1/2"
     },
     {
         "answer": 
@@ -30,7 +30,7 @@
     },
     {
         "answer": 
-         "Solution:\n        1. Find critical points:\n        → f'(x) = 3x² - 6x\n  1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n      → Set derivative to zero: 3x(x-2) = 0 ⇒ x=0, x=2\n    \n        2. Evaluate function at critical points and endpoints:\n        → f(-1) = (-1)^3 - 3(-1)^2 + 4 = -1 -3 +4 = 0.0000\n        → f(0) = 0³ - 3(0)² +4 = 4.0000\n        → f(2) = 8 - 12 +4 = 0.0000\n        → f(3) = 27 - 27 +4 = 4.0000\n    \n        3. Compare values:\n        → Minimum occurs at x=-1 and x=2\n    \n        Verification:\n        → Second derivative test: f''(x) = 6x-6\n        → f''(-1) = -12 < 0 (local max)\n        → f''(2) = 6 > 0 (local min)\n    \n        \\boxed{0}"
+         "Solution:\n        1. Find critical points:\n        → f'(x) = 3x² - 6x\n  1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n1. Find critical points:\n        → f'(x) = 3x² - 6x\n      → Set derivative to zero: 3x(x-2) = 0 ⇒ x=0, x=2  → Set derivative to zero: 3x(x-2) = 0 ⇒ x=0, x=2 → Set derivative to zero: 3x(x-2) = 0 ⇒ x=0, x=2 → Set derivative to zero: 3x(x-2) = 0 ⇒ x=0, x=2 → Set derivative to zero: 3x(x-2) = 0 ⇒ x=0, x=2 → Set derivative to zero: 3x(x-2) = 0 ⇒ x=0, x=2 → Set derivative to zero: 3x(x-2) = 0 ⇒ x=0, x=2 → Set derivative to zero: 3x(x-2) = 0 ⇒ x=0, x=2 → Set derivative to zero: 3x(x-2) = 0 ⇒ x=0, x=2 → Set derivative to zero: 3x(x-2) = 0 ⇒ x=0, x=2 → Set derivative to zero: 3x(x-2) = 0 ⇒ x=0, x=2\n    \n        2. Evaluate function at critical points and endpoints:\n        → f(-1) = (-1)^3 - 3(-1)^2 + 4 = -1 -3 +4 = 0.0000\n        → f(0) = 0³ - 3(0)² +4 = 4.0000\n        → f(2) = 8 - 12 +4 = 0.0000\n        → f(3) = 27 - 27 +4 = 4.0000\n    \n        3. Compare values:\n        → Minimum occurs at x=-1 and x=2\n    \n        Verification:\n        → Second derivative test: f''(x) = 6x-6\n        → f''(-1) = -12 < 0 (local max)\n        → f''(2) = 6 > 0 (local min)\n    \n        \\boxed{0}"
         ,
         "ground_truth_answer": "0"
     }