revert to origin

Author: MOLYHECI

.gitignore

@@ -20,16 +20,7 @@ configs/process/experiments/
 **/meteorscorer/data/meteor-1.5.jar
 **/ciderscorer/coco-val-df.p
 **/data
-!dataflow/data/
 **/ckpt
-**/ckpr
 tmp.*
 configs/process/text_process_reasoner.yaml
-docs/src/getting_started/Dev.md
-configs/process/test_process_math.yaml
-Text2SqlPipeline/bird_example/
-
-*.jsonl
-*.pt
-*.pth
-./.git/*
+docs/src/getting_started/Dev.md

configs/process/text_process_reasoner_ansfilter.yaml

@@ -0,0 +1,29 @@
+model_cache_path: '../ckpt' # Path to cache models
+dependencies: [text]
+save_path: "../dataflow-develop/processed.jsonl"
+
+data:
+  text:
+    use_hf: False # Whether to use huggingface_dataset, if used, ignore the local data path below
+    dataset_name: 'yahma/alpaca-cleaned'
+    dataset_split: 'train'  
+    name: 'default' 
+    revision: null
+    data_path: './demos/text_process/reasoners/math_5_samples.json'  # Local data path, supports json, jsonl, parquet formats
+    formatter: "TextFormatter" # Data loader type
+    keys: 'answer' # Key name to be processed, for sft data, it can be specified as ['instruction','input','output']
+
+processors:
+  AnswerFormatterFilter: {}
+  AnswerNgramFilter:
+    min_score: 0.1
+    max_score: 1.0
+    ngrams: 5
+  AnswerGroundTruthFilter: 
+    compare_method: math_verify # exact or math_verify
+  AnswerTokenLengthFilter:
+    max_answer_token_length: 512
+    tokenizer_dir: '../Qwen2.5-0.5B-Instruct'
+
+  
+  

dataflow/Eval/Text/__init__.py

@@ -110,7 +110,6 @@
     'DeitaComplexityScorer': ('dataflow/Eval/Text/models/deita_complexity_scorer.py', 'DeitaComplexityScorer'),  
     'PresidioScorer': ('dataflow/Eval/Text/models/presidio_scorer.py', 'PresidioScorer'),  
     'RMScorer': ('dataflow/Eval/Text/models/rm_scorer.py', 'RMScorer'),  
-    'PairQualScorer': ('dataflow/Eval/Text/models/pair_qual_scorer.py','PairQualScorer'),
     'VendiScorer': ('dataflow/Eval/Text/diversity/vendi_scorer.py', 'VendiScorer'),  
     'Task2VecScorer': ('dataflow/Eval/Text/diversity/task2vec_scorer.py', 'Task2VecScorer'),  
     'LangkitScorer': ('dataflow/Eval/Text/statistics/langkit_scorer.py', 'LangkitScorer'),  

dataflow/Eval/Text/models/Qurating/modeling/modeling_flash_llama.py

@@ -525,10 +525,7 @@ def forward(
             ((attention_mask is not None) and (not attention_mask.all().item()))
             and not use_cache
         ):
-            try: # for flash-attn latest version
-                hidden_states, unpad_indices, cu_seqlens, max_seqlen, _ = unpad_input(hidden_states, attention_mask)
-            except: # for flash-attn 2.3.3 verstion
-                hidden_states, unpad_indices, cu_seqlens, max_seqlen = unpad_input(hidden_states, attention_mask)
+            hidden_states, unpad_indices, cu_seqlens, max_seqlen = unpad_input(hidden_states, attention_mask)
             unpadded_lengths = (cu_seqlens, max_seqlen)
         else:
             unpadded_lengths = None

dataflow/Eval/Text/models/pair_qual_scorer.py

@@ -1,6 +1,4 @@
 from datasets import Dataset
-from dataflow.format import TextFormatter
-from dataflow.utils.utils import get_logger
 
 class Deduplicator:
 
@@ -13,20 +11,15 @@ def dedup_func(self, dataset):
     def __call__(self, dataset):
         init_len = len(dataset)
         deduped_dataset = self.dedup_func(dataset)
-        print(f'Implemented {self.__class__.__name__}. Data Number: {init_len} -> {len(deduped_dataset)}')
+        print(f'Implemented {self.__class__.__name__}. Data Number: {init_len} -> {len(deduped_dataset)}', flush=True)
 
         return deduped_dataset
 
 class TextDeduplicator(Deduplicator):
 
     def __init__(self, args=None):
         self.data_type = "text"
-        self.logger = get_logger()
-        if "input_file" in args.keys():
-            self.config = args
-            self.formatter = TextFormatter(args)
-            self.dataset = self.formatter.load_dataset()
-
+        
     def __call__(self, dataset):
         init_len = len(dataset)
         labels = self.dedup_func(dataset)
@@ -37,12 +30,8 @@ def filter_by_labels(example, index):
             deduped_dataset = dataset
         else:
             deduped_dataset = dataset.filter(labels)
-        self.logger.info(f'Implemented {self.dedupliactor_name}. Data Number: {init_len} -> {len(deduped_dataset)}')
+        print(f'Implemented {self.dedupliactor_name}. Data Number: {init_len} -> {len(deduped_dataset)}')
         return deduped_dataset
-    
-    def run(self):
-        deduplicated_dataset = self.__call__(self.dataset)
-        deduplicated_dataset.dump(self.config['output_file'])
 
 class ImageDeduplicator(Deduplicator):
 

dataflow/core/generator/reasoner.py

@@ -1,9 +1,6 @@
 from dataflow.data import DataFlowDataset
 from dataflow.core import ScoreRecord
-from dataflow.format import TextFormatter
 from datasets import Dataset
-from dataflow.utils.utils import get_logger
-
 class Filter():
 
     def __init__(self, args=None):
@@ -19,11 +16,6 @@ class TextFilter(Filter):
 
     def __init__(self, args=None):
         self.data_type = "text"
-        self.logger = get_logger()
-        if "input_file" in args.keys():
-            self.config = args
-            self.formatter = TextFormatter(args)
-            self.dataset = self.formatter.load_dataset()
 
     def __call__(self, dataset):
         init_len = len(dataset)
@@ -37,12 +29,9 @@ def filter_by_labels(example, index):
             filtered_dataset = dataset
         else:
             filtered_dataset = dataset.filter(labels)
-        self.logger.info(f'Implemented {self.filter_name}. Data Number: {init_len} -> {len(filtered_dataset)}')
+
+        print(f'Implemented {self.filter_name}. Data Number: {init_len} -> {len(filtered_dataset)}', flush=True)
         return filtered_dataset
-    
-    def run(self):
-        filtered_dataset = self.__call__(self.dataset)
-        filtered_dataset.dump(save_path=self.config['output_file'])
 
 class ImageFilter(Filter):
 

dataflow/core/process/deduplicator.py

@@ -1,6 +1,5 @@
 from dataflow.data import DataFlowDataset
 from dataflow.core import ScoreRecord
-from dataflow.format import TextFormatter
 from datasets import Dataset
 
 class Reasoner():
@@ -20,33 +19,11 @@ def __init__(self, args=None):
         self.filter_name = "ReasonerFilter"
         self.args = args
 
-        self.input_question_key = args.get("input_question_key","")
-        self.max_worker = args.get("max_worker",1)
-        
-        # answer format filter
-        self.keys = args.get("input_keys","")
-        # self.output_question_key = args.get("output_question_key","")
-        
-        # answer gt verification
-        self.test_answer_key = args.get("test_answer_key","")
-        self.gt_answer_key = args.get("gt_answer_key","")
-        
-        # ngram filter
-        self.question_key = args.get("question_key","")
-        self.answer_key = args.get("answer_key","")
-        
-        # api args
         api_args = args.get('api_args', None)
         if api_args is not None:
             self.model_name = api_args['model_name']
             self.api_url = api_args['api_url']
             self.mode_test = api_args['mode_test']
-            
-        if "input_file" in args.keys():
-            self.formatter = TextFormatter(args)
-            self.dataset = self.formatter.load_dataset()
-
-            
     def filter_func(self, dataset):
         pass
 
@@ -56,6 +33,7 @@ def __call__(self, dataset: DataFlowDataset):
         score_record = ScoreRecord()
         dataset.set_score_record(score_record)
         labels = self.filter_func(dataset)
+        
         if isinstance(dataset.dataset, Dataset):
             def filter_by_labels(example, index):
                 return labels[index] == 1
@@ -65,8 +43,4 @@ def filter_by_labels(example, index):
             filtered_dataset = dataset.filter(labels)
 
         print(f'Implemented {self.filter_name}. Data Number: {init_len} -> {len(filtered_dataset)}', flush=True)
-        return filtered_dataset
-    
-    def run(self):
-        filtered_dataset = self.__call__(self.dataset)
-        filtered_dataset.dump(save_path=self.args['output_file'])
+        return filtered_dataset

dataflow/core/process/filter.py

@@ -1,6 +1,3 @@
-from dataflow.format import TextFormatter
-from dataflow.utils.utils import get_logger
-
 class Refiner():
 
     def __init__(self, args):
@@ -13,20 +10,9 @@ class TextRefiner(Refiner):
 
     def __init__(self, args=None):
         self.data_type = "text"
-        self.logger = get_logger()
-        if "input_file" in args.keys():
-            self.config = args
-            self.formatter = TextFormatter(args)
-            self.dataset = self.formatter.load_dataset()
-        
-
 
     def __call__(self, dataset):
         refined_dataset, numbers = self.refine_func(dataset)
-        self.logger.info(f'Implemented {self.refiner_name}. {numbers} data refined.')
+        print(f'Implemented {self.refiner_name}. {numbers} data refined.', flush=True)
 
         return refined_dataset
-    
-    def run(self):
-        refined_dataset = self.__call__(self.dataset)
-        refined_dataset.dump(self.config['output_file'])