Merge branch 'API' of github.com:Open-DataFlow/DataFlow-Eval-Process into API

Author: MOLYHECI

dataflow/process/text/deduplicators/ccnet_deduplicator.py

@@ -25,7 +25,8 @@ def dedup_func(self, dataset):
                 text = str(sample[dataset.keys]).encode('utf-8')
             hash_value = self._compute_hash(text)
             hash_values.append(hash_value)
-        print(json.dumps({"hash_values": hash_values}))
-        return hash_values
+        # print(json.dumps({"ccnet_hash_values": hash_values}))
+
+        return json.dumps({"ccnet_hash_values": hash_values})
 
 

dataflow/process/text/deduplicators/hash_deduplicator.py

@@ -30,5 +30,5 @@ def dedup_func(self, dataset):
 
             hash_value = self._compute_hash(text)
             hash_values.append(hash_value)
-        print(json.dumps({"hash_values": hash_values}))
-        return json.dumps({"hash_values": hash_values})
+        # print(json.dumps({"hash_values": hash_values}))
+        return json.dumps({"exact_hash_values": hash_values})

dataflow/process/text/deduplicators/minhash_deduplicator.py

@@ -35,8 +35,8 @@ def dedup_func(self, dataset):
                 minhash = self.create_minhash(text)
                 result = lsh.query(minhash)
                 hash_values.append(result)
-        print(json.dumps({"hash_values": hash_values}))
-        return json.dumps({"hash_values": hash_values})
+        # print(json.dumps({"hash_values": hash_values}))
+        return json.dumps({"minhash_hash_values": hash_values})
 
 
 

dataflow/process/text/deduplicators/ngramhash_deduplicator.py

@@ -32,8 +32,8 @@ def dedup_func(self, dataset):
             ngrams = [text[i*gram_length:(i+1)*gram_length] for i in range(self.n_gram)]
             hash_value = set(self._compute_hash(ngram) for ngram in ngrams)
             hash_values.append(hash_value)
-        print(json.dumps({"hash_values": hash_values}))
-        return json.dumps({"hash_values": hash_values})
+        # print(json.dumps({"hash_values": hash_values}))
+        return json.dumps({"ngram_hash_values": hash_values})
 
 
 

dataflow/process/text/deduplicators/sem_deduplicator.py

@@ -87,5 +87,5 @@ def dedup_func(self, dataset):
         # Compute embeddings for the dataset texts
         embeddings = get_text_embedding(texts, self.tokenizer, self.model, self.device)
         embeddings = normalize(torch.tensor(embeddings), dim=1)
-        print(json.dumps({"embeddings": embeddings.tolist()}))
-        return json.dumps({"embeddings": embeddings.tolist()})
+        # print(json.dumps({"embeddings": embeddings.tolist()}))
+        return json.dumps({"semhash_embeddings": embeddings.tolist()})

dataflow/process/text/deduplicators/simhash_deduplicator.py

@@ -31,7 +31,7 @@ def get_similarity(simhash, another_simhash):
                 text = str(sample[dataset.keys]) 
             simhash = Simhash(text, f=self.fingerprint_size)
             simhashes.append(simhash)
-        print(json.dumps({"hash_values": [simhash.value for simhash in simhashes]}))
-        return json.dumps({"hash_values": [simhash.value for simhash in simhashes]})
+        # print(json.dumps({"hash_values": [simhash.value for simhash in simhashes]}))
+        return json.dumps({"simhash_values": [simhash.value for simhash in simhashes]})
 
 

dataflow/utils/utils.py

@@ -317,3 +317,35 @@ def refine():
     save_path = cfg['save_path']
     for dataset in dataset_dict.values():
         dataset.dump(save_path)
+
+def deduplicate():
+    from ..config import api_init_config
+    from dataflow.data import DataFlowDSDict
+    from dataflow.utils.registry import FORMATTER_REGISTRY
+    from dataflow.core import ScoreRecord
+    cfg = api_init_config()
+    dataset_dict = DataFlowDSDict()
+
+    if isinstance(cfg.yaml, str):
+        with open(cfg.yaml, 'r') as f:
+            cfg.yaml = yaml.safe_load(f)  # 解析成字典
+    
+    for scorer_name, args in cfg.yaml.items():
+        if "num_workers" in cfg:
+            args["num_workers"] = cfg.num_workers
+        if "model_cache_path" in cfg:
+            args["model_cache_dir"] = cfg.model_cache_path
+        processor = get_processor(scorer_name, args)
+        if processor.data_type not in dataset_dict.keys():
+            formatter = FORMATTER_REGISTRY.get('TextFormatter')(cfg['data'], cfg['key'], cfg['sft_single_round'], cfg['sft_multi_round'], cfg['RLHF'])
+            datasets = formatter.load_dataset()
+            dataset_dict[processor.data_type] = datasets
+        else:
+            datasets = dataset_dict[processor.data_type]
+        processed_dataset = processor(datasets)
+        dataset_dict[processor.data_type] = processed_dataset
+        print(processed_dataset)
+
+
+            
+            

deduplicator.py

@@ -0,0 +1,3 @@
+from dataflow.utils.utils import deduplicate
+
+deduplicate()