wip: perf evaluate_service

Author: ChenZiHong-Gavin

examples/evaluate/evaluate_kg/evaluate_kg.sh

@@ -1,2 +1,2 @@
 python3 -m graphgen.run \
---config_file examples/evaluate/evaluate_kg/evaluate_kg_config.yaml
+--config_file examples/evaluate/evaluate_kg/kg_evaluation_config.yaml

examples/evaluate/evaluate_qa/evaluate.sh

@@ -0,0 +1,2 @@
+python3 -m graphgen.run \
+--config_file examples/evaluate/evaluate_qa/qa_evaluation_config.yaml

examples/evaluate/evaluate_qa/evaluate_qa.sh

@@ -92,5 +92,7 @@ nodes:
       metrics:
         - qa_length
         - qa_mtld
-        - qa_reward_score
-        - qa_uni_score
+        # - qa_reward_score
+        # - qa_uni_score
+      mtld_params:
+        threshold: 0.7

examples/evaluate/evaluate_qa/qa_evaluation_config.yaml

@@ -1,5 +1,4 @@
 from .evaluator import (
-    KGQualityEvaluator,
     LengthEvaluator,
     MTLDEvaluator,
     RewardEvaluator,

graphgen/models/__init__.py

@@ -152,7 +152,9 @@ async def _evaluate_entity_extraction(
     ) -> Dict[str, Any]:
         """Use LLM to evaluate entity extraction quality."""
         try:
-            prompt = ENTITY_EVALUATION_PROMPT.format(
+            lang = detect_main_language(chunk.content)
+
+            prompt = ACCURACY_EVALUATION_PROMPT[lang]["ENTITY"].format(
                 chunk_content=chunk.content,
                 extracted_entities=json.dumps(
                     extracted_entities, ensure_ascii=False, indent=2
@@ -225,7 +227,8 @@ async def _evaluate_relation_extraction(
     ) -> Dict[str, Any]:
         """Use LLM to evaluate relation extraction quality."""
         try:
-            prompt = RELATION_EVALUATION_PROMPT.format(
+            lang = detect_main_language(chunk.content)
+            prompt = ACCURACY_EVALUATION_PROMPT[lang]["RELATION"].format(
                 chunk_content=chunk.content,
                 extracted_relations=json.dumps(
                     extracted_relations, ensure_ascii=False, indent=2

graphgen/models/evaluator/kg/accuracy_evaluator.py

@@ -1,10 +1,12 @@
+
+import os
 from graphgen.bases import BaseEvaluator, QAPair
 from graphgen.models.tokenizer import Tokenizer
 
 
 class LengthEvaluator(BaseEvaluator):
-    def __init__(self, tokenizer: Tokenizer):
-        self.tokenizer = tokenizer
+    def __init__(self):
+        self.tokenizer: Tokenizer = Tokenizer(os.environ["TOKENIZER_MODEL"] or "cl100k_base")
 
     def evaluate(self, pair: QAPair) -> float:
         """

graphgen/models/evaluator/qa/length_evaluator.py

@@ -7,6 +7,8 @@
 from .quiz import QuizService
 from .read import read
 from .search import SearchService
+from .evaluate import EvaluateService
+
 
 operators = {
     "read": read,
@@ -18,4 +20,5 @@
     "search": SearchService,
     "partition": PartitionService,
     "generate": GenerateService,
+    "evaluate": EvaluateService,
 }

graphgen/operators/__init__.py

@@ -0,0 +1 @@
+from .evaluate_service import EvaluateService

graphgen/operators/evaluate/__init__.py

@@ -10,27 +10,35 @@ class EvaluateService(BaseOperator):
     2. QA Quality Evaluation
     """
 
-    def __init__(self, working_dir: str = "cache", metrics: list[str] = None):
+    def __init__(self, working_dir: str = "cache", metrics: list[str] = None, **kwargs):
         super().__init__(working_dir=working_dir, op_name="evaluate_service")
         self.llm_client: BaseLLMWrapper = init_llm("synthesizer")
         self.metrics = metrics
-
-        self.evaluators = {
-            "xxx": "xxxEvaluator"
-        }
-
-        self.graph_storage = init_storage(
-            xx, xx, xx
-        )
+        self.kwargs = kwargs
+        self.evaluators = {}
 
     def _init_evaluators(self):
         for metric in self.metrics:
-
+            if metric == "qa_length":
+                from graphgen.models import LengthEvaluator
+                self.evaluators[metric] = LengthEvaluator()
+            elif metric == "qa_mtld":
+                from graphgen.models import MTLDEvaluator
+                self.evaluators[metric] = MTLDEvaluator(self.kwargs.get("mtld_params", {}))
+            elif metric == "qa_reward_score":
+                from graphgen.models import RewardEvaluator
+                self.evaluators[metric] = RewardEvaluator(self.kwargs.get("reward_params", {}))
+            elif metric == "qa_uni_score":
+                from graphgen.models import UniEvaluator
+                self.evaluators[metric] = UniEvaluator(self.kwargs.get("uni_params", {}))
+            else:
+                raise ValueError(f"Unknown metric: {metric}")
 
     def process(self, batch: pd.DataFrame) -> pd.DataFrame:
         items = batch.to_dict(orient="records")
         return pd.DataFrame(self.evaluate(items))
 
     def evaluate(self, items: list[dict]) -> list[dict]:
-        # 用evaluators 评估 items
+        print(items)
         pass
+