refator: refactor LengthEvaluator

Author: ChenZiHong-Gavin

graphgen/models/evaluator/qa/length_evaluator.py

@@ -1,19 +1,16 @@
 from graphgen.bases.base_evaluator import BaseEvaluator
 from graphgen.bases.datatypes import QAPair
 from graphgen.models.tokenizer import Tokenizer
-from graphgen.utils import create_event_loop
 
 
 class LengthEvaluator(BaseEvaluator):
-    def __init__(self, tokenizer_name: str = "cl100k_base", max_concurrent: int = 100):
-        super().__init__(max_concurrent)
-        self.tokenizer_name = tokenizer_name
-        self.tokenizer = Tokenizer(model_name=self.tokenizer_name)
+    def __init__(self, tokenizer: Tokenizer):
+        self.tokenizer = tokenizer
 
-    async def evaluate_single(self, pair: QAPair) -> float:
-        loop = create_event_loop()
-        return await loop.run_in_executor(None, self._calculate_length, pair.answer)
-
-    def _calculate_length(self, text: str) -> float:
-        tokens = self.tokenizer.encode(text)
+    def evaluate(self, pair: QAPair) -> float:
+        """
+        Evaluate the length of the qa pair.
+        """
+        content = pair.question + pair.answer
+        tokens = self.tokenizer.encode(content)
         return len(tokens)