Merge pull request #1534 from hanhainebula/master

Support evaluation: BRIGHT

Author: hanhainebula

FlagEmbedding/evaluation/bright/__init__.py

@@ -0,0 +1,17 @@
+from FlagEmbedding.abc.evaluation import (
+    AbsEvalModelArgs as BrightEvalModelArgs,
+)
+
+from .data_loader import BrightShortEvalDataLoader, BrightLongEvalDataLoader
+from .arguments import BrightEvalArgs
+from .runner import BrightEvalRunner
+from .searcher import BrightEvalDenseRetriever
+
+__all__ = [
+    "BrightEvalArgs",
+    "BrightEvalModelArgs",
+    "BrightEvalRunner",
+    "BrightEvalDenseRetriever",
+    "BrightShortEvalDataLoader",
+    "BrightLongEvalDataLoader",
+]

FlagEmbedding/evaluation/bright/__main__.py

@@ -0,0 +1,28 @@
+from transformers import HfArgumentParser
+
+from FlagEmbedding.evaluation.bright import (
+    BrightEvalArgs, BrightEvalModelArgs,
+    BrightEvalRunner
+)
+
+
+def main():
+    parser = HfArgumentParser((
+        BrightEvalArgs,
+        BrightEvalModelArgs
+    ))
+
+    eval_args, model_args = parser.parse_args_into_dataclasses()
+    eval_args: BrightEvalArgs
+    model_args: BrightEvalModelArgs
+
+    runner = BrightEvalRunner(
+        eval_args=eval_args,
+        model_args=model_args
+    )
+
+    runner.run()
+
+
+if __name__ == "__main__":
+    main()

FlagEmbedding/evaluation/bright/arguments.py

@@ -0,0 +1,16 @@
+from dataclasses import dataclass, field
+
+from FlagEmbedding.abc.evaluation.arguments import AbsEvalArgs
+
+
+@dataclass
+class BrightEvalArgs(AbsEvalArgs):
+    """
+    Argument class for Bright evaluation.
+    """
+    task_type: str = field(
+        default="short", metadata={"help": "The task type to evaluate on. Available options: ['short', 'long']. Default: short", "choices": ["short", "long"]}
+    )
+    use_special_instructions: bool = field(
+        default=True, metadata={"help": "Whether to use specific instructions in `prompts.py` for evaluation. Default: True"}
+    )