refactor: simplify TaskBasedRubricGenerator API and update docs

- Remove RubricGenerationConfig class, pass parameters directly to constructor
- Add DEFAULT_RUBRICS constant for fallback rubrics
- Update zero_shot_evaluation docs with pre-defined queries usage guide
- Simplify related tests

Author: XiaoBoAI

cookbooks/zero_shot_evaluation/zero_shot_pipeline.py

@@ -34,7 +34,7 @@
 
 # OpenJudge core components
 from openjudge.analyzer import PairwiseAnalyzer, PairwiseAnalysisResult
-from openjudge.generator.simple_rubric import RubricGenerationConfig, TaskBasedRubricGenerator
+from openjudge.generator.simple_rubric import TaskBasedRubricGenerator
 from openjudge.graders.llm_grader import GraderMode, LLMGrader
 from openjudge.graders.schema import GraderResult
 from openjudge.models.openai_chat_model import OpenAIChatModel
@@ -415,13 +415,10 @@ async def generate_rubrics(
             sample_queries = [q.query for q in self._queries[:5]]
 
         # Use OpenJudge's TaskBasedRubricGenerator
-        rubric_config = RubricGenerationConfig(
-            task_description=self.config.task.description,
-            scenario=self.config.task.scenario,
-        )
         generator = TaskBasedRubricGenerator(
-            config=rubric_config,
             model=self._create_judge_model(),
+            task_description=self.config.task.description,
+            scenario=self.config.task.scenario,
         )
         self._rubrics = await generator.generate(sample_queries)
         return self._rubrics

docs/applications/zero_shot_evaluation.md

@@ -73,6 +73,36 @@ python -m cookbooks.zero_shot_evaluation --config config.yaml --fresh --save
 python -m cookbooks.zero_shot_evaluation --config config.yaml --queries_file queries.json --save
 ```
 
+### Using Pre-defined Queries
+
+Skip query generation by providing your own queries file. This is useful when you want to evaluate models on a specific set of questions.
+
+**Create a queries file** (`queries.json`):
+
+```json
+[
+  {"query": "Translate: AI is transforming industries."},
+  {"query": "Translate: The weather is nice today."},
+  {"query": "Translate: How to learn programming effectively?"}
+]
+```
+
+The `category` and `difficulty` fields are optional:
+
+```json
+[
+  {"query": "Your question here", "category": "general", "difficulty": "easy"}
+]
+```
+
+**Run evaluation**:
+
+```bash
+python -m cookbooks.zero_shot_evaluation --config config.yaml --queries_file queries.json --save
+```
+
+The pipeline will skip query generation and directly use your queries for model comparison.
+
 
 ## Configuration
 
@@ -189,13 +219,13 @@ responses = await collector.collect(queries)
 ### Step 3: Generate Evaluation Rubrics
 
 ```python
-from openjudge.generator.simple_rubric import TaskBasedRubricGenerator, RubricGenerationConfig
+from openjudge.generator.simple_rubric import TaskBasedRubricGenerator
 
-rubric_config = RubricGenerationConfig(
+rubric_gen = TaskBasedRubricGenerator(
+    model=judge_model,
     task_description=task.description,
     scenario=task.scenario,
 )
-rubric_gen = TaskBasedRubricGenerator(config=rubric_config, model=judge_model)
 rubrics = await rubric_gen.generate(
     sample_queries=[q.query for q in queries[:5]]
 )

openjudge/generator/__init__.py

@@ -18,15 +18,17 @@
     SimpleRubricsGenerator: Main generator for simple rubric-based graders
     SimpleRubricsGeneratorConfig: Configuration for simple rubric generation
     TaskBasedRubricGenerator: Core rubric generation logic
-    RubricGenerationConfig: Configuration for rubric generation
+
+Constants:
+    DEFAULT_RUBRICS: Default fallback rubrics if generation fails
 """
 
 from openjudge.generator.base_generator import BaseGraderGenerator, GraderGeneratorConfig
 from openjudge.generator.llm_grader_generator import LLMGraderGenerator, LLMGraderGeneratorConfig
 
 # Simple rubric generation
 from openjudge.generator.simple_rubric import (
-    RubricGenerationConfig,
+    DEFAULT_RUBRICS,
     SimpleRubricsGenerator,
     SimpleRubricsGeneratorConfig,
     TaskBasedRubricGenerator,
@@ -42,5 +44,5 @@
     "SimpleRubricsGenerator",
     "SimpleRubricsGeneratorConfig",
     "TaskBasedRubricGenerator",
-    "RubricGenerationConfig",
+    "DEFAULT_RUBRICS",
 ]

openjudge/generator/simple_rubric/__init__.py

@@ -12,15 +12,17 @@
     SimpleRubricsGenerator: Main generator class that creates LLMGrader instances
     SimpleRubricsGeneratorConfig: Configuration for the generator
     TaskBasedRubricGenerator: Core rubric generation logic
-    RubricGenerationConfig: Configuration for rubric generation
+
+Constants:
+    DEFAULT_RUBRICS: Default fallback rubrics if generation fails
 """
 
 from openjudge.generator.simple_rubric.generator import (
     SimpleRubricsGenerator,
     SimpleRubricsGeneratorConfig,
 )
 from openjudge.generator.simple_rubric.rubric_generator import (
-    RubricGenerationConfig,
+    DEFAULT_RUBRICS,
     TaskBasedRubricGenerator,
 )
 
@@ -30,6 +32,5 @@
     "SimpleRubricsGeneratorConfig",
     # Core rubric generation logic
     "TaskBasedRubricGenerator",
-    "RubricGenerationConfig",
+    "DEFAULT_RUBRICS",
 ]
-

openjudge/generator/simple_rubric/generator.py

@@ -36,12 +36,11 @@
     LLMGraderGeneratorConfig,
 )
 from openjudge.generator.simple_rubric.rubric_generator import (
-    RubricGenerationConfig,
+    DEFAULT_RUBRICS,
     TaskBasedRubricGenerator,
 )
 from openjudge.graders.llm_grader import LLMGrader
 from openjudge.graders.schema import GraderMode
-from openjudge.models.base_chat_model import BaseChatModel
 from openjudge.models.openai_chat_model import OpenAIChatModel
 from openjudge.models.schema.prompt_template import LanguageEnum
 
@@ -55,15 +54,12 @@ class SimpleRubricsGeneratorConfig(LLMGraderGeneratorConfig):
 
     Attributes:
         task_description: Description of the task for evaluation.
-                         Should describe what kind of queries and responses are expected.
         scenario: Optional usage scenario for context.
-                 Helps the generator understand the evaluation context.
         language: Language for prompts (ZH or EN). Defaults to EN.
         default_rubrics: Fallback rubrics if generation fails.
-                        These are used when LLM generation fails.
-        max_retries: Maximum number of retry attempts for LLM calls. Defaults to 3.
-        min_score: Minimum score for pointwise evaluation. Defaults to 0.
-        max_score: Maximum score for pointwise evaluation. Defaults to 1.
+        max_retries: Maximum number of retry attempts for LLM calls.
+        min_score: Minimum score for pointwise evaluation.
+        max_score: Maximum score for pointwise evaluation.
 
     Inherited from LLMGraderGeneratorConfig:
         grader_name: Human-readable name for the generated grader.
@@ -72,24 +68,11 @@ class SimpleRubricsGeneratorConfig(LLMGraderGeneratorConfig):
         custom_evaluation_prompt: Custom template for evaluation.
     """
 
-    # Task description parameters
     task_description: str = ""
     scenario: Optional[str] = None
     language: LanguageEnum = LanguageEnum.EN
-
-    # Fallback configuration
-    default_rubrics: List[str] = field(
-        default_factory=lambda: [
-            "Accuracy: Whether the response is factually correct",
-            "Relevance: Whether the response addresses the query",
-            "Completeness: Whether the response is comprehensive",
-        ]
-    )
-
-    # Generation parameters
+    default_rubrics: List[str] = field(default_factory=lambda: DEFAULT_RUBRICS.copy())
     max_retries: int = 3
-
-    # Pointwise-specific parameters
     min_score: int = 0
     max_score: int = 1
 
@@ -107,14 +90,6 @@ class SimpleRubricsGenerator(LLMGraderGenerator):
     2. Uses an LLM to generate relevant evaluation criteria
     3. Creates an LLMGrader configured with these rubrics
 
-    This is suitable for scenarios where:
-    - You have a clear task description
-    - You don't have labeled preference data for rubric learning
-    - You want a quick way to set up evaluation
-
-    For more sophisticated rubric generation from preference data,
-    see the iterative_rubric module.
-
     Example:
         >>> config = SimpleRubricsGeneratorConfig(
         ...     grader_name="Medical QA Grader",
@@ -127,38 +102,25 @@ class SimpleRubricsGenerator(LLMGraderGenerator):
         ...     dataset=[],
         ...     sample_queries=["What are the symptoms of flu?"]
         ... )
-        >>> # Now use the grader to evaluate responses
-        >>> result = await grader.aevaluate(query="...", response="...")
     """
 
     def __init__(self, config: SimpleRubricsGeneratorConfig) -> None:
         """Initialize the simple rubrics generator.
 
         Args:
-            config: Configuration for rubric generation. Includes:
-                - grader_name: Name for the generated grader
-                - model: Language model for generation and evaluation
-                - task_description: Description of the evaluation task
-                - scenario: Optional usage scenario
-                - language: Language for prompts (ZH or EN)
-                - grader_mode: POINTWISE or LISTWISE
-                - default_rubrics: Fallback rubrics if generation fails
+            config: Configuration for rubric generation.
         """
         super().__init__(config)
         self.config: SimpleRubricsGeneratorConfig = config
 
-        # Initialize the rubric generator
-        rubric_config = RubricGenerationConfig(
+        self._rubric_generator = TaskBasedRubricGenerator(
+            model=config.model,
             task_description=config.task_description,
             scenario=config.scenario,
             language=config.language,
             default_rubrics=config.default_rubrics,
             max_retries=config.max_retries,
         )
-        self._rubric_generator = TaskBasedRubricGenerator(
-            config=rubric_config,
-            model=config.model,
-        )
 
     async def generate(
         self,
@@ -168,28 +130,20 @@ async def generate(
     ) -> LLMGrader:
         """Generate an LLMGrader with rubrics from task description.
 
-        This method generates evaluation rubrics based on the task description
-        and creates an LLMGrader instance configured with these rubrics.
-
         Args:
-            dataset: List of data dictionaries. For this generator, the dataset
-                    is optional and only used to extract sample queries if
-                    sample_queries is not provided.
+            dataset: List of data dictionaries (used to extract sample queries
+                    if sample_queries is not provided).
             sample_queries: Optional list of sample queries for context.
-                           If not provided, queries may be extracted from dataset.
             **kwargs: Additional arguments (currently unused).
 
         Returns:
             LLMGrader: Configured grader instance with generated rubrics.
         """
-        # Extract sample queries from dataset if not provided
         if sample_queries is None and dataset:
             sample_queries = [d.get("query", "") for d in dataset[:5] if d.get("query")]
 
-        # Generate rubrics
-        rubrics = await self._generate_rubrics(dataset, sample_queries=sample_queries, **kwargs)
+        rubrics = await self._generate_rubrics(sample_queries)
 
-        # Prepare grader kwargs
         grader_kwargs = {
             "name": self.config.grader_name,
             "model": self.config.model,
@@ -198,16 +152,13 @@ async def generate(
             "language": self.config.language,
         }
 
-        # Add min_score and max_score only for pointwise mode
         if self.config.grader_mode == GraderMode.POINTWISE:
             grader_kwargs["min_score"] = self.config.min_score
             grader_kwargs["max_score"] = self.config.max_score
 
-        # Add template: use custom if provided, otherwise use default based on mode
         if self.config.custom_evaluation_prompt is not None:
             grader_kwargs["template"] = self.config.custom_evaluation_prompt
         else:
-            # Use default evaluation template based on grader mode
             if self.config.grader_mode == GraderMode.POINTWISE:
                 grader_kwargs["template"] = POINTWISE_EVALUATION_TEMPLATE
             else:
@@ -217,35 +168,22 @@ async def generate(
 
     async def _generate_rubrics(
         self,
-        dataset: List[dict],
         sample_queries: Optional[List[str]] = None,
-        **kwargs,
     ) -> str:
         """Generate rubrics from task description.
 
-        This method uses the TaskBasedRubricGenerator to create rubrics
-        based on the task description and sample queries.
-
         Args:
-            dataset: List of data dictionaries (used for extracting sample queries
-                    if sample_queries is not provided).
             sample_queries: Optional list of sample queries for context.
-            **kwargs: Additional arguments (currently unused).
 
         Returns:
             str: Formatted string containing evaluation rubrics.
         """
-        # Generate rubrics as list
-        rubrics_list = await self._rubric_generator.generate(
-            sample_queries=sample_queries,
-        )
+        rubrics_list = await self._rubric_generator.generate(sample_queries=sample_queries)
 
-        # Format rubrics into a string
         formatted_rubrics = "\n\n".join(
             [f"{i + 1}. {rubric}" for i, rubric in enumerate(rubrics_list)]
         )
 
         logger.info(f"Generated {len(rubrics_list)} rubrics from task description")
 
         return formatted_rubrics
-