refactor: move rubric_generator to simple_rubric module and update docs

Author: XiaoBoAI

cookbooks/zero_shot_evaluation/zero_shot_pipeline.py

@@ -34,7 +34,7 @@
 
 # OpenJudge core components
 from openjudge.analyzer import PairwiseAnalyzer, PairwiseAnalysisResult
-from openjudge.generator import RubricGenerationConfig, RubricGenerator
+from openjudge.generator.simple_rubric import RubricGenerationConfig, TaskBasedRubricGenerator
 from openjudge.graders.llm_grader import GraderMode, LLMGrader
 from openjudge.graders.schema import GraderResult
 from openjudge.models.openai_chat_model import OpenAIChatModel
@@ -288,7 +288,7 @@ class ZeroShotPipeline:
     5. Analyze results and rank models
 
     The pipeline integrates with OpenJudge's core components:
-    - Uses RubricGenerator from openjudge.generator for rubric generation
+    - Uses TaskBasedRubricGenerator from openjudge.generator.simple_rubric for rubric generation
     - Uses PairwiseAnalyzer from openjudge.analyzer for result analysis
     - Uses LLMGrader and GradingRunner for pairwise evaluation
 
@@ -408,18 +408,18 @@ async def generate_rubrics(
         self,
         sample_queries: Optional[List[str]] = None,
     ) -> List[str]:
-        """Step 3: Generate evaluation rubrics using OpenJudge's RubricGenerator."""
+        """Step 3: Generate evaluation rubrics using OpenJudge's TaskBasedRubricGenerator."""
         logger.info("Step 3: Generating evaluation rubrics...")
 
         if not sample_queries and self._queries:
             sample_queries = [q.query for q in self._queries[:5]]
 
-        # Use OpenJudge's RubricGenerator
+        # Use OpenJudge's TaskBasedRubricGenerator
         rubric_config = RubricGenerationConfig(
             task_description=self.config.task.description,
             scenario=self.config.task.scenario,
         )
-        generator = RubricGenerator(
+        generator = TaskBasedRubricGenerator(
             config=rubric_config,
             model=self._create_judge_model(),
         )

docs/applications/zero_shot_evaluation.md

@@ -33,9 +33,9 @@ Zero-shot evaluation automates the entire evaluation pipeline:
 |------|-----------|-------------|
 | 1 | `QueryGenerator` | Generate diverse test queries from task description |
 | 2 | `ResponseCollector` | Collect responses from all target endpoints |
-| 3 | `RubricGenerator` | Generate evaluation criteria for the task |
+| 3 | `TaskBasedRubricGenerator` | Generate evaluation criteria for the task |
 | 4 | `GradingRunner` | Run pairwise comparisons with judge model |
-| 5 | `ZeroShotEvaluator` | Analyze results and produce rankings |
+| 5 | `ZeroShotPipeline` | Analyze results and produce rankings |
 
 
 ## Quick Start
@@ -44,11 +44,11 @@ Zero-shot evaluation automates the entire evaluation pipeline:
 
 ```python
 import asyncio
-from cookbooks.zero_shot_evaluation import ZeroShotEvaluator
+from cookbooks.zero_shot_evaluation.zero_shot_pipeline import ZeroShotPipeline
 
 async def main():
-    evaluator = ZeroShotEvaluator.from_config("config.yaml")
-    result = await evaluator.evaluate()
+    pipeline = ZeroShotPipeline.from_config("config.yaml")
+    result = await pipeline.evaluate()
 
     print(f"Best Model: {result.best_pipeline}")
     for rank, (model, win_rate) in enumerate(result.rankings, 1):
@@ -137,7 +137,8 @@ For fine-grained control, use individual components directly:
 ### Step 1: Generate Test Queries
 
 ```python
-from cookbooks.zero_shot_evaluation import QueryGenerator, TaskConfig, QueryGenerationConfig, OpenAIEndpoint
+from cookbooks.zero_shot_evaluation.query_generator import QueryGenerator
+from cookbooks.zero_shot_evaluation.schema import TaskConfig, QueryGenerationConfig, OpenAIEndpoint
 
 # Configure task and endpoint
 task = TaskConfig(
@@ -171,7 +172,8 @@ queries = await generator.generate()
 ### Step 2: Collect Responses
 
 ```python
-from cookbooks.zero_shot_evaluation import ResponseCollector, EvaluationConfig
+from cookbooks.zero_shot_evaluation.response_collector import ResponseCollector
+from cookbooks.zero_shot_evaluation.schema import EvaluationConfig
 
 collector = ResponseCollector(
     target_endpoints={
@@ -187,9 +189,13 @@ responses = await collector.collect(queries)
 ### Step 3: Generate Evaluation Rubrics
 
 ```python
-from cookbooks.zero_shot_evaluation import RubricGenerator
+from openjudge.generator.simple_rubric import TaskBasedRubricGenerator, RubricGenerationConfig
 
-rubric_gen = RubricGenerator(judge_endpoint, task)
+rubric_config = RubricGenerationConfig(
+    task_description=task.description,
+    scenario=task.scenario,
+)
+rubric_gen = TaskBasedRubricGenerator(config=rubric_config, model=judge_model)
 rubrics = await rubric_gen.generate(
     sample_queries=[q.query for q in queries[:5]]
 )
@@ -203,16 +209,16 @@ rubrics = await rubric_gen.generate(
 ### Step 4: Run Full Evaluation
 
 ```python
-from cookbooks.zero_shot_evaluation import ZeroShotEvaluator
+from cookbooks.zero_shot_evaluation.zero_shot_pipeline import ZeroShotPipeline
 
-evaluator = ZeroShotEvaluator(
+pipeline = ZeroShotPipeline(
     task_description="Code review assistant",
     target_endpoints=target_endpoints,
     judge_endpoint=judge_endpoint,
     num_queries=20
 )
 
-result = await evaluator.evaluate()
+result = await pipeline.evaluate()
 ```
 
 

openjudge/generator/__init__.py

@@ -4,22 +4,43 @@
 This module provides generators for automatically creating graders and
 evaluation criteria based on data or task descriptions.
 
+Submodules:
+    simple_rubric: Task-description-based rubric generation (zero-shot)
+    iterative_rubric: Preference-data-based rubric generation (iterative refinement)
+
 Classes:
     BaseGraderGenerator: Abstract base class for grader generators
     GraderGeneratorConfig: Configuration for grader generation
-    RubricGenerator: Generator for evaluation rubrics
+    LLMGraderGenerator: Base class for LLM-based grader generators
+    LLMGraderGeneratorConfig: Configuration for LLM grader generation
+
+    # Simple rubric generation (from task description)
+    SimpleRubricsGenerator: Main generator for simple rubric-based graders
+    SimpleRubricsGeneratorConfig: Configuration for simple rubric generation
+    TaskBasedRubricGenerator: Core rubric generation logic
     RubricGenerationConfig: Configuration for rubric generation
 """
 
 from openjudge.generator.base_generator import BaseGraderGenerator, GraderGeneratorConfig
-from openjudge.generator.rubric_generator import RubricGenerationConfig, RubricGenerator
+from openjudge.generator.llm_grader_generator import LLMGraderGenerator, LLMGraderGeneratorConfig
+
+# Simple rubric generation
+from openjudge.generator.simple_rubric import (
+    RubricGenerationConfig,
+    SimpleRubricsGenerator,
+    SimpleRubricsGeneratorConfig,
+    TaskBasedRubricGenerator,
+)
 
 __all__ = [
-    # Grader Generator
+    # Base classes
     "BaseGraderGenerator",
     "GraderGeneratorConfig",
-    # Rubric Generator
-    "RubricGenerator",
+    "LLMGraderGenerator",
+    "LLMGraderGeneratorConfig",
+    # Simple rubric generation
+    "SimpleRubricsGenerator",
+    "SimpleRubricsGeneratorConfig",
+    "TaskBasedRubricGenerator",
     "RubricGenerationConfig",
 ]
-

openjudge/generator/simple_rubric/__init__.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+"""Simple rubric generator module for automatic evaluation criteria generation.
+
+This module provides a simple, task-description-based approach to generating
+evaluation rubrics. It generates rubrics from task descriptions and sample
+queries, without requiring labeled training data.
+
+This is in contrast to the iterative_rubric module which learns rubrics from
+preference data through an iterative refinement process.
+
+Classes:
+    SimpleRubricsGenerator: Main generator class that creates LLMGrader instances
+    SimpleRubricsGeneratorConfig: Configuration for the generator
+    TaskBasedRubricGenerator: Core rubric generation logic
+    RubricGenerationConfig: Configuration for rubric generation
+"""
+
+from openjudge.generator.simple_rubric.generator import (
+    SimpleRubricsGenerator,
+    SimpleRubricsGeneratorConfig,
+)
+from openjudge.generator.simple_rubric.rubric_generator import (
+    RubricGenerationConfig,
+    TaskBasedRubricGenerator,
+)
+
+__all__ = [
+    # Main generator (creates LLMGrader)
+    "SimpleRubricsGenerator",
+    "SimpleRubricsGeneratorConfig",
+    # Core rubric generation logic
+    "TaskBasedRubricGenerator",
+    "RubricGenerationConfig",
+]
+