feat(paper_review): add rebuttal generation and assessment support

Add two new capabilities to the paper review pipeline:
- Rebuttal Generation: drafts point-by-point rebuttals with [TODO] placeholders
  for items requiring new experiments, proofs, or data the author must provide
- Rebuttal Assessment: evaluates whether an author's rebuttal adequately addresses
  reviewer concerns and provides an updated recommendation score (1-6)

New files:
- graders/rebuttal_generation.py, graders/rebuttal_assessment.py
- prompts/rebuttal_generation.py, prompts/rebuttal_assessment.py
- examples/rebuttal_workflow.py

Modified files:
- schema.py: RebuttalResult, RebuttalAssessmentResult, new ReviewStage values
- pipeline.py: generate_rebuttal(), assess_rebuttal(), review_and_report() extension
- report.py: rebuttal draft and assessment report sections
- __init__.py files: export new classes

Made-with: Cursor

Author: XiaoBoAI

cookbooks/paper_review/__init__.py

@@ -21,6 +21,8 @@
     CriticalityGrader,
     FormatGrader,
     JailbreakingGrader,
+    RebuttalAssessmentGrader,
+    RebuttalGenerationGrader,
     ReviewGrader,
 )
 from cookbooks.paper_review.pipeline import PaperReviewPipeline, PipelineConfig
@@ -32,6 +34,10 @@
     CriticalityResult,
     PaperReviewResult,
     ProgressCallback,
+    RebuttalAssessmentResult,
+    RebuttalConcern,
+    RebuttalPointAssessment,
+    RebuttalResult,
     ReviewProgress,
     ReviewResult,
     ReviewStage,
@@ -49,6 +55,8 @@
     "CriticalityGrader",
     "FormatGrader",
     "JailbreakingGrader",
+    "RebuttalGenerationGrader",
+    "RebuttalAssessmentGrader",
     # Processors
     "BibChecker",
     "TexPackageProcessor",
@@ -58,6 +66,10 @@
     "ReviewResult",
     "CriticalityResult",
     "BibVerificationSummary",
+    "RebuttalResult",
+    "RebuttalConcern",
+    "RebuttalAssessmentResult",
+    "RebuttalPointAssessment",
     # Progress
     "ReviewStage",
     "ReviewProgress",

cookbooks/paper_review/examples/rebuttal_workflow.py

@@ -0,0 +1,174 @@
+# -*- coding: utf-8 -*-
+"""Example: rebuttal generation and assessment workflows.
+
+Usage:
+    # Generate a rebuttal draft after reviewing a paper
+    python -m cookbooks.paper_review.examples.rebuttal_workflow generate \
+        --pdf_path paper.pdf --api_key YOUR_KEY
+
+    # Assess an existing rebuttal against a review
+    python -m cookbooks.paper_review.examples.rebuttal_workflow assess \
+        --pdf_path paper.pdf --rebuttal_path rebuttal.txt --api_key YOUR_KEY
+
+    # Full pipeline: review + generate rebuttal in one call
+    python -m cookbooks.paper_review.examples.rebuttal_workflow full \
+        --pdf_path paper.pdf --api_key YOUR_KEY
+"""
+
+import asyncio
+from pathlib import Path
+
+import fire
+
+from cookbooks.paper_review.pipeline import PaperReviewPipeline, PipelineConfig
+from cookbooks.paper_review.report import generate_report
+
+
+async def _generate_rebuttal(
+    pdf_path: str,
+    model_name: str = "gpt-4o",
+    api_key: str = "",
+    base_url: str | None = None,
+    discipline: str | None = None,
+    venue: str | None = None,
+    language: str | None = None,
+    output_path: str | None = None,
+):
+    """Review a paper then generate a rebuttal draft with [TODO] placeholders."""
+    config = PipelineConfig(
+        model_name=model_name,
+        api_key=api_key,
+        base_url=base_url,
+        discipline=discipline,
+        venue=venue,
+        language=language,
+    )
+    pipeline = PaperReviewPipeline(config)
+
+    print("Step 1: Reviewing paper...")
+    result = await pipeline.review_paper(pdf_path)
+
+    if not result.review:
+        print("Review failed or was disabled. Cannot generate rebuttal.")
+        return
+
+    print(f"Review score: {result.review.score}/6")
+    print("\nStep 2: Generating rebuttal draft...")
+    rebuttal = await pipeline.generate_rebuttal(pdf_path, review_result=result)
+    result.rebuttal = rebuttal
+
+    report = generate_report(result, Path(pdf_path).stem, output_path)
+    if output_path:
+        print(f"\nReport saved to: {output_path}")
+    else:
+        print("\n" + report)
+
+    todo_count = rebuttal.rebuttal_text.count("[TODO:")
+    if todo_count:
+        print(f"\n>>> {todo_count} [TODO] placeholder(s) require your attention. <<<")
+
+
+async def _assess_rebuttal(
+    pdf_path: str,
+    rebuttal_path: str,
+    review_text: str | None = None,
+    model_name: str = "gpt-4o",
+    api_key: str = "",
+    base_url: str | None = None,
+    discipline: str | None = None,
+    language: str | None = None,
+    output_path: str | None = None,
+):
+    """Review a paper then assess an existing rebuttal."""
+    config = PipelineConfig(
+        model_name=model_name,
+        api_key=api_key,
+        base_url=base_url,
+        discipline=discipline,
+        language=language,
+    )
+    pipeline = PaperReviewPipeline(config)
+
+    rebuttal_text = Path(rebuttal_path).read_text(encoding="utf-8")
+
+    if review_text:
+        print("Using provided review text.")
+        result = await pipeline.review_paper(pdf_path)
+    else:
+        print("Step 1: Reviewing paper...")
+        result = await pipeline.review_paper(pdf_path)
+        if result.review:
+            review_text = result.review.review
+        else:
+            print("Review failed. Cannot assess rebuttal without review text.")
+            return
+
+    print("\nStep 2: Assessing rebuttal...")
+    assessment = await pipeline.assess_rebuttal(
+        pdf_path,
+        rebuttal_text=rebuttal_text,
+        review_text=review_text,
+        review_result=result,
+    )
+    result.rebuttal_assessment = assessment
+
+    report = generate_report(result, Path(pdf_path).stem, output_path)
+    if output_path:
+        print(f"\nReport saved to: {output_path}")
+    else:
+        print("\n" + report)
+
+    print(f"\nScore: {assessment.original_score}/6 -> {assessment.updated_score}/6")
+
+
+async def _full_pipeline(
+    pdf_path: str,
+    model_name: str = "gpt-4o",
+    api_key: str = "",
+    base_url: str | None = None,
+    discipline: str | None = None,
+    venue: str | None = None,
+    language: str | None = None,
+    output_path: str | None = None,
+):
+    """Review + generate rebuttal in a single review_and_report call."""
+    config = PipelineConfig(
+        model_name=model_name,
+        api_key=api_key,
+        base_url=base_url,
+        discipline=discipline,
+        venue=venue,
+        language=language,
+        enable_rebuttal_generation=True,
+    )
+    pipeline = PaperReviewPipeline(config)
+    result, report = await pipeline.review_and_report(
+        pdf_path,
+        paper_name=Path(pdf_path).stem,
+        output_path=output_path,
+    )
+
+    if output_path:
+        print(f"Report saved to: {output_path}")
+    else:
+        print(report)
+
+
+class RebuttalCLI:
+    """CLI for rebuttal workflows."""
+
+    def generate(self, **kwargs):
+        """Generate a rebuttal draft."""
+        asyncio.run(_generate_rebuttal(**kwargs))
+
+    def assess(self, **kwargs):
+        """Assess an existing rebuttal."""
+        asyncio.run(_assess_rebuttal(**kwargs))
+
+    def full(self, **kwargs):
+        """Full pipeline: review + generate rebuttal."""
+        asyncio.run(_full_pipeline(**kwargs))
+
+
+if __name__ == "__main__":
+    fire.Fire(RebuttalCLI)

cookbooks/paper_review/graders/__init__.py

@@ -5,6 +5,8 @@
 from cookbooks.paper_review.graders.criticality import CriticalityGrader
 from cookbooks.paper_review.graders.format import FormatGrader
 from cookbooks.paper_review.graders.jailbreaking import JailbreakingGrader
+from cookbooks.paper_review.graders.rebuttal_assessment import RebuttalAssessmentGrader
+from cookbooks.paper_review.graders.rebuttal_generation import RebuttalGenerationGrader
 from cookbooks.paper_review.graders.review import ReviewGrader
 
 __all__ = [
@@ -13,4 +15,6 @@
     "CriticalityGrader",
     "FormatGrader",
     "JailbreakingGrader",
+    "RebuttalGenerationGrader",
+    "RebuttalAssessmentGrader",
 ]

cookbooks/paper_review/graders/rebuttal_assessment.py

@@ -0,0 +1,154 @@
+# -*- coding: utf-8 -*-
+"""Rebuttal assessment grader for academic papers."""
+
+import json
+import re
+from typing import List, Optional
+
+from cookbooks.paper_review.disciplines.base import DisciplineConfig
+from cookbooks.paper_review.prompts.rebuttal_assessment import (
+    REBUTTAL_ASSESSMENT_USER_PROMPT,
+    get_rebuttal_assessment_system_prompt,
+)
+from cookbooks.paper_review.utils import extract_response_content
+from openjudge.graders.base_grader import GraderError, GraderMode, GraderScore
+from openjudge.graders.llm_grader import LLMGrader
+from openjudge.models.base_chat_model import BaseChatModel
+
+
+def parse_rebuttal_assessment_response(text: str) -> dict:
+    """Parse JSON-formatted rebuttal assessment response."""
+    json_match = re.search(r"\{[\s\S]*\}", text)
+    if json_match:
+        try:
+            data = json.loads(json_match.group())
+            point_assessments = []
+            for p in data.get("point_assessments", []):
+                point_assessments.append(
+                    {
+                        "concern": p.get("concern", ""),
+                        "author_response_summary": p.get("author_response_summary", ""),
+                        "adequacy": p.get("adequacy", "not_addressed"),
+                        "reasoning": p.get("reasoning", ""),
+                    }
+                )
+            return {
+                "updated_score": int(data.get("updated_score", 3)),
+                "score_change_reasoning": data.get("score_change_reasoning", ""),
+                "overall_assessment": data.get("overall_assessment", ""),
+                "point_assessments": point_assessments,
+                "unresolved_concerns": data.get("unresolved_concerns", []),
+                "rebuttal_strengths": data.get("rebuttal_strengths", []),
+            }
+        except (json.JSONDecodeError, ValueError, TypeError):
+            pass
+
+    return {
+        "updated_score": 3,
+        "score_change_reasoning": "",
+        "overall_assessment": text,
+        "point_assessments": [],
+        "unresolved_concerns": [],
+        "rebuttal_strengths": [],
+    }
+
+
+def build_rebuttal_assessment_messages(
+    pdf_data: str,
+    review_text: str,
+    rebuttal_text: str,
+    original_score: int,
+    discipline: Optional[DisciplineConfig] = None,
+    language: Optional[str] = None,
+) -> List[dict]:
+    """Build messages for rebuttal assessment."""
+    user_prompt = REBUTTAL_ASSESSMENT_USER_PROMPT.format(
+        original_score=original_score,
+        review_text=review_text,
+        rebuttal_text=rebuttal_text,
+    )
+    return [
+        {
+            "role": "system",
+            "content": get_rebuttal_assessment_system_prompt(
+                discipline=discipline,
+                language=language,
+            ),
+        },
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": user_prompt},
+                {"type": "file", "file": {"file_data": pdf_data}},
+            ],
+        },
+    ]
+
+
+class RebuttalAssessmentGrader(LLMGrader):
+    """Grader that assesses whether a rebuttal adequately addresses reviewer concerns.
+
+    Score range: 1-6 (updated recommendation after reading the rebuttal)
+    """
+
+    def __init__(
+        self,
+        model: BaseChatModel | dict,
+        discipline: Optional[DisciplineConfig] = None,
+        language: Optional[str] = None,
+    ):
+        super().__init__(
+            name="rebuttal_assessment",
+            mode=GraderMode.POINTWISE,
+            description="Assess rebuttal adequacy and update recommendation",
+            model=model,
+            template="",
+        )
+        self.discipline = discipline
+        self.language = language
+
+    async def aevaluate(
+        self,
+        pdf_data: str,
+        review_text: str,
+        rebuttal_text: str,
+        original_score: int,
+    ) -> GraderScore:
+        """Assess a rebuttal.
+
+        Args:
+            pdf_data: Base64 encoded PDF data URL
+            review_text: The original reviewer comments
+            rebuttal_text: The author's rebuttal text
+            original_score: The original recommendation score (1-6)
+
+        Returns:
+            GraderScore with updated score and assessment details in metadata
+        """
+        try:
+            messages = build_rebuttal_assessment_messages(
+                pdf_data,
+                review_text,
+                rebuttal_text,
+                original_score,
+                discipline=self.discipline,
+                language=self.language,
+            )
+            response = await self.model.achat(messages=messages)
+            content = await extract_response_content(response)
+            parsed = parse_rebuttal_assessment_response(content)
+
+            return GraderScore(
+                name=self.name,
+                score=parsed["updated_score"],
+                reason=parsed["overall_assessment"],
+                metadata={
+                    "original_score": original_score,
+                    "score_change_reasoning": parsed["score_change_reasoning"],
+                    "point_assessments": parsed["point_assessments"],
+                    "unresolved_concerns": parsed["unresolved_concerns"],
+                    "rebuttal_strengths": parsed["rebuttal_strengths"],
+                },
+            )
+        except Exception as e:
+            return GraderError(name=self.name, error=str(e))