Skip to content

A common util method of formatting history for agent graders. #42

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ployts merged 1 commit into from
Jan 8, 2026
Merged

A common util method of formatting history for agent graders. #42

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 2 additions & 24 deletions openjudge/graders/agent/action/action_alignment.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

from loguru import logger

from openjudge.graders.agent.utils import format_history
from openjudge.graders.base_grader import GraderMode, GraderScore
from openjudge.graders.llm_grader import LLMGrader
from openjudge.models.base_chat_model import BaseChatModel
Expand Down Expand Up @@ -189,29 +190,6 @@ def __init__(
language=language,
)

def _format_history(self, history: Optional[list] = None) -> str:
"""Format history steps for evaluation.

Args:
history: Optional list of previous step dictionaries

Returns:
Formatted history string, or empty string if no history
"""
if not history:
return ""

lines = ["<History Steps>"]
for i, hist_step in enumerate(history):
lines.append(f"Step {i + 1}:")
for key, value in hist_step.items():
if value:
lines.append(f"{key.capitalize()}: {value}")
lines.append("")
lines.append("</History Steps>")

return "\n".join(lines)

async def aevaluate(
self,
plan: str,
Expand Down Expand Up @@ -244,7 +222,7 @@ async def aevaluate(
context_str = f"<context>\n{context}\n</context>"

# Format history
history_str = self._format_history(history)
history_str = format_history(history)

try:
result = await super().aevaluate(
Expand Down
26 changes: 2 additions & 24 deletions openjudge/graders/agent/memory/memory_accuracy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

from loguru import logger

from openjudge.graders.agent.utils import format_history
from openjudge.graders.base_grader import GraderMode, GraderScore
from openjudge.graders.llm_grader import LLMGrader
from openjudge.models.base_chat_model import BaseChatModel
Expand Down Expand Up @@ -180,29 +181,6 @@ def __init__(
language=language,
)

def _format_history(self, history: Optional[list] = None) -> str:
"""Format history steps for evaluation.

Args:
history: Optional list of previous step dictionaries

Returns:
Formatted history string, or empty string if no history
"""
if not history:
return ""

lines = ["<History Steps>"]
for i, hist_step in enumerate(history):
lines.append(f"Step {i + 1}:")
for key, value in hist_step.items():
if value:
lines.append(f"{key.capitalize()}: {value}")
lines.append("")
lines.append("</History Steps>")

return "\n".join(lines)

async def aevaluate(
self,
observation: str,
Expand Down Expand Up @@ -237,7 +215,7 @@ async def aevaluate(
context_str = f"<context>\n{context}\n</context>"

# Format history
history_str = self._format_history(history)
history_str = format_history(history)

try:
result = await super().aevaluate(
Expand Down
26 changes: 2 additions & 24 deletions openjudge/graders/agent/memory/memory_detail_preservation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

from loguru import logger

from openjudge.graders.agent.utils import format_history
from openjudge.graders.base_grader import GraderMode, GraderScore
from openjudge.graders.llm_grader import LLMGrader
from openjudge.models.base_chat_model import BaseChatModel
Expand Down Expand Up @@ -180,29 +181,6 @@ def __init__(
language=language,
)

def _format_history(self, history: Optional[list] = None) -> str:
"""Format history steps for evaluation.

Args:
history: Optional list of previous step dictionaries

Returns:
Formatted history string, or empty string if no history
"""
if not history:
return ""

lines = ["<History Steps>"]
for i, hist_step in enumerate(history):
lines.append(f"Step {i + 1}:")
for key, value in hist_step.items():
if value:
lines.append(f"{key.capitalize()}: {value}")
lines.append("")
lines.append("</History Steps>")

return "\n".join(lines)

async def aevaluate(
self,
observation: str,
Expand Down Expand Up @@ -237,7 +215,7 @@ async def aevaluate(
context_str = f"<context>\n{context}\n</context>"

# Format history
history_str = self._format_history(history)
history_str = format_history(history)

try:
result = await super().aevaluate(
Expand Down
26 changes: 2 additions & 24 deletions openjudge/graders/agent/memory/memory_retrieval_effectiveness.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

from loguru import logger

from openjudge.graders.agent.utils import format_history
from openjudge.graders.base_grader import GraderMode, GraderScore
from openjudge.graders.llm_grader import LLMGrader
from openjudge.models.base_chat_model import BaseChatModel
Expand Down Expand Up @@ -183,29 +184,6 @@ def __init__(
language=language,
)

def _format_history(self, history: Optional[list] = None) -> str:
"""Format history steps for evaluation.

Args:
history: Optional list of previous step dictionaries

Returns:
Formatted history string, or empty string if no history
"""
if not history:
return ""

lines = ["<History Steps>"]
for i, hist_step in enumerate(history):
lines.append(f"Step {i + 1}:")
for key, value in hist_step.items():
if value:
lines.append(f"{key.capitalize()}: {value}")
lines.append("")
lines.append("</History Steps>")

return "\n".join(lines)

async def aevaluate(
self,
plan: str,
Expand Down Expand Up @@ -243,7 +221,7 @@ async def aevaluate(
context_str = f"<context>\n{context}\n</context>"

# Format history
history_str = self._format_history(history)
history_str = format_history(history)

try:
result = await super().aevaluate(
Expand Down
26 changes: 2 additions & 24 deletions openjudge/graders/agent/plan/plan_feasibility.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

from loguru import logger

from openjudge.graders.agent.utils import format_history
from openjudge.graders.base_grader import GraderMode, GraderScore
from openjudge.graders.llm_grader import LLMGrader
from openjudge.models.base_chat_model import BaseChatModel
Expand Down Expand Up @@ -183,29 +184,6 @@ def __init__(
language=language,
)

def _format_history(self, history: Optional[list] = None) -> str:
"""Format history steps for evaluation.

Args:
history: Optional list of previous step dictionaries

Returns:
Formatted history string, or empty string if no history
"""
if not history:
return ""

lines = ["<History Steps>"]
for i, hist_step in enumerate(history):
lines.append(f"Step {i + 1}:")
for key, value in hist_step.items():
if value:
lines.append(f"{key.capitalize()}: {value}")
lines.append("")
lines.append("</History Steps>")

return "\n".join(lines)

async def aevaluate(
self,
plan: str,
Expand Down Expand Up @@ -243,7 +221,7 @@ async def aevaluate(
context_str = f"<context>\n{context}\n</context>"

# Format history
history_str = self._format_history(history)
history_str = format_history(history)

try:
result = await super().aevaluate(
Expand Down
26 changes: 2 additions & 24 deletions openjudge/graders/agent/reflection/reflection_accuracy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

from loguru import logger

from openjudge.graders.agent.utils import format_history
from openjudge.graders.base_grader import GraderMode, GraderScore
from openjudge.graders.llm_grader import LLMGrader
from openjudge.models.base_chat_model import BaseChatModel
Expand Down Expand Up @@ -180,29 +181,6 @@ def __init__(
language=language,
)

def _format_history(self, history: Optional[list] = None) -> str:
"""Format history steps for evaluation.

Args:
history: Optional list of previous step dictionaries

Returns:
Formatted history string, or empty string if no history
"""
if not history:
return ""

lines = ["<History Steps>"]
for i, hist_step in enumerate(history):
lines.append(f"Step {i + 1}:")
for key, value in hist_step.items():
if value:
lines.append(f"{key.capitalize()}: {value}")
lines.append("")
lines.append("</History Steps>")

return "\n".join(lines)

async def aevaluate(
self,
observation: str,
Expand Down Expand Up @@ -237,7 +215,7 @@ async def aevaluate(
context_str = f"<context>\n{context}\n</context>"

# Format history
history_str = self._format_history(history)
history_str = format_history(history)

try:
result = await super().aevaluate(
Expand Down
26 changes: 2 additions & 24 deletions openjudge/graders/agent/reflection/reflection_outcome_understanding.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

from loguru import logger

from openjudge.graders.agent.utils import format_history
from openjudge.graders.base_grader import GraderMode, GraderScore
from openjudge.graders.llm_grader import LLMGrader
from openjudge.models.base_chat_model import BaseChatModel
Expand Down Expand Up @@ -304,29 +305,6 @@ def __init__(
language=language,
)

def _format_history(self, history: Optional[list] = None) -> str:
"""Format history steps for evaluation.

Args:
history: Optional list of previous step dictionaries

Returns:
Formatted history string, or empty string if no history
"""
if not history:
return ""

lines = ["<History Steps>"]
for i, hist_step in enumerate(history):
lines.append(f"Step {i + 1}:")
for key, value in hist_step.items():
if value:
lines.append(f"{key.capitalize()}: {value}")
lines.append("")
lines.append("</History Steps>")

return "\n".join(lines)

async def aevaluate(
self,
observation: str,
Expand Down Expand Up @@ -361,7 +339,7 @@ async def aevaluate(
context_str = f"<context>\n{context}\n</context>"

# Format history
history_str = self._format_history(history)
history_str = format_history(history)

try:
result = await super().aevaluate(
Expand Down
26 changes: 2 additions & 24 deletions openjudge/graders/agent/reflection/reflection_progress_awareness.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

from loguru import logger

from openjudge.graders.agent.utils import format_history
from openjudge.graders.base_grader import GraderMode, GraderScore
from openjudge.graders.llm_grader import LLMGrader
from openjudge.models.base_chat_model import BaseChatModel
Expand Down Expand Up @@ -221,29 +222,6 @@ def __init__(
language=language,
)

def _format_history(self, history: Optional[list] = None) -> str:
"""Format history steps for evaluation.

Args:
history: Optional list of previous step dictionaries

Returns:
Formatted history string, or empty string if no history
"""
if not history:
return ""

lines = ["<History Steps>"]
for i, hist_step in enumerate(history):
lines.append(f"Step {i + 1}:")
for key, value in hist_step.items():
if value:
lines.append(f"{key.capitalize()}: {value}")
lines.append("")
lines.append("</History Steps>")

return "\n".join(lines)

async def aevaluate(
self,
observation: str,
Expand Down Expand Up @@ -278,7 +256,7 @@ async def aevaluate(
context_str = f"<context>\n{context}\n</context>"

# Format history
history_str = self._format_history(history)
history_str = format_history(history)

try:
result = await super().aevaluate(
Expand Down
Loading