[feat]add skills graders (#152)

* add skills graders

* Skills graders: streamline tests and fix pre-commit hygiene

- Refine skill grader modules (comprehensive, pairwise, relevance, safety) and exports
- Simplify skill grader test suites with less boilerplate
- Ignore tests/graders/skills/*.json artifacts in .gitignore
- Run pytest in pre-commit via uv so the project environment provides pytest
- Fix flake8 E226 spacing and long-line issues in cookbooks, experiments, UI batch panels

Made-with: Cursor

* feat(graders): add LLMGrader system prompts to skill PromptTemplates

Align skill graders with tool_call_accuracy: each default template now uses
a system message (LLMGrader.SYSTEM_PROMPT_EN/ZH) plus the existing user rubric.

Made-with: Cursor

* chore: align .pre-commit-config.yaml with main

Use python -m pytest for the local pytest hook (no uv in CI). Match main
hooks: drop gitleaks and detect-aws-credentials extras from the feature branch.

Made-with: Cursor

Author: helloml0326

.gitignore

@@ -38,6 +38,7 @@ local/
 
 # Test Artifacts
 tests/plt_*
+tests/graders/skills/*.json
 
 # Security Files
 *.pem

.pre-commit-config.yaml

@@ -1,8 +1,4 @@
 repos:
-  - repo: https://github.com/gitleaks/gitleaks
-    rev: v8.18.4
-    hooks:
-      - id: gitleaks
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v6.0.0
     hooks:
@@ -12,8 +8,6 @@ repos:
       - id: check-toml
       - id: check-json
       - id: detect-private-key
-      - id: detect-aws-credentials
-        args: ['--allow-missing-credentials']
       - id: trailing-whitespace
   - repo: https://github.com/PyCQA/autoflake
     rev: v2.2.1
@@ -111,4 +105,4 @@ repos:
           "--ignore=tests/generator",
           "--ignore=data",
           "--ignore=examples"
-        ]
+        ]

cookbooks/auto_arena/report_generator.py

@@ -69,7 +69,7 @@ def _prepare_context(
         original_details = [d for d in details if d.order == "original"]
 
         # Format rankings
-        rankings_text = "\n".join(f"{i+1}. {name}: {rate:.1%}" for i, (name, rate) in enumerate(result.rankings))
+        rankings_text = "\n".join(f"{i + 1}. {name}: {rate:.1%}" for i, (name, rate) in enumerate(result.rankings))
 
         # Format rubrics
         rubrics_text = "\n".join(f"- {r}" for r in rubrics)

cookbooks/ref_hallucination_arena/collectors/bib_extractor.py

@@ -293,7 +293,7 @@ def _parse_plain_text(self, text: str) -> List[Reference]:
 
                 refs.append(
                     Reference(
-                        key=f"ref_{len(refs)+1}",
+                        key=f"ref_{len(refs) + 1}",
                         title=title.strip(),
                         authors=authors.strip() if authors else None,
                         year=year.strip(),

cookbooks/ref_hallucination_arena/pipeline.py

@@ -545,7 +545,7 @@ def _verify_one(model_name: str, idx: int) -> None:
             total_v = sum(m.verified for m in model_results)
             total_r = sum(m.total_refs for m in model_results)
             if total_r > 0:
-                logger.info(f"  {model_name}: {total_v}/{total_r} verified " f"({total_v/total_r:.1%})")
+                logger.info(f"  {model_name}: {total_v}/{total_r} verified " f"({total_v / total_r:.1%})")
             else:
                 logger.info(f"  {model_name}: 0 refs")
 
@@ -735,7 +735,7 @@ def _on_query_complete(local_idx: int, result_dict: dict) -> None:
             tv = sum(m.verified for m in mr)
             tr = sum(m.total_refs for m in mr)
             if tr > 0:
-                logger.info(f"  {mn}: {tv}/{tr} verified ({tv/tr:.1%})")
+                logger.info(f"  {mn}: {tv}/{tr} verified ({tv / tr:.1%})")
             else:
                 logger.info(f"  {mn}: 0 refs")
 

experiments/run_grader_evaluations.py

@@ -342,16 +342,16 @@ async def run_all_evaluations(
         "agent": agent_model,
     }
 
-    print(f"\n{'='*70}")
+    print(f"\n{'=' * 70}")
     print("OpenJudge Grader Evaluation Suite")
-    print(f"{'='*70}")
+    print(f"{'=' * 70}")
     print(f"Categories: {', '.join(categories)}")
     print(f"Text Model: {text_model}")
     print(f"Multimodal Model: {multimodal_model}")
     print(f"Agent Model: {agent_model}")
     print(f"Max Workers: {max_workers}")
     print(f"Total Evaluations: {len(configs_to_run)}")
-    print(f"{'='*70}\n")
+    print(f"{'=' * 70}\n")
 
     # Create semaphore for concurrency control
     semaphore = asyncio.Semaphore(max_workers)

openjudge/graders/common/search_correctness.py

@@ -73,7 +73,7 @@ async def aexecute(self, query: str, search_depth: str = "advanced", **kwargs) -
             for i, r in enumerate(results[:5]):
                 content = r.get("content", "")[:1500]
                 summary_parts.append(
-                    f"[{i+1}] {r.get('title', '')}\n" f"URL: {r.get('url', '')}\n" f"Content: {content}"
+                    f"[{i + 1}] {r.get('title', '')}\n" f"URL: {r.get('url', '')}\n" f"Content: {content}"
                 )
 
             return ToolResult(

openjudge/graders/skills/__init__.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+"""
+Skill Graders
+
+This module contains graders for evaluating AI Agent Skill packages:
+- Safety evaluation: detects dangerous operations, overly broad permissions, and missing safeguards
+- Relevance evaluation: measures how well a skill's capabilities address a task description
+- Completeness evaluation: measures whether a skill provides sufficient detail to accomplish a task
+- Structure evaluation: assesses structural design quality across anti-pattern quality,
+  specification compliance, progressive disclosure, and freedom calibration
+- Comprehensive evaluation: holistic multi-dimensional assessment combining all four dimensions
+- Comprehensive pairwise evaluation: head-to-head comparison of two skill candidates
+"""
+
+from openjudge.graders.skills.completeness import SkillCompletenessGrader
+from openjudge.graders.skills.comprehensive import SkillComprehensiveGrader
+from openjudge.graders.skills.comprehensive_pairwise import (
+    SkillComprehensivePairwiseGrader,
+)
+from openjudge.graders.skills.relevance import SkillRelevanceGrader
+from openjudge.graders.skills.safety import SkillSafetyGrader
+from openjudge.graders.skills.structure import SkillStructureGrader
+
+__all__ = [
+    "SkillSafetyGrader",
+    "SkillRelevanceGrader",
+    "SkillCompletenessGrader",
+    "SkillStructureGrader",
+    "SkillComprehensiveGrader",
+    "SkillComprehensivePairwiseGrader",
+]