evals: log reasoning and extend max_tokens for chat completions (#62)

Author: volsgd

gpt_oss/evals/__main__.py

@@ -3,6 +3,7 @@
 from datetime import datetime
 
 from . import report
+from .basic_eval import BasicEval
 from .gpqa_eval import GPQAEval
 from .aime_eval import AIME25Eval
 from .healthbench_eval import HealthBenchEval
@@ -81,6 +82,7 @@ def main():
                 reasoning_effort=reasoning_effort,
                 temperature=args.temperature,
                 base_url=args.base_url,
+                max_tokens=131_072,
             )
 
     print(f"Running with args {args}")
@@ -98,9 +100,11 @@ def get_evals(eval_name, debug_mode):
         )
         # Set num_examples = None to reproduce full evals
         match eval_name:
+            case "basic":
+                return BasicEval()
             case "gpqa":
                 return GPQAEval(
-                    n_repeats=8,
+                    n_repeats=1 if args.debug else 8,
                     num_examples=num_examples,
                     debug=debug_mode,
                     n_threads=args.n_threads or 1,
@@ -131,7 +135,7 @@ def get_evals(eval_name, debug_mode):
                 )
             case "aime25":
                 return AIME25Eval(
-                    n_repeats=8,
+                    n_repeats=1 if args.debug else 8,
                     num_examples=num_examples,
                     n_threads=args.n_threads or 1,
                 )

gpt_oss/evals/basic_eval.py

@@ -0,0 +1,38 @@
+"""
+Basic eval
+"""
+from . import report
+
+from .types import Eval, EvalResult, SamplerBase, SingleEvalResult
+
+class BasicEval(Eval):
+    def __init__(self,):
+        self.examples = [{
+            "question": "hi",
+            "answer": "hi, how can i help?",
+        }]
+
+    def __call__(self, sampler: SamplerBase) -> EvalResult:
+        def fn(row: dict):
+            sampler_response = sampler([
+                sampler._pack_message(content=row["question"], role="user")
+            ])
+            response_text = sampler_response.response_text
+            extracted_answer = response_text
+            actual_queried_prompt_messages = sampler_response.actual_queried_message_list
+            score = 1.0 if len(extracted_answer) > 0 else 0.0
+            html = report.jinja_env.from_string(report.HTML_JINJA).render(
+                prompt_messages=actual_queried_prompt_messages,
+                next_message=dict(content=response_text, role="assistant"),
+                score=score,
+                correct_answer=row["answer"],
+                extracted_answer=extracted_answer,
+            )
+            convo = actual_queried_prompt_messages + [dict(content=response_text, role="assistant")]
+            return SingleEvalResult(
+                html=html, score=score, convo=convo, metrics={"chars": len(response_text)}
+            )
+
+        results = report.map_with_progress(fn, self.examples, num_threads=1)
+        return report.aggregate_results(results)
+

gpt_oss/evals/chat_completions_sampler.py

@@ -27,7 +27,6 @@ def __init__(
         reasoning_effort: str | None = None,
         base_url: str = "http://localhost:8000/v1",
     ):
-        self.api_key_name = "OPENAI_API_KEY"
         self.client = OpenAI(base_url=base_url, timeout=24 * 60 * 60)
         self.model = model
         self.system_message = system_message
@@ -63,8 +62,13 @@ def __call__(self, message_list: MessageList) -> SamplerResponse:
                         temperature=self.temperature,
                         max_tokens=self.max_tokens,
                     )
-                content = response.choices[0].message.content
-                if content is None:
+
+                choice = response.choices[0]
+                content = choice.message.content
+                if getattr(choice.message, "reasoning", None):
+                    message_list.append(self._pack_message("assistant", choice.message.reasoning))
+
+                if not content:
                     raise ValueError("OpenAI API returned empty response; retrying")
                 return SamplerResponse(
                     response_text=content,

gpt_oss/evals/responses_sampler.py

@@ -17,12 +17,11 @@ def __init__(
         model: str,
         developer_message: str | None = None,
         temperature: float = 1.0,
-        max_tokens: int = 1024,
+        max_tokens: int = 131_072,
         reasoning_model: bool = False,
         reasoning_effort: str | None = None,
         base_url: str = "http://localhost:8000/v1",
     ):
-        self.api_key_name = "OPENAI_API_KEY"
         self.client = OpenAI(base_url=base_url, timeout=24*60*60)
         self.model = model
         self.developer_message = developer_message