evals: add chat completions API sampler (#59)

* evals: admit --sampler chat_completions

* gpt_oss.evals: allow modifying the model names

Author: volsgd

gpt_oss/evals/__main__.py

@@ -6,9 +6,9 @@
 from .gpqa_eval import GPQAEval
 from .aime_eval import AIME25Eval
 from .healthbench_eval import HealthBenchEval
-from .chat_completion_sampler import (
+from .chat_completions_sampler import (
     OPENAI_SYSTEM_MESSAGE_API,
-    ChatCompletionSampler,
+    ChatCompletionsSampler,
 )
 from .responses_sampler import ResponsesSampler
 
@@ -19,12 +19,23 @@ def main():
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
     parser.add_argument(
-        "--list-models", action="store_true", help="List available models"
+        "--model",
+        type=str,
+        default="gpt-oss-120b,gpt-oss-20b",
+        help="Select a model by name. Accepts a comma-separated list.",
     )
     parser.add_argument(
-        "--model",
+        "--reasoning-effort",
+        type=str,
+        default="low,medium,high",
+        help="Reasoning effort (low, medium, high). Accepts a comma-separated list.",
+    )
+    parser.add_argument(
+        "--sampler",
         type=str,
-        help="Select a model by name. Also accepts a comma-separated list of models.",
+        choices=["responses", "chat_completions"],
+        default="responses",
+        help="Sampler backend to use for models.",
     )
     parser.add_argument(
         "--base-url",
@@ -36,7 +47,7 @@ def main():
         "--eval",
         type=str,
         default="gpqa,healthbench,healthbench_hard,healthbench_consensus,aime25",
-        help="Select an eval by name. Also accepts a comma-separated list of evals.",
+        help="Select an eval by name. Accepts a comma-separated list.",
     )
     parser.add_argument(
         "--temperature",
@@ -59,71 +70,26 @@ def main():
 
     args = parser.parse_args()
 
-    models = {
-        "120b-low": ResponsesSampler(
-            model="gpt-oss-120b",
-            reasoning_model=True,
-            reasoning_effort="low",
-            temperature=args.temperature,
-            base_url=args.base_url,
-        ),
-        "120b": ResponsesSampler(
-            model="gpt-oss-120b",
-            reasoning_model=True,
-            reasoning_effort="medium",
-            temperature=args.temperature,
-            base_url=args.base_url,
-        ),
-        "120b-high": ResponsesSampler(
-            model="gpt-oss-120b",
-            reasoning_model=True,
-            reasoning_effort="high",
-            temperature=args.temperature,
-            base_url=args.base_url,
-        ),
-        "20b-low": ResponsesSampler(
-            model="gpt-oss-20b",
-            reasoning_model=True,
-            reasoning_effort="low",
-            temperature=args.temperature,
-            base_url=args.base_url,
-        ),
-        "20b": ResponsesSampler(
-            model="gpt-oss-20b",
-            reasoning_model=True,
-            reasoning_effort="medium",
-            temperature=args.temperature,
-            base_url=args.base_url,
-        ),
-        "20b-high": ResponsesSampler(
-            model="gpt-oss-20b",
-            reasoning_model=True,
-            reasoning_effort="high",
-            temperature=args.temperature,
-            base_url=args.base_url,
-        ),
-    }
-
-    if args.list_models:
-        print("Available models:")
-        for model_name in models.keys():
-            print(f" - {model_name}")
-        return
-
-    if args.model:
-        models_chosen = args.model.split(",")
-        for model_name in models_chosen:
-            if model_name not in models:
-                print(f"Error: Model '{model_name}' not found.")
-                return
-        models = {model_name: models[model_name] for model_name in models_chosen}
+    sampler_cls = ResponsesSampler if args.sampler == "responses" else ChatCompletionsSampler
+
+    models = {}
+    for model_name in args.model.split(","):
+        for reasoning_effort in args.reasoning_effort.split(","):
+            models[f"{model_name}-{reasoning_effort}"] = sampler_cls(
+                model=model_name,
+                reasoning_model=True,
+                reasoning_effort=reasoning_effort,
+                temperature=args.temperature,
+                base_url=args.base_url,
+            )
 
     print(f"Running with args {args}")
 
-    grading_sampler = ChatCompletionSampler(
+    grading_sampler = ChatCompletionsSampler(
         model="gpt-4.1-2025-04-14",
         system_message=OPENAI_SYSTEM_MESSAGE_API,
         max_tokens=2048,
+        base_url="https://api.openai.com/v1",
     )
 
     def get_evals(eval_name, debug_mode):
@@ -172,17 +138,15 @@ def get_evals(eval_name, debug_mode):
             case _:
                 raise Exception(f"Unrecognized eval type: {eval_name}")
 
-    evals_list = args.eval.split(",")
     evals = {}
-    for eval_name in evals_list:
+    for eval_name in args.eval.split(","):
         evals[eval_name] = get_evals(eval_name, args.debug)
 
-    print(evals)
     debug_suffix = "_DEBUG" if args.debug else ""
     print(debug_suffix)
     mergekey2resultpath = {}
-    print(f"Running the following evals: {list(evals.keys())}")
-    print(f"Running evals for the following models: {list(models.keys())}")
+    print(f"Running the following evals: {evals}")
+    print(f"Running evals for the following models: {models}")
 
     now = datetime.now()
     date_str = now.strftime("%Y%m%d_%H%M%S")
@@ -220,6 +184,7 @@ def get_evals(eval_name, debug_mode):
                 print(f"Writing all results to {full_result_filename}")
 
             mergekey2resultpath[f"{file_stem}"] = result_filename
+
     merge_metrics = []
     for eval_model_name, result_filename in mergekey2resultpath.items():
         try:

gpt_oss/evals/chat_completion_sampler.py renamed to gpt_oss/evals/chat_completions_sampler.py

@@ -6,35 +6,38 @@
 
 from .types import MessageList, SamplerBase, SamplerResponse
 
+
 OPENAI_SYSTEM_MESSAGE_API = "You are a helpful assistant."
 OPENAI_SYSTEM_MESSAGE_CHATGPT = (
     "You are ChatGPT, a large language model trained by OpenAI, based on the GPT-4 architecture."
     + "\nKnowledge cutoff: 2023-12\nCurrent date: 2024-04-01"
 )
 
 
-class ChatCompletionSampler(SamplerBase):
-    """
-    Sample from OpenAI's chat completion API
-    """
+class ChatCompletionsSampler(SamplerBase):
+    """Sample from a Chat Completions compatible API."""
 
     def __init__(
         self,
         model: str = "gpt-3.5-turbo",
         system_message: str | None = None,
         temperature: float = 0.5,
         max_tokens: int = 1024,
+        reasoning_model: bool = False,
+        reasoning_effort: str | None = None,
+        base_url: str = "http://localhost:8000/v1",
     ):
         self.api_key_name = "OPENAI_API_KEY"
-        self.client = OpenAI()
-        # using api_key=os.environ.get("OPENAI_API_KEY")  # please set your API_KEY
+        self.client = OpenAI(base_url=base_url, timeout=24 * 60 * 60)
         self.model = model
         self.system_message = system_message
         self.temperature = temperature
         self.max_tokens = max_tokens
+        self.reasoning_model = reasoning_model
+        self.reasoning_effort = reasoning_effort
         self.image_format = "url"
 
-    def _pack_message(self, role: str, content: Any):
+    def _pack_message(self, role: str, content: Any) -> dict[str, Any]:
         return {"role": str(role), "content": content}
 
     def __call__(self, message_list: MessageList) -> SamplerResponse:
@@ -45,12 +48,21 @@ def __call__(self, message_list: MessageList) -> SamplerResponse:
         trial = 0
         while True:
             try:
-                response = self.client.chat.completions.create(
-                    model=self.model,
-                    messages=message_list,
-                    temperature=self.temperature,
-                    max_tokens=self.max_tokens,
-                )
+                if self.reasoning_model:
+                    response = self.client.chat.completions.create(
+                        model=self.model,
+                        messages=message_list,
+                        reasoning_effort=self.reasoning_effort,
+                        temperature=self.temperature,
+                        max_tokens=self.max_tokens,
+                    )
+                else:
+                    response = self.client.chat.completions.create(
+                        model=self.model,
+                        messages=message_list,
+                        temperature=self.temperature,
+                        max_tokens=self.max_tokens,
+                    )
                 content = response.choices[0].message.content
                 if content is None:
                     raise ValueError("OpenAI API returned empty response; retrying")
@@ -59,7 +71,6 @@ def __call__(self, message_list: MessageList) -> SamplerResponse:
                     response_metadata={"usage": response.usage},
                     actual_queried_message_list=message_list,
                 )
-            # NOTE: BadRequestError is triggered once for MMMU, please uncomment if you are reruning MMMU
             except openai.BadRequestError as e:
                 print("Bad Request Error", e)
                 return SamplerResponse(
@@ -68,7 +79,7 @@ def __call__(self, message_list: MessageList) -> SamplerResponse:
                     actual_queried_message_list=message_list,
                 )
             except Exception as e:
-                exception_backoff = 2**trial  # expontial back off
+                exception_backoff = 2 ** trial  # exponential back off
                 print(
                     f"Rate limit exception so wait and retry {trial} after {exception_backoff} sec",
                     e,

gpt_oss/evals/healthbench_eval.py

@@ -26,9 +26,9 @@
 import numpy as np
 
 from . import report
-from .chat_completion_sampler import (
+from .chat_completions_sampler import (
     OPENAI_SYSTEM_MESSAGE_API,
-    ChatCompletionSampler,
+    ChatCompletionsSampler,
 )
 from .types import Eval, EvalResult, MessageList, SamplerBase, SingleEvalResult
 
@@ -540,10 +540,11 @@ def physician_completions_main(
     now = datetime.now()
     date_str = now.strftime("%Y%m%d_%H%M")
 
-    grading_sampler = ChatCompletionSampler(
+    grading_sampler = ChatCompletionsSampler(
         model="gpt-4.1-2025-04-14",
         system_message=OPENAI_SYSTEM_MESSAGE_API,
         max_tokens=2048,
+        base_url="https://api.openai.com/v1",
     )
     dummy_sampler = SamplerBase()