fix scripts

Author: codelion

scripts/eval_aime_benchmark.py

@@ -19,9 +19,9 @@
 logger = logging.getLogger(__name__)
 
 # Initialize OpenAI client
-# client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"), base_url="https://openrouter.ai/api/v1")
+client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"), base_url="https://openrouter.ai/api/v1")
 
-client = OpenAI(api_key="optillm", base_url="http://localhost:8001/v1")
+# client = OpenAI(api_key="optillm", base_url="http://localhost:8001/v1")
 
 SYSTEM_PROMPT = '''You are solving AIME (American Invitational Mathematics Examination) problems.
 
@@ -282,12 +282,12 @@ def get_llm_response(problem: str, model: str, analyze_logits: bool = False) ->
                 {"role": "user", "content": SYSTEM_PROMPT + problem}
             ],
             max_tokens=8192,
-            extra_body={
-                "decoding": "thinkdeeper",
-                "min_thinking_tokens" : 0,
-                "max_thinking_tokens" : 8000,
-                "max_thoughts": 100,
-            },
+            # extra_body={
+            #     "decoding": "thinkdeeper",
+            #     "min_thinking_tokens" : 0,
+            #     "max_thinking_tokens" : 8000,
+            #     "max_thoughts": 100,
+            # },
             **kwargs
         )
 

scripts/eval_optillmbench.py

@@ -36,7 +36,7 @@
 ]
 
 def load_optillm_bench() -> datasets.Dataset:
-    """Load the OptILLM Bench dataset."""
+    """Load the OptiLLM Bench dataset."""
     try:
         dataset = load_dataset("codelion/optillmbench")
         return dataset["test"]  # We use the test split for evaluation
@@ -161,10 +161,6 @@ def evaluate_model(
                 ],
                 temperature=0.2,
                 max_tokens=4096,
-                reasoning_effort="low",
-                extra_body = {
-                    "decoding" : "thinkdeeper",
-                }
             )
 
             # Calculate time taken
@@ -255,7 +251,7 @@ def generate_report(all_metrics: Dict[str, Dict[str, float]], output_dir: str):
     report = []
 
     # Header
-    report.append("# OptILLM Bench Evaluation Report")
+    report.append("# OptiLLM Bench Evaluation Report")
     report.append(f"Generated on: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n")
 
     # Overall Results Table
@@ -303,7 +299,7 @@ def generate_report(all_metrics: Dict[str, Dict[str, float]], output_dir: str):
     logger.info(f"Report saved to {report_path}")
 
 def main():
-    parser = argparse.ArgumentParser(description="Evaluate a model on OptILLM Bench")
+    parser = argparse.ArgumentParser(description="Evaluate a model on OptiLLM Bench")
     parser.add_argument("--model", required=True, help="Model identifier")
     parser.add_argument("--base-url", default="http://localhost:8000/v1", 
                         help="Base URL for API endpoint")