Skip to content

Commit 7f1f294

Browse files
codelioncodelion
committed
fix
1 parent 24ff60a commit 7f1f294

File tree

4 files changed

+198
-3
lines changed

4 files changed

+198
-3
lines changed

‎openevolve/_version.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
"""Version information for openevolve package."""
22

3-
__version__ = "0.2.2"
3+
__version__ = "0.2.3"

‎openevolve/llm/openai.py‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,14 +66,38 @@ async def generate_with_context(
6666
formatted_messages.extend(messages)
6767

6868
# Set up generation parameters
69-
if self.api_base == "https://api.openai.com/v1" and str(self.model).lower().startswith("o"):
70-
# For o-series models
69+
# Define OpenAI reasoning models that require max_completion_tokens
70+
# These models don't support temperature/top_p and use different parameters
71+
OPENAI_REASONING_MODEL_PREFIXES = (
72+
# O-series reasoning models
73+
"o1-", "o1", # o1, o1-mini, o1-preview
74+
"o3-", "o3", # o3, o3-mini, o3-pro
75+
"o4-", # o4-mini
76+
# GPT-5 series are also reasoning models
77+
"gpt-5-", "gpt-5" # gpt-5, gpt-5-mini, gpt-5-nano
78+
)
79+
80+
# Check if this is an OpenAI reasoning model
81+
model_lower = str(self.model).lower()
82+
is_openai_reasoning_model = (
83+
self.api_base == "https://api.openai.com/v1" and
84+
model_lower.startswith(OPENAI_REASONING_MODEL_PREFIXES)
85+
)
86+
87+
if is_openai_reasoning_model:
88+
# For OpenAI reasoning models
7189
params = {
7290
"model": self.model,
7391
"messages": formatted_messages,
7492
"max_completion_tokens": kwargs.get("max_tokens", self.max_tokens),
7593
}
94+
# Add optional reasoning parameters if provided
95+
if "reasoning_effort" in kwargs:
96+
params["reasoning_effort"] = kwargs["reasoning_effort"]
97+
if "verbosity" in kwargs:
98+
params["verbosity"] = kwargs["verbosity"]
7699
else:
100+
# Standard parameters for all other models
77101
params = {
78102
"model": self.model,
79103
"messages": formatted_messages,

‎tests/test_model_parameter_demo.py‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
"""
2+
Demonstration of fixed OpenAI model parameter handling
3+
"""
4+
5+
def demo_model_parameter_selection():
6+
"""Demonstrate how different models get different parameters"""
7+
8+
# Mock the logic from openai.py
9+
OPENAI_REASONING_MODEL_PREFIXES = (
10+
# O-series reasoning models
11+
"o1-", "o1", # o1, o1-mini, o1-preview
12+
"o3-", "o3", # o3, o3-mini, o3-pro
13+
"o4-", # o4-mini
14+
# GPT-5 series are also reasoning models
15+
"gpt-5-", "gpt-5" # gpt-5, gpt-5-mini, gpt-5-nano
16+
)
17+
18+
def get_params_for_model(model_name, api_base="https://api.openai.com/v1"):
19+
"""Show what parameters would be used for each model"""
20+
model_lower = str(model_name).lower()
21+
is_openai_reasoning_model = (
22+
api_base == "https://api.openai.com/v1" and
23+
model_lower.startswith(OPENAI_REASONING_MODEL_PREFIXES)
24+
)
25+
26+
if is_openai_reasoning_model:
27+
return {
28+
"type": "reasoning_model",
29+
"uses": "max_completion_tokens",
30+
"supports": ["reasoning_effort", "verbosity"],
31+
"excludes": ["temperature", "top_p"]
32+
}
33+
else:
34+
return {
35+
"type": "standard_model",
36+
"uses": "max_tokens",
37+
"supports": ["temperature", "top_p"],
38+
"excludes": []
39+
}
40+
41+
print("🔧 OpenAI Model Parameter Selection Demo")
42+
print("=" * 50)
43+
44+
test_models = [
45+
# Reasoning models
46+
("o1-mini", "✅ Reasoning"),
47+
("o1-preview", "✅ Reasoning"),
48+
("o3-mini-2025-01-31", "✅ Reasoning (with date)"),
49+
("gpt-5-nano", "✅ Reasoning (GPT-5 series)"),
50+
51+
# Standard models
52+
("gpt-4o-mini", "❌ Standard (not reasoning)"),
53+
("gpt-4o", "❌ Standard"),
54+
("gpt-4-turbo", "❌ Standard"),
55+
]
56+
57+
for model, description in test_models:
58+
params = get_params_for_model(model)
59+
print(f"\n📋 Model: {model}")
60+
print(f" Type: {description}")
61+
print(f" Uses: {params['uses']}")
62+
print(f" Supports: {', '.join(params['supports'])}")
63+
if params['excludes']:
64+
print(f" Excludes: {', '.join(params['excludes'])}")
65+
66+
print("\n" + "=" * 50)
67+
print("✅ Fix successful! No more false positives/negatives.")
68+
69+
if __name__ == "__main__":
70+
demo_model_parameter_selection()

‎tests/test_openai_model_detection.py‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
"""
2+
Test OpenAI reasoning model detection logic
3+
"""
4+
5+
import unittest
6+
from unittest.mock import MagicMock
7+
8+
9+
class TestOpenAIReasoningModelDetection(unittest.TestCase):
10+
"""Test that OpenAI reasoning models are correctly identified"""
11+
12+
def test_reasoning_model_detection(self):
13+
"""Test various model names to ensure correct reasoning model detection"""
14+
15+
# Define the same constants as in the code
16+
OPENAI_REASONING_MODEL_PREFIXES = (
17+
# O-series reasoning models
18+
"o1-", "o1", # o1, o1-mini, o1-preview
19+
"o3-", "o3", # o3, o3-mini, o3-pro
20+
"o4-", # o4-mini
21+
# GPT-5 series are also reasoning models
22+
"gpt-5-", "gpt-5" # gpt-5, gpt-5-mini, gpt-5-nano
23+
)
24+
25+
def is_reasoning_model(model_name, api_base="https://api.openai.com/v1"):
26+
"""Test function that mimics the logic in openai.py"""
27+
model_lower = str(model_name).lower()
28+
return (
29+
api_base == "https://api.openai.com/v1" and
30+
model_lower.startswith(OPENAI_REASONING_MODEL_PREFIXES)
31+
)
32+
33+
# Test cases: (model_name, expected_result, description)
34+
test_cases = [
35+
# Reasoning models - should return True
36+
("o1", True, "Base o1 model"),
37+
("o1-mini", True, "o1-mini model"),
38+
("o1-preview", True, "o1-preview model"),
39+
("o1-mini-2025-01-31", True, "o1-mini with date"),
40+
("o3", True, "Base o3 model"),
41+
("o3-mini", True, "o3-mini model"),
42+
("o3-pro", True, "o3-pro model"),
43+
("o4-mini", True, "o4-mini model"),
44+
("gpt-5", True, "Base gpt-5 model"),
45+
("gpt-5-mini", True, "gpt-5-mini model"),
46+
("gpt-5-nano", True, "gpt-5-nano model"),
47+
48+
# Non-reasoning models - should return False
49+
("gpt-4o-mini", False, "gpt-4o-mini (not reasoning)"),
50+
("gpt-4o", False, "gpt-4o (not reasoning)"),
51+
("gpt-4", False, "gpt-4 (not reasoning)"),
52+
("gpt-3.5-turbo", False, "gpt-3.5-turbo (not reasoning)"),
53+
("claude-3", False, "Non-OpenAI model"),
54+
("gemini-pro", False, "Non-OpenAI model"),
55+
56+
# Edge cases
57+
("O1-MINI", True, "Uppercase o1-mini"),
58+
("GPT-5-MINI", True, "Uppercase gpt-5-mini"),
59+
]
60+
61+
for model_name, expected, description in test_cases:
62+
with self.subTest(model=model_name, desc=description):
63+
result = is_reasoning_model(model_name)
64+
self.assertEqual(
65+
result,
66+
expected,
67+
f"Model '{model_name}' ({description}): expected {expected}, got {result}"
68+
)
69+
70+
def test_non_openai_api_base(self):
71+
"""Test that non-OpenAI API bases don't trigger reasoning model logic"""
72+
OPENAI_REASONING_MODEL_PREFIXES = (
73+
"o1-", "o1", "o3-", "o3", "o4-", "gpt-5-", "gpt-5"
74+
)
75+
76+
def is_reasoning_model(model_name, api_base):
77+
model_lower = str(model_name).lower()
78+
return (
79+
api_base == "https://api.openai.com/v1" and
80+
model_lower.startswith(OPENAI_REASONING_MODEL_PREFIXES)
81+
)
82+
83+
# Even reasoning model names should return False for non-OpenAI APIs
84+
test_cases = [
85+
("o1-mini", "https://api.anthropic.com/v1", False),
86+
("gpt-5", "https://generativelanguage.googleapis.com/v1beta/openai/", False),
87+
("o3-mini", "https://api.deepseek.com/v1", False),
88+
]
89+
90+
for model_name, api_base, expected in test_cases:
91+
with self.subTest(model=model_name, api=api_base):
92+
result = is_reasoning_model(model_name, api_base)
93+
self.assertEqual(
94+
result,
95+
expected,
96+
f"Model '{model_name}' with API '{api_base}' should return {expected}"
97+
)
98+
99+
100+
if __name__ == "__main__":
101+
unittest.main()

0 commit comments

Comments
 (0)