Skip to content

Commit 2331fb4

Browse files
ishaan-jaffishaan-jaff
authored
[Bug]: Gemini 2.5 Pro – schema validation fails with OpenAI-style type arrays in tools (#14154)
* fix: _convert_schema_types * fix recursive detector * test_convert_schema_types_type_array_conversion * fix: DEFAULT_NUM_WORKERS_LITELLM_PROXY
1 parent 98d57b5 commit 2331fb4

File tree

6 files changed

+292
-22
lines changed

6 files changed

+292
-22
lines changed

‎docs/my-website/docs/proxy/config_settings.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -431,6 +431,7 @@ router_settings:
431431
| DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT | Default token count for mock response completions. Default is 20
432432
| DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT | Default token count for mock response prompts. Default is 10
433433
| DEFAULT_MODEL_CREATED_AT_TIME | Default creation timestamp for models. Default is 1677610602
434+
| DEFAULT_NUM_WORKERS_LITELLM_PROXY | Default number of workers for LiteLLM proxy. Default is 4. **We strongly recommend setting NUM Workers to Number of vCPUs available** |
434435
| DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD | Default threshold for prompt injection similarity. Default is 0.7
435436
| DEFAULT_POLLING_INTERVAL | Default polling interval for schedulers in seconds. Default is 0.03
436437
| DEFAULT_REASONING_EFFORT_DISABLE_THINKING_BUDGET | Default reasoning effort disable thinking budget. Default is 0

‎litellm/llms/vertex_ai/common_utils.py‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,8 @@ def _build_vertex_schema(parameters: dict, add_property_ordering: bool = False):
215215
# * https://github.com/pydantic/pydantic/discussions/4872
216216
convert_anyof_null_to_nullable(parameters)
217217

218+
_convert_schema_types(parameters)
219+
218220
# Handle empty items objects
219221
process_items(parameters)
220222
add_object_type(parameters)
@@ -439,6 +441,47 @@ def _convert_vertex_datetime_to_openai_datetime(vertex_datetime: str) -> int:
439441
return int(dt.timestamp())
440442

441443

444+
def _convert_schema_types(schema, depth=0):
445+
"""
446+
Convert type arrays and lowercase types for Vertex AI compatibility.
447+
448+
Transforms OpenAI-style schemas to Vertex AI format by converting type arrays
449+
like ["string", "number"] to anyOf format and converting all types to uppercase.
450+
"""
451+
if depth > DEFAULT_MAX_RECURSE_DEPTH:
452+
raise ValueError(
453+
f"Max depth of {DEFAULT_MAX_RECURSE_DEPTH} exceeded while processing schema. Please check the schema for excessive nesting."
454+
)
455+
456+
if not isinstance(schema, dict):
457+
return
458+
459+
460+
# Handle type field
461+
if "type" in schema:
462+
type_val = schema["type"]
463+
if isinstance(type_val, list) and len(type_val) > 1:
464+
# Convert ["string", "number"] -> {"anyOf": [{"type": "STRING"}, {"type": "NUMBER"}]}
465+
schema["anyOf"] = [{"type": t} for t in type_val if isinstance(t, str)]
466+
schema.pop("type")
467+
elif isinstance(type_val, list) and len(type_val) == 1:
468+
schema["type"] = type_val[0]
469+
elif isinstance(type_val, str):
470+
schema["type"] = type_val
471+
472+
# Recursively process nested properties, items, and anyOf
473+
for key in ["properties", "items", "anyOf"]:
474+
if key in schema:
475+
value = schema[key]
476+
if key == "properties" and isinstance(value, dict):
477+
for prop_schema in value.values():
478+
_convert_schema_types(prop_schema, depth + 1)
479+
elif key == "items":
480+
_convert_schema_types(value, depth + 1)
481+
elif key == "anyOf" and isinstance(value, list):
482+
for anyof_schema in value:
483+
_convert_schema_types(anyof_schema, depth + 1)
484+
442485
def get_vertex_project_id_from_url(url: str) -> Optional[str]:
443486
"""
444487
Get the vertex project id from the url

‎litellm/model_prices_and_context_window_backup.json‎

Lines changed: 135 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -11991,6 +11991,108 @@
1199111991
"mode": "chat",
1199211992
"supports_tool_choice": true
1199311993
},
11994+
"openrouter/openai/gpt-4.1": {
11995+
"max_tokens": 32768,
11996+
"max_input_tokens": 1047576,
11997+
"max_output_tokens": 32768,
11998+
"input_cost_per_token": 2e-06,
11999+
"output_cost_per_token": 8e-06,
12000+
"cache_read_input_token_cost": 5e-07,
12001+
"litellm_provider": "openrouter",
12002+
"mode": "chat",
12003+
"supports_function_calling": true,
12004+
"supports_parallel_function_calling": true,
12005+
"supports_response_schema": true,
12006+
"supports_vision": true,
12007+
"supports_prompt_caching": true,
12008+
"supports_system_messages": true,
12009+
"supports_tool_choice": true
12010+
},
12011+
"openrouter/openai/gpt-4.1-2025-04-14": {
12012+
"max_tokens": 32768,
12013+
"max_input_tokens": 1047576,
12014+
"max_output_tokens": 32768,
12015+
"input_cost_per_token": 2e-06,
12016+
"output_cost_per_token": 8e-06,
12017+
"cache_read_input_token_cost": 5e-07,
12018+
"litellm_provider": "openrouter",
12019+
"mode": "chat",
12020+
"supports_function_calling": true,
12021+
"supports_parallel_function_calling": true,
12022+
"supports_response_schema": true,
12023+
"supports_vision": true,
12024+
"supports_prompt_caching": true,
12025+
"supports_system_messages": true,
12026+
"supports_tool_choice": true
12027+
},
12028+
"openrouter/openai/gpt-4.1-mini": {
12029+
"max_tokens": 32768,
12030+
"max_input_tokens": 1047576,
12031+
"max_output_tokens": 32768,
12032+
"input_cost_per_token": 4e-07,
12033+
"output_cost_per_token": 1.6e-06,
12034+
"cache_read_input_token_cost": 1e-07,
12035+
"litellm_provider": "openrouter",
12036+
"mode": "chat",
12037+
"supports_function_calling": true,
12038+
"supports_parallel_function_calling": true,
12039+
"supports_response_schema": true,
12040+
"supports_vision": true,
12041+
"supports_prompt_caching": true,
12042+
"supports_system_messages": true,
12043+
"supports_tool_choice": true
12044+
},
12045+
"openrouter/openai/gpt-4.1-mini-2025-04-14": {
12046+
"max_tokens": 32768,
12047+
"max_input_tokens": 1047576,
12048+
"max_output_tokens": 32768,
12049+
"input_cost_per_token": 4e-07,
12050+
"output_cost_per_token": 1.6e-06,
12051+
"cache_read_input_token_cost": 1e-07,
12052+
"litellm_provider": "openrouter",
12053+
"mode": "chat",
12054+
"supports_function_calling": true,
12055+
"supports_parallel_function_calling": true,
12056+
"supports_response_schema": true,
12057+
"supports_vision": true,
12058+
"supports_prompt_caching": true,
12059+
"supports_system_messages": true,
12060+
"supports_tool_choice": true
12061+
},
12062+
"openrouter/openai/gpt-4.1-nano": {
12063+
"max_tokens": 32768,
12064+
"max_input_tokens": 1047576,
12065+
"max_output_tokens": 32768,
12066+
"input_cost_per_token": 1e-07,
12067+
"output_cost_per_token": 4e-07,
12068+
"cache_read_input_token_cost": 2.5e-08,
12069+
"litellm_provider": "openrouter",
12070+
"mode": "chat",
12071+
"supports_function_calling": true,
12072+
"supports_parallel_function_calling": true,
12073+
"supports_response_schema": true,
12074+
"supports_vision": true,
12075+
"supports_prompt_caching": true,
12076+
"supports_system_messages": true,
12077+
"supports_tool_choice": true
12078+
},
12079+
"openrouter/openai/gpt-4.1-nano-2025-04-14": {
12080+
"max_tokens": 32768,
12081+
"max_input_tokens": 1047576,
12082+
"max_output_tokens": 32768,
12083+
"input_cost_per_token": 1e-07,
12084+
"output_cost_per_token": 4e-07,
12085+
"cache_read_input_token_cost": 2.5e-08,
12086+
"litellm_provider": "openrouter",
12087+
"mode": "chat",
12088+
"supports_function_calling": true,
12089+
"supports_parallel_function_calling": true,
12090+
"supports_response_schema": true,
12091+
"supports_vision": true,
12092+
"supports_prompt_caching": true,
12093+
"supports_system_messages": true,
12094+
"supports_tool_choice": true
12095+
},
1199412096
"openrouter/openai/gpt-5-mini": {
1199512097
"max_tokens": 128000,
1199612098
"max_input_tokens": 400000,
@@ -14970,32 +15072,32 @@
1497015072
"output_cost_per_token": 6e-06,
1497115073
"max_input_tokens": 262000,
1497215074
"litellm_provider": "together_ai",
14973-
"supports_function_calling": false,
14974-
"supports_parallel_function_calling": false,
15075+
"supports_function_calling": true,
15076+
"supports_parallel_function_calling": true,
1497515077
"mode": "chat",
14976-
"supports_tool_choice": false,
15078+
"supports_tool_choice": true,
1497715079
"source": "https://www.together.ai/models/qwen3-235b-a22b-instruct-2507-fp8"
1497815080
},
1497915081
"together_ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": {
1498015082
"input_cost_per_token": 2e-06,
1498115083
"output_cost_per_token": 2e-06,
1498215084
"max_input_tokens": 256000,
1498315085
"litellm_provider": "together_ai",
14984-
"supports_function_calling": false,
14985-
"supports_parallel_function_calling": false,
15086+
"supports_function_calling": true,
15087+
"supports_parallel_function_calling": true,
1498615088
"mode": "chat",
14987-
"supports_tool_choice": false,
15089+
"supports_tool_choice": true,
1498815090
"source": "https://www.together.ai/models/qwen3-coder-480b-a35b-instruct"
1498915091
},
1499015092
"together_ai/Qwen/Qwen3-235B-A22B-Thinking-2507": {
1499115093
"input_cost_per_token": 6.5e-07,
1499215094
"output_cost_per_token": 3e-06,
1499315095
"max_input_tokens": 256000,
1499415096
"litellm_provider": "together_ai",
14995-
"supports_function_calling": false,
14996-
"supports_parallel_function_calling": false,
15097+
"supports_function_calling": true,
15098+
"supports_parallel_function_calling": true,
1499715099
"mode": "chat",
14998-
"supports_tool_choice": false,
15100+
"supports_tool_choice": true,
1499915101
"source": "https://www.together.ai/models/qwen3-235b-a22b-thinking-2507"
1500015102
},
1500115103
"together_ai/Qwen/Qwen3-235B-A22B-fp8-tput": {
@@ -15038,10 +15140,10 @@
1503815140
"output_cost_per_token": 2.19e-06,
1503915141
"max_input_tokens": 128000,
1504015142
"litellm_provider": "together_ai",
15041-
"supports_function_calling": false,
15042-
"supports_parallel_function_calling": false,
15143+
"supports_function_calling": true,
15144+
"supports_parallel_function_calling": true,
1504315145
"mode": "chat",
15044-
"supports_tool_choice": false,
15146+
"supports_tool_choice": true,
1504515147
"source": "https://www.together.ai/models/deepseek-r1-0528-throughput"
1504615148
},
1504715149
"together_ai/mistralai/Mistral-Small-24B-Instruct-2501": {
@@ -15066,9 +15168,9 @@
1506615168
"output_cost_per_token": 6e-07,
1506715169
"max_input_tokens": 128000,
1506815170
"litellm_provider": "together_ai",
15069-
"supports_function_calling": false,
15070-
"supports_tool_choice": false,
15071-
"supports_parallel_function_calling": false,
15171+
"supports_function_calling": true,
15172+
"supports_tool_choice": true,
15173+
"supports_parallel_function_calling": true,
1507215174
"mode": "chat",
1507315175
"source": "https://www.together.ai/models/gpt-oss-120b"
1507415176
},
@@ -15077,9 +15179,9 @@
1507715179
"output_cost_per_token": 2e-07,
1507815180
"max_input_tokens": 128000,
1507915181
"litellm_provider": "together_ai",
15080-
"supports_function_calling": false,
15081-
"supports_tool_choice": false,
15082-
"supports_parallel_function_calling": false,
15182+
"supports_function_calling": true,
15183+
"supports_tool_choice": true,
15184+
"supports_parallel_function_calling": true,
1508315185
"mode": "chat",
1508415186
"source": "https://www.together.ai/models/gpt-oss-20b"
1508515187
},
@@ -15088,12 +15190,24 @@
1508815190
"output_cost_per_token": 1.1e-06,
1508915191
"max_input_tokens": 128000,
1509015192
"litellm_provider": "together_ai",
15091-
"supports_function_calling": false,
15092-
"supports_tool_choice": false,
15093-
"supports_parallel_function_calling": false,
15193+
"supports_function_calling": true,
15194+
"supports_tool_choice": true,
15195+
"supports_parallel_function_calling": true,
1509415196
"mode": "chat",
1509515197
"source": "https://www.together.ai/models/glm-4-5-air"
1509615198
},
15199+
"together_ai/deepseek-ai/DeepSeek-V3.1": {
15200+
"input_cost_per_token": 0.6e-06,
15201+
"output_cost_per_token": 1.7e-06,
15202+
"max_tokens": 128000,
15203+
"litellm_provider": "together_ai",
15204+
"supports_function_calling": true,
15205+
"supports_parallel_function_calling": true,
15206+
"supports_reasoning": true,
15207+
"mode": "chat",
15208+
"supports_tool_choice": true,
15209+
"source": "https://www.together.ai/models/deepseek-v3-1"
15210+
},
1509715211
"ollama/codegemma": {
1509815212
"max_tokens": 8192,
1509915213
"max_input_tokens": 8192,

‎tests/code_coverage_tests/recursive_detector.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,8 @@
2525
"filter_value_from_dict", # max depth set.
2626
"normalize_json_schema_types", # max depth set.
2727
"_extract_fields_recursive", # max depth set.
28-
"_remove_json_schema_refs", # max depth set.
28+
"_remove_json_schema_refs", # max depth set.,
29+
"_convert_schema_types", # max depth set.,
2930
]
3031

3132

‎tests/llm_translation/base_llm_unit_tests.py‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,52 @@ def test_content_list_handling(self):
141141
# for OpenAI the content contains the JSON schema, so we need to assert that the content is not None
142142
assert response.choices[0].message.content is not None
143143

144+
145+
def test_tool_call_with_property_type_array(self):
146+
litellm._turn_on_debug()
147+
from litellm.utils import supports_function_calling
148+
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
149+
litellm.model_cost = litellm.get_model_cost_map(url="")
150+
151+
base_completion_call_args = self.get_base_completion_call_args()
152+
if not supports_function_calling(base_completion_call_args["model"], None):
153+
print("Model does not support function calling")
154+
pytest.skip("Model does not support function calling")
155+
base_completion_call_args = self.get_base_completion_call_args()
156+
response = self.completion_function(
157+
**base_completion_call_args,
158+
messages = [
159+
{
160+
"role": "user",
161+
"content": "Tell me if the shoe brand Air Jordan has more models than the shoe brand Nike."
162+
}
163+
],
164+
tools = [
165+
{
166+
"type": "function",
167+
"function": {
168+
"name": "shoe_get_id",
169+
"description": "Get information about a show by its ID or name",
170+
"parameters": {
171+
"type": "object",
172+
"properties": {
173+
"shoe_id": {
174+
"type": ["string", "number"],
175+
"description": "The shoe ID or name"
176+
}
177+
},
178+
"required": ["shoe_id"],
179+
"additionalProperties": False,
180+
"$schema": "http://json-schema.org/draft-07/schema#"
181+
}
182+
}
183+
},
184+
]
185+
)
186+
print(response)
187+
print(json.dumps(response, indent=4, default=str))
188+
189+
144190
def test_streaming(self):
145191
"""Check if litellm handles streaming correctly"""
146192
from litellm.types.utils import ModelResponseStream

0 commit comments

Comments
 (0)