fix: Validate the completion params that are passed as input to the API #3

- added hardcoded validation for openai_args in inference.
reference:
openai https://platform.openai.com/docs/api-reference/chat/create
anthropic https://platform.claude.com/docs/en/api/python/messages/create
microsoft https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/chatgpt?view=foundry-classic&tabs=python-secure

Author: katwinkl3

tlm/api.py

@@ -4,6 +4,7 @@
 from tlm.config.presets import WorkflowType
 from tlm.inference import InferenceResult, tlm_inference
 from tlm.types import SemanticEval, CompletionParams
+from tlm.utils.chat_completion_validation import _validate_chat_completion_params
 
 
 async def inference(
@@ -21,6 +22,7 @@ async def inference(
         constrain_outputs=config_input.constrain_outputs,
     )
     config = Config.from_input(config_input, workflow_type)
+    _validate_chat_completion_params(openai_args, config.provider)
     return await tlm_inference(
         completion_params=openai_args,
         response=response,

tlm/config/provider.py

@@ -14,6 +14,11 @@ class APICredentials(BaseModel):
     api_base: str | None = None
     api_version: str | None = None
 
+OPENAI_PROVIDER = "openai"
+BEDROCK_PROVIDER = "bedrock"
+GOOGLE_PROVIDER = "google"
+AZURE_PROVIDER = "azure"
+
 
 class ModelProvider(APICredentials):
     model: str
@@ -24,13 +29,13 @@ def set_provider_from_model(self):
         """Automatically set provider based on model name if provider is None."""
         if self.provider is None:
             if self.model in OPENAI_MODELS:
-                self.provider = "openai"
+                self.provider = OPENAI_PROVIDER
             elif self.model in BEDROCK_MODELS:
-                self.provider = "bedrock"
+                self.provider = BEDROCK_PROVIDER
             elif self.model in GOOGLE_MODELS:
-                self.provider = "google"
+                self.provider = GOOGLE_PROVIDER
             elif self.model in AZURE_MODELS:
-                self.provider = "azure"
+                self.provider = AZURE_PROVIDER
 
         if self.model in BEDROCK_MODELS:
             self.model = BEDROCK_MODEL_TO_INFERENCE_PROFILE_ID[self.model]

tlm/utils/chat_completion_validation.py

@@ -0,0 +1,99 @@
+"""Validation helpers for chat completion parameter dictionaries."""
+
+from __future__ import annotations
+from tlm.types.base import CompletionParams
+from tlm.config.provider import AZURE_PROVIDER, BEDROCK_PROVIDER, GOOGLE_PROVIDER, OPENAI_PROVIDER
+from typing import FrozenSet
+
+VALID_OPENAI_CHAT_COMPLETION_PARAMS: FrozenSet[str] = frozenset(
+    {
+        "audio",
+        "function_call",
+        "functions",
+        "frequency_penalty",
+        "logit_bias",
+        "logprobs",
+        "max_completion_tokens",
+        "max_tokens",
+        "messages",
+        "metadata",
+        "model",
+        "modalities",
+        "n",
+        "parallel_tool_calls",
+        "prediction",
+        "presence_penalty",
+        "prompt_cache_key",
+        "prompt_cache_retention",
+        "reasoning",
+        "reasoning_effort",
+        "response_format",
+        "safety_identifier",
+        "seed",
+        "service_tier",
+        "stop",
+        "store",
+        "stream",
+        "stream_options",
+        "temperature",
+        "tool_choice",
+        "tools",
+        "top_logprobs",
+        "top_p",
+        "user",
+        "verbosity",
+        "web_search_options",
+    }
+)
+VALID_AZURE_CHAT_COMPLETION_PARAMS: FrozenSet[str] = VALID_OPENAI_CHAT_COMPLETION_PARAMS
+VALID_GOOGLE_CHAT_COMPLETION_PARAMS: FrozenSet[str] = VALID_OPENAI_CHAT_COMPLETION_PARAMS
+VALID_BEDROCK_CHAT_COMPLETION_PARAMS: FrozenSet[str] = frozenset(
+    {
+        *VALID_OPENAI_CHAT_COMPLETION_PARAMS,
+        "betas",
+        "container",
+        "context_management",
+        "mcp_servers",
+        "output_config",
+        "output_format",
+        "stop_sequences",
+        "system",
+        "thinking",
+        "top_k",
+        "top_p",
+        "tool_config",
+    }
+)
+
+def _resolve_valid_chat_completion_params(provider: str | None) -> FrozenSet[str]:
+    if provider == OPENAI_PROVIDER:
+        return VALID_OPENAI_CHAT_COMPLETION_PARAMS
+    elif provider == BEDROCK_PROVIDER:
+        return VALID_BEDROCK_CHAT_COMPLETION_PARAMS
+    elif provider == GOOGLE_PROVIDER:
+        return VALID_GOOGLE_CHAT_COMPLETION_PARAMS
+    elif provider == AZURE_PROVIDER:
+        return VALID_AZURE_CHAT_COMPLETION_PARAMS
+    else:
+        return VALID_OPENAI_CHAT_COMPLETION_PARAMS
+
+
+REQUIRED_CHAT_COMPLETION_PARAMS: FrozenSet[str] = frozenset({"messages"})
+
+
+def _validate_chat_completion_params(params: CompletionParams, provider: str | None) -> None:
+    """Ensure only supported chat completion params are passed into inference."""
+
+    missing_required = [param for param in REQUIRED_CHAT_COMPLETION_PARAMS if param not in params]
+    if missing_required:
+        required_str = ", ".join(sorted(REQUIRED_CHAT_COMPLETION_PARAMS))
+        raise ValueError(f"openai_args must include the following parameter(s): {required_str}")
+
+    valid_params = _resolve_valid_chat_completion_params(provider)
+
+    invalid_keys = sorted(set(params.keys()) - valid_params)
+    if invalid_keys:
+        raise ValueError(
+            f"Unsupported chat completion parameter(s) for provider {provider}: "
+            + ", ".join(invalid_keys)
+        )