Fix max token input (#478)

Signed-off-by: Onur Yilmaz <[email protected]>

Author: oyilmaz-nvidia

nemo_export/tensorrt_llm_deployable_ray.py

@@ -123,7 +123,7 @@ async def completions(self, request: Dict[Any, Any]):
             # Prepare inference inputs with proper parameter mapping
             inference_inputs = {
                 "prompts": request.get("prompts", []),
-                "max_length": request.get("max_tokens", 256),
+                "max_output_len": request.get("max_tokens", 256),
                 "temperature": request.get("temperature", 1.0),
                 "top_k": request.get("top_k", 0),
                 "top_p": request.get("top_p", 0.0),
@@ -197,7 +197,7 @@ async def chat_completions(self, request: Dict[Any, Any]):
 
             inference_inputs = {
                 "prompts": [messages],  # Wrap messages in a list so apply_chat_template gets the full conversation
-                "max_length": request.get("max_tokens", 256),
+                "max_output_len": request.get("max_tokens", 256),
                 "temperature": request.get("temperature", 1.0),
                 "top_k": request.get("top_k", 0),
                 "top_p": request.get("top_p", 0.0),
@@ -248,7 +248,7 @@ async def chat_completions(self, request: Dict[Any, Any]):
                         ),
                         "finish_reason": (
                             "length"
-                            if generated_texts and len(str(generated_texts[0])) >= inference_inputs["max_length"]
+                            if generated_texts and len(str(generated_texts[0])) >= inference_inputs["max_output_len"]
                             else "stop"
                         ),
                     }