fix: always include temperature parameter for OpenAI compatible providers

- Modified base-openai-compatible-provider.ts to always include temperature with fallback to defaultTemperature
- Modified openai.ts to always include temperature with fallback to 0 (or DEEP_SEEK_DEFAULT_TEMPERATURE for DeepSeek reasoner models)
- Updated tests to expect temperature to always be included with appropriate defaults
- This fixes TabbyApi/ExLlamaV2 crashes when temperature is undefined/null

Fixes #7581

Author: roomote

src/api/providers/__tests__/groq.spec.ts

@@ -195,7 +195,7 @@ describe("GroqHandler", () => {
 		)
 	})
 
-	it("should omit temperature when modelTemperature is undefined", async () => {
+	it("should include default temperature of 0.5 when modelTemperature is undefined", async () => {
 		const modelId: GroqModelId = "llama-3.1-8b-instant"
 		const handlerWithoutTemp = new GroqHandler({
 			apiModelId: modelId,
@@ -224,13 +224,10 @@ describe("GroqHandler", () => {
 				model: modelId,
 				messages: expect.arrayContaining([{ role: "system", content: systemPrompt }]),
 				stream: true,
+				temperature: 0.5, // Should include Groq's default temperature of 0.5
 			}),
 			undefined,
 		)
-
-		// Verify temperature is NOT included
-		const callArgs = mockCreate.mock.calls[0][0]
-		expect(callArgs).not.toHaveProperty("temperature")
 	})
 
 	it("should include temperature when modelTemperature is explicitly set", async () => {

src/api/providers/__tests__/openai.spec.ts

@@ -316,20 +316,20 @@ describe("OpenAiHandler", () => {
 			expect(callArgs.max_completion_tokens).toBe(4096)
 		})
 
-		it("should omit temperature when modelTemperature is undefined", async () => {
+		it("should include default temperature of 0 when modelTemperature is undefined", async () => {
 			const optionsWithoutTemperature: ApiHandlerOptions = {
 				...mockOptions,
-				// modelTemperature is not set, should not include temperature
+				// modelTemperature is not set, should include default temperature of 0
 			}
 			const handlerWithoutTemperature = new OpenAiHandler(optionsWithoutTemperature)
 			const stream = handlerWithoutTemperature.createMessage(systemPrompt, messages)
 			// Consume the stream to trigger the API call
 			for await (const _chunk of stream) {
 			}
-			// Assert the mockCreate was called without temperature
+			// Assert the mockCreate was called with default temperature of 0
 			expect(mockCreate).toHaveBeenCalled()
 			const callArgs = mockCreate.mock.calls[0][0]
-			expect(callArgs).not.toHaveProperty("temperature")
+			expect(callArgs.temperature).toBe(0)
 		})
 
 		it("should include temperature when modelTemperature is explicitly set to 0", async () => {
@@ -431,6 +431,7 @@ describe("OpenAiHandler", () => {
 				{
 					model: mockOptions.openAiModelId,
 					messages: [{ role: "user", content: "Test prompt" }],
+					temperature: 0, // temperature should always be included with default of 0
 				},
 				{},
 			)
@@ -515,7 +516,7 @@ describe("OpenAiHandler", () => {
 					],
 					stream: true,
 					stream_options: { include_usage: true },
-					// temperature should be omitted when not set
+					temperature: 0, // temperature should always be included with default of 0
 				},
 				{ path: "/models/chat/completions" },
 			)
@@ -562,6 +563,7 @@ describe("OpenAiHandler", () => {
 						{ role: "user", content: systemPrompt },
 						{ role: "user", content: "Hello!" },
 					],
+					temperature: 0, // temperature should always be included with default of 0
 				},
 				{ path: "/models/chat/completions" },
 			)
@@ -579,6 +581,7 @@ describe("OpenAiHandler", () => {
 				{
 					model: azureOptions.openAiModelId,
 					messages: [{ role: "user", content: "Test prompt" }],
+					temperature: 0, // temperature should always be included with default of 0
 				},
 				{ path: "/models/chat/completions" },
 			)

src/api/providers/__tests__/roo.spec.ts

@@ -354,16 +354,16 @@ describe("RooHandler", () => {
 	})
 
 	describe("temperature and model configuration", () => {
-		it("should omit temperature when not explicitly set", async () => {
+		it("should include default temperature of 0.7 when not explicitly set", async () => {
 			handler = new RooHandler(mockOptions)
 			const stream = handler.createMessage(systemPrompt, messages)
 			for await (const _chunk of stream) {
 				// Consume stream
 			}
 
 			expect(mockCreate).toHaveBeenCalledWith(
-				expect.not.objectContaining({
-					temperature: expect.anything(),
+				expect.objectContaining({
+					temperature: 0.7, // Should include Roo's default temperature of 0.7
 				}),
 				undefined,
 			)

src/api/providers/base-openai-compatible-provider.ts

@@ -79,11 +79,9 @@ export abstract class BaseOpenAiCompatibleProvider<ModelName extends string>
 			messages: [{ role: "system", content: systemPrompt }, ...convertToOpenAiMessages(messages)],
 			stream: true,
 			stream_options: { include_usage: true },
-		}
-
-		// Only include temperature if explicitly set
-		if (this.options.modelTemperature !== undefined) {
-			params.temperature = this.options.modelTemperature
+			// Always include temperature to prevent TabbyApi/ExLlamaV2 crashes
+			// Use explicitly set temperature, or fall back to defaultTemperature (which defaults to 0)
+			temperature: this.options.modelTemperature ?? this.defaultTemperature,
 		}
 
 		return this.client.chat.completions.create(params, requestOptions)

src/api/providers/openai.ts

@@ -161,14 +161,9 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl
 				stream: true as const,
 				...(isGrokXAI ? {} : { stream_options: { include_usage: true } }),
 				...(reasoning && reasoning),
-			}
-
-			// Only include temperature if explicitly set
-			if (this.options.modelTemperature !== undefined) {
-				requestOptions.temperature = this.options.modelTemperature
-			} else if (deepseekReasoner) {
-				// DeepSeek Reasoner has a specific default temperature
-				requestOptions.temperature = DEEP_SEEK_DEFAULT_TEMPERATURE
+				// Always include temperature to prevent TabbyApi/ExLlamaV2 crashes
+				// Use explicitly set temperature, or DeepSeek default for reasoner models, or fall back to 0
+				temperature: this.options.modelTemperature ?? (deepseekReasoner ? DEEP_SEEK_DEFAULT_TEMPERATURE : 0),
 			}
 
 			// Add max_tokens if needed
@@ -231,6 +226,9 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl
 					: enabledLegacyFormat
 						? [systemMessage, ...convertToSimpleMessages(messages)]
 						: [systemMessage, ...convertToOpenAiMessages(messages)],
+				// Always include temperature to prevent TabbyApi/ExLlamaV2 crashes
+				// Use explicitly set temperature, or DeepSeek default for reasoner models, or fall back to 0
+				temperature: this.options.modelTemperature ?? (deepseekReasoner ? DEEP_SEEK_DEFAULT_TEMPERATURE : 0),
 			}
 
 			// Add max_tokens if needed
@@ -276,6 +274,8 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl
 			const requestOptions: OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming = {
 				model: model.id,
 				messages: [{ role: "user", content: prompt }],
+				// Always include temperature to prevent TabbyApi/ExLlamaV2 crashes
+				temperature: this.options.modelTemperature ?? 0,
 			}
 
 			// Add max_tokens if needed