diff --git a/src/api/providers/__tests__/groq.spec.ts b/src/api/providers/__tests__/groq.spec.ts index 52846617f4..4392f5f137 100644 --- a/src/api/providers/__tests__/groq.spec.ts +++ b/src/api/providers/__tests__/groq.spec.ts @@ -195,7 +195,7 @@ describe("GroqHandler", () => { ) }) - it("should omit temperature when modelTemperature is undefined", async () => { + it("should include default temperature of 0.5 when modelTemperature is undefined", async () => { const modelId: GroqModelId = "llama-3.1-8b-instant" const handlerWithoutTemp = new GroqHandler({ apiModelId: modelId, @@ -224,13 +224,10 @@ describe("GroqHandler", () => { model: modelId, messages: expect.arrayContaining([{ role: "system", content: systemPrompt }]), stream: true, + temperature: 0.5, // Should include Groq's default temperature of 0.5 }), undefined, ) - - // Verify temperature is NOT included - const callArgs = mockCreate.mock.calls[0][0] - expect(callArgs).not.toHaveProperty("temperature") }) it("should include temperature when modelTemperature is explicitly set", async () => { diff --git a/src/api/providers/__tests__/openai.spec.ts b/src/api/providers/__tests__/openai.spec.ts index 14ed35430a..9d01ae68f0 100644 --- a/src/api/providers/__tests__/openai.spec.ts +++ b/src/api/providers/__tests__/openai.spec.ts @@ -316,20 +316,20 @@ describe("OpenAiHandler", () => { expect(callArgs.max_completion_tokens).toBe(4096) }) - it("should omit temperature when modelTemperature is undefined", async () => { + it("should include default temperature of 0 when modelTemperature is undefined", async () => { const optionsWithoutTemperature: ApiHandlerOptions = { ...mockOptions, - // modelTemperature is not set, should not include temperature + // modelTemperature is not set, should include default temperature of 0 } const handlerWithoutTemperature = new OpenAiHandler(optionsWithoutTemperature) const stream = handlerWithoutTemperature.createMessage(systemPrompt, messages) // Consume the stream to trigger the API call for await (const _chunk of stream) { } - // Assert the mockCreate was called without temperature + // Assert the mockCreate was called with default temperature of 0 expect(mockCreate).toHaveBeenCalled() const callArgs = mockCreate.mock.calls[0][0] - expect(callArgs).not.toHaveProperty("temperature") + expect(callArgs.temperature).toBe(0) }) it("should include temperature when modelTemperature is explicitly set to 0", async () => { @@ -431,6 +431,7 @@ describe("OpenAiHandler", () => { { model: mockOptions.openAiModelId, messages: [{ role: "user", content: "Test prompt" }], + temperature: 0, // temperature should always be included with default of 0 }, {}, ) @@ -515,7 +516,7 @@ describe("OpenAiHandler", () => { ], stream: true, stream_options: { include_usage: true }, - // temperature should be omitted when not set + temperature: 0, // temperature should always be included with default of 0 }, { path: "/models/chat/completions" }, ) @@ -562,6 +563,7 @@ describe("OpenAiHandler", () => { { role: "user", content: systemPrompt }, { role: "user", content: "Hello!" }, ], + temperature: 0, // temperature should always be included with default of 0 }, { path: "/models/chat/completions" }, ) @@ -579,6 +581,7 @@ describe("OpenAiHandler", () => { { model: azureOptions.openAiModelId, messages: [{ role: "user", content: "Test prompt" }], + temperature: 0, // temperature should always be included with default of 0 }, { path: "/models/chat/completions" }, ) diff --git a/src/api/providers/__tests__/roo.spec.ts b/src/api/providers/__tests__/roo.spec.ts index 5509c51e02..d9f726cffb 100644 --- a/src/api/providers/__tests__/roo.spec.ts +++ b/src/api/providers/__tests__/roo.spec.ts @@ -354,7 +354,7 @@ describe("RooHandler", () => { }) describe("temperature and model configuration", () => { - it("should omit temperature when not explicitly set", async () => { + it("should include default temperature of 0.7 when not explicitly set", async () => { handler = new RooHandler(mockOptions) const stream = handler.createMessage(systemPrompt, messages) for await (const _chunk of stream) { @@ -362,8 +362,8 @@ describe("RooHandler", () => { } expect(mockCreate).toHaveBeenCalledWith( - expect.not.objectContaining({ - temperature: expect.anything(), + expect.objectContaining({ + temperature: 0.7, // Should include Roo's default temperature of 0.7 }), undefined, ) diff --git a/src/api/providers/base-openai-compatible-provider.ts b/src/api/providers/base-openai-compatible-provider.ts index d079e22a1c..3ee7eead5e 100644 --- a/src/api/providers/base-openai-compatible-provider.ts +++ b/src/api/providers/base-openai-compatible-provider.ts @@ -79,11 +79,9 @@ export abstract class BaseOpenAiCompatibleProvider messages: [{ role: "system", content: systemPrompt }, ...convertToOpenAiMessages(messages)], stream: true, stream_options: { include_usage: true }, - } - - // Only include temperature if explicitly set - if (this.options.modelTemperature !== undefined) { - params.temperature = this.options.modelTemperature + // Always include temperature to prevent TabbyApi/ExLlamaV2 crashes + // Use explicitly set temperature, or fall back to defaultTemperature (which defaults to 0) + temperature: this.options.modelTemperature ?? this.defaultTemperature, } return this.client.chat.completions.create(params, requestOptions) diff --git a/src/api/providers/openai.ts b/src/api/providers/openai.ts index 36158d770c..93ebc8e54a 100644 --- a/src/api/providers/openai.ts +++ b/src/api/providers/openai.ts @@ -161,14 +161,9 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl stream: true as const, ...(isGrokXAI ? {} : { stream_options: { include_usage: true } }), ...(reasoning && reasoning), - } - - // Only include temperature if explicitly set - if (this.options.modelTemperature !== undefined) { - requestOptions.temperature = this.options.modelTemperature - } else if (deepseekReasoner) { - // DeepSeek Reasoner has a specific default temperature - requestOptions.temperature = DEEP_SEEK_DEFAULT_TEMPERATURE + // Always include temperature to prevent TabbyApi/ExLlamaV2 crashes + // Use explicitly set temperature, or DeepSeek default for reasoner models, or fall back to 0 + temperature: this.options.modelTemperature ?? (deepseekReasoner ? DEEP_SEEK_DEFAULT_TEMPERATURE : 0), } // Add max_tokens if needed @@ -231,6 +226,9 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl : enabledLegacyFormat ? [systemMessage, ...convertToSimpleMessages(messages)] : [systemMessage, ...convertToOpenAiMessages(messages)], + // Always include temperature to prevent TabbyApi/ExLlamaV2 crashes + // Use explicitly set temperature, or DeepSeek default for reasoner models, or fall back to 0 + temperature: this.options.modelTemperature ?? (deepseekReasoner ? DEEP_SEEK_DEFAULT_TEMPERATURE : 0), } // Add max_tokens if needed @@ -276,6 +274,8 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl const requestOptions: OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming = { model: model.id, messages: [{ role: "user", content: prompt }], + // Always include temperature to prevent TabbyApi/ExLlamaV2 crashes + temperature: this.options.modelTemperature ?? 0, } // Add max_tokens if needed