fix: Possibly wrong temperature used when prompt as temperature specified

Author: Veikkosuhonen

src/client/components/ChatV2/ChatV2.tsx

@@ -194,6 +194,7 @@ const ChatV2Content = () => {
 
     const generationInfo: MessageGenerationInfo = {
       model: acualModel,
+      temperature: modelTemperature,
       promptInfo,
     }
 
@@ -204,7 +205,6 @@ const ChatV2Content = () => {
           options: {
             generationInfo,
             chatMessages: newMessages,
-            modelTemperature,
             saveConsent,
           },
           courseId,

src/server/routes/ai/v3.ts

@@ -142,7 +142,7 @@ router.post('/stream', upload.single('file'), async (r, res) => {
     systemMessage,
     promptMessages: prompt?.messages,
     model: prompt?.model ?? generationInfo.model,
-    temperature: prompt?.temperature ?? options.modelTemperature,
+    temperature: prompt?.temperature ?? generationInfo.temperature ?? undefined,
     tools,
     writeEvent: async (event: ChatEvent) => {
       await new Promise((resolve) => {

src/server/services/langchain/MockModel.ts

@@ -11,9 +11,9 @@ import { StructuredTool } from '@langchain/core/tools'
  * FakeStreamingChatModel echoes the first input message out of the box.
  */
 export class MockModel extends FakeStreamingChatModel {
-  temperature: number
+  temperature?: number | null
 
-  constructor({ tools, temperature }: { tools: StructuredTool[]; temperature: number }) {
+  constructor({ tools, temperature }: { tools: StructuredTool[]; temperature?: number | null }) {
     super({
       sleep: 5,
     })

src/server/services/langchain/chat.ts

@@ -23,7 +23,7 @@ type ChatModel = Runnable<BaseLanguageModelInput, AIMessageChunk, BaseChatModelC
  * @param temperature The temperature for the model's responses.
  * @returns A chat model instance.
  */
-const getChatModel = (modelConfig: (typeof validModels)[number], tools: StructuredTool[], temperature: number): ChatModel => {
+const getChatModel = (modelConfig: (typeof validModels)[number], tools: StructuredTool[], temperature?: number): ChatModel => {
   const chatModel =
     modelConfig.name === 'mock'
       ? new MockModel({ tools, temperature })
@@ -78,7 +78,7 @@ export const streamChat = async ({
   user,
 }: {
   model: ValidModelName
-  temperature: number
+  temperature?: number
   systemMessage: string
   chatMessages: ChatMessage[]
   promptMessages?: Message[]

src/shared/chat.ts

@@ -58,7 +58,10 @@ export type ChatMessage = UserMessage | AssistantMessage
 export type ChatRole = ChatMessage['role']
 
 export const MessageGenerationInfoSchema = z.object({
-  model: z.union(validModels.map((m) => z.literal(m.name))),
+  // May be overridden by prompt
+  model: ValidModelNameSchema,
+  // May be overridden by prompt
+  temperature: z.number().min(0).max(1).optional().nullable(),
   promptInfo: z.discriminatedUnion('type', [
     z.object({
       type: z.literal('saved'),
@@ -87,7 +90,6 @@ export const PostStreamSchemaV3 = z.object({
     chatMessages: z.array(ChatMessageSchema),
     generationInfo: MessageGenerationInfoSchema,
     userConsent: z.boolean().optional(),
-    modelTemperature: z.number().min(0).max(2),
     saveConsent: z.boolean().optional(),
     courseId: z.string().optional(),
   }),