From 93dc32d17fd6a048428b68fefa95b8db39d4cb0a Mon Sep 17 00:00:00 2001 From: Roo Code Date: Tue, 5 Aug 2025 20:39:49 +0000 Subject: [PATCH] fix: parse gpt-oss special token format in LM Studio responses - Add detection for gpt-oss models in LmStudioHandler - Implement parseGptOssFormat method to extract actual content from special token format - Add comprehensive tests for the new parsing logic - Fixes #6739 --- src/api/providers/__tests__/lmstudio.spec.ts | 129 +++++++++++++++++++ src/api/providers/lm-studio.ts | 46 ++++++- 2 files changed, 172 insertions(+), 3 deletions(-) diff --git a/src/api/providers/__tests__/lmstudio.spec.ts b/src/api/providers/__tests__/lmstudio.spec.ts index 0adebdeea7..f03755fdcf 100644 --- a/src/api/providers/__tests__/lmstudio.spec.ts +++ b/src/api/providers/__tests__/lmstudio.spec.ts @@ -164,4 +164,133 @@ describe("LmStudioHandler", () => { expect(modelInfo.info.contextWindow).toBe(128_000) }) }) + + describe("gpt-oss special token parsing", () => { + it("should parse gpt-oss format with special tokens", async () => { + // Mock gpt-oss model response with special tokens + mockCreate.mockImplementationOnce(async (options) => { + return { + [Symbol.asyncIterator]: async function* () { + yield { + choices: [ + { + delta: { + content: + '<|start|>assistant<|channel|>commentary to=read_file <|constrain|>json<|message|>{"args":[{"file":{"path":"documentation/program_analysis.md"}}]}', + }, + index: 0, + }, + ], + usage: null, + } + }, + } + }) + + // Create handler with gpt-oss model + const gptOssHandler = new LmStudioHandler({ + apiModelId: "gpt-oss-20b", + lmStudioModelId: "gpt-oss-20b", + lmStudioBaseUrl: "http://localhost:1234", + }) + + const systemPrompt = "You are a helpful assistant." + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: "Read the file", + }, + ] + + const stream = gptOssHandler.createMessage(systemPrompt, messages) + const chunks: any[] = [] + for await (const chunk of stream) { + chunks.push(chunk) + } + + const textChunks = chunks.filter((chunk) => chunk.type === "text") + expect(textChunks).toHaveLength(1) + // Should extract just the JSON message content + expect(textChunks[0].text).toBe('{"args":[{"file":{"path":"documentation/program_analysis.md"}}]}') + }) + + it("should handle gpt-oss format without message token", async () => { + mockCreate.mockImplementationOnce(async (options) => { + return { + [Symbol.asyncIterator]: async function* () { + yield { + choices: [ + { + delta: { + content: + "<|start|>assistant<|channel|>commentary to=analyze_code <|constrain|>text", + }, + index: 0, + }, + ], + usage: null, + } + }, + } + }) + + const gptOssHandler = new LmStudioHandler({ + apiModelId: "gpt-oss-20b", + lmStudioModelId: "gpt-oss-20b", + lmStudioBaseUrl: "http://localhost:1234", + }) + + const systemPrompt = "You are a helpful assistant." + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: "Analyze the code", + }, + ] + + const stream = gptOssHandler.createMessage(systemPrompt, messages) + const chunks: any[] = [] + for await (const chunk of stream) { + chunks.push(chunk) + } + + const textChunks = chunks.filter((chunk) => chunk.type === "text") + expect(textChunks).toHaveLength(1) + // Should clean up special tokens and function patterns + expect(textChunks[0].text).toBe("assistant commentary text") + }) + + it("should not parse special tokens for non-gpt-oss models", async () => { + // Mock response with special-looking content + mockCreate.mockImplementationOnce(async (options) => { + return { + [Symbol.asyncIterator]: async function* () { + yield { + choices: [ + { + delta: { + content: + "Here is some content with <|special|> tokens that should not be parsed", + }, + index: 0, + }, + ], + usage: null, + } + }, + } + }) + + const stream = handler.createMessage("System prompt", [{ role: "user", content: "Test" }]) + const chunks: any[] = [] + for await (const chunk of stream) { + chunks.push(chunk) + } + + const textChunks = chunks.filter((chunk) => chunk.type === "text") + expect(textChunks).toHaveLength(1) + // Should keep the content as-is for non-gpt-oss models + expect(textChunks[0].text).toBe("Here is some content with <|special|> tokens that should not be parsed") + }) + }) }) diff --git a/src/api/providers/lm-studio.ts b/src/api/providers/lm-studio.ts index 6c49920bd1..78209bfbac 100644 --- a/src/api/providers/lm-studio.ts +++ b/src/api/providers/lm-studio.ts @@ -100,9 +100,24 @@ export class LmStudioHandler extends BaseProvider implements SingleCompletionHan const delta = chunk.choices[0]?.delta if (delta?.content) { - assistantText += delta.content - for (const processedChunk of matcher.update(delta.content)) { - yield processedChunk + // Check if this is a gpt-oss model with special token format + const isGptOss = this.getModel().id?.toLowerCase().includes("gpt-oss") + + if (isGptOss && delta.content.includes("<|") && delta.content.includes("|>")) { + // Parse gpt-oss special token format + // Format: <|start|>assistant<|channel|>commentary to=read_file <|constrain|>json<|message|>{"args":[...]} + const cleanedContent = this.parseGptOssFormat(delta.content) + if (cleanedContent) { + assistantText += cleanedContent + for (const processedChunk of matcher.update(cleanedContent)) { + yield processedChunk + } + } + } else { + assistantText += delta.content + for (const processedChunk of matcher.update(delta.content)) { + yield processedChunk + } } } } @@ -169,6 +184,31 @@ export class LmStudioHandler extends BaseProvider implements SingleCompletionHan ) } } + + /** + * Parse gpt-oss special token format + * Format example: <|start|>assistant<|channel|>commentary to=read_file <|constrain|>json<|message|>{"args":[...]} + * We want to extract just the actual message content + */ + private parseGptOssFormat(content: string): string { + // Remove all special tokens and extract the actual message + // Pattern: <|token|> where token can be any word + const specialTokenPattern = /<\|[^|]+\|>/g + + // First, check if this contains the message token + const messageMatch = content.match(/<\|message\|>(.+)$/s) + if (messageMatch) { + // Extract content after <|message|> token + return messageMatch[1].trim() + } + + // Otherwise, just remove all special tokens + const cleaned = content.replace(specialTokenPattern, " ").trim() + + // Also clean up any "to=function_name" patterns that might remain + const functionPattern = /\s*to=\w+\s*/g + return cleaned.replace(functionPattern, " ").trim() + } } export async function getLmStudioModels(baseUrl = "http://localhost:1234") {