Skip to content

Commit f37e6f6

Browse files
ctecte
authored
Fix Requesty extended thinking (#4051)
1 parent dfacdb3 commit f37e6f6

File tree

5 files changed

+83
-116
lines changed

5 files changed

+83
-116
lines changed

‎src/api/providers/__tests__/deepseek.test.ts‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -140,12 +140,8 @@ describe("DeepSeekHandler", () => {
140140

141141
it("should set includeMaxTokens to true", () => {
142142
// Create a new handler and verify OpenAI client was called with includeMaxTokens
143-
new DeepSeekHandler(mockOptions)
144-
expect(OpenAI).toHaveBeenCalledWith(
145-
expect.objectContaining({
146-
apiKey: mockOptions.deepSeekApiKey,
147-
}),
148-
)
143+
const _handler = new DeepSeekHandler(mockOptions)
144+
expect(OpenAI).toHaveBeenCalledWith(expect.objectContaining({ apiKey: mockOptions.deepSeekApiKey }))
149145
})
150146
})
151147

‎src/api/providers/__tests__/requesty.test.ts‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,9 @@ import { RequestyHandler } from "../requesty"
77
import { ApiHandlerOptions } from "../../../shared/api"
88

99
jest.mock("openai")
10+
1011
jest.mock("delay", () => jest.fn(() => Promise.resolve()))
12+
1113
jest.mock("../fetchers/modelCache", () => ({
1214
getModels: jest.fn().mockImplementation(() => {
1315
return Promise.resolve({
@@ -150,7 +152,7 @@ describe("RequestyHandler", () => {
150152
// Verify OpenAI client was called with correct parameters
151153
expect(mockCreate).toHaveBeenCalledWith(
152154
expect.objectContaining({
153-
max_tokens: undefined,
155+
max_tokens: 8192,
154156
messages: [
155157
{
156158
role: "system",
@@ -164,7 +166,7 @@ describe("RequestyHandler", () => {
164166
model: "coding/claude-4-sonnet",
165167
stream: true,
166168
stream_options: { include_usage: true },
167-
temperature: undefined,
169+
temperature: 0,
168170
}),
169171
)
170172
})
@@ -198,9 +200,9 @@ describe("RequestyHandler", () => {
198200

199201
expect(mockCreate).toHaveBeenCalledWith({
200202
model: mockOptions.requestyModelId,
201-
max_tokens: undefined,
203+
max_tokens: 8192,
202204
messages: [{ role: "system", content: "test prompt" }],
203-
temperature: undefined,
205+
temperature: 0,
204206
})
205207
})
206208

‎src/api/providers/mistral.ts‎

Lines changed: 31 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -18,60 +18,50 @@ export class MistralHandler extends BaseProvider implements SingleCompletionHand
1818

1919
constructor(options: ApiHandlerOptions) {
2020
super()
21+
2122
if (!options.mistralApiKey) {
2223
throw new Error("Mistral API key is required")
2324
}
2425

25-
// Set default model ID if not provided
26-
this.options = {
27-
...options,
28-
apiModelId: options.apiModelId || mistralDefaultModelId,
29-
}
26+
// Set default model ID if not provided.
27+
const apiModelId = options.apiModelId || mistralDefaultModelId
28+
this.options = { ...options, apiModelId }
3029

31-
const baseUrl = this.getBaseUrl()
32-
console.debug(`[Roo Code] MistralHandler using baseUrl: ${baseUrl}`)
3330
this.client = new Mistral({
34-
serverURL: baseUrl,
31+
serverURL: apiModelId.startsWith("codestral-")
32+
? this.options.mistralCodestralUrl || "https://codestral.mistral.ai"
33+
: "https://api.mistral.ai",
3534
apiKey: this.options.mistralApiKey,
3635
})
3736
}
3837

39-
private getBaseUrl(): string {
40-
const modelId = this.options.apiModelId ?? mistralDefaultModelId
41-
console.debug(`[Roo Code] MistralHandler using modelId: ${modelId}`)
42-
if (modelId?.startsWith("codestral-")) {
43-
return this.options.mistralCodestralUrl || "https://codestral.mistral.ai"
44-
}
45-
return "https://api.mistral.ai"
46-
}
47-
4838
override async *createMessage(
4939
systemPrompt: string,
5040
messages: Anthropic.Messages.MessageParam[],
5141
metadata?: ApiHandlerCreateMessageMetadata,
5242
): ApiStream {
53-
const { id: model } = this.getModel()
43+
const { id: model, maxTokens, temperature } = this.getModel()
5444

5545
const response = await this.client.chat.stream({
56-
model: this.options.apiModelId || mistralDefaultModelId,
46+
model,
5747
messages: [{ role: "system", content: systemPrompt }, ...convertToMistralMessages(messages)],
58-
maxTokens: this.options.includeMaxTokens ? this.getModel().info.maxTokens : undefined,
59-
temperature: this.options.modelTemperature ?? MISTRAL_DEFAULT_TEMPERATURE,
48+
maxTokens,
49+
temperature,
6050
})
6151

6252
for await (const chunk of response) {
6353
const delta = chunk.data.choices[0]?.delta
54+
6455
if (delta?.content) {
6556
let content: string = ""
57+
6658
if (typeof delta.content === "string") {
6759
content = delta.content
6860
} else if (Array.isArray(delta.content)) {
6961
content = delta.content.map((c) => (c.type === "text" ? c.text : "")).join("")
7062
}
71-
yield {
72-
type: "text",
73-
text: content,
74-
}
63+
64+
yield { type: "text", text: content }
7565
}
7666

7767
if (chunk.data.usage) {
@@ -84,35 +74,39 @@ export class MistralHandler extends BaseProvider implements SingleCompletionHand
8474
}
8575
}
8676

87-
override getModel(): { id: MistralModelId; info: ModelInfo } {
88-
const modelId = this.options.apiModelId
89-
if (modelId && modelId in mistralModels) {
90-
const id = modelId as MistralModelId
91-
return { id, info: mistralModels[id] }
92-
}
93-
return {
94-
id: mistralDefaultModelId,
95-
info: mistralModels[mistralDefaultModelId],
96-
}
77+
override getModel() {
78+
const id = this.options.apiModelId ?? mistralDefaultModelId
79+
const info = mistralModels[id as MistralModelId] ?? mistralModels[mistralDefaultModelId]
80+
81+
// @TODO: Move this to the `getModelParams` function.
82+
const maxTokens = this.options.includeMaxTokens ? info.maxTokens : undefined
83+
const temperature = this.options.modelTemperature ?? MISTRAL_DEFAULT_TEMPERATURE
84+
85+
return { id, info, maxTokens, temperature }
9786
}
9887

9988
async completePrompt(prompt: string): Promise<string> {
10089
try {
90+
const { id: model, temperature } = this.getModel()
91+
10192
const response = await this.client.chat.complete({
102-
model: this.options.apiModelId || mistralDefaultModelId,
93+
model,
10394
messages: [{ role: "user", content: prompt }],
104-
temperature: this.options.modelTemperature ?? MISTRAL_DEFAULT_TEMPERATURE,
95+
temperature,
10596
})
10697

10798
const content = response.choices?.[0]?.message.content
99+
108100
if (Array.isArray(content)) {
109101
return content.map((c) => (c.type === "text" ? c.text : "")).join("")
110102
}
103+
111104
return content || ""
112105
} catch (error) {
113106
if (error instanceof Error) {
114107
throw new Error(`Mistral completion error: ${error.message}`)
115108
}
109+
116110
throw error
117111
}
118112
}

‎src/api/providers/openai.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,7 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl
154154
...(reasoning && reasoning),
155155
}
156156

157+
// @TODO: Move this to the `getModelParams` function.
157158
if (this.options.includeMaxTokens) {
158159
requestOptions.max_tokens = modelInfo.maxTokens
159160
}

‎src/api/providers/requesty.ts‎

Lines changed: 43 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,13 @@ import { calculateApiCostOpenAI } from "../../shared/cost"
88

99
import { convertToOpenAiMessages } from "../transform/openai-format"
1010
import { ApiStream, ApiStreamUsageChunk } from "../transform/stream"
11+
import { getModelParams } from "../transform/model-params"
12+
import { AnthropicReasoningParams } from "../transform/reasoning"
1113

1214
import { DEFAULT_HEADERS } from "./constants"
1315
import { getModels } from "./fetchers/modelCache"
1416
import { BaseProvider } from "./base-provider"
15-
import type { SingleCompletionHandler, ApiHandlerCreateMessageMetadata } from "../"
17+
import type { SingleCompletionHandler, ApiHandlerCreateMessageMetadata } from "../index"
1618

1719
// Requesty usage includes an extra field for Anthropic use cases.
1820
// Safely cast the prompt token details section to the appropriate structure.
@@ -31,10 +33,7 @@ type RequestyChatCompletionParams = OpenAI.Chat.ChatCompletionCreateParams & {
3133
mode?: string
3234
}
3335
}
34-
thinking?: {
35-
type: string
36-
budget_tokens?: number
37-
}
36+
thinking?: AnthropicReasoningParams
3837
}
3938

4039
export class RequestyHandler extends BaseProvider implements SingleCompletionHandler {
@@ -44,25 +43,33 @@ export class RequestyHandler extends BaseProvider implements SingleCompletionHan
4443

4544
constructor(options: ApiHandlerOptions) {
4645
super()
47-
this.options = options
48-
49-
const apiKey = this.options.requestyApiKey ?? "not-provided"
50-
const baseURL = "https://router.requesty.ai/v1"
5146

52-
const defaultHeaders = DEFAULT_HEADERS
47+
this.options = options
5348

54-
this.client = new OpenAI({ baseURL, apiKey, defaultHeaders })
49+
this.client = new OpenAI({
50+
baseURL: "https://router.requesty.ai/v1",
51+
apiKey: this.options.requestyApiKey ?? "not-provided",
52+
defaultHeaders: DEFAULT_HEADERS,
53+
})
5554
}
5655

5756
public async fetchModel() {
5857
this.models = await getModels({ provider: "requesty" })
5958
return this.getModel()
6059
}
6160

62-
override getModel(): { id: string; info: ModelInfo } {
61+
override getModel() {
6362
const id = this.options.requestyModelId ?? requestyDefaultModelId
6463
const info = this.models[id] ?? requestyDefaultModelInfo
65-
return { id, info }
64+
65+
const params = getModelParams({
66+
format: "anthropic",
67+
modelId: id,
68+
model: info,
69+
settings: this.options,
70+
})
71+
72+
return { id, info, ...params }
6673
}
6774

6875
protected processUsageMetrics(usage: any, modelInfo?: ModelInfo): ApiStreamUsageChunk {
@@ -90,70 +97,44 @@ export class RequestyHandler extends BaseProvider implements SingleCompletionHan
9097
messages: Anthropic.Messages.MessageParam[],
9198
metadata?: ApiHandlerCreateMessageMetadata,
9299
): ApiStream {
93-
const model = await this.fetchModel()
94-
95-
let openAiMessages: OpenAI.Chat.ChatCompletionMessageParam[] = [
100+
const {
101+
id: model,
102+
info,
103+
maxTokens: max_tokens,
104+
temperature,
105+
reasoningEffort: reasoning_effort,
106+
reasoning: thinking,
107+
} = await this.fetchModel()
108+
109+
const openAiMessages: OpenAI.Chat.ChatCompletionMessageParam[] = [
96110
{ role: "system", content: systemPrompt },
97111
...convertToOpenAiMessages(messages),
98112
]
99113

100-
let maxTokens = undefined
101-
if (this.options.modelMaxTokens) {
102-
maxTokens = this.options.modelMaxTokens
103-
} else if (this.options.includeMaxTokens) {
104-
maxTokens = model.info.maxTokens
105-
}
106-
107-
let reasoningEffort = undefined
108-
if (this.options.reasoningEffort) {
109-
reasoningEffort = this.options.reasoningEffort
110-
}
111-
112-
let thinking = undefined
113-
if (this.options.modelMaxThinkingTokens) {
114-
thinking = {
115-
type: "enabled",
116-
budget_tokens: this.options.modelMaxThinkingTokens,
117-
}
118-
}
119-
120-
const temperature = this.options.modelTemperature
121-
122114
const completionParams: RequestyChatCompletionParams = {
123-
model: model.id,
124-
max_tokens: maxTokens,
125115
messages: openAiMessages,
126-
temperature: temperature,
116+
model,
117+
max_tokens,
118+
temperature,
119+
...(reasoning_effort && { reasoning_effort }),
120+
...(thinking && { thinking }),
127121
stream: true,
128122
stream_options: { include_usage: true },
129-
reasoning_effort: reasoningEffort,
130-
thinking: thinking,
131-
requesty: {
132-
trace_id: metadata?.taskId,
133-
extra: {
134-
mode: metadata?.mode,
135-
},
136-
},
123+
requesty: { trace_id: metadata?.taskId, extra: { mode: metadata?.mode } },
137124
}
138125

139126
const stream = await this.client.chat.completions.create(completionParams)
140-
141127
let lastUsage: any = undefined
142128

143129
for await (const chunk of stream) {
144130
const delta = chunk.choices[0]?.delta
131+
145132
if (delta?.content) {
146-
yield {
147-
type: "text",
148-
text: delta.content,
149-
}
133+
yield { type: "text", text: delta.content }
150134
}
151135

152136
if (delta && "reasoning_content" in delta && delta.reasoning_content) {
153-
yield {
154-
type: "reasoning",
155-
text: (delta.reasoning_content as string | undefined) || "",
156-
}
137+
yield { type: "reasoning", text: (delta.reasoning_content as string | undefined) || "" }
157138
}
158139

159140
if (chunk.usage) {
@@ -162,25 +143,18 @@ export class RequestyHandler extends BaseProvider implements SingleCompletionHan
162143
}
163144

164145
if (lastUsage) {
165-
yield this.processUsageMetrics(lastUsage, model.info)
146+
yield this.processUsageMetrics(lastUsage, info)
166147
}
167148
}
168149

169150
async completePrompt(prompt: string): Promise<string> {
170-
const model = await this.fetchModel()
151+
const { id: model, maxTokens: max_tokens, temperature } = await this.fetchModel()
171152

172153
let openAiMessages: OpenAI.Chat.ChatCompletionMessageParam[] = [{ role: "system", content: prompt }]
173154

174-
let maxTokens = undefined
175-
if (this.options.includeMaxTokens) {
176-
maxTokens = model.info.maxTokens
177-
}
178-
179-
const temperature = this.options.modelTemperature
180-
181155
const completionParams: RequestyChatCompletionParams = {
182-
model: model.id,
183-
max_tokens: maxTokens,
156+
model,
157+
max_tokens,
184158
messages: openAiMessages,
185159
temperature: temperature,
186160
}

0 commit comments

Comments
 (0)