Skip to content

Commit 91ef9fb

Browse files
mrubensmrubens
committed
Clean up maxTokens logic
1 parent d8cafbc commit 91ef9fb

File tree

5 files changed

+64
-58
lines changed

5 files changed

+64
-58
lines changed

‎src/api/providers/glama.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ export class GlamaHandler implements ApiHandler, SingleCompletionHandler {
6969
let maxTokens: number | undefined
7070

7171
if (this.getModel().id.startsWith("anthropic/")) {
72-
maxTokens = 8_192
72+
maxTokens = this.getModel().info.maxTokens
7373
}
7474

7575
const requestOptions: OpenAI.Chat.ChatCompletionCreateParams = {
@@ -177,7 +177,7 @@ export class GlamaHandler implements ApiHandler, SingleCompletionHandler {
177177
}
178178

179179
if (this.getModel().id.startsWith("anthropic/")) {
180-
requestOptions.max_tokens = 8192
180+
requestOptions.max_tokens = this.getModel().info.maxTokens
181181
}
182182

183183
const response = await this.client.chat.completions.create(requestOptions)

‎src/api/providers/openrouter.ts‎

Lines changed: 3 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -54,20 +54,8 @@ export class OpenRouterHandler implements ApiHandler, SingleCompletionHandler {
5454

5555
// prompt caching: https://openrouter.ai/docs/prompt-caching
5656
// this is specifically for claude models (some models may 'support prompt caching' automatically without this)
57-
switch (this.getModel().id) {
58-
case "anthropic/claude-3.7-sonnet":
59-
case "anthropic/claude-3.5-sonnet":
60-
case "anthropic/claude-3.5-sonnet:beta":
61-
case "anthropic/claude-3.5-sonnet-20240620":
62-
case "anthropic/claude-3.5-sonnet-20240620:beta":
63-
case "anthropic/claude-3-5-haiku":
64-
case "anthropic/claude-3-5-haiku:beta":
65-
case "anthropic/claude-3-5-haiku-20241022":
66-
case "anthropic/claude-3-5-haiku-20241022:beta":
67-
case "anthropic/claude-3-haiku":
68-
case "anthropic/claude-3-haiku:beta":
69-
case "anthropic/claude-3-opus":
70-
case "anthropic/claude-3-opus:beta":
57+
switch (true) {
58+
case this.getModel().id.startsWith("anthropic/"):
7159
openAiMessages[0] = {
7260
role: "system",
7361
content: [
@@ -103,23 +91,6 @@ export class OpenRouterHandler implements ApiHandler, SingleCompletionHandler {
10391
break
10492
}
10593

106-
// Not sure how openrouter defaults max tokens when no value is provided, but the anthropic api requires this value and since they offer both 4096 and 8192 variants, we should ensure 8192.
107-
// (models usually default to max tokens allowed)
108-
let maxTokens: number | undefined
109-
switch (this.getModel().id) {
110-
case "anthropic/claude-3.7-sonnet":
111-
case "anthropic/claude-3.5-sonnet":
112-
case "anthropic/claude-3.5-sonnet:beta":
113-
case "anthropic/claude-3.5-sonnet-20240620":
114-
case "anthropic/claude-3.5-sonnet-20240620:beta":
115-
case "anthropic/claude-3-5-haiku":
116-
case "anthropic/claude-3-5-haiku:beta":
117-
case "anthropic/claude-3-5-haiku-20241022":
118-
case "anthropic/claude-3-5-haiku-20241022:beta":
119-
maxTokens = 8_192
120-
break
121-
}
122-
12394
let defaultTemperature = OPENROUTER_DEFAULT_TEMPERATURE
12495
let topP: number | undefined = undefined
12596

@@ -140,7 +111,7 @@ export class OpenRouterHandler implements ApiHandler, SingleCompletionHandler {
140111
let fullResponseText = ""
141112
const stream = await this.client.chat.completions.create({
142113
model: this.getModel().id,
143-
max_tokens: maxTokens,
114+
max_tokens: this.getModel().info.maxTokens,
144115
temperature: this.options.modelTemperature ?? defaultTemperature,
145116
top_p: topP,
146117
messages: openAiMessages,

‎src/api/providers/unbound.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ export class UnboundHandler implements ApiHandler, SingleCompletionHandler {
7171
let maxTokens: number | undefined
7272

7373
if (this.getModel().id.startsWith("anthropic/")) {
74-
maxTokens = 8_192
74+
maxTokens = this.getModel().info.maxTokens
7575
}
7676

7777
const { data: completion, response } = await this.client.chat.completions
@@ -150,7 +150,7 @@ export class UnboundHandler implements ApiHandler, SingleCompletionHandler {
150150
}
151151

152152
if (this.getModel().id.startsWith("anthropic/")) {
153-
requestOptions.max_tokens = 8192
153+
requestOptions.max_tokens = this.getModel().info.maxTokens
154154
}
155155

156156
const response = await this.client.chat.completions.create(requestOptions)

‎src/core/webview/ClineProvider.ts‎

Lines changed: 56 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1926,6 +1926,17 @@ export class ClineProvider implements vscode.WebviewViewProvider {
19261926
cacheReadsPrice: parsePrice(rawModel.cached_price),
19271927
}
19281928

1929+
switch (rawModel.id) {
1930+
case rawModel.id.startsWith("anthropic/claude-3-7-sonnet"):
1931+
modelInfo.maxTokens = 16384
1932+
break
1933+
case rawModel.id.startsWith("anthropic/"):
1934+
modelInfo.maxTokens = 8192
1935+
break
1936+
default:
1937+
break
1938+
}
1939+
19291940
models[rawModel.id] = modelInfo
19301941
}
19311942
} else {
@@ -2076,6 +2087,17 @@ export class ClineProvider implements vscode.WebviewViewProvider {
20762087
cacheReadsPrice: parsePrice(rawModel.pricePerToken?.cacheRead),
20772088
}
20782089

2090+
switch (rawModel.id) {
2091+
case rawModel.id.startsWith("anthropic/claude-3-7-sonnet"):
2092+
modelInfo.maxTokens = 16384
2093+
break
2094+
case rawModel.id.startsWith("anthropic/"):
2095+
modelInfo.maxTokens = 8192
2096+
break
2097+
default:
2098+
break
2099+
}
2100+
20792101
models[rawModel.id] = modelInfo
20802102
}
20812103
} else {
@@ -2127,46 +2149,46 @@ export class ClineProvider implements vscode.WebviewViewProvider {
21272149
description: rawModel.description,
21282150
}
21292151

2130-
switch (rawModel.id) {
2131-
case "anthropic/claude-3.7-sonnet":
2132-
case "anthropic/claude-3.7-sonnet:beta":
2133-
case "anthropic/claude-3.5-sonnet":
2134-
case "anthropic/claude-3.5-sonnet:beta":
2135-
// NOTE: this needs to be synced with api.ts/openrouter default model info.
2152+
// NOTE: this needs to be synced with api.ts/openrouter default model info.
2153+
switch (true) {
2154+
case rawModel.id.startsWith("anthropic/claude-3.7-sonnet"):
21362155
modelInfo.supportsComputerUse = true
21372156
modelInfo.supportsPromptCache = true
21382157
modelInfo.cacheWritesPrice = 3.75
21392158
modelInfo.cacheReadsPrice = 0.3
2159+
modelInfo.maxTokens = 16384
21402160
break
2141-
case "anthropic/claude-3.5-sonnet-20240620":
2142-
case "anthropic/claude-3.5-sonnet-20240620:beta":
2161+
case rawModel.id.startsWith("anthropic/claude-3.5-sonnet-20240620"):
21432162
modelInfo.supportsPromptCache = true
21442163
modelInfo.cacheWritesPrice = 3.75
21452164
modelInfo.cacheReadsPrice = 0.3
2165+
modelInfo.maxTokens = 8192
21462166
break
2147-
case "anthropic/claude-3-5-haiku":
2148-
case "anthropic/claude-3-5-haiku:beta":
2149-
case "anthropic/claude-3-5-haiku-20241022":
2150-
case "anthropic/claude-3-5-haiku-20241022:beta":
2151-
case "anthropic/claude-3.5-haiku":
2152-
case "anthropic/claude-3.5-haiku:beta":
2153-
case "anthropic/claude-3.5-haiku-20241022":
2154-
case "anthropic/claude-3.5-haiku-20241022:beta":
2167+
case rawModel.id.startsWith("anthropic/claude-3.5-sonnet"):
2168+
modelInfo.supportsComputerUse = true
2169+
modelInfo.supportsPromptCache = true
2170+
modelInfo.cacheWritesPrice = 3.75
2171+
modelInfo.cacheReadsPrice = 0.3
2172+
modelInfo.maxTokens = 8192
2173+
break
2174+
case rawModel.id.startsWith("anthropic/claude-3-5-haiku"):
21552175
modelInfo.supportsPromptCache = true
21562176
modelInfo.cacheWritesPrice = 1.25
21572177
modelInfo.cacheReadsPrice = 0.1
2178+
modelInfo.maxTokens = 8192
21582179
break
2159-
case "anthropic/claude-3-opus":
2160-
case "anthropic/claude-3-opus:beta":
2180+
case rawModel.id.startsWith("anthropic/claude-3-opus"):
21612181
modelInfo.supportsPromptCache = true
21622182
modelInfo.cacheWritesPrice = 18.75
21632183
modelInfo.cacheReadsPrice = 1.5
2184+
modelInfo.maxTokens = 8192
21642185
break
2165-
case "anthropic/claude-3-haiku":
2166-
case "anthropic/claude-3-haiku:beta":
2186+
case rawModel.id.startsWith("anthropic/claude-3-haiku"):
2187+
default:
21672188
modelInfo.supportsPromptCache = true
21682189
modelInfo.cacheWritesPrice = 0.3
21692190
modelInfo.cacheReadsPrice = 0.03
2191+
modelInfo.maxTokens = 8192
21702192
break
21712193
}
21722194

@@ -2200,7 +2222,7 @@ export class ClineProvider implements vscode.WebviewViewProvider {
22002222
if (response.data) {
22012223
const rawModels: Record<string, any> = response.data
22022224
for (const [modelId, model] of Object.entries(rawModels)) {
2203-
models[modelId] = {
2225+
const modelInfo: ModelInfo = {
22042226
maxTokens: model?.maxTokens ? parseInt(model.maxTokens) : undefined,
22052227
contextWindow: model?.contextWindow ? parseInt(model.contextWindow) : 0,
22062228
supportsImages: model?.supportsImages ?? false,
@@ -2211,6 +2233,19 @@ export class ClineProvider implements vscode.WebviewViewProvider {
22112233
cacheWritesPrice: model?.cacheWritePrice ? parseFloat(model.cacheWritePrice) : undefined,
22122234
cacheReadsPrice: model?.cacheReadPrice ? parseFloat(model.cacheReadPrice) : undefined,
22132235
}
2236+
2237+
switch (true) {
2238+
case modelId.startsWith("anthropic/claude-3-7-sonnet"):
2239+
modelInfo.maxTokens = 16384
2240+
break
2241+
case modelId.startsWith("anthropic/"):
2242+
modelInfo.maxTokens = 8192
2243+
break
2244+
default:
2245+
break
2246+
}
2247+
2248+
models[modelId] = modelInfo
22142249
}
22152250
}
22162251
await fs.writeFile(unboundModelsFilePath, JSON.stringify(models))

‎src/shared/api.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ export type AnthropicModelId = keyof typeof anthropicModels
9797
export const anthropicDefaultModelId: AnthropicModelId = "claude-3-7-sonnet-20250219"
9898
export const anthropicModels = {
9999
"claude-3-7-sonnet-20250219": {
100-
maxTokens: 64_000,
100+
maxTokens: 16384,
101101
contextWindow: 200_000,
102102
supportsImages: true,
103103
supportsComputerUse: true,

0 commit comments

Comments
 (0)