Skip to content

Commit 1b56c3f

Browse files
ocasta181ocasta181
committed
new attempt
1 parent df7b458 commit 1b56c3f

File tree

10 files changed

+376
-39
lines changed

10 files changed

+376
-39
lines changed

‎src/core/Cline.ts‎

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ import { getNextTruncationRange, getTruncatedMessages } from "./sliding-window"
6161
import { ClineProvider, GlobalFileNames } from "./webview/ClineProvider"
6262
import { DEFAULT_LANGUAGE_SETTINGS, getLanguageKey, LanguageDisplay, LanguageKey } from "../shared/Languages"
6363
import { telemetryService } from "../services/telemetry/TelemetryService"
64+
import { getMaxAllowedSize } from "../utils/content-size"
6465

6566
const cwd = vscode.workspace.workspaceFolders?.map((folder) => folder.uri.fsPath).at(0) ?? path.join(os.homedir(), "Desktop") // may or may not exist but fs checking existence would immediately ask for permission which would be bad UX, need to come up with a better solution
6667

@@ -1166,9 +1167,26 @@ export class Cline {
11661167
// Tools
11671168

11681169
async executeCommandTool(command: string): Promise<[boolean, ToolResponse]> {
1170+
const contextWindow = this.api.getModel().info.contextWindow || 128_000
1171+
const maxAllowedSize = getMaxAllowedSize(contextWindow)
1172+
const usedContext = this.apiConversationHistory.reduce((total, msg) => {
1173+
if (Array.isArray(msg.content)) {
1174+
return (
1175+
total +
1176+
msg.content.reduce((acc, block) => {
1177+
if (block.type === "text") {
1178+
return acc + block.text.length / 4 // Rough estimate of tokens
1179+
}
1180+
return acc
1181+
}, 0)
1182+
)
1183+
}
1184+
return total + (typeof msg.content === "string" ? msg.content.length / 4 : 0)
1185+
}, 0)
1186+
11691187
const terminalInfo = await this.terminalManager.getOrCreateTerminal(cwd)
11701188
terminalInfo.terminal.show() // weird visual bug when creating new terminals (even manually) where there's an empty space at the top.
1171-
const process = this.terminalManager.runCommand(terminalInfo, command)
1189+
const process = this.terminalManager.runCommand(terminalInfo, command, maxAllowedSize, usedContext)
11721190

11731191
let userFeedback: { text?: string; images?: string[] } | undefined
11741192
let didContinue = false
@@ -1994,8 +2012,28 @@ export class Cline {
19942012
break
19952013
}
19962014
}
2015+
// Get context window and used context from API model
2016+
const contextWindow = this.api.getModel().info.contextWindow || 128_000
2017+
const maxAllowedSize = getMaxAllowedSize(contextWindow)
2018+
2019+
// Calculate used context from current conversation
2020+
const usedContext = this.apiConversationHistory.reduce((total, msg) => {
2021+
if (Array.isArray(msg.content)) {
2022+
return (
2023+
total +
2024+
msg.content.reduce((acc, block) => {
2025+
if (block.type === "text") {
2026+
return acc + block.text.length / 4 // Rough estimate of tokens
2027+
}
2028+
return acc
2029+
}, 0)
2030+
)
2031+
}
2032+
return total + (typeof msg.content === "string" ? msg.content.length / 4 : 0)
2033+
}, 0)
2034+
19972035
// now execute the tool like normal
1998-
const content = await extractTextFromFile(absolutePath)
2036+
const content = await extractTextFromFile(absolutePath, maxAllowedSize, usedContext)
19992037
pushToolResult(content)
20002038

20012039
break

‎src/core/mentions/index.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ async function getFileOrFolderContent(mentionPath: string, cwd: string): Promise
156156
if (isBinary) {
157157
return "(Binary file, unable to display content)"
158158
}
159-
const content = await extractTextFromFile(absPath)
159+
const content = await extractTextFromFile(absPath, 128_000) // Use standard context window size
160160
return content
161161
} else if (stats.isDirectory()) {
162162
const entries = await fs.readdir(absPath, { withFileTypes: true })
@@ -177,7 +177,7 @@ async function getFileOrFolderContent(mentionPath: string, cwd: string): Promise
177177
if (isBinary) {
178178
return undefined
179179
}
180-
const content = await extractTextFromFile(absoluteFilePath)
180+
const content = await extractTextFromFile(absoluteFilePath, 128_000) // Use standard context window size
181181
return `<file_content path="${filePath.toPosix()}">\n${content}\n</file_content>`
182182
} catch (error) {
183183
return undefined

‎src/integrations/misc/extract-text.test.ts‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
import { expect } from "chai"
2+
import { extractTextFromFile } from "./extract-text"
3+
import fs from "fs/promises"
4+
import path from "path"
5+
import os from "os"
6+
import { ContentTooLargeError } from "../../shared/errors"
7+
8+
const CONTEXT_LIMIT = 1000
9+
const USED_CONTEXT = 200
10+
11+
describe("extract-text", () => {
12+
let tempFilePath: string
13+
14+
beforeEach(async () => {
15+
tempFilePath = path.join(os.tmpdir(), "test-file.txt")
16+
})
17+
18+
afterEach(async () => {
19+
await fs.unlink(tempFilePath).catch(() => {})
20+
})
21+
22+
it("throws error for non-existent file", async () => {
23+
const nonExistentPath = path.join(os.tmpdir(), "non-existent.txt")
24+
try {
25+
await extractTextFromFile(nonExistentPath, CONTEXT_LIMIT, USED_CONTEXT)
26+
throw new Error("Should have thrown error")
27+
} catch (error) {
28+
expect(error.message).to.include("File not found")
29+
}
30+
})
31+
32+
it("throws ContentTooLargeError when file would exceed limit", async () => {
33+
const largeContent = "x".repeat(3000) // 3000 bytes = ~750 tokens
34+
await fs.writeFile(tempFilePath, largeContent)
35+
36+
try {
37+
await extractTextFromFile(tempFilePath, CONTEXT_LIMIT, USED_CONTEXT)
38+
throw new Error("Should have thrown error")
39+
} catch (error) {
40+
expect(error).to.be.instanceOf(ContentTooLargeError)
41+
expect(error.details.type).to.equal("file")
42+
expect(error.details.path).to.equal(tempFilePath)
43+
expect(error.details.size.wouldExceedLimit).to.equal(true)
44+
}
45+
})
46+
47+
it("reads text file content when within size limit", async () => {
48+
const content = "Hello world"
49+
await fs.writeFile(tempFilePath, content)
50+
51+
const result = await extractTextFromFile(tempFilePath, CONTEXT_LIMIT, USED_CONTEXT)
52+
expect(result).to.equal(content)
53+
})
54+
55+
it("throws error for binary files", async () => {
56+
// Create a simple binary file
57+
const buffer = new Uint8Array([0x89, 0x50, 0x4e, 0x47]) // PNG file header
58+
await fs.writeFile(tempFilePath, buffer, { encoding: "binary" })
59+
60+
try {
61+
await extractTextFromFile(tempFilePath, CONTEXT_LIMIT, USED_CONTEXT)
62+
throw new Error("Should have thrown error")
63+
} catch (error) {
64+
expect(error.message).to.include("Cannot read text for file type")
65+
}
66+
})
67+
})

‎src/integrations/misc/extract-text.ts‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,25 @@ import pdf from "pdf-parse/lib/pdf-parse"
44
import mammoth from "mammoth"
55
import fs from "fs/promises"
66
import { isBinaryFile } from "isbinaryfile"
7+
import { estimateFileSize } from "../../utils/content-size"
8+
import { ContentTooLargeError } from "../../shared/errors"
79

8-
export async function extractTextFromFile(filePath: string): Promise<string> {
10+
export async function extractTextFromFile(filePath: string, contextLimit: number, usedContext: number = 0): Promise<string> {
911
try {
1012
await fs.access(filePath)
1113
} catch (error) {
1214
throw new Error(`File not found: ${filePath}`)
1315
}
16+
17+
// Check file size before attempting to read
18+
const sizeEstimate = await estimateFileSize(filePath, contextLimit, usedContext)
19+
if (sizeEstimate.wouldExceedLimit) {
20+
throw new ContentTooLargeError({
21+
type: "file",
22+
path: filePath,
23+
size: sizeEstimate,
24+
})
25+
}
1426
const fileExtension = path.extname(filePath).toLowerCase()
1527
switch (fileExtension) {
1628
case ".pdf":

‎src/integrations/terminal/TerminalManager.ts‎

Lines changed: 36 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,33 @@
11
import pWaitFor from "p-wait-for"
22
import * as vscode from "vscode"
3+
4+
/*
5+
The new shellIntegration API gives us access to terminal command execution output handling.
6+
However, we don't update our VSCode type definitions or engine requirements to maintain compatibility
7+
with older VSCode versions. Users on older versions will automatically fall back to using sendText
8+
for terminal command execution.
9+
Interestingly, some environments like Cursor enable these APIs even without the latest VSCode engine.
10+
This approach allows us to leverage advanced features when available while ensuring broad compatibility.
11+
*/
12+
declare module "vscode" {
13+
// https://github.com/microsoft/vscode/blob/f0417069c62e20f3667506f4b7e53ca0004b4e3e/src/vscode-dts/vscode.d.ts#L7442
14+
interface Terminal {
15+
shellIntegration?: {
16+
cwd?: vscode.Uri
17+
executeCommand?: (command: string) => {
18+
read: () => AsyncIterable<string>
19+
}
20+
}
21+
}
22+
// https://github.com/microsoft/vscode/blob/f0417069c62e20f3667506f4b7e53ca0004b4e3e/src/vscode-dts/vscode.d.ts#L10794
23+
interface Window {
24+
onDidStartTerminalShellExecution?: (
25+
listener: (e: any) => any,
26+
thisArgs?: any,
27+
disposables?: vscode.Disposable[],
28+
) => vscode.Disposable
29+
}
30+
}
331
import { arePathsEqual } from "../../utils/path"
432
import { mergePromise, TerminalProcess, TerminalProcessResultPromise } from "./TerminalProcess"
533
import { TerminalInfo, TerminalRegistry } from "./TerminalRegistry"
@@ -61,34 +89,6 @@ Resources:
6189
- https://github.com/microsoft/vscode-extension-samples/blob/main/shell-integration-sample/src/extension.ts
6290
*/
6391

64-
/*
65-
The new shellIntegration API gives us access to terminal command execution output handling.
66-
However, we don't update our VSCode type definitions or engine requirements to maintain compatibility
67-
with older VSCode versions. Users on older versions will automatically fall back to using sendText
68-
for terminal command execution.
69-
Interestingly, some environments like Cursor enable these APIs even without the latest VSCode engine.
70-
This approach allows us to leverage advanced features when available while ensuring broad compatibility.
71-
*/
72-
declare module "vscode" {
73-
// https://github.com/microsoft/vscode/blob/f0417069c62e20f3667506f4b7e53ca0004b4e3e/src/vscode-dts/vscode.d.ts#L7442
74-
interface Terminal {
75-
shellIntegration?: {
76-
cwd?: vscode.Uri
77-
executeCommand?: (command: string) => {
78-
read: () => AsyncIterable<string>
79-
}
80-
}
81-
}
82-
// https://github.com/microsoft/vscode/blob/f0417069c62e20f3667506f4b7e53ca0004b4e3e/src/vscode-dts/vscode.d.ts#L10794
83-
interface Window {
84-
onDidStartTerminalShellExecution?: (
85-
listener: (e: any) => any,
86-
thisArgs?: any,
87-
disposables?: vscode.Disposable[],
88-
) => vscode.Disposable
89-
}
90-
}
91-
9292
export class TerminalManager {
9393
private terminalIds: Set<number> = new Set()
9494
private processes: Map<number, TerminalProcess> = new Map()
@@ -109,7 +109,12 @@ export class TerminalManager {
109109
}
110110
}
111111

112-
runCommand(terminalInfo: TerminalInfo, command: string): TerminalProcessResultPromise {
112+
runCommand(
113+
terminalInfo: TerminalInfo,
114+
command: string,
115+
contextLimit?: number,
116+
usedContext?: number,
117+
): TerminalProcessResultPromise {
113118
terminalInfo.busy = true
114119
terminalInfo.lastCommand = command
115120
const process = new TerminalProcess()
@@ -141,14 +146,14 @@ export class TerminalManager {
141146
// if shell integration is already active, run the command immediately
142147
if (terminalInfo.terminal.shellIntegration) {
143148
process.waitForShellIntegration = false
144-
process.run(terminalInfo.terminal, command)
149+
process.run(terminalInfo.terminal, command, contextLimit, usedContext)
145150
} else {
146151
// docs recommend waiting 3s for shell integration to activate
147152
pWaitFor(() => terminalInfo.terminal.shellIntegration !== undefined, { timeout: 4000 }).finally(() => {
148153
const existingProcess = this.processes.get(terminalInfo.id)
149154
if (existingProcess && existingProcess.waitForShellIntegration) {
150155
existingProcess.waitForShellIntegration = false
151-
existingProcess.run(terminalInfo.terminal, command)
156+
existingProcess.run(terminalInfo.terminal, command, contextLimit, usedContext)
152157
}
153158
})
154159
}

‎src/integrations/terminal/TerminalProcess.ts‎

Lines changed: 48 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
import { EventEmitter } from "events"
2-
import stripAnsi from "strip-ansi"
2+
import * as stripAnsi from "strip-ansi"
33
import * as vscode from "vscode"
4+
import { ContentTooLargeError } from "../../shared/errors"
5+
import { estimateContentSize } from "../../utils/content-size"
46

57
export interface TerminalProcessEvents {
68
line: [line: string]
@@ -22,11 +24,22 @@ export class TerminalProcess extends EventEmitter<TerminalProcessEvents> {
2224
private lastRetrievedIndex: number = 0
2325
isHot: boolean = false
2426
private hotTimer: NodeJS.Timeout | null = null
27+
private totalBytes: number = 0
28+
private contextLimit: number = 100000 // Default context window size
29+
private usedContext: number = 0
30+
private lastCommand: string = ""
2531

2632
// constructor() {
2733
// super()
2834

29-
async run(terminal: vscode.Terminal, command: string) {
35+
async run(terminal: vscode.Terminal, command: string, contextLimit?: number, usedContext?: number) {
36+
if (contextLimit) {
37+
this.contextLimit = contextLimit
38+
}
39+
if (usedContext) {
40+
this.usedContext = usedContext
41+
}
42+
this.lastCommand = command
3043
if (terminal.shellIntegration && terminal.shellIntegration.executeCommand) {
3144
const execution = terminal.shellIntegration.executeCommand(command)
3245
const stream = execution.read()
@@ -35,6 +48,24 @@ export class TerminalProcess extends EventEmitter<TerminalProcessEvents> {
3548
let didOutputNonCommand = false
3649
let didEmitEmptyLine = false
3750
for await (let data of stream) {
51+
// Add to total bytes before checking size
52+
const dataBytes = Buffer.from(data).length
53+
this.totalBytes += dataBytes
54+
55+
// Check total accumulated size
56+
const sizeEstimate = estimateContentSize(Buffer.alloc(this.totalBytes), this.contextLimit, this.usedContext)
57+
if (sizeEstimate.wouldExceedLimit) {
58+
this.emit(
59+
"error",
60+
new ContentTooLargeError({
61+
type: "terminal",
62+
command,
63+
size: sizeEstimate,
64+
}),
65+
)
66+
return
67+
}
68+
3869
// 1. Process chunk and remove artifacts
3970
if (isFirstChunk) {
4071
/*
@@ -184,6 +215,21 @@ export class TerminalProcess extends EventEmitter<TerminalProcessEvents> {
184215

185216
// Inspired by https://github.com/sindresorhus/execa/blob/main/lib/transform/split.js
186217
private emitIfEol(chunk: string) {
218+
// Check size before adding to buffer
219+
const newBufferSize = this.buffer.length + chunk.length
220+
const sizeEstimate = estimateContentSize(Buffer.alloc(newBufferSize), this.contextLimit, this.usedContext)
221+
if (sizeEstimate.wouldExceedLimit) {
222+
this.emit(
223+
"error",
224+
new ContentTooLargeError({
225+
type: "terminal",
226+
command: this.lastCommand,
227+
size: sizeEstimate,
228+
}),
229+
)
230+
return
231+
}
232+
187233
this.buffer += chunk
188234
let lineEndIndex: number
189235
while ((lineEndIndex = this.buffer.indexOf("\n")) !== -1) {

‎src/shared/errors.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
import { SizeEstimate } from "../utils/content-size"
2+
3+
/**
4+
* Error thrown when content would exceed the model's context window limit
5+
*/
6+
export class ContentTooLargeError extends Error {
7+
constructor(
8+
public details: {
9+
type: "file" | "terminal"
10+
path?: string
11+
command?: string
12+
size: SizeEstimate
13+
},
14+
) {
15+
super("Content too large for context window")
16+
this.name = "ContentTooLargeError"
17+
}
18+
}

0 commit comments

Comments
 (0)