Skip to content

Commit c5ccc42

Browse files
authored
feat: add file read character limit (#461)
1 parent 2ef8f83 commit c5ccc42

File tree

16 files changed

+274
-12
lines changed

16 files changed

+274
-12
lines changed

packages/types/src/global-settings.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ export const DEFAULT_WRITE_DELAY_MS = 1000
2828
* while preventing context window explosions from extremely long lines.
2929
*/
3030
export const DEFAULT_TERMINAL_OUTPUT_CHARACTER_LIMIT = 50_000
31+
export const DEFAULT_FILE_READ_CHARACTER_LIMIT = 20_000
3132

3233
/**
3334
* GlobalSettings
@@ -114,6 +115,7 @@ export const globalSettingsSchema = z.object({
114115

115116
terminalOutputLineLimit: z.number().optional(),
116117
terminalOutputCharacterLimit: z.number().optional(),
118+
maxReadCharacterLimit: z.number().optional(),
117119
terminalShellIntegrationTimeout: z.number().optional(),
118120
terminalShellIntegrationDisabled: z.boolean().optional(),
119121
terminalCommandDelay: z.number().optional(),
@@ -290,6 +292,7 @@ export const EVALS_SETTINGS: RooCodeSettings = {
290292

291293
terminalOutputLineLimit: 500,
292294
terminalOutputCharacterLimit: DEFAULT_TERMINAL_OUTPUT_CHARACTER_LIMIT,
295+
maxReadCharacterLimit: DEFAULT_FILE_READ_CHARACTER_LIMIT,
293296
terminalShellIntegrationTimeout: 30000,
294297
terminalCommandDelay: 0,
295298
terminalPowershellCounter: false,

src/core/mentions/index.ts

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ export async function parseMentions(
8282
includeDiagnosticMessages: boolean = true,
8383
maxDiagnosticMessages: number = 50,
8484
maxReadFileLine?: number,
85+
maxReadCharacterLimit?: number,
8586
): Promise<string> {
8687
const mentions: Set<string> = new Set()
8788
const validCommands: Map<string, Command> = new Map()
@@ -189,6 +190,7 @@ export async function parseMentions(
189190
rooIgnoreController,
190191
showRooIgnoredFiles,
191192
maxReadFileLine,
193+
maxReadCharacterLimit,
192194
)
193195
if (mention.endsWith("/")) {
194196
parsedText += `\n\n<folder_content path="${mentionPath}">\n${content}\n</folder_content>`
@@ -267,6 +269,7 @@ async function getFileOrFolderContent(
267269
rooIgnoreController?: any,
268270
showRooIgnoredFiles: boolean = false,
269271
maxReadFileLine?: number,
272+
maxReadCharacterLimit?: number,
270273
): Promise<string> {
271274
const unescapedPath = unescapeSpaces(mentionPath)
272275
const absPath = path.resolve(cwd, unescapedPath)
@@ -279,7 +282,7 @@ async function getFileOrFolderContent(
279282
return `(File ${mentionPath} is ignored by .rooignore)`
280283
}
281284
try {
282-
const content = await extractTextFromFile(absPath, maxReadFileLine)
285+
const content = await extractTextFromFile(absPath, maxReadFileLine, maxReadCharacterLimit)
283286
return content
284287
} catch (error) {
285288
return `(Failed to read contents of ${mentionPath}): ${error.message}`
@@ -319,7 +322,11 @@ async function getFileOrFolderContent(
319322
if (isBinary) {
320323
return undefined
321324
}
322-
const content = await extractTextFromFile(absoluteFilePath, maxReadFileLine)
325+
const content = await extractTextFromFile(
326+
absoluteFilePath,
327+
maxReadFileLine,
328+
maxReadCharacterLimit,
329+
)
323330
return `<file_content path="${filePath.toPosix()}">\n${content}\n</file_content>`
324331
} catch (error) {
325332
return undefined

src/core/mentions/processUserContentMentions.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ export async function processUserContentMentions({
1818
includeDiagnosticMessages = true,
1919
maxDiagnosticMessages = 50,
2020
maxReadFileLine,
21+
maxReadCharacterLimit,
2122
}: {
2223
userContent: Anthropic.Messages.ContentBlockParam[]
2324
cwd: string
@@ -29,6 +30,7 @@ export async function processUserContentMentions({
2930
includeDiagnosticMessages?: boolean
3031
maxDiagnosticMessages?: number
3132
maxReadFileLine?: number
33+
maxReadCharacterLimit?: number
3234
}) {
3335
// Process userContent array, which contains various block types:
3436
// TextBlockParam, ImageBlockParam, ToolUseBlockParam, and ToolResultBlockParam.
@@ -62,6 +64,7 @@ export async function processUserContentMentions({
6264
includeDiagnosticMessages,
6365
maxDiagnosticMessages,
6466
maxReadFileLine,
67+
maxReadCharacterLimit,
6568
),
6669
}
6770
}
@@ -82,6 +85,7 @@ export async function processUserContentMentions({
8285
includeDiagnosticMessages,
8386
maxDiagnosticMessages,
8487
maxReadFileLine,
88+
maxReadCharacterLimit,
8589
),
8690
}
8791
}
@@ -103,6 +107,7 @@ export async function processUserContentMentions({
103107
includeDiagnosticMessages,
104108
maxDiagnosticMessages,
105109
maxReadFileLine,
110+
maxReadCharacterLimit,
106111
),
107112
}
108113
}

src/core/task/Task.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1812,6 +1812,7 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
18121812
includeDiagnosticMessages = true,
18131813
maxDiagnosticMessages = 50,
18141814
maxReadFileLine = -1,
1815+
maxReadCharacterLimit = 20000,
18151816
} = (await this.providerRef.deref()?.getState()) ?? {}
18161817

18171818
const parsedUserContent = await processUserContentMentions({
@@ -1824,6 +1825,7 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
18241825
includeDiagnosticMessages,
18251826
maxDiagnosticMessages,
18261827
maxReadFileLine,
1828+
maxReadCharacterLimit,
18271829
})
18281830

18291831
const environmentDetails = await getEnvironmentDetails(this, currentIncludeFileDetails)

src/core/webview/ClineProvider.ts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ import {
4141
DEFAULT_WRITE_DELAY_MS,
4242
ORGANIZATION_ALLOW_ALL,
4343
DEFAULT_MODES,
44+
DEFAULT_FILE_READ_CHARACTER_LIMIT,
4445
} from "@roo-code/types"
4546
import { TelemetryService } from "@roo-code/telemetry"
4647
import { CloudService, BridgeOrchestrator, getRooCodeApiUrl } from "@roo-code/cloud"
@@ -1765,6 +1766,7 @@ export class ClineProvider
17651766
writeDelayMs,
17661767
terminalOutputLineLimit,
17671768
terminalOutputCharacterLimit,
1769+
maxReadCharacterLimit,
17681770
terminalShellIntegrationTimeout,
17691771
terminalShellIntegrationDisabled,
17701772
terminalCommandDelay,
@@ -1882,6 +1884,7 @@ export class ClineProvider
18821884
writeDelayMs: writeDelayMs ?? DEFAULT_WRITE_DELAY_MS,
18831885
terminalOutputLineLimit: terminalOutputLineLimit ?? 500,
18841886
terminalOutputCharacterLimit: terminalOutputCharacterLimit ?? DEFAULT_TERMINAL_OUTPUT_CHARACTER_LIMIT,
1887+
maxReadCharacterLimit: maxReadCharacterLimit ?? DEFAULT_FILE_READ_CHARACTER_LIMIT,
18851888
terminalShellIntegrationTimeout: terminalShellIntegrationTimeout ?? Terminal.defaultShellIntegrationTimeout,
18861889
terminalShellIntegrationDisabled: terminalShellIntegrationDisabled ?? false,
18871890
terminalCommandDelay: terminalCommandDelay ?? 0,
@@ -2102,6 +2105,7 @@ export class ClineProvider
21022105
terminalOutputLineLimit: stateValues.terminalOutputLineLimit ?? 500,
21032106
terminalOutputCharacterLimit:
21042107
stateValues.terminalOutputCharacterLimit ?? DEFAULT_TERMINAL_OUTPUT_CHARACTER_LIMIT,
2108+
maxReadCharacterLimit: stateValues.maxReadCharacterLimit ?? DEFAULT_FILE_READ_CHARACTER_LIMIT,
21052109
terminalShellIntegrationTimeout:
21062110
stateValues.terminalShellIntegrationTimeout ?? Terminal.defaultShellIntegrationTimeout,
21072111
terminalShellIntegrationDisabled: stateValues.terminalShellIntegrationDisabled ?? false,

src/core/webview/webviewMessageHandler.ts

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1462,16 +1462,21 @@ export const webviewMessageHandler = async (
14621462
)
14631463
}
14641464
break
1465+
case "maxReadCharacterLimit":
14651466
case "terminalOutputCharacterLimit":
14661467
// Validate that the character limit is a positive number
14671468
const charLimit = message.value
14681469
if (typeof charLimit === "number" && charLimit > 0) {
1469-
await updateGlobalState("terminalOutputCharacterLimit", charLimit)
1470+
await updateGlobalState(
1471+
message.type as "terminalOutputCharacterLimit" | "maxReadCharacterLimit",
1472+
charLimit,
1473+
)
14701474
await provider.postStateToWebview()
14711475
} else {
14721476
vscode.window.showErrorMessage(
1473-
t("common:errors.invalid_character_limit") ||
1474-
"Terminal output character limit must be a positive number",
1477+
t("common:errors.invalid_character_limit") || message.type === "terminalOutputCharacterLimit"
1478+
? "Terminal output character limit must be a positive number"
1479+
: "File read character limit must be a positive number",
14751480
)
14761481
}
14771482
break

src/integrations/misc/__tests__/extract-text.spec.ts

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -709,3 +709,169 @@ describe("processCarriageReturns", () => {
709709
expect(processCarriageReturns(input)).toBe(expected)
710710
})
711711
})
712+
713+
describe("extractTextFromFile with character limit", () => {
714+
const fs = require("fs/promises")
715+
const path = require("path")
716+
const os = require("os")
717+
718+
let tempDir: string
719+
let testFilePath: string
720+
721+
beforeEach(async () => {
722+
tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "extract-text-test-"))
723+
testFilePath = path.join(tempDir, "test.txt")
724+
})
725+
726+
afterEach(async () => {
727+
try {
728+
await fs.rm(tempDir, { recursive: true, force: true })
729+
} catch (error) {
730+
// Ignore cleanup errors
731+
}
732+
})
733+
734+
it("should apply character limit when file content exceeds limit", async () => {
735+
// 创建一个包含大量字符的文件
736+
const longContent = "a".repeat(1000)
737+
await fs.writeFile(testFilePath, longContent)
738+
739+
const { extractTextFromFile } = await import("../extract-text")
740+
const result = await extractTextFromFile(testFilePath, undefined, 100)
741+
742+
// 应该被字符限制截断
743+
expect(result.length).toBeLessThan(longContent.length + 50) // 加上行号和截断信息
744+
expect(result).toContain("[...") // 应该包含截断标识
745+
expect(result).toContain("characters omitted...]")
746+
})
747+
748+
it("should apply character limit even when line limit is not exceeded", async () => {
749+
// 创建少量行但每行很长的文件
750+
const longLine = "x".repeat(500)
751+
const content = `${longLine}\n${longLine}\n${longLine}`
752+
await fs.writeFile(testFilePath, content)
753+
754+
const { extractTextFromFile } = await import("../extract-text")
755+
const result = await extractTextFromFile(testFilePath, 10, 200) // 行数限制10,字符限制200
756+
757+
// 字符限制应该优先生效
758+
expect(result).toContain("characters omitted")
759+
expect(result).not.toContain("lines omitted")
760+
})
761+
762+
it("should apply both line and character limits when line limit is exceeded first", async () => {
763+
// 创建很多短行的文件
764+
const lines = Array.from({ length: 50 }, (_, i) => `line${i + 1}`)
765+
const content = lines.join("\n")
766+
await fs.writeFile(testFilePath, content)
767+
768+
const { extractTextFromFile } = await import("../extract-text")
769+
const result = await extractTextFromFile(testFilePath, 10, 10000) // 行数限制10,字符限制很大
770+
771+
// 行数限制应该先生效,应该只显示行数截断信息
772+
expect(result).toContain("showing 10 of 50 total lines")
773+
expect(result).not.toContain("character limit")
774+
})
775+
776+
it("should show different truncation messages for different scenarios", async () => {
777+
// 测试场景1:只有行数限制
778+
const lines = Array.from({ length: 20 }, (_, i) => `line${i + 1}`)
779+
const content1 = lines.join("\n")
780+
await fs.writeFile(testFilePath, content1)
781+
782+
const { extractTextFromFile } = await import("../extract-text")
783+
const result1 = await extractTextFromFile(testFilePath, 5, undefined)
784+
expect(result1).toContain("showing 5 of 20 total lines")
785+
expect(result1).not.toContain("character limit")
786+
787+
// 测试场景2:行数限制 + 字符限制都生效
788+
const longLines = Array.from({ length: 20 }, (_, i) => `${"x".repeat(100)}_line${i + 1}`)
789+
const content2 = longLines.join("\n")
790+
await fs.writeFile(testFilePath, content2)
791+
792+
const result2 = await extractTextFromFile(testFilePath, 5, 200)
793+
expect(result2).toContain("showing 5 of 20 total lines")
794+
expect(result2).toContain("character limit (200)")
795+
796+
// 测试场景3:只有字符限制
797+
const longContent = "a".repeat(1000)
798+
await fs.writeFile(testFilePath, longContent)
799+
800+
const result3 = await extractTextFromFile(testFilePath, undefined, 100)
801+
expect(result3).toContain("characters omitted")
802+
expect(result3).toContain("character limit (100)")
803+
expect(result3).not.toContain("total lines")
804+
})
805+
806+
it("should not apply character limit when content is within limit", async () => {
807+
const shortContent = "short content"
808+
await fs.writeFile(testFilePath, shortContent)
809+
810+
const { extractTextFromFile } = await import("../extract-text")
811+
const result = await extractTextFromFile(testFilePath, undefined, 1000)
812+
813+
// 内容应该完整保留,只添加行号
814+
expect(result).toBe("1 | short content\n")
815+
expect(result).not.toContain("characters omitted")
816+
})
817+
818+
it("should handle character limit with line limit when both are exceeded", async () => {
819+
// 创建很多长行的文件
820+
const longLine = "y".repeat(100)
821+
const lines = Array.from({ length: 30 }, (_, i) => `${longLine}_${i + 1}`)
822+
const content = lines.join("\n")
823+
await fs.writeFile(testFilePath, content)
824+
825+
const { extractTextFromFile } = await import("../extract-text")
826+
const result = await extractTextFromFile(testFilePath, 5, 500) // 行数限制5,字符限制500
827+
828+
// 行数限制先生效,然后字符限制应用到截断后的内容
829+
expect(result).toContain("showing 5 of 30 total lines")
830+
// 字符限制也应该应用
831+
expect(result).toContain("characters omitted")
832+
})
833+
834+
it("should validate maxReadCharacterLimit parameter", async () => {
835+
await fs.writeFile(testFilePath, "test content")
836+
837+
const { extractTextFromFile } = await import("../extract-text")
838+
839+
// 测试无效的字符限制参数
840+
await expect(extractTextFromFile(testFilePath, undefined, 0)).rejects.toThrow(
841+
"Invalid maxReadCharacterLimit: 0. Must be a positive integer or undefined for unlimited.",
842+
)
843+
844+
await expect(extractTextFromFile(testFilePath, undefined, -1)).rejects.toThrow(
845+
"Invalid maxReadCharacterLimit: -1. Must be a positive integer or undefined for unlimited.",
846+
)
847+
})
848+
849+
it("should work correctly when maxReadCharacterLimit is undefined", async () => {
850+
const content = "test content without limit"
851+
await fs.writeFile(testFilePath, content)
852+
853+
const { extractTextFromFile } = await import("../extract-text")
854+
const result = await extractTextFromFile(testFilePath, undefined, undefined)
855+
856+
// 应该返回完整内容加行号
857+
expect(result).toBe("1 | test content without limit\n")
858+
})
859+
860+
it("should apply character limit to line-limited content correctly", async () => {
861+
// 创建内容,行数超限但字符数在限制内
862+
const lines = Array.from({ length: 20 }, (_, i) => `line${i + 1}`)
863+
const content = lines.join("\n")
864+
await fs.writeFile(testFilePath, content)
865+
866+
const { extractTextFromFile } = await import("../extract-text")
867+
const result = await extractTextFromFile(testFilePath, 5, 200) // 行数限制5,字符限制200
868+
869+
// 行数限制先生效
870+
expect(result).toContain("showing 5 of 20 total lines")
871+
872+
// 然后字符限制应用到结果上
873+
if (result.length > 200) {
874+
expect(result).toContain("characters omitted")
875+
}
876+
})
877+
})

0 commit comments

Comments
 (0)