collapse char reads into one file

Author: liwilliam2021

src/core/tools/__tests__/readFileTool.spec.ts

@@ -4,7 +4,6 @@ import * as path from "path"
 
 import { countFileLines } from "../../../integrations/misc/line-counter"
 import { readLines } from "../../../integrations/misc/read-lines"
-import { readPartialContent } from "../../../integrations/misc/read-partial-content"
 import { extractTextFromFile, addLineNumbers, getSupportedBinaryFormats } from "../../../integrations/misc/extract-text"
 import { parseSourceCodeDefinitionsForFile } from "../../../services/tree-sitter"
 import { isBinaryFile } from "isbinaryfile"
@@ -35,17 +34,6 @@ vi.mock("../../../integrations/misc/line-counter")
 vi.mock("../../../integrations/misc/read-lines", () => ({
 	readLines: vi.fn().mockResolvedValue("mocked line content"),
 }))
-vi.mock("../../../integrations/misc/read-partial-content", () => ({
-	readPartialSingleLineContent: vi.fn().mockResolvedValue("mocked partial content"),
-	readPartialContent: vi.fn().mockResolvedValue({
-		content: "mocked partial content",
-		charactersRead: 100,
-		totalCharacters: 1000,
-		linesRead: 5,
-		totalLines: 50,
-		lastLineRead: 5,
-	}),
-}))
 vi.mock("../contextValidator")
 
 // Mock fs/promises readFile for image tests
@@ -1379,15 +1367,8 @@ describe("read_file tool XML output structure", () => {
 				reason: "This is a partial read - the remaining content cannot be accessed due to context limitations.",
 			})
 
-			// Mock readPartialContent to return truncated content
-			vi.mocked(readPartialContent).mockResolvedValue({
-				content: "Line 1\nLine 2\n...truncated...",
-				charactersRead: 2000,
-				totalCharacters: 500000,
-				linesRead: 100,
-				totalLines: 10000,
-				lastLineRead: 100,
-			})
+			// Mock readLines to return truncated content with maxChars
+			vi.mocked(readLines).mockResolvedValue("Line 1\nLine 2\n...truncated...")
 
 			const result = await executeReadFileTool(
 				{ args: `<file><path>large-file.ts</path></file>` },
@@ -1430,15 +1411,8 @@ describe("read_file tool XML output structure", () => {
 				reason: "This is a partial read - the remaining content cannot be accessed due to context limitations.",
 			})
 
-			// Mock readPartialContent to return truncated content for single-line file
-			vi.mocked(readPartialContent).mockResolvedValue({
-				content: "const a=1;const b=2;...truncated",
-				charactersRead: 5000,
-				totalCharacters: 10000,
-				linesRead: 1,
-				totalLines: 1,
-				lastLineRead: 1,
-			})
+			// Mock readLines to return truncated content for single-line file with maxChars
+			vi.mocked(readLines).mockResolvedValue("const a=1;const b=2;...truncated")
 
 			const result = await executeReadFileTool(
 				{ args: `<file><path>minified.js</path></file>` },
@@ -1463,15 +1437,8 @@ describe("read_file tool XML output structure", () => {
 				reason: "This is a partial read - the remaining content cannot be accessed due to context limitations.",
 			})
 
-			// Mock readPartialContent to return truncated content
-			vi.mocked(readPartialContent).mockResolvedValue({
-				content: "Line 1\nLine 2\n...truncated...",
-				charactersRead: 50000,
-				totalCharacters: 250000,
-				linesRead: 1000,
-				totalLines: 5000,
-				lastLineRead: 1000,
-			})
+			// Mock readLines to return truncated content with maxChars
+			vi.mocked(readLines).mockResolvedValue("Line 1\nLine 2\n...truncated...")
 
 			const result = await executeReadFileTool(
 				{ args: `<file><path>large-file.ts</path></file>` },
@@ -1496,15 +1463,8 @@ describe("read_file tool XML output structure", () => {
 				reason: "This is a partial read - the remaining content cannot be accessed due to context limitations.",
 			})
 
-			// Mock readPartialContent for single-line file
-			vi.mocked(readPartialContent).mockResolvedValue({
-				content: "const a=1;const b=2;const c=3;",
-				charactersRead: 8000,
-				totalCharacters: 10000,
-				linesRead: 1,
-				totalLines: 1,
-				lastLineRead: 1,
-			})
+			// Mock readLines for single-line file with maxChars
+			vi.mocked(readLines).mockResolvedValue("const a=1;const b=2;const c=3;")
 
 			const result = await executeReadFileTool(
 				{ args: `<file><path>semi-large.js</path></file>` },

src/core/tools/readFileTool.ts

@@ -12,7 +12,6 @@ import { isPathOutsideWorkspace } from "../../utils/pathUtils"
 import { getReadablePath } from "../../utils/path"
 import { countFileLines } from "../../integrations/misc/line-counter"
 import { readLines } from "../../integrations/misc/read-lines"
-import { readPartialContent } from "../../integrations/misc/read-partial-content"
 import { extractTextFromFile, addLineNumbers, getSupportedBinaryFormats } from "../../integrations/misc/extract-text"
 import { parseSourceCodeDefinitionsForFile } from "../../services/tree-sitter"
 import { parseXml } from "../../utils/xml"
@@ -578,12 +577,15 @@ export async function readFileTool(
 
 				// Handle files with validation limits (character-based reading)
 				if (shouldApplyValidation) {
-					const result = await readPartialContent(fullPath, validation.safeContentLimit)
+					const partialContent = await readLines(fullPath, undefined, undefined, validation.safeContentLimit)
+
+					// Count lines in the partial content
+					const linesRead = partialContent ? (partialContent.match(/\n/g) || []).length + 1 : 0
 
 					// Generate line range attribute based on what was read
-					const lineRangeAttr = result.linesRead === 1 ? ` lines="1"` : ` lines="1-${result.lastLineRead}"`
+					const lineRangeAttr = linesRead === 1 ? ` lines="1"` : ` lines="1-${linesRead}"`
 
-					const content = addLineNumbers(result.content, 1)
+					const content = addLineNumbers(partialContent, 1)
 					let xmlInfo = `<content${lineRangeAttr}>\n${content}</content>\n`
 
 					// Add simple notice about partial read

src/integrations/misc/__tests__/read-lines.spec.ts

@@ -128,5 +128,79 @@ describe("nthline", () => {
 				expect(lines).toEqual("\n\n\n")
 			})
 		})
+
+		describe("maxChars parameter", () => {
+			it("should limit output to maxChars when reading entire file", async () => {
+				const content = await readLines(testFile, undefined, undefined, 20)
+				expect(content).toEqual("Line 1\nLine 2\nLine 3")
+				expect(content.length).toBe(20)
+			})
+
+			it("should limit output to maxChars when reading a range", async () => {
+				const content = await readLines(testFile, 5, 1, 15)
+				// When maxChars cuts off in the middle of a line, we get partial content
+				expect(content).toEqual("Line 2\nLine 3\nL")
+				expect(content.length).toBe(15)
+			})
+
+			it("should return empty string when maxChars is 0", async () => {
+				const content = await readLines(testFile, undefined, undefined, 0)
+				expect(content).toEqual("")
+			})
+
+			it("should handle maxChars smaller than first line", async () => {
+				const content = await readLines(testFile, undefined, undefined, 3)
+				expect(content).toEqual("Lin")
+				expect(content.length).toBe(3)
+			})
+
+			it("should handle maxChars that cuts off in the middle of a line", async () => {
+				const content = await readLines(testFile, 2, 0, 10)
+				expect(content).toEqual("Line 1\nLin")
+				expect(content.length).toBe(10)
+			})
+
+			it("should respect both line limits and maxChars", async () => {
+				// This should read lines 2-4, but stop at 25 chars
+				const content = await readLines(testFile, 3, 1, 25)
+				expect(content).toEqual("Line 2\nLine 3\nLine 4\n")
+				expect(content.length).toBeLessThanOrEqual(25)
+			})
+
+			it("should handle maxChars with single line file", async () => {
+				await withTempFile(
+					"single-line-maxchars.txt",
+					"This is a long single line of text",
+					async (filepath) => {
+						const content = await readLines(filepath, undefined, undefined, 10)
+						expect(content).toEqual("This is a ")
+						expect(content.length).toBe(10)
+					},
+				)
+			})
+
+			it("should handle maxChars with Unicode characters", async () => {
+				await withTempFile("unicode-maxchars.txt", "Hello 😀 World\nLine 2", async (filepath) => {
+					// Note: The emoji counts as 2 chars in JavaScript strings
+					const content = await readLines(filepath, undefined, undefined, 10)
+					expect(content).toEqual("Hello 😀 W")
+					expect(content.length).toBe(10)
+				})
+			})
+
+			it("should handle maxChars larger than file size", async () => {
+				const content = await readLines(testFile, undefined, undefined, 1000)
+				const fullContent = await readLines(testFile)
+				expect(content).toEqual(fullContent)
+			})
+
+			it("should handle maxChars with empty lines", async () => {
+				await withTempFile("empty-lines-maxchars.txt", "Line 1\n\n\nLine 4\n", async (filepath) => {
+					const content = await readLines(filepath, undefined, undefined, 10)
+					expect(content).toEqual("Line 1\n\n\nL")
+					expect(content.length).toBe(10)
+				})
+			})
+		})
 	})
 })

src/integrations/misc/read-lines.ts

@@ -18,10 +18,11 @@ const outOfRangeError = (filepath: string, n: number) => {
  * @param filepath - Path to the file to read
  * @param endLine - Optional. The line number to stop reading at (inclusive). If undefined, reads to the end of file.
  * @param startLine - Optional. The line number to start reading from (inclusive). If undefined, starts from line 0.
+ * @param maxChars - Optional. Maximum number of characters to read. If specified, reading stops when this limit is reached.
  * @returns Promise resolving to a string containing the read lines joined with newlines
  * @throws {RangeError} If line numbers are invalid or out of range
  */
-export function readLines(filepath: string, endLine?: number, startLine?: number): Promise<string> {
+export function readLines(filepath: string, endLine?: number, startLine?: number, maxChars?: number): Promise<string> {
 	return new Promise((resolve, reject) => {
 		// Reject if startLine is defined but not a number
 		if (startLine !== undefined && typeof startLine !== "number") {
@@ -52,11 +53,16 @@ export function readLines(filepath: string, endLine?: number, startLine?: number
 			)
 		}
 
-		// Set up stream
-		const input = createReadStream(filepath)
+		// Set up stream - only add 'end' option when maxChars is specified to avoid reading entire file
+		const streamOptions =
+			maxChars !== undefined
+				? { end: Math.min(maxChars * 2, maxChars + 1024 * 1024) } // Read at most 2x maxChars or maxChars + 1MB
+				: undefined
+		const input = createReadStream(filepath, streamOptions)
 		let buffer = ""
 		let lineCount = 0
 		let result = ""
+		let totalCharsRead = 0
 
 		// Handle errors
 		input.on("error", reject)
@@ -73,7 +79,24 @@ export function readLines(filepath: string, endLine?: number, startLine?: number
 			while (nextNewline !== -1) {
 				// If we're in the target range, add this line to the result
 				if (lineCount >= effectiveStartLine && (endLine === undefined || lineCount <= endLine)) {
-					result += buffer.substring(pos, nextNewline + 1) // Include the newline
+					const lineToAdd = buffer.substring(pos, nextNewline + 1) // Include the newline
+
+					// Check if adding this line would exceed maxChars (only if maxChars is specified)
+					if (maxChars !== undefined && totalCharsRead + lineToAdd.length > maxChars) {
+						// Add only the portion that fits within maxChars
+						const remainingChars = maxChars - totalCharsRead
+						if (remainingChars > 0) {
+							result += lineToAdd.substring(0, remainingChars)
+						}
+						input.destroy()
+						resolve(result)
+						return
+					}
+
+					result += lineToAdd
+					if (maxChars !== undefined) {
+						totalCharsRead += lineToAdd.length
+					}
 				}
 
 				// Move position and increment line counter
@@ -100,7 +123,15 @@ export function readLines(filepath: string, endLine?: number, startLine?: number
 			// Process any remaining data in buffer (last line without newline)
 			if (buffer.length > 0) {
 				if (lineCount >= effectiveStartLine && (endLine === undefined || lineCount <= endLine)) {
-					result += buffer
+					// Check if adding this would exceed maxChars (only if maxChars is specified)
+					if (maxChars !== undefined && totalCharsRead + buffer.length > maxChars) {
+						const remainingChars = maxChars - totalCharsRead
+						if (remainingChars > 0) {
+							result += buffer.substring(0, remainingChars)
+						}
+					} else {
+						result += buffer
+					}
 				}
 				lineCount++
 			}