feat: Add file context tracking system (#2440)

* feat: Add file context tracking system

This commit adds a comprehensive file context tracking system that monitors file operations (reads, edits) by both Roo and users. The system helps prevent stale context issues and improves checkpoint management.

Key features:
- Track files accessed via tools, mentions, or edits
- Monitor file changes outside of Roo using file watchers
- Store file operation metadata with timestamps
- Trigger checkpoints automatically when files are modified
- Prevent false positives by distinguishing between Roo and user edits

The implementation includes:
- New FileContextTracker class to manage file operations
- Type definitions for file metadata tracking
- Integration with all file-related tools
- File mention tracking in the mentions system
- Improved checkpoint triggering based on file modifications

* Update src/core/context-tracking/FileContextTracker.ts

Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com>

* Update src/core/context-tracking/FileContextTracker.ts

Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com>

* test: Add mocks for getFileContextTracker in Cline tests

---------

Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com>
Co-authored-by: Matt Rubens <[email protected]>

Author: 3 people

src/core/Cline.ts

@@ -53,6 +53,7 @@ import { calculateApiCostAnthropic } from "../utils/cost"
 import { fileExistsAtPath } from "../utils/fs"
 import { arePathsEqual } from "../utils/path"
 import { parseMentions } from "./mentions"
+import { FileContextTracker } from "./context-tracking/FileContextTracker"
 import { RooIgnoreController } from "./ignore/RooIgnoreController"
 import { AssistantMessageContent, parseAssistantMessage, ToolParamName, ToolUseName } from "./assistant-message"
 import { formatResponse } from "./prompts/responses"
@@ -130,6 +131,7 @@ export class Cline extends EventEmitter<ClineEvents> {
 
 	readonly apiConfiguration: ApiConfiguration
 	api: ApiHandler
+	private fileContextTracker: FileContextTracker
 	private urlContentFetcher: UrlContentFetcher
 	browserSession: BrowserSession
 	didEditFile: boolean = false
@@ -201,14 +203,15 @@ export class Cline extends EventEmitter<ClineEvents> {
 			throw new Error("Either historyItem or task/images must be provided")
 		}
 
+		this.taskId = historyItem ? historyItem.id : crypto.randomUUID()
+		this.instanceId = crypto.randomUUID().slice(0, 8)
+		this.taskNumber = -1
+
 		this.rooIgnoreController = new RooIgnoreController(this.cwd)
+		this.fileContextTracker = new FileContextTracker(provider, this.taskId)
 		this.rooIgnoreController.initialize().catch((error) => {
 			console.error("Failed to initialize RooIgnoreController:", error)
 		})
-
-		this.taskId = historyItem ? historyItem.id : crypto.randomUUID()
-		this.instanceId = crypto.randomUUID().slice(0, 8)
-		this.taskNumber = -1
 		this.apiConfiguration = apiConfiguration
 		this.api = buildApiHandler(apiConfiguration)
 		this.urlContentFetcher = new UrlContentFetcher(provider.context)
@@ -929,6 +932,7 @@ export class Cline extends EventEmitter<ClineEvents> {
 		this.urlContentFetcher.closeBrowser()
 		this.browserSession.closeBrowser()
 		this.rooIgnoreController?.dispose()
+		this.fileContextTracker.dispose()
 
 		// If we're not streaming then `abortStream` (which reverts the diff
 		// view changes) won't be called, so we need to revert the changes here.
@@ -1322,8 +1326,6 @@ export class Cline extends EventEmitter<ClineEvents> {
 
 		const block = cloneDeep(this.assistantMessageContent[this.currentStreamingContentIndex]) // need to create copy bc while stream is updating the array, it could be updating the reference block properties too
 
-		let isCheckpointPossible = false
-
 		switch (block.type) {
 			case "text": {
 				if (this.didRejectTool || this.didAlreadyUseTool) {
@@ -1460,7 +1462,6 @@ export class Cline extends EventEmitter<ClineEvents> {
 
 					// Flag a checkpoint as possible since we've used a tool
 					// which may have changed the file system.
-					isCheckpointPossible = true
 				}
 
 				const askApproval = async (
@@ -1583,6 +1584,7 @@ export class Cline extends EventEmitter<ClineEvents> {
 						break
 					case "read_file":
 						await readFileTool(this, block, askApproval, handleError, pushToolResult, removeClosingTag)
+
 						break
 					case "fetch_instructions":
 						await fetchInstructionsTool(this, block, askApproval, handleError, pushToolResult)
@@ -1662,7 +1664,9 @@ export class Cline extends EventEmitter<ClineEvents> {
 				break
 		}
 
-		if (isCheckpointPossible) {
+		const recentlyModifiedFiles = this.fileContextTracker.getAndClearCheckpointPossibleFile()
+		if (recentlyModifiedFiles.length > 0) {
+			// TODO: we can track what file changes were made and only checkpoint those files, this will be save storage
 			this.checkpointSave()
 		}
 
@@ -1783,18 +1787,17 @@ export class Cline extends EventEmitter<ClineEvents> {
 		)
 
 		const [parsedUserContent, environmentDetails] = await this.loadContext(userContent, includeFileDetails)
-		userContent = parsedUserContent
 		// add environment details as its own text block, separate from tool results
-		userContent.push({ type: "text", text: environmentDetails })
+		const finalUserContent = [...parsedUserContent, { type: "text", text: environmentDetails }] as UserContent
 
-		await this.addToApiConversationHistory({ role: "user", content: userContent })
+		await this.addToApiConversationHistory({ role: "user", content: finalUserContent })
 		telemetryService.captureConversationMessage(this.taskId, "user")
 
 		// since we sent off a placeholder api_req_started message to update the webview while waiting to actually start the API request (to load potential details for example), we need to update the text of that message
 		const lastApiReqIndex = findLastIndex(this.clineMessages, (m) => m.say === "api_req_started")
 
 		this.clineMessages[lastApiReqIndex].text = JSON.stringify({
-			request: userContent.map((block) => formatContentBlockToMarkdown(block)).join("\n\n"),
+			request: finalUserContent.map((block) => formatContentBlockToMarkdown(block)).join("\n\n"),
 		} satisfies ClineApiReqInfo)
 
 		await this.saveClineMessages()
@@ -2045,62 +2048,73 @@ export class Cline extends EventEmitter<ClineEvents> {
 	}
 
 	async loadContext(userContent: UserContent, includeFileDetails: boolean = false) {
-		return await Promise.all([
-			// Process userContent array, which contains various block types:
-			// TextBlockParam, ImageBlockParam, ToolUseBlockParam, and ToolResultBlockParam.
-			// We need to apply parseMentions() to:
-			// 1. All TextBlockParam's text (first user message with task)
-			// 2. ToolResultBlockParam's content/context text arrays if it contains "<feedback>" (see formatToolDeniedFeedback, attemptCompletion, executeCommand, and consecutiveMistakeCount >= 3) or "<answer>" (see askFollowupQuestion), we place all user generated content in these tags so they can effectively be used as markers for when we should parse mentions)
-			Promise.all(
-				userContent.map(async (block) => {
-					const shouldProcessMentions = (text: string) =>
-						text.includes("<task>") || text.includes("<feedback>")
-
-					if (block.type === "text") {
-						if (shouldProcessMentions(block.text)) {
+		// Process userContent array, which contains various block types:
+		// TextBlockParam, ImageBlockParam, ToolUseBlockParam, and ToolResultBlockParam.
+		// We need to apply parseMentions() to:
+		// 1. All TextBlockParam's text (first user message with task)
+		// 2. ToolResultBlockParam's content/context text arrays if it contains "<feedback>" (see formatToolDeniedFeedback, attemptCompletion, executeCommand, and consecutiveMistakeCount >= 3) or "<answer>" (see askFollowupQuestion), we place all user generated content in these tags so they can effectively be used as markers for when we should parse mentions)
+		const parsedUserContent = await Promise.all(
+			userContent.map(async (block) => {
+				const shouldProcessMentions = (text: string) => text.includes("<task>") || text.includes("<feedback>")
+
+				if (block.type === "text") {
+					if (shouldProcessMentions(block.text)) {
+						return {
+							...block,
+							text: await parseMentions(
+								block.text,
+								this.cwd,
+								this.urlContentFetcher,
+								this.fileContextTracker,
+							),
+						}
+					}
+					return block
+				} else if (block.type === "tool_result") {
+					if (typeof block.content === "string") {
+						if (shouldProcessMentions(block.content)) {
 							return {
 								...block,
-								text: await parseMentions(block.text, this.cwd, this.urlContentFetcher),
+								content: await parseMentions(
+									block.content,
+									this.cwd,
+									this.urlContentFetcher,
+									this.fileContextTracker,
+								),
 							}
 						}
 						return block
-					} else if (block.type === "tool_result") {
-						if (typeof block.content === "string") {
-							if (shouldProcessMentions(block.content)) {
-								return {
-									...block,
-									content: await parseMentions(block.content, this.cwd, this.urlContentFetcher),
-								}
-							}
-							return block
-						} else if (Array.isArray(block.content)) {
-							const parsedContent = await Promise.all(
-								block.content.map(async (contentBlock) => {
-									if (contentBlock.type === "text" && shouldProcessMentions(contentBlock.text)) {
-										return {
-											...contentBlock,
-											text: await parseMentions(
-												contentBlock.text,
-												this.cwd,
-												this.urlContentFetcher,
-											),
-										}
+					} else if (Array.isArray(block.content)) {
+						const parsedContent = await Promise.all(
+							block.content.map(async (contentBlock) => {
+								if (contentBlock.type === "text" && shouldProcessMentions(contentBlock.text)) {
+									return {
+										...contentBlock,
+										text: await parseMentions(
+											contentBlock.text,
+											this.cwd,
+											this.urlContentFetcher,
+											this.fileContextTracker,
+										),
 									}
-									return contentBlock
-								}),
-							)
-							return {
-								...block,
-								content: parsedContent,
-							}
+								}
+								return contentBlock
+							}),
+						)
+						return {
+							...block,
+							content: parsedContent,
 						}
-						return block
 					}
 					return block
-				}),
-			),
-			this.getEnvironmentDetails(includeFileDetails),
-		])
+				}
+				return block
+			}),
+		)
+
+		const environmentDetails = await this.getEnvironmentDetails(includeFileDetails)
+
+		return [parsedUserContent, environmentDetails]
 	}
 
 	async getEnvironmentDetails(includeFileDetails: boolean = false) {
@@ -2251,6 +2265,16 @@ export class Cline extends EventEmitter<ClineEvents> {
 		// 	details += "\n(No errors detected)"
 		// }
 
+		// Add recently modified files section
+		const recentlyModifiedFiles = this.fileContextTracker.getAndClearRecentlyModifiedFiles()
+		if (recentlyModifiedFiles.length > 0) {
+			details +=
+				"\n\n# Recently Modified Files\nThese files have been modified since you last accessed them (file was just edited so you may need to re-read it before editing):"
+			for (const filePath of recentlyModifiedFiles) {
+				details += `\n${filePath}`
+			}
+		}
+
 		if (terminalDetails) {
 			details += terminalDetails
 		}
@@ -2619,4 +2643,9 @@ export class Cline extends EventEmitter<ClineEvents> {
 			this.enableCheckpoints = false
 		}
 	}
+
+	// Public accessor for fileContextTracker
+	public getFileContextTracker(): FileContextTracker {
+		return this.fileContextTracker
+	}
 }

src/core/__tests__/Cline.test.ts

@@ -16,6 +16,16 @@ import { ApiStreamChunk } from "../../api/transform/stream"
 // Mock RooIgnoreController
 jest.mock("../ignore/RooIgnoreController")
 
+// Mock storagePathManager to prevent dynamic import issues
+jest.mock("../../shared/storagePathManager", () => ({
+	getTaskDirectoryPath: jest.fn().mockImplementation((globalStoragePath, taskId) => {
+		return Promise.resolve(`${globalStoragePath}/tasks/${taskId}`)
+	}),
+	getSettingsDirectoryPath: jest.fn().mockImplementation((globalStoragePath) => {
+		return Promise.resolve(`${globalStoragePath}/settings`)
+	}),
+}))
+
 // Mock fileExistsAtPath
 jest.mock("../../utils/fs", () => ({
 	fileExistsAtPath: jest.fn().mockImplementation((filePath) => {
@@ -941,6 +951,7 @@ describe("Cline", () => {
 						"<task>Text with @/some/path in task tags</task>",
 						expect.any(String),
 						expect.any(Object),
+						expect.any(Object),
 					)
 
 					// Feedback tag content should be processed
@@ -951,6 +962,7 @@ describe("Cline", () => {
 						"<feedback>Check @/some/path</feedback>",
 						expect.any(String),
 						expect.any(Object),
+						expect.any(Object),
 					)
 
 					// Regular tool result should not be processed

src/core/__tests__/read-file-maxReadFileLine.test.ts

@@ -122,6 +122,9 @@ describe("read_file tool with maxReadFileLine setting", () => {
 		mockCline.say = jest.fn().mockResolvedValue(undefined)
 		mockCline.ask = jest.fn().mockResolvedValue(true)
 		mockCline.presentAssistantMessage = jest.fn()
+		mockCline.getFileContextTracker = jest.fn().mockReturnValue({
+			trackFileContext: jest.fn().mockResolvedValue(undefined),
+		})
 
 		// Reset tool result
 		toolResult = undefined

src/core/__tests__/read-file-xml.test.ts

@@ -114,6 +114,10 @@ describe("read_file tool XML output structure", () => {
 		mockCline.ask = jest.fn().mockResolvedValue(true)
 		mockCline.presentAssistantMessage = jest.fn()
 		mockCline.sayAndCreateMissingParamError = jest.fn().mockResolvedValue("Missing required parameter")
+		// Add mock for getFileContextTracker method
+		mockCline.getFileContextTracker = jest.fn().mockReturnValue({
+			trackFileContext: jest.fn().mockResolvedValue(undefined),
+		})
 
 		// Reset tool result
 		toolResult = undefined