Split api and chat message persistence into a separate module

Author: cte

src/core/Cline.ts

@@ -1,4 +1,3 @@
-import fs from "fs/promises"
 import * as path from "path"
 import os from "os"
 import crypto from "crypto"
@@ -8,7 +7,6 @@ import { Anthropic } from "@anthropic-ai/sdk"
 import cloneDeep from "clone-deep"
 import delay from "delay"
 import pWaitFor from "p-wait-for"
-import getFolderSize from "get-folder-size"
 import { serializeError } from "serialize-error"
 import * as vscode from "vscode"
 
@@ -58,7 +56,6 @@ import { TerminalRegistry } from "../integrations/terminal/TerminalRegistry"
 
 // utils
 import { calculateApiCostAnthropic } from "../utils/cost"
-import { fileExistsAtPath } from "../utils/fs"
 import { arePathsEqual, getWorkspacePath } from "../utils/path"
 
 // tools
@@ -93,6 +90,7 @@ import { truncateConversationIfNeeded } from "./sliding-window"
 import { ClineProvider } from "./webview/ClineProvider"
 import { validateToolUse } from "./mode-validator"
 import { MultiSearchReplaceDiffStrategy } from "./diff/strategies/multi-search-replace"
+import { readApiMessages, saveApiMessages, readTaskMessages, saveTaskMessages } from "./task-persistence"
 
 type UserContent = Array<Anthropic.Messages.ContentBlockParam>
 
@@ -164,6 +162,7 @@ export class Cline extends EventEmitter<ClineEvents> {
 	consecutiveMistakeCountForApplyDiff: Map<string, number> = new Map()
 	// Not private since it needs to be accessible by tools.
 	providerRef: WeakRef<ClineProvider>
+	private readonly globalStoragePath: string
 	private abort: boolean = false
 	didFinishAbortingStream = false
 	abandoned = false
@@ -203,7 +202,6 @@ export class Cline extends EventEmitter<ClineEvents> {
 		task,
 		images,
 		historyItem,
-		experiments,
 		startTask = true,
 		rootTask,
 		parentTask,
@@ -222,9 +220,11 @@ export class Cline extends EventEmitter<ClineEvents> {
 
 		this.rooIgnoreController = new RooIgnoreController(this.cwd)
 		this.fileContextTracker = new FileContextTracker(provider, this.taskId)
+
 		this.rooIgnoreController.initialize().catch((error) => {
 			console.error("Failed to initialize RooIgnoreController:", error)
 		})
+
 		this.apiConfiguration = apiConfiguration
 		this.api = buildApiHandler(apiConfiguration)
 		this.urlContentFetcher = new UrlContentFetcher(provider.context)
@@ -234,6 +234,7 @@ export class Cline extends EventEmitter<ClineEvents> {
 		this.fuzzyMatchThreshold = fuzzyMatchThreshold
 		this.consecutiveMistakeLimit = consecutiveMistakeLimit
 		this.providerRef = new WeakRef(provider)
+		this.globalStoragePath = provider.context.globalStorageUri.fsPath
 		this.diffViewProvider = new DiffViewProvider(this.cwd)
 		this.enableCheckpoints = enableCheckpoints
 
@@ -284,26 +285,8 @@ export class Cline extends EventEmitter<ClineEvents> {
 
 	// Storing task to disk for history
 
-	private async ensureTaskDirectoryExists(): Promise<string> {
-		const globalStoragePath = this.providerRef.deref()?.context.globalStorageUri.fsPath
-		if (!globalStoragePath) {
-			throw new Error("Global storage uri is invalid")
-		}
-
-		// Use storagePathManager to retrieve the task storage directory
-		const { getTaskDirectoryPath } = await import("../shared/storagePathManager")
-		return getTaskDirectoryPath(globalStoragePath, this.taskId)
-	}
-
 	private async getSavedApiConversationHistory(): Promise<Anthropic.MessageParam[]> {
-		const filePath = path.join(await this.ensureTaskDirectoryExists(), GlobalFileNames.apiConversationHistory)
-		const fileExists = await fileExistsAtPath(filePath)
-
-		if (fileExists) {
-			return JSON.parse(await fs.readFile(filePath, "utf8"))
-		}
-
-		return []
+		return readApiMessages({ taskId: this.taskId, globalStoragePath: this.globalStoragePath })
 	}
 
 	private async addToApiConversationHistory(message: Anthropic.MessageParam) {
@@ -319,30 +302,19 @@ export class Cline extends EventEmitter<ClineEvents> {
 
 	private async saveApiConversationHistory() {
 		try {
-			const filePath = path.join(await this.ensureTaskDirectoryExists(), GlobalFileNames.apiConversationHistory)
-			await fs.writeFile(filePath, JSON.stringify(this.apiConversationHistory))
+			await saveApiMessages({
+				messages: this.apiConversationHistory,
+				taskId: this.taskId,
+				globalStoragePath: this.globalStoragePath,
+			})
 		} catch (error) {
 			// in the off chance this fails, we don't want to stop the task
 			console.error("Failed to save API conversation history:", error)
 		}
 	}
 
 	private async getSavedClineMessages(): Promise<ClineMessage[]> {
-		const filePath = path.join(await this.ensureTaskDirectoryExists(), GlobalFileNames.uiMessages)
-
-		if (await fileExistsAtPath(filePath)) {
-			return JSON.parse(await fs.readFile(filePath, "utf8"))
-		} else {
-			// check old location
-			const oldPath = path.join(await this.ensureTaskDirectoryExists(), "claude_messages.json")
-			if (await fileExistsAtPath(oldPath)) {
-				const data = JSON.parse(await fs.readFile(oldPath, "utf8"))
-				await fs.unlink(oldPath) // remove old file
-				return data
-			}
-		}
-
-		return []
+		return readTaskMessages({ taskId: this.taskId, globalStoragePath: this.globalStoragePath })
 	}
 
 	private async addToClineMessages(message: ClineMessage) {
@@ -364,46 +336,17 @@ export class Cline extends EventEmitter<ClineEvents> {
 
 	private async saveClineMessages() {
 		try {
-			const taskDir = await this.ensureTaskDirectoryExists()
-			const filePath = path.join(taskDir, GlobalFileNames.uiMessages)
-			await fs.writeFile(filePath, JSON.stringify(this.clineMessages))
+			const { historyItem, tokenUsage } = await saveTaskMessages({
+				messages: this.clineMessages,
+				taskId: this.taskId,
+				taskNumber: this.taskNumber,
+				globalStoragePath: this.globalStoragePath,
+				workspace: this.cwd,
+			})
 
-			const tokenUsage = this.getTokenUsage()
 			this.emit("taskTokenUsageUpdated", this.taskId, tokenUsage)
 
-			const taskMessage = this.clineMessages[0] // First message is always the task say
-
-			const lastRelevantMessage =
-				this.clineMessages[
-					findLastIndex(
-						this.clineMessages,
-						(m) => !(m.ask === "resume_task" || m.ask === "resume_completed_task"),
-					)
-				]
-
-			let taskDirSize = 0
-
-			try {
-				taskDirSize = await getFolderSize.loose(taskDir)
-			} catch (err) {
-				console.error(
-					`[saveClineMessages] failed to get task directory size (${taskDir}): ${err instanceof Error ? err.message : String(err)}`,
-				)
-			}
-
-			await this.providerRef.deref()?.updateTaskHistory({
-				id: this.taskId,
-				number: this.taskNumber,
-				ts: lastRelevantMessage.ts,
-				task: taskMessage.text ?? "",
-				tokensIn: tokenUsage.totalTokensIn,
-				tokensOut: tokenUsage.totalTokensOut,
-				cacheWrites: tokenUsage.totalCacheWrites,
-				cacheReads: tokenUsage.totalCacheReads,
-				totalCost: tokenUsage.totalCost,
-				size: taskDirSize,
-				workspace: this.cwd,
-			})
+			await this.providerRef.deref()?.updateTaskHistory(historyItem)
 		} catch (error) {
 			console.error("Failed to save cline messages:", error)
 		}
@@ -853,7 +796,7 @@ export class Cline extends EventEmitter<ClineEvents> {
 		}
 
 		const wasRecent = lastClineMessage?.ts && Date.now() - lastClineMessage.ts < 30_000
-		
+
 		newUserContent.push({
 			type: "text",
 			text:

src/core/task-persistence/apiMessages.ts

@@ -0,0 +1,50 @@
+import * as path from "path"
+import * as fs from "fs/promises"
+
+import { Anthropic } from "@anthropic-ai/sdk"
+
+import { fileExistsAtPath } from "../../utils/fs"
+
+import { GlobalFileNames } from "../../shared/globalFileNames"
+import { getTaskDirectoryPath } from "../../shared/storagePathManager"
+
+export type ApiMessage = Anthropic.MessageParam & { ts?: number }
+
+export async function readApiMessages({
+	taskId,
+	globalStoragePath,
+}: {
+	taskId: string
+	globalStoragePath: string
+}): Promise<ApiMessage[]> {
+	const taskDir = await getTaskDirectoryPath(globalStoragePath, taskId)
+	const filePath = path.join(taskDir, GlobalFileNames.uiMessages)
+
+	if (await fileExistsAtPath(filePath)) {
+		return JSON.parse(await fs.readFile(filePath, "utf8"))
+	} else {
+		const oldPath = path.join(taskDir, "claude_messages.json")
+
+		if (await fileExistsAtPath(oldPath)) {
+			const data = JSON.parse(await fs.readFile(oldPath, "utf8"))
+			await fs.unlink(oldPath)
+			return data
+		}
+	}
+
+	return []
+}
+
+export async function saveApiMessages({
+	messages,
+	taskId,
+	globalStoragePath,
+}: {
+	messages: ApiMessage[]
+	taskId: string
+	globalStoragePath: string
+}) {
+	const taskDir = await getTaskDirectoryPath(globalStoragePath, taskId)
+	const filePath = path.join(taskDir, GlobalFileNames.apiConversationHistory)
+	await fs.writeFile(filePath, JSON.stringify(messages))
+}

src/core/task-persistence/index.ts

@@ -0,0 +1,2 @@
+export { readApiMessages, saveApiMessages } from "./apiMessages"
+export { readTaskMessages, saveTaskMessages } from "./taskMessages"

src/core/task-persistence/taskMessages.ts

@@ -0,0 +1,84 @@
+import * as path from "path"
+import * as fs from "fs/promises"
+
+import getFolderSize from "get-folder-size"
+
+import { fileExistsAtPath } from "../../utils/fs"
+
+import { GlobalFileNames } from "../../shared/globalFileNames"
+import { ClineMessage } from "../../shared/ExtensionMessage"
+import { combineApiRequests } from "../../shared/combineApiRequests"
+import { combineCommandSequences } from "../../shared/combineCommandSequences"
+import { getApiMetrics } from "../../shared/getApiMetrics"
+import { findLastIndex } from "../../shared/array"
+import { HistoryItem } from "../../shared/HistoryItem"
+import { getTaskDirectoryPath } from "../../shared/storagePathManager"
+
+export async function readTaskMessages({
+	taskId,
+	globalStoragePath,
+}: {
+	taskId: string
+	globalStoragePath: string
+}): Promise<ClineMessage[]> {
+	const taskDir = await getTaskDirectoryPath(globalStoragePath, taskId)
+	const filePath = path.join(taskDir, GlobalFileNames.apiConversationHistory)
+	const fileExists = await fileExistsAtPath(filePath)
+
+	if (fileExists) {
+		return JSON.parse(await fs.readFile(filePath, "utf8"))
+	}
+
+	return []
+}
+
+export async function saveTaskMessages({
+	messages,
+	taskId,
+	taskNumber,
+	globalStoragePath,
+	workspace,
+}: {
+	messages: ClineMessage[]
+	taskId: string
+	taskNumber: number
+	globalStoragePath: string
+	workspace: string
+}) {
+	const taskDir = await getTaskDirectoryPath(globalStoragePath, taskId)
+	const filePath = path.join(taskDir, GlobalFileNames.uiMessages)
+	await fs.writeFile(filePath, JSON.stringify(messages))
+
+	const taskMessage = messages[0] // First message is always the task say.
+
+	const lastRelevantMessage =
+		messages[findLastIndex(messages, (m) => !(m.ask === "resume_task" || m.ask === "resume_completed_task"))]
+
+	let taskDirSize = 0
+
+	try {
+		taskDirSize = await getFolderSize.loose(taskDir)
+	} catch (error) {
+		console.error(
+			`[saveTaskMessages] getFolderSize.loose failed -> ${error instanceof Error ? error.message : String(error)}`,
+		)
+	}
+
+	const tokenUsage = getApiMetrics(combineApiRequests(combineCommandSequences(messages.slice(1))))
+
+	const historyItem: HistoryItem = {
+		id: taskId,
+		number: taskNumber,
+		ts: lastRelevantMessage.ts,
+		task: taskMessage.text ?? "",
+		tokensIn: tokenUsage.totalTokensIn,
+		tokensOut: tokenUsage.totalTokensOut,
+		cacheWrites: tokenUsage.totalCacheWrites,
+		cacheReads: tokenUsage.totalCacheReads,
+		totalCost: tokenUsage.totalCost,
+		size: taskDirSize,
+		workspace,
+	}
+
+	return { historyItem, tokenUsage }
+}