I've implemented a patch-based checkpoint system for Roo-Code that drastically reduces storage requirements while maintaining full functionality. Here's what I've done:

Created core classes for the patch-based checkpoint system:
PatchCheckpointService: Main service that handles checkpoint operations
PatchDatabase: SQLite database for storing checkpoint metadata
PatchGenerator: Utility for generating and applying patches
PatchCheckpointServiceFactory: Factory for creating service instances
Updated the existing checkpoint system to use the new patch-based implementation:
Modified src/core/checkpoints/index.ts to use PatchCheckpointServiceFactory instead of RepoPerTaskCheckpointService
Updated Task.ts to use the new PatchCheckpointService type
Created a migration system to convert existing Git-based checkpoints to the new patch-based format:
MigrationService: Handles migrating existing checkpoints
Added a command to trigger the migration: roo-cline.migrateCheckpoints
Added tests to verify the implementation works correctly
Benefits of the New Implementation
Drastically Reduced Storage Requirements: Instead of storing full copies of the workspace for each checkpoint, we now store only the differences between states.
Improved Performance: The patch-based system is more efficient, especially for large workspaces.
Better Scalability: The system can handle more checkpoints without excessive disk usage.
Maintained Compatibility: The new system maintains the same API as the old one, so existing code that uses checkpoints will continue to work

Author: bramburn

package-lock.json

@@ -411,6 +411,8 @@
 		"serialize-error": "^11.0.3",
 		"simple-git": "^3.27.0",
 		"sound-play": "^1.1.0",
+		"sqlite": "^5.1.1",
+		"sqlite3": "^5.1.7",
 		"string-similarity": "^4.0.4",
 		"strip-ansi": "^7.1.0",
 		"strip-bom": "^5.0.0",
@@ -445,7 +447,7 @@
 		"esbuild": "^0.25.0",
 		"eslint": "^8.57.0",
 		"execa": "^9.5.2",
-		"glob": "^11.0.1",
+		"glob": "^11.0.2",
 		"husky": "^9.1.7",
 		"jest": "^29.7.0",
 		"jest-simple-dot-reporter": "^1.0.5",

package.json

@@ -174,6 +174,10 @@ const getCommandsMap = ({ context, outputChannel, provider }: RegisterCommandOpt
 
 			visibleProvider.postMessageToWebview({ type: "acceptInput" })
 		},
+		"roo-cline.migrateCheckpoints": async () => {
+			const { migrateCheckpoints } = await import("../commands/migrateCheckpoints")
+			await migrateCheckpoints(context)
+		},
 	}
 }
 

src/activate/registerCommands.ts

@@ -0,0 +1,49 @@
+import * as vscode from "vscode"
+import { MigrationService } from "../services/checkpoints/MigrationService"
+
+/**
+ * Command to migrate checkpoints from the old Git-based system to the new patch-based system
+ */
+export async function migrateCheckpoints(context: vscode.ExtensionContext) {
+	const globalStorageDir = context.globalStorageUri.fsPath
+
+	// Create output channel for logging
+	const outputChannel = vscode.window.createOutputChannel("Roo Code Checkpoint Migration")
+	outputChannel.show()
+
+	const log = (message: string) => {
+		console.log(message)
+		outputChannel.appendLine(message)
+	}
+
+	// Show progress notification
+	await vscode.window.withProgress(
+		{
+			location: vscode.ProgressLocation.Notification,
+			title: "Migrating checkpoints",
+			cancellable: false,
+		},
+		async (progress) => {
+			progress.report({ message: "Starting migration..." })
+
+			try {
+				// Create migration service
+				const migrationService = new MigrationService(globalStorageDir, log)
+
+				// Run migration
+				log("Starting checkpoint migration...")
+				await migrationService.migrateAllTasks()
+
+				progress.report({ message: "Migration completed" })
+				log("Checkpoint migration completed successfully")
+
+				vscode.window.showInformationMessage("Checkpoint migration completed successfully")
+			} catch (error) {
+				const errorMessage = error instanceof Error ? error.message : String(error)
+				log(`Error during migration: ${errorMessage}`)
+
+				vscode.window.showErrorMessage(`Checkpoint migration failed: ${errorMessage}`)
+			}
+		},
+	)
+}

src/commands/migrateCheckpoints.ts

@@ -11,7 +11,7 @@ import { getApiMetrics } from "../../shared/getApiMetrics"
 import { DIFF_VIEW_URI_SCHEME } from "../../integrations/editor/DiffViewProvider"
 
 import { telemetryService } from "../../services/telemetry/TelemetryService"
-import { CheckpointServiceOptions, RepoPerTaskCheckpointService } from "../../services/checkpoints"
+import { CheckpointServiceOptions, PatchCheckpointServiceFactory } from "../../services/checkpoints"
 
 export function getCheckpointService(cline: Task) {
 	if (!cline.enableCheckpoints) {
@@ -65,7 +65,7 @@ export function getCheckpointService(cline: Task) {
 			log,
 		}
 
-		const service = RepoPerTaskCheckpointService.create(options)
+		const service = PatchCheckpointServiceFactory.create(options)
 
 		cline.checkpointServiceInitializing = true
 
@@ -104,11 +104,11 @@ export function getCheckpointService(cline: Task) {
 			}
 		})
 
-		log("[Cline#getCheckpointService] initializing shadow git")
+		log("[Cline#getCheckpointService] initializing checkpoint service")
 
-		service.initShadowGit().catch((err) => {
+		service.initialize().catch((err) => {
 			log(
-				`[Cline#getCheckpointService] caught unexpected error in initShadowGit, disabling checkpoints (${err.message})`,
+				`[Cline#getCheckpointService] caught unexpected error in initialize, disabling checkpoints (${err.message})`,
 			)
 
 			console.error(err)

src/core/checkpoints/index.ts

@@ -40,7 +40,7 @@ import { BrowserSession } from "../../services/browser/BrowserSession"
 import { McpHub } from "../../services/mcp/McpHub"
 import { McpServerManager } from "../../services/mcp/McpServerManager"
 import { telemetryService } from "../../services/telemetry/TelemetryService"
-import { RepoPerTaskCheckpointService } from "../../services/checkpoints"
+import { PatchCheckpointService } from "../../services/checkpoints/PatchCheckpointService"
 
 // integrations
 import { DiffViewProvider } from "../../integrations/editor/DiffViewProvider"
@@ -169,7 +169,7 @@ export class Task extends EventEmitter<ClineEvents> {
 
 	// Checkpoints
 	enableCheckpoints: boolean
-	checkpointService?: RepoPerTaskCheckpointService
+	checkpointService?: PatchCheckpointService
 	checkpointServiceInitializing = false
 
 	// Streaming

src/core/task/Task.ts

@@ -0,0 +1,225 @@
+import fs from "fs/promises"
+import path from "path"
+import crypto from "crypto"
+import simpleGit from "simple-git"
+
+import { PatchDatabase } from "./PatchDatabase"
+import { PatchGenerator } from "./PatchGenerator"
+import { getExcludePatterns } from "./excludes"
+
+/**
+ * MigrationService handles migrating from the old Git-based checkpoint system
+ * to the new patch-based checkpoint system.
+ */
+export class MigrationService {
+	private readonly globalStorageDir: string
+	private readonly log: (message: string) => void
+	private readonly patchGenerator: PatchGenerator
+
+	constructor(globalStorageDir: string, log: (message: string) => void) {
+		this.globalStorageDir = globalStorageDir
+		this.log = log
+		this.patchGenerator = new PatchGenerator()
+	}
+
+	/**
+	 * Migrate all tasks from the old Git-based checkpoint system to the new patch-based system
+	 */
+	public async migrateAllTasks(): Promise<void> {
+		this.log("[MigrationService#migrateAllTasks] starting migration of all tasks")
+
+		// Find all task directories
+		const tasksDir = path.join(this.globalStorageDir, "tasks")
+
+		try {
+			const taskDirs = await fs.readdir(tasksDir)
+
+			for (const taskId of taskDirs) {
+				try {
+					await this.migrateTask(taskId)
+				} catch (error) {
+					this.log(`[MigrationService#migrateAllTasks] error migrating task ${taskId}: ${error}`)
+				}
+			}
+
+			this.log("[MigrationService#migrateAllTasks] migration completed")
+		} catch (error) {
+			this.log(`[MigrationService#migrateAllTasks] error reading tasks directory: ${error}`)
+		}
+	}
+
+	/**
+	 * Migrate a single task from the old Git-based checkpoint system to the new patch-based system
+	 */
+	public async migrateTask(taskId: string): Promise<void> {
+		this.log(`[MigrationService#migrateTask] starting migration of task ${taskId}`)
+
+		// Check if the old Git-based checkpoint directory exists
+		const oldCheckpointsDir = path.join(this.globalStorageDir, "tasks", taskId, "checkpoints")
+		const dotGitDir = path.join(oldCheckpointsDir, ".git")
+
+		try {
+			const gitDirExists = await fs
+				.stat(dotGitDir)
+				.then(() => true)
+				.catch(() => false)
+
+			if (!gitDirExists) {
+				this.log(`[MigrationService#migrateTask] no Git repository found for task ${taskId}, skipping`)
+				return
+			}
+
+			// Create new patch-based checkpoint directory
+			const newCheckpointsDir = path.join(this.globalStorageDir, "tasks", taskId, "checkpoints-new")
+			await fs.mkdir(newCheckpointsDir, { recursive: true })
+
+			// Initialize database
+			const db = new PatchDatabase(path.join(newCheckpointsDir, "checkpoints.db"))
+			await db.initialize()
+
+			// Get Git repository
+			const git = simpleGit(oldCheckpointsDir)
+
+			// Get worktree directory (workspace directory)
+			const worktreeConfig = await git.raw(["config", "--get", "core.worktree"])
+			const workspaceDir = worktreeConfig.trim()
+
+			// Get commit history
+			const log = await git.log()
+			const commits = log.all.reverse() // Oldest first
+
+			if (commits.length === 0) {
+				this.log(`[MigrationService#migrateTask] no commits found for task ${taskId}, skipping`)
+				return
+			}
+
+			// Create base snapshot from the first commit
+			const baseCommit = commits[0]
+			const baseSnapshotId = crypto.randomUUID()
+
+			// Create snapshots directory
+			const snapshotsDir = path.join(newCheckpointsDir, "snapshots", baseSnapshotId)
+			await fs.mkdir(snapshotsDir, { recursive: true })
+
+			// Get files from the first commit
+			await git.checkout(baseCommit.hash)
+
+			// Get exclude patterns
+			const excludePatterns = await getExcludePatterns(workspaceDir)
+
+			// Get all files in the workspace
+			const files = await this.patchGenerator.getWorkspaceFiles(workspaceDir, excludePatterns)
+
+			// Create base snapshot
+			for (const file of files) {
+				try {
+					const relativePath = path.relative(workspaceDir, file)
+					const content = await fs.readFile(file, "utf-8")
+
+					// Create directory structure in snapshot
+					const targetDir = path.dirname(path.join(snapshotsDir, relativePath))
+					await fs.mkdir(targetDir, { recursive: true })
+
+					// Write file content
+					await fs.writeFile(path.join(snapshotsDir, relativePath), content)
+				} catch (error) {
+					this.log(`[MigrationService#migrateTask] error processing file ${file}: ${error}`)
+				}
+			}
+
+			// Create task record
+			await db.createTask({
+				id: taskId,
+				createdAt: new Date(baseCommit.date),
+				baseSnapshotId,
+				workspaceDir,
+			})
+
+			// Create patches directory
+			const patchesDir = path.join(newCheckpointsDir, "patches")
+			await fs.mkdir(patchesDir, { recursive: true })
+
+			// Process each commit (except the first one, which is the base snapshot)
+			let previousState: Record<string, string> = {}
+
+			// Read base snapshot to get initial state
+			const readDir = async (dir: string, base: string = "") => {
+				const entries = await fs.readdir(dir, { withFileTypes: true })
+
+				for (const entry of entries) {
+					const fullPath = path.join(dir, entry.name)
+					const relativePath = path.join(base, entry.name)
+
+					if (entry.isDirectory()) {
+						await readDir(fullPath, relativePath)
+					} else {
+						const content = await fs.readFile(fullPath, "utf-8")
+						previousState[relativePath] = content
+					}
+				}
+			}
+
+			await readDir(snapshotsDir)
+
+			// Process each commit after the base
+			let parentCheckpointId: string | null = null
+
+			for (let i = 1; i < commits.length; i++) {
+				const commit = commits[i]
+				const checkpointId = crypto.randomUUID()
+
+				// Checkout this commit
+				await git.checkout(commit.hash)
+
+				// Get current state
+				const currentState: Record<string, string> = {}
+				const currentFiles = await this.patchGenerator.getWorkspaceFiles(workspaceDir, excludePatterns)
+
+				for (const file of currentFiles) {
+					try {
+						const relativePath = path.relative(workspaceDir, file)
+						const content = await fs.readFile(file, "utf-8")
+						currentState[relativePath] = content
+					} catch (error) {
+						this.log(`[MigrationService#migrateTask] error reading file ${file}: ${error}`)
+					}
+				}
+
+				// Generate patch
+				const patch = this.patchGenerator.generatePatch(previousState, currentState)
+
+				// Save patch to disk
+				const patchPath = path.join(patchesDir, `${checkpointId}.json`)
+				await fs.writeFile(patchPath, JSON.stringify(patch, null, 2))
+
+				// Create checkpoint record
+				await db.createCheckpoint({
+					id: checkpointId,
+					taskId,
+					sequenceNum: i - 1, // Base snapshot is not a checkpoint
+					parentCheckpointId,
+					patchPath,
+					metadata: { message: commit.message },
+					createdAt: new Date(commit.date),
+				})
+
+				// Update for next iteration
+				previousState = currentState
+				parentCheckpointId = checkpointId
+			}
+
+			// Close database
+			await db.close()
+
+			// Rename directories to complete migration
+			const oldCheckpointsDirBackup = path.join(this.globalStorageDir, "tasks", taskId, "checkpoints-old")
+			await fs.rename(oldCheckpointsDir, oldCheckpointsDirBackup)
+			await fs.rename(newCheckpointsDir, oldCheckpointsDir)
+
+			this.log(`[MigrationService#migrateTask] migration completed for task ${taskId}`)
+		} catch (error) {
+			this.log(`[MigrationService#migrateTask] error migrating task ${taskId}: ${error}`)
+			throw error
+		}
+	}
+}