feat: add retry mechanism with exponential backoff

- Track retryCount and lastErrorTime in state for crash recovery
- Implement exponential backoff with jitter (max 5 min wait)
- Skip failed tasks after maxRetries exceeded
- Add alert() method to logger for critical errors
- Reset retry count on successful phase completion
- Add comprehensive tests for backoff calculation and retry tracking

Signed-off-by: leocavalcante <[email protected]>

Author: leocavalcante

src/logger.ts

@@ -116,6 +116,16 @@ export class Logger {
 		this.writeToAlerts(formatted)
 	}
 
+	/**
+	 * Log a critical alert - always written to alerts file and shown prominently
+	 */
+	alert(message: string): void {
+		const formatted = `[ALERT] ${message}`
+		console.error(`${ANSI.red}${ANSI.bold}${formatted}${ANSI.reset}`)
+		this.writeToBuffer(this.formatForFile(formatted))
+		this.writeToAlerts(formatted)
+	}
+
 	/**
 	 * Log only if verbose mode enabled
 	 */

src/loop.ts

@@ -12,6 +12,7 @@ import { Builder } from "./builder.ts"
 import { extractEvaluationReason, isComplete, parseEvaluation } from "./evaluator.ts"
 import {
 	ensureDirectories,
+	getISOTimestamp,
 	getTimestampForFilename,
 	initializePaths,
 	readFileOrNull,
@@ -87,14 +88,48 @@ export async function runLoop(config: Config): Promise<void> {
 						await runEvaluationPhase(state, builder, paths, logger, config)
 						break
 				}
+
+				// Success - reset retry count
+				state.retryCount = 0
+				state.lastErrorTime = undefined
 			} catch (err) {
-				logger.logError(`Error in ${state.phase} phase: ${err}`)
+				// Track the error
+				state.retryCount++
+				state.lastErrorTime = getISOTimestamp()
+
+				logger.logError(
+					`Error in ${state.phase} phase (attempt ${state.retryCount}/${config.maxRetries}): ${err}`,
+				)
 
-				// Retry with backoff
-				if (!shutdownRequested) {
-					const backoffMs = config.backoffBase * 1000
-					logger.say(`Retrying in ${config.backoffBase} seconds...`)
-					await sleep(backoffMs)
+				// Check if we've exceeded max retries
+				if (state.retryCount >= config.maxRetries) {
+					logger.logError(`Max retries (${config.maxRetries}) exceeded for ${state.phase} phase`)
+					logger.alert(`CRITICAL: ${state.phase} phase failed after ${config.maxRetries} attempts`)
+
+					// Reset retry count and move to next phase or skip task
+					state.retryCount = 0
+					state.lastErrorTime = undefined
+
+					if (state.phase === "build") {
+						// Skip the failed task and continue
+						logger.warn("Skipping failed task and continuing...")
+						await skipCurrentTask(state, paths, logger)
+					} else if (state.phase === "plan") {
+						// Clear any stuck idea and retry planning
+						state.currentIdeaPath = undefined
+						state.currentIdeaFilename = undefined
+						logger.warn("Clearing idea state and retrying plan phase...")
+					}
+					// For evaluation, just retry - it will eventually succeed or the user will intervene
+				} else {
+					// Retry with exponential backoff
+					const backoffMs = calculateBackoff(state.retryCount, config.backoffBase)
+					const backoffSec = Math.round(backoffMs / 1000)
+					logger.say(`Retrying in ${backoffSec} seconds...`)
+
+					if (!shutdownRequested) {
+						await sleep(backoffMs)
+					}
 				}
 			}
 
@@ -460,6 +495,71 @@ export function sleep(ms: number): Promise<void> {
 	return new Promise((resolve) => setTimeout(resolve, ms))
 }
 
+/**
+ * Calculate exponential backoff with jitter.
+ * @param retryCount - Current retry attempt (1-based)
+ * @param baseSeconds - Base delay in seconds
+ * @returns Delay in milliseconds
+ */
+export function calculateBackoff(retryCount: number, baseSeconds: number): number {
+	// Exponential backoff: base * 2^(retry-1) with max of 5 minutes
+	const exponentialDelay = baseSeconds * 2 ** (retryCount - 1)
+	const cappedDelay = Math.min(exponentialDelay, 300) // Max 5 minutes
+
+	// Add jitter (up to 20% randomness) to prevent thundering herd
+	const jitter = cappedDelay * 0.2 * Math.random()
+
+	return Math.round((cappedDelay + jitter) * 1000)
+}
+
+/**
+ * Skip the current task in build phase when it has failed too many times.
+ * Marks the task as completed (with a note) and moves to the next task.
+ */
+async function skipCurrentTask(state: RuntimeState, paths: Paths, logger: Logger): Promise<void> {
+	const planContent = await readFileOrNull(paths.currentPlan)
+	if (!planContent) return
+
+	const uncompletedTasks = getUncompletedTasks(planContent)
+
+	if (uncompletedTasks.length === 0) {
+		state.phase = "evaluation"
+		return
+	}
+
+	const currentTask = uncompletedTasks[0]
+	if (!currentTask) {
+		state.phase = "evaluation"
+		return
+	}
+
+	// Mark task as completed (even though it failed - to allow progress)
+	const updatedPlan = markTaskComplete(planContent, currentTask.lineNumber)
+
+	// Add a note about the skipped task
+	const noteComment = `<!-- SKIPPED: Task failed after max retries -->`
+	const planWithNote = updatedPlan.replace(
+		new RegExp(`(- \\[x\\] ${escapeRegExp(currentTask.description)})`),
+		`$1 ${noteComment}`,
+	)
+
+	await writeFile(paths.currentPlan, planWithNote)
+	logger.warn(`Skipped failed task: ${currentTask.description}`)
+
+	// Check if this was the last task
+	const remainingTasks = getUncompletedTasks(planWithNote)
+	if (remainingTasks.length === 0) {
+		state.phase = "evaluation"
+	}
+}
+
+/**
+ * Escape special regex characters in a string.
+ */
+function escapeRegExp(str: string): string {
+	return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")
+}
+
 /**
  * Check if shutdown has been requested.
  * Exported for testing and external shutdown checks.

src/state.ts

@@ -13,6 +13,8 @@ const DEFAULT_STATE: State = {
 	lastUpdate: "",
 	currentIdeaPath: undefined,
 	currentIdeaFilename: undefined,
+	retryCount: 0,
+	lastErrorTime: undefined,
 }
 
 /**
@@ -57,6 +59,14 @@ export async function loadState(stateFile: string): Promise<RuntimeState> {
 			)
 		}
 
+		// Validate retryCount is a non-negative number
+		if (
+			parsed.retryCount !== undefined &&
+			(typeof parsed.retryCount !== "number" || parsed.retryCount < 0)
+		) {
+			parsed.retryCount = DEFAULT_STATE.retryCount
+		}
+
 		// Merge with defaults to handle missing fields
 		const state: State = {
 			cycle: parsed.cycle ?? DEFAULT_STATE.cycle,
@@ -66,6 +76,8 @@ export async function loadState(stateFile: string): Promise<RuntimeState> {
 			lastUpdate: parsed.lastUpdate ?? DEFAULT_STATE.lastUpdate,
 			currentIdeaPath: parsed.currentIdeaPath,
 			currentIdeaFilename: parsed.currentIdeaFilename,
+			retryCount: parsed.retryCount ?? DEFAULT_STATE.retryCount,
+			lastErrorTime: parsed.lastErrorTime,
 		}
 
 		return toRuntimeState(state)
@@ -100,6 +112,8 @@ export async function saveState(stateFile: string, state: RuntimeState): Promise
 		lastUpdate: getISOTimestamp(),
 		currentIdeaPath: state.currentIdeaPath,
 		currentIdeaFilename: state.currentIdeaFilename,
+		retryCount: state.retryCount,
+		lastErrorTime: state.lastErrorTime,
 	}
 
 	const content = JSON.stringify(persistedState, null, 2)
@@ -156,5 +170,7 @@ export function newCycleState(currentCycle: number): Partial<RuntimeState> {
 		currentTaskDesc: "",
 		currentIdeaPath: undefined,
 		currentIdeaFilename: undefined,
+		retryCount: 0,
+		lastErrorTime: undefined,
 	}
 }

src/types.ts

@@ -49,6 +49,10 @@ export interface State {
 	currentIdeaPath?: string
 	/** Filename of the idea currently being processed (for display) */
 	currentIdeaFilename?: string
+	/** Number of consecutive failures in current phase */
+	retryCount: number
+	/** ISO timestamp of last error (for backoff calculation) */
+	lastErrorTime?: string
 }
 
 /** Runtime state with additional non-persisted fields */

tests/loop.test.ts

@@ -8,6 +8,7 @@ import { join } from "node:path"
 import { Logger } from "../src/logger.ts"
 import {
 	archivePlan,
+	calculateBackoff,
 	isShutdownRequested,
 	logStartupInfo,
 	requestShutdown,
@@ -44,6 +45,9 @@ function createTestConfig(overrides?: Partial<Config>): Config {
 		backoffBase: 10,
 		logRetention: 30,
 		taskPauseSeconds: 2,
+		autoCommit: true,
+		autoPush: true,
+		commitSignoff: false,
 		...overrides,
 	}
 }
@@ -117,6 +121,55 @@ describe("loop", () => {
 		})
 	})
 
+	describe("calculateBackoff", () => {
+		test("first retry uses base delay", () => {
+			const delay = calculateBackoff(1, 10)
+			// Base is 10 seconds = 10000ms, with up to 20% jitter
+			expect(delay).toBeGreaterThanOrEqual(10000)
+			expect(delay).toBeLessThanOrEqual(12000)
+		})
+
+		test("second retry doubles delay", () => {
+			const delay = calculateBackoff(2, 10)
+			// 10 * 2^1 = 20 seconds, with jitter
+			expect(delay).toBeGreaterThanOrEqual(20000)
+			expect(delay).toBeLessThanOrEqual(24000)
+		})
+
+		test("third retry quadruples delay", () => {
+			const delay = calculateBackoff(3, 10)
+			// 10 * 2^2 = 40 seconds, with jitter
+			expect(delay).toBeGreaterThanOrEqual(40000)
+			expect(delay).toBeLessThanOrEqual(48000)
+		})
+
+		test("caps at 5 minutes maximum", () => {
+			const delay = calculateBackoff(10, 10)
+			// Should cap at 300 seconds (5 min), with jitter
+			expect(delay).toBeGreaterThanOrEqual(300000)
+			expect(delay).toBeLessThanOrEqual(360000)
+		})
+
+		test("works with different base values", () => {
+			const delay = calculateBackoff(1, 5)
+			// Base is 5 seconds = 5000ms, with jitter
+			expect(delay).toBeGreaterThanOrEqual(5000)
+			expect(delay).toBeLessThanOrEqual(6000)
+		})
+
+		test("returns milliseconds", () => {
+			const delay = calculateBackoff(1, 1)
+			// 1 second base = ~1000-1200ms with jitter
+			expect(delay).toBeGreaterThanOrEqual(1000)
+			expect(delay).toBeLessThanOrEqual(1200)
+		})
+
+		test("is non-negative", () => {
+			const delay = calculateBackoff(1, 0)
+			expect(delay).toBeGreaterThanOrEqual(0)
+		})
+	})
+
 	describe("isShutdownRequested", () => {
 		test("returns false initially", () => {
 			resetShutdownFlags()

tests/state.test.ts

@@ -149,5 +149,77 @@ describe("state", () => {
 			expect(newState.taskIndex).toBe(0)
 			expect(newState.totalTasks).toBe(0)
 		})
+
+		test("resets retry count", () => {
+			const newState = newCycleState(5)
+
+			expect(newState.retryCount).toBe(0)
+			expect(newState.lastErrorTime).toBeUndefined()
+		})
+	})
+
+	describe("retryCount tracking", () => {
+		test("loadState returns default retryCount of 0", async () => {
+			const stateFile = join(TEST_DIR, "nonexistent.json")
+			const state = await loadState(stateFile)
+
+			expect(state.retryCount).toBe(0)
+		})
+
+		test("loadState preserves retryCount from file", async () => {
+			const stateFile = join(TEST_DIR, "retry-state.json")
+			const savedState = {
+				cycle: 1,
+				phase: "build",
+				taskIndex: 0,
+				retryCount: 2,
+				lastErrorTime: "2024-01-01T00:00:00Z",
+			}
+			await Bun.write(stateFile, JSON.stringify(savedState))
+
+			const state = await loadState(stateFile)
+
+			expect(state.retryCount).toBe(2)
+			expect(state.lastErrorTime).toBe("2024-01-01T00:00:00Z")
+		})
+
+		test("saveState persists retryCount", async () => {
+			const stateFile = join(TEST_DIR, "save-retry.json")
+			const state = {
+				cycle: 1,
+				phase: "build" as const,
+				taskIndex: 0,
+				totalTasks: 3,
+				currentTaskNum: 1,
+				currentTaskDesc: "Test",
+				lastUpdate: "",
+				retryCount: 3,
+				lastErrorTime: "2024-01-01T12:00:00Z",
+			}
+
+			await saveState(stateFile, state)
+
+			const content = await Bun.file(stateFile).text()
+			const saved = JSON.parse(content)
+
+			expect(saved.retryCount).toBe(3)
+			expect(saved.lastErrorTime).toBe("2024-01-01T12:00:00Z")
+		})
+
+		test("invalid retryCount defaults to 0", async () => {
+			const stateFile = join(TEST_DIR, "invalid-retry.json")
+			await Bun.write(stateFile, JSON.stringify({ cycle: 1, retryCount: -1 }))
+
+			const state = await loadState(stateFile)
+
+			expect(state.retryCount).toBe(0)
+		})
+
+		test("resetState clears retryCount", () => {
+			const state = resetState()
+
+			expect(state.retryCount).toBe(0)
+			expect(state.lastErrorTime).toBeUndefined()
+		})
 	})
 })