fix: resolve Windows ENAMETOOLONG error in Claude Code integration (#5631)

- Use environment variable CLAUDE_CODE_SYSTEM_PROMPT for long system prompts (>7000 chars)
- Prevents Windows command line length limit (~8191 chars) from causing ENAMETOOLONG errors
- Maintains backward compatibility by using command line args for short prompts
- Add comprehensive tests for both short and long system prompt scenarios
- Follows existing pattern used for messages parameter (stdin vs command line)

Author: roomote

src/integrations/claude-code/__tests__/run.spec.ts

@@ -287,4 +287,168 @@ describe("runClaudeCode", () => {
 		consoleErrorSpy.mockRestore()
 		await generator.return(undefined)
 	})
+
+	test("should use command line argument for short system prompts", async () => {
+		const { runClaudeCode } = await import("../run")
+		const shortSystemPrompt = "You are a helpful assistant"
+		const options = {
+			systemPrompt: shortSystemPrompt,
+			messages: [{ role: "user" as const, content: "Hello" }],
+		}
+
+		const generator = runClaudeCode(options)
+
+		// Consume at least one item to trigger process spawn
+		await generator.next()
+
+		// Clean up the generator
+		await generator.return(undefined)
+
+		// Verify execa was called with system prompt as command line argument
+		const [, args, execaOptions] = mockExeca.mock.calls[0]
+		expect(args).toContain("--system-prompt")
+		expect(args).toContain(shortSystemPrompt)
+
+		// Verify no environment variable was set for short prompt
+		expect(execaOptions.env?.CLAUDE_CODE_SYSTEM_PROMPT).toBeUndefined()
+	})
+
+	test("should use environment variable for long system prompts to avoid Windows ENAMETOOLONG error", async () => {
+		const { runClaudeCode } = await import("../run")
+		// Create a system prompt longer than MAX_COMMAND_LINE_LENGTH (7000 chars)
+		const longSystemPrompt = "You are a helpful assistant. " + "A".repeat(7000)
+		const options = {
+			systemPrompt: longSystemPrompt,
+			messages: [{ role: "user" as const, content: "Hello" }],
+		}
+
+		const generator = runClaudeCode(options)
+
+		// Consume at least one item to trigger process spawn
+		await generator.next()
+
+		// Clean up the generator
+		await generator.return(undefined)
+
+		// Verify execa was called without --system-prompt in command line arguments
+		const [, args, execaOptions] = mockExeca.mock.calls[0]
+		expect(args).not.toContain("--system-prompt")
+		expect(args).not.toContain(longSystemPrompt)
+
+		// Verify environment variable was set with the long system prompt
+		expect(execaOptions.env?.CLAUDE_CODE_SYSTEM_PROMPT).toBe(longSystemPrompt)
+	})
+
+	test("should handle exactly MAX_COMMAND_LINE_LENGTH system prompt using command line", async () => {
+		const { runClaudeCode } = await import("../run")
+		// Create a system prompt exactly at the threshold (7000 chars)
+		const exactLengthPrompt = "A".repeat(7000)
+		const options = {
+			systemPrompt: exactLengthPrompt,
+			messages: [{ role: "user" as const, content: "Hello" }],
+		}
+
+		const generator = runClaudeCode(options)
+
+		// Consume at least one item to trigger process spawn
+		await generator.next()
+
+		// Clean up the generator
+		await generator.return(undefined)
+
+		// Verify execa was called with system prompt as command line argument (at threshold)
+		const [, args, execaOptions] = mockExeca.mock.calls[0]
+		expect(args).toContain("--system-prompt")
+		expect(args).toContain(exactLengthPrompt)
+
+		// Verify no environment variable was set
+		expect(execaOptions.env?.CLAUDE_CODE_SYSTEM_PROMPT).toBeUndefined()
+	})
+
+	test("should handle system prompt one character over threshold using environment variable", async () => {
+		const { runClaudeCode } = await import("../run")
+		// Create a system prompt one character over the threshold (7001 chars)
+		const overThresholdPrompt = "A".repeat(7001)
+		const options = {
+			systemPrompt: overThresholdPrompt,
+			messages: [{ role: "user" as const, content: "Hello" }],
+		}
+
+		const generator = runClaudeCode(options)
+
+		// Consume at least one item to trigger process spawn
+		await generator.next()
+
+		// Clean up the generator
+		await generator.return(undefined)
+
+		// Verify execa was called without --system-prompt in command line arguments
+		const [, args, execaOptions] = mockExeca.mock.calls[0]
+		expect(args).not.toContain("--system-prompt")
+		expect(args).not.toContain(overThresholdPrompt)
+
+		// Verify environment variable was set
+		expect(execaOptions.env?.CLAUDE_CODE_SYSTEM_PROMPT).toBe(overThresholdPrompt)
+	})
+
+	test("should preserve existing environment variables when using CLAUDE_CODE_SYSTEM_PROMPT", async () => {
+		const { runClaudeCode } = await import("../run")
+
+		// Mock process.env to have some existing variables
+		const originalEnv = process.env
+		process.env = {
+			...originalEnv,
+			EXISTING_VAR: "existing_value",
+			PATH: "/usr/bin:/bin",
+		}
+
+		const longSystemPrompt = "You are a helpful assistant. " + "A".repeat(7000)
+		const options = {
+			systemPrompt: longSystemPrompt,
+			messages: [{ role: "user" as const, content: "Hello" }],
+		}
+
+		const generator = runClaudeCode(options)
+
+		// Consume at least one item to trigger process spawn
+		await generator.next()
+
+		// Clean up the generator
+		await generator.return(undefined)
+
+		// Verify environment variables include both existing and new ones
+		const [, , execaOptions] = mockExeca.mock.calls[0]
+		expect(execaOptions.env).toEqual({
+			...process.env,
+			CLAUDE_CODE_MAX_OUTPUT_TOKENS: expect.any(String), // Always set by the implementation
+			CLAUDE_CODE_SYSTEM_PROMPT: longSystemPrompt,
+		})
+
+		// Restore original environment
+		process.env = originalEnv
+	})
+
+	test("should work with empty system prompt", async () => {
+		const { runClaudeCode } = await import("../run")
+		const options = {
+			systemPrompt: "",
+			messages: [{ role: "user" as const, content: "Hello" }],
+		}
+
+		const generator = runClaudeCode(options)
+
+		// Consume at least one item to trigger process spawn
+		await generator.next()
+
+		// Clean up the generator
+		await generator.return(undefined)
+
+		// Verify execa was called with empty system prompt as command line argument
+		const [, args, execaOptions] = mockExeca.mock.calls[0]
+		expect(args).toContain("--system-prompt")
+		expect(args).toContain("")
+
+		// Verify no environment variable was set
+		expect(execaOptions.env?.CLAUDE_CODE_SYSTEM_PROMPT).toBeUndefined()
+	})
 })

src/integrations/claude-code/run.ts

@@ -110,6 +110,10 @@ const claudeCodeTools = [
 
 const CLAUDE_CODE_TIMEOUT = 600000 // 10 minutes
 
+// Windows has a command line length limit of ~8191 characters
+// If the system prompt is too long, we'll use an environment variable instead
+const MAX_COMMAND_LINE_LENGTH = 7000 // Conservative limit to account for other arguments
+
 function runProcess({
 	systemPrompt,
 	messages,
@@ -119,10 +123,17 @@ function runProcess({
 }: ClaudeCodeOptions & { maxOutputTokens?: number }) {
 	const claudePath = path || "claude"
 
-	const args = [
-		"-p",
-		"--system-prompt",
-		systemPrompt,
+	// Check if system prompt is too long for command line
+	const useEnvForSystemPrompt = systemPrompt.length > MAX_COMMAND_LINE_LENGTH
+
+	const args = ["-p"]
+
+	// Only add --system-prompt to command line if it's short enough
+	if (!useEnvForSystemPrompt) {
+		args.push("--system-prompt", systemPrompt)
+	}
+
+	args.push(
 		"--verbose",
 		"--output-format",
 		"stream-json",
@@ -131,32 +142,41 @@ function runProcess({
 		// Roo Code will handle recursive calls
 		"--max-turns",
 		"1",
-	]
+	)
 
 	if (modelId) {
 		args.push("--model", modelId)
 	}
 
+	const env: Record<string, string> = {
+		...process.env,
+		// Use the configured value, or the environment variable, or default to CLAUDE_CODE_DEFAULT_MAX_OUTPUT_TOKENS
+		CLAUDE_CODE_MAX_OUTPUT_TOKENS:
+			maxOutputTokens?.toString() ||
+			process.env.CLAUDE_CODE_MAX_OUTPUT_TOKENS ||
+			CLAUDE_CODE_DEFAULT_MAX_OUTPUT_TOKENS.toString(),
+	}
+
+	// If system prompt is too long, pass it via environment variable
+	if (useEnvForSystemPrompt) {
+		env.CLAUDE_CODE_SYSTEM_PROMPT = systemPrompt
+	}
+
 	const child = execa(claudePath, args, {
 		stdin: "pipe",
 		stdout: "pipe",
 		stderr: "pipe",
-		env: {
-			...process.env,
-			// Use the configured value, or the environment variable, or default to CLAUDE_CODE_DEFAULT_MAX_OUTPUT_TOKENS
-			CLAUDE_CODE_MAX_OUTPUT_TOKENS:
-				maxOutputTokens?.toString() ||
-				process.env.CLAUDE_CODE_MAX_OUTPUT_TOKENS ||
-				CLAUDE_CODE_DEFAULT_MAX_OUTPUT_TOKENS.toString(),
-		},
+		env,
 		cwd,
 		maxBuffer: 1024 * 1024 * 1000,
 		timeout: CLAUDE_CODE_TIMEOUT,
 	})
 
 	// Write messages to stdin after process is spawned
-	// This avoids the E2BIG error on Linux when passing large messages as command line arguments
+	// This avoids the E2BIG error on Linux and ENAMETOOLONG error on Windows when passing large data as command line arguments
 	// Linux has a per-argument limit of ~128KiB for execve() system calls
+	// Windows has a total command line length limit of ~8191 characters
+	// For system prompts, we use environment variables when they exceed the safe limit
 	const messagesJson = JSON.stringify(messages)
 
 	// Use setImmediate to ensure the process has been spawned before writing to stdin