-
Notifications
You must be signed in to change notification settings - Fork 11k
feat: preserve EOL in files #16087
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat: preserve EOL in files #16087
Changes from 4 commits
ff93cfe
f50e7cf
3414bb7
d7e47f2
0a163ea
80815fc
83e7f88
79a59a6
e5910c4
2a212af
6a162ee
a63c258
399a11e
7ac4b79
769ea88
409b11b
3545e90
5126e7a
2a6eb2c
c03a809
dc6c5d3
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,287 @@ | ||
| /** | ||
| * @license | ||
| * Copyright 2025 Google LLC | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| import { | ||
| describe, | ||
| it, | ||
| expect, | ||
| beforeEach, | ||
| afterEach, | ||
| vi, | ||
| type Mocked, | ||
| } from 'vitest'; | ||
| import { detectLineEnding } from '../utils/textUtils.js'; | ||
| import { WriteFileTool } from './write-file.js'; | ||
| import { EditTool } from './edit.js'; | ||
| import type { Config } from '../config/config.js'; | ||
| import { ApprovalMode } from '../policy/types.js'; | ||
| import { ToolConfirmationOutcome } from './tools.js'; | ||
| import type { ToolRegistry } from './tool-registry.js'; | ||
| import path from 'node:path'; | ||
| import fs from 'node:fs'; | ||
| import os from 'node:os'; | ||
| import { GeminiClient } from '../core/client.js'; | ||
| import type { BaseLlmClient } from '../core/baseLlmClient.js'; | ||
| import { | ||
| ensureCorrectEdit, | ||
| ensureCorrectFileContent, | ||
| } from '../utils/editCorrector.js'; | ||
| import { StandardFileSystemService } from '../services/fileSystemService.js'; | ||
| import { WorkspaceContext } from '../utils/workspaceContext.js'; | ||
| import { | ||
| createMockMessageBus, | ||
| getMockMessageBusInstance, | ||
| } from '../test-utils/mock-message-bus.js'; | ||
|
|
||
| const rootDir = path.resolve(os.tmpdir(), 'gemini-cli-line-ending-test-root'); | ||
|
|
||
| // --- MOCKS --- | ||
| vi.mock('../core/client.js'); | ||
| vi.mock('../utils/editCorrector.js'); | ||
| vi.mock('../ide/ide-client.js', () => ({ | ||
| IdeClient: { | ||
| getInstance: vi.fn().mockResolvedValue({ | ||
| openDiff: vi.fn(), | ||
| isDiffingEnabled: vi.fn().mockReturnValue(false), | ||
| }), | ||
| }, | ||
| })); | ||
|
|
||
| let mockGeminiClientInstance: Mocked<GeminiClient>; | ||
| let mockBaseLlmClientInstance: Mocked<BaseLlmClient>; | ||
| const mockEnsureCorrectEdit = vi.fn<typeof ensureCorrectEdit>(); | ||
| const mockEnsureCorrectFileContent = vi.fn<typeof ensureCorrectFileContent>(); | ||
|
|
||
| // Mock Config | ||
| const fsService = new StandardFileSystemService(); | ||
| const mockConfigInternal = { | ||
| getTargetDir: () => rootDir, | ||
| getApprovalMode: vi.fn(() => ApprovalMode.DEFAULT), | ||
| setApprovalMode: vi.fn(), | ||
| getGeminiClient: vi.fn(), | ||
| getBaseLlmClient: vi.fn(), | ||
| getFileSystemService: () => fsService, | ||
| getIdeMode: vi.fn(() => false), | ||
| getWorkspaceContext: () => new WorkspaceContext(rootDir), | ||
| getApiKey: () => 'test-key', | ||
| getModel: () => 'test-model', | ||
| getSandbox: () => false, | ||
| getDebugMode: () => false, | ||
| getQuestion: () => undefined, | ||
| getToolDiscoveryCommand: () => undefined, | ||
| getToolCallCommand: () => undefined, | ||
| getMcpServerCommand: () => undefined, | ||
| getMcpServers: () => undefined, | ||
| getUserAgent: () => 'test-agent', | ||
| getUserMemory: () => '', | ||
| setUserMemory: vi.fn(), | ||
| getGeminiMdFileCount: () => 0, | ||
| setGeminiMdFileCount: vi.fn(), | ||
| getToolRegistry: () => | ||
| ({ | ||
| registerTool: vi.fn(), | ||
| discoverTools: vi.fn(), | ||
| }) as unknown as ToolRegistry, | ||
| isInteractive: () => false, | ||
| }; | ||
| const mockConfig = mockConfigInternal as unknown as Config; | ||
|
|
||
| vi.mock('../telemetry/loggers.js', () => ({ | ||
| logFileOperation: vi.fn(), | ||
| logEditStrategy: vi.fn(), | ||
| logEditCorrectionEvent: vi.fn(), | ||
| })); | ||
|
|
||
| // --- END MOCKS --- | ||
|
|
||
| describe('Line Ending Preservation', () => { | ||
| let tempDir: string; | ||
|
|
||
| beforeEach(() => { | ||
| vi.clearAllMocks(); | ||
| tempDir = fs.mkdtempSync( | ||
| path.join(os.tmpdir(), 'line-ending-test-external-'), | ||
| ); | ||
| if (!fs.existsSync(rootDir)) { | ||
| fs.mkdirSync(rootDir, { recursive: true }); | ||
| } | ||
|
|
||
| mockGeminiClientInstance = new (vi.mocked(GeminiClient))( | ||
| mockConfig, | ||
| ) as Mocked<GeminiClient>; | ||
| vi.mocked(GeminiClient).mockImplementation(() => mockGeminiClientInstance); | ||
|
|
||
| mockBaseLlmClientInstance = { | ||
| generateJson: vi.fn(), | ||
| } as unknown as Mocked<BaseLlmClient>; | ||
|
|
||
| vi.mocked(ensureCorrectEdit).mockImplementation(mockEnsureCorrectEdit); | ||
| vi.mocked(ensureCorrectFileContent).mockImplementation( | ||
| mockEnsureCorrectFileContent, | ||
| ); | ||
|
|
||
| mockConfigInternal.getGeminiClient.mockReturnValue( | ||
| mockGeminiClientInstance, | ||
| ); | ||
| mockConfigInternal.getBaseLlmClient.mockReturnValue( | ||
| mockBaseLlmClientInstance, | ||
| ); | ||
| }); | ||
|
|
||
| afterEach(() => { | ||
| if (fs.existsSync(tempDir)) { | ||
| fs.rmSync(tempDir, { recursive: true, force: true }); | ||
| } | ||
| if (fs.existsSync(rootDir)) { | ||
| fs.rmSync(rootDir, { recursive: true, force: true }); | ||
| } | ||
| vi.clearAllMocks(); | ||
| }); | ||
|
|
||
| describe('detectLineEnding', () => { | ||
| it('should detect CRLF', () => { | ||
| expect(detectLineEnding('line1\r\nline2')).toBe('\r\n'); | ||
| expect(detectLineEnding('line1\r\n')).toBe('\r\n'); | ||
| }); | ||
|
|
||
| it('should detect LF', () => { | ||
| expect(detectLineEnding('line1\nline2')).toBe('\n'); | ||
| expect(detectLineEnding('line1\n')).toBe('\n'); | ||
| expect(detectLineEnding('line1')).toBe('\n'); // Default to LF if no newline | ||
| }); | ||
| }); | ||
|
|
||
| describe('WriteFileTool', () => { | ||
| let tool: WriteFileTool; | ||
| const abortSignal = new AbortController().signal; | ||
|
|
||
| beforeEach(() => { | ||
| const bus = createMockMessageBus(); | ||
| getMockMessageBusInstance(bus).defaultToolDecision = 'ask_user'; | ||
| tool = new WriteFileTool(mockConfig, bus); | ||
| }); | ||
|
|
||
| it('should preserve CRLF when overwriting an existing file', async () => { | ||
| const filePath = path.join(rootDir, 'crlf_file.txt'); | ||
| const originalContent = 'line1\r\nline2\r\n'; | ||
| fs.writeFileSync(filePath, originalContent); // Write with CRLF (or however Node writes binary buffer) | ||
| // Ensure strictly CRLF | ||
| fs.writeFileSync(filePath, Buffer.from('line1\r\nline2\r\n')); | ||
|
|
||
| // Proposed content from LLM (usually LF) | ||
| const proposedContent = 'line1\nline2\nline3\n'; | ||
|
|
||
| // Mock corrections to return proposed content as-is (but usually normalized) | ||
| mockEnsureCorrectEdit.mockResolvedValue({ | ||
| params: { | ||
| file_path: filePath, | ||
| old_string: originalContent, | ||
| new_string: proposedContent, | ||
| }, | ||
| occurrences: 1, | ||
| }); | ||
|
|
||
| const params = { file_path: filePath, content: proposedContent }; | ||
| const invocation = tool.build(params); | ||
|
|
||
| // Force approval | ||
| const confirmDetails = await invocation.shouldConfirmExecute(abortSignal); | ||
| if ( | ||
| confirmDetails && | ||
| typeof confirmDetails === 'object' && | ||
| 'onConfirm' in confirmDetails | ||
| ) { | ||
| await confirmDetails.onConfirm(ToolConfirmationOutcome.ProceedOnce); | ||
| } | ||
|
|
||
| await invocation.execute(abortSignal); | ||
|
|
||
| const writtenContent = fs.readFileSync(filePath, 'utf8'); | ||
| // Expect all newlines to be CRLF | ||
| expect(writtenContent).toBe('line1\r\nline2\r\nline3\r\n'); | ||
| }); | ||
|
|
||
| it('should use OS EOL for new files', async () => { | ||
| const filePath = path.join(rootDir, 'new_os_eol_file.txt'); | ||
| const proposedContent = 'line1\nline2\n'; | ||
|
|
||
| mockEnsureCorrectFileContent.mockResolvedValue(proposedContent); | ||
|
|
||
| const params = { file_path: filePath, content: proposedContent }; | ||
| const invocation = tool.build(params); | ||
|
|
||
| const confirmDetails = await invocation.shouldConfirmExecute(abortSignal); | ||
| if ( | ||
| confirmDetails && | ||
| typeof confirmDetails === 'object' && | ||
| 'onConfirm' in confirmDetails | ||
| ) { | ||
| await confirmDetails.onConfirm(ToolConfirmationOutcome.ProceedOnce); | ||
| } | ||
|
|
||
| await invocation.execute(abortSignal); | ||
|
|
||
| const writtenContent = fs.readFileSync(filePath, 'utf8'); | ||
|
|
||
| if (os.EOL === '\r\n') { | ||
| expect(writtenContent).toBe('line1\r\nline2\r\n'); | ||
| } else { | ||
| expect(writtenContent).toBe('line1\nline2\n'); | ||
| } | ||
| }); | ||
| }); | ||
|
|
||
| describe('EditTool', () => { | ||
| let tool: EditTool; | ||
| const abortSignal = new AbortController().signal; | ||
|
|
||
| beforeEach(() => { | ||
| const bus = createMockMessageBus(); | ||
| getMockMessageBusInstance(bus).defaultToolDecision = 'ask_user'; | ||
| tool = new EditTool(mockConfig, bus); | ||
| }); | ||
|
|
||
| it('should preserve CRLF when editing a file', async () => { | ||
| const filePath = path.join(rootDir, 'edit_crlf.txt'); | ||
| const originalContent = 'line1\r\nline2\r\nline3\r\n'; | ||
| fs.writeFileSync(filePath, Buffer.from(originalContent)); | ||
|
|
||
| const oldString = 'line2'; | ||
| const newString = 'modified'; | ||
|
|
||
| const params = { | ||
| file_path: filePath, | ||
| old_string: oldString, | ||
| new_string: newString, | ||
| instruction: 'Change line2 to modified', | ||
| }; | ||
| const invocation = tool.build(params); | ||
|
|
||
| // Force approval | ||
| const confirmDetails = await invocation.shouldConfirmExecute(abortSignal); | ||
| if ( | ||
| confirmDetails && | ||
| typeof confirmDetails === 'object' && | ||
| 'onConfirm' in confirmDetails | ||
| ) { | ||
| await confirmDetails.onConfirm(ToolConfirmationOutcome.ProceedOnce); | ||
| } | ||
|
|
||
| await invocation.execute(abortSignal); | ||
|
|
||
| const writtenContent = fs.readFileSync(filePath, 'utf8'); | ||
| // Expect 'line2' to be replaced by 'modified', and all endings to be CRLF | ||
| // Logic: | ||
| // 1. Read: line1\r\nline2\r\nline3\r\n | ||
| // 2. Detect: CRLF | ||
| // 3. Normalize: line1\nline2\nline3\n | ||
| // 4. Replace: line1\nmodified\nline3\n (assuming safeLiteralReplace works on normalized strings) | ||
| // 5. Restore: line1\r\nmodified\r\nline3\r\n | ||
|
|
||
| expect(writtenContent).toBe('line1\r\nmodified\r\nline3\r\n'); | ||
| }); | ||
| }); | ||
| }); | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -6,6 +6,7 @@ | |
|
|
||
| import fs from 'node:fs'; | ||
| import path from 'node:path'; | ||
| import os from 'node:os'; | ||
| import * as Diff from 'diff'; | ||
| import { WRITE_FILE_TOOL_NAME } from './tool-names.js'; | ||
| import type { Config } from '../config/config.js'; | ||
|
|
@@ -32,6 +33,7 @@ import { | |
| ensureCorrectEdit, | ||
| ensureCorrectFileContent, | ||
| } from '../utils/editCorrector.js'; | ||
| import { detectLineEnding } from '../utils/textUtils.js'; | ||
| import { DEFAULT_DIFF_OPTIONS, getDiffStat } from './diffOptions.js'; | ||
| import type { | ||
| ModifiableDeclarativeTool, | ||
|
|
@@ -284,9 +286,19 @@ class WriteFileToolInvocation extends BaseToolInvocation< | |
| fs.mkdirSync(dirName, { recursive: true }); | ||
| } | ||
|
|
||
| let finalContent = fileContent; | ||
| const useCRLF = | ||
| !isNewFile && originalContent | ||
| ? detectLineEnding(originalContent) === '\r\n' | ||
| : os.EOL === '\r\n'; | ||
|
|
||
| if (useCRLF) { | ||
| finalContent = finalContent.replace(/\r?\n/g, '\r\n'); | ||
| } | ||
|
Comment on lines
306
to
314
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This logic for handling line endings is correct, but it contains duplicated code which can make it harder to maintain. You can simplify this block by combining the conditions for converting to CRLF, which improves readability and maintainability. let finalContent = fileContent;
const preserveCRLF = !isNewFile && originalContent && detectLineEnding(originalContent) === '\r\n';
// The `else` branch in the original logic covers new files or existing empty files.
const useOSEOLForNewFile = (isNewFile || !originalContent) && os.EOL === '\r\n';
if (preserveCRLF || useOSEOLForNewFile) {
finalContent = finalContent.replace(/\r?\n/g, '\r\n');
}
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Fixed :) |
||
|
|
||
| await this.config | ||
| .getFileSystemService() | ||
| .writeTextFile(this.resolvedPath, fileContent); | ||
| .writeTextFile(this.resolvedPath, finalContent); | ||
|
|
||
| // Generate diff for display result | ||
| const fileName = path.basename(this.resolvedPath); | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
remove this comment block
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Removed