diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index 0fa911ca91e..0c8d1ae9c53 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -132,6 +132,7 @@ jobs: NODE_OPTIONS: '--max-old-space-size=8192' AWS_TOOLKIT_TEST_CACHE_DIR: '/tmp/.vscode-test/' AWS_TOOLKIT_TEST_USER_DIR: '/tmp/.vscode-test/user-data/' + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - uses: actions/checkout@v4 - name: Use Node.js ${{ matrix.node-version }} @@ -169,6 +170,7 @@ jobs: NODE_OPTIONS: '--max-old-space-size=8192' AWS_TOOLKIT_TEST_CACHE_DIR: '/tmp/.vscode-test/' AWS_TOOLKIT_TEST_USER_DIR: '/tmp/.vscode-test/user-data/' + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - uses: actions/checkout@v4 - name: Use Node.js ${{ matrix.node-version }} diff --git a/buildspec/linuxTests.yml b/buildspec/linuxTests.yml index 900b720e61a..9e787406813 100644 --- a/buildspec/linuxTests.yml +++ b/buildspec/linuxTests.yml @@ -13,6 +13,7 @@ env: # followed by Error: Could not delete obsolete instance handle Error: ENOENT: no such file or directory, unlink AWS_TOOLKIT_TEST_CACHE_DIR: '/tmp/.vscode-test/' AWS_TOOLKIT_TEST_USER_DIR: '/tmp/.vscode-test/user-data/' + VSCODE_RIPGREP_TOKEN: ${GITHUB_READONLY_TOKEN} phases: install: diff --git a/buildspec/packageTestVsix.yml b/buildspec/packageTestVsix.yml index 73f15c0ba10..ee9ad7b7034 100644 --- a/buildspec/packageTestVsix.yml +++ b/buildspec/packageTestVsix.yml @@ -8,6 +8,7 @@ env: AWS_TOOLKIT_TEST_USER_DIR: '/tmp/' # needed or else webpack will cause it to run out of memory NODE_OPTIONS: '--max-old-space-size=8192' + # VSCODE_RIPGREP_TOKEN will be set in pre_build phase phases: install: @@ -20,6 +21,9 @@ phases: pre_build: commands: - export HOME=/home/codebuild-user + # Set up VSCODE_RIPGREP_TOKEN for GitHub API access + - export VSCODE_RIPGREP_TOKEN=${GITHUB_READONLY_TOKEN} + - echo "Setting up VSCODE_RIPGREP_TOKEN for GitHub API access" - bash buildspec/shared/linux-pre_build.sh build: diff --git a/package-lock.json b/package-lock.json index a52329b07aa..ba9fe4d1a88 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ ], "dependencies": { "@types/node": "^22.7.5", + "@vscode/ripgrep": "1.15.11", "vscode-nls": "^5.2.0", "vscode-nls-dev": "^4.0.4" }, @@ -15022,6 +15023,40 @@ "version": "1.64.0", "license": "MIT" }, + "node_modules/@vscode/ripgrep": { + "version": "1.15.11", + "resolved": "https://registry.npmjs.org/@vscode/ripgrep/-/ripgrep-1.15.11.tgz", + "integrity": "sha512-G/VqtA6kR50mJkIH4WA+I2Q78V5blovgPPq0VPYM0QIRp57lYMkdV+U9VrY80b3AvaC72A1z8STmyxc8ZKiTsw==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "https-proxy-agent": "^7.0.2", + "proxy-from-env": "^1.1.0", + "yauzl": "^2.9.2" + } + }, + "node_modules/@vscode/ripgrep/node_modules/agent-base": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.3.tgz", + "integrity": "sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/@vscode/ripgrep/node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/@vscode/test-electron": { "version": "2.3.8", "dev": true, @@ -16514,7 +16549,6 @@ }, "node_modules/buffer-crc32": { "version": "0.2.13", - "dev": true, "license": "MIT", "engines": { "node": "*" @@ -17509,7 +17543,6 @@ }, "node_modules/debug": { "version": "4.3.4", - "dev": true, "license": "MIT", "dependencies": { "ms": "2.1.2" @@ -18902,7 +18935,6 @@ }, "node_modules/fd-slicer": { "version": "1.1.0", - "dev": true, "license": "MIT", "dependencies": { "pend": "~1.2.0" @@ -22507,7 +22539,6 @@ }, "node_modules/pend": { "version": "1.2.0", - "dev": true, "license": "MIT" }, "node_modules/picocolors": { @@ -22965,7 +22996,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", - "dev": true, "license": "MIT" }, "node_modules/psl": { @@ -26690,7 +26720,6 @@ }, "node_modules/yauzl": { "version": "2.10.0", - "dev": true, "license": "MIT", "dependencies": { "buffer-crc32": "~0.2.3", diff --git a/package.json b/package.json index 68463ece306..8d86b4a5b0b 100644 --- a/package.json +++ b/package.json @@ -75,6 +75,7 @@ "dependencies": { "@types/node": "^22.7.5", "vscode-nls": "^5.2.0", - "vscode-nls-dev": "^4.0.4" + "vscode-nls-dev": "^4.0.4", + "@vscode/ripgrep": "1.15.11" } } diff --git a/packages/amazonq/scripts/build/copyFiles.ts b/packages/amazonq/scripts/build/copyFiles.ts index 45b1d263f0b..42cda370e80 100644 --- a/packages/amazonq/scripts/build/copyFiles.ts +++ b/packages/amazonq/scripts/build/copyFiles.ts @@ -60,6 +60,11 @@ const tasks: CopyTask[] = [ target: path.join('../../node_modules', 'web-tree-sitter', 'tree-sitter.wasm'), destination: path.join('src', 'tree-sitter.wasm'), }, + // ripgrep binary + { + target: path.join('../../node_modules', '@vscode/ripgrep', 'bin'), + destination: 'bin/', + }, ] function copy(task: CopyTask): void { diff --git a/packages/core/src/codewhispererChat/tools/fileSearch.ts b/packages/core/src/codewhispererChat/tools/fileSearch.ts new file mode 100644 index 00000000000..c346f4664b1 --- /dev/null +++ b/packages/core/src/codewhispererChat/tools/fileSearch.ts @@ -0,0 +1,111 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +import * as vscode from 'vscode' +import { getLogger } from '../../shared/logger/logger' +import { readDirectoryRecursively } from '../../shared/utilities/workspaceUtils' +import fs from '../../shared/fs/fs' +import { Writable } from 'stream' +import path from 'path' +import { InvokeOutput, OutputKind, sanitizePath, CommandValidation } from './toolShared' +import { isInDirectory } from '../../shared/filesystemUtilities' + +export interface FileSearchParams { + path: string + pattern: string + maxDepth?: number + caseSensitive?: boolean +} + +export class FileSearch { + private fsPath: string + private pattern: RegExp + private maxDepth?: number + private readonly logger = getLogger('fileSearch') + + constructor(params: FileSearchParams) { + this.fsPath = params.path + // Create RegExp with case sensitivity option + this.pattern = new RegExp(params.pattern, params.caseSensitive ? '' : 'i') + this.maxDepth = params.maxDepth + } + + public async validate(): Promise { + if (!this.fsPath || this.fsPath.trim().length === 0) { + throw new Error('Path cannot be empty.') + } + if (this.maxDepth !== undefined && this.maxDepth < 0) { + throw new Error('MaxDepth cannot be negative.') + } + + const sanitized = sanitizePath(this.fsPath) + this.fsPath = sanitized + + const pathUri = vscode.Uri.file(this.fsPath) + let pathExists: boolean + try { + pathExists = await fs.existsDir(pathUri) + if (!pathExists) { + throw new Error(`Path: "${this.fsPath}" does not exist or cannot be accessed.`) + } + } catch (err) { + throw new Error(`Path: "${this.fsPath}" does not exist or cannot be accessed. (${err})`) + } + } + + public queueDescription(updates: Writable): void { + const fileName = path.basename(this.fsPath) + if (this.maxDepth === undefined) { + updates.write(`Searching for files matching pattern: ${this.pattern} in ${fileName} recursively`) + } else if (this.maxDepth === 0) { + updates.write(`Searching for files matching pattern: ${this.pattern} in ${fileName}`) + } else { + const level = this.maxDepth > 1 ? 'levels' : 'level' + updates.write( + `Searching for files matching pattern: ${this.pattern} in ${fileName} limited to ${this.maxDepth} subfolder ${level}` + ) + } + updates.end() + } + + public requiresAcceptance(): CommandValidation { + const workspaceFolders = vscode.workspace.workspaceFolders + if (!workspaceFolders || workspaceFolders.length === 0) { + return { requiresAcceptance: true } + } + const isInWorkspace = workspaceFolders.some((folder) => isInDirectory(folder.uri.fsPath, this.fsPath)) + if (!isInWorkspace) { + return { requiresAcceptance: true } + } + return { requiresAcceptance: false } + } + + public async invoke(updates?: Writable): Promise { + try { + const fileUri = vscode.Uri.file(this.fsPath) + const allFiles = await readDirectoryRecursively(fileUri, this.maxDepth) + + // Filter files by regex pattern + const matchedFiles = allFiles.filter((filePath) => { + // Extract just the filename from the path + const fileName = path.basename(filePath.split(' ').slice(1).join(' ')) + return this.pattern.test(fileName) + }) + + return this.createOutput(matchedFiles.join('\n')) + } catch (error: any) { + this.logger.error(`Failed to search files in "${this.fsPath}": ${error.message || error}`) + throw new Error(`Failed to search files in "${this.fsPath}": ${error.message || error}`) + } + } + + private createOutput(content: string): InvokeOutput { + return { + output: { + kind: OutputKind.Text, + content: content, + }, + } + } +} diff --git a/packages/core/src/codewhispererChat/tools/grepSearch.ts b/packages/core/src/codewhispererChat/tools/grepSearch.ts new file mode 100644 index 00000000000..6de3a04453a --- /dev/null +++ b/packages/core/src/codewhispererChat/tools/grepSearch.ts @@ -0,0 +1,246 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +import * as vscode from 'vscode' +import { getLogger } from '../../shared/logger/logger' +import { sanitizePath, InvokeOutput, OutputKind } from './toolShared' +import fs from '../../shared/fs/fs' +import { Writable } from 'stream' +import { ChildProcess, ChildProcessOptions } from '../../shared/utilities/processUtils' +import { rgPath } from '@vscode/ripgrep' +import path from 'path' + +export interface GrepSearchParams { + path?: string + query: string + caseSensitive?: boolean + excludePattern?: string + includePattern?: string + explanation?: string +} + +export class GrepSearch { + private path: string + private query: string + private caseSensitive: boolean + private excludePattern?: string + private includePattern?: string + private readonly logger = getLogger('grepSearch') + + constructor(params: GrepSearchParams) { + this.path = this.getSearchDirectory(params.path) + this.query = params.query + this.caseSensitive = params.caseSensitive ?? false + this.excludePattern = params.excludePattern + this.includePattern = params.includePattern + } + + public async validate(): Promise { + if (!this.query || this.query.trim().length === 0) { + throw new Error('Grep search query cannot be empty.') + } + + if (this.path.trim().length === 0) { + throw new Error('Path cannot be empty and no workspace folder is available.') + } + + const sanitized = sanitizePath(this.path) + this.path = sanitized + + const pathUri = vscode.Uri.file(this.path) + let pathExists: boolean + try { + pathExists = await fs.existsDir(pathUri) + if (!pathExists) { + throw new Error(`Path: "${this.path}" does not exist or cannot be accessed.`) + } + } catch (err) { + throw new Error(`Path: "${this.path}" does not exist or cannot be accessed. (${err})`) + } + } + + public queueDescription(updates: Writable): void { + updates.write(`Grepping for "${this.query}" in directory: ${this.path}`) + updates.end() + } + + public async invoke(updates?: Writable): Promise { + try { + const results = await this.executeRipgrep(updates) + return this.createOutput(results) + } catch (error: any) { + this.logger.error(`Failed to search in "${this.path}": ${error.message || error}`) + throw new Error(`Failed to search in "${this.path}": ${error.message || error}`) + } + } + + private getSearchDirectory(path?: string): string { + let searchLocation = '' + if (path && path.trim().length !== 0) { + searchLocation = path + } else { + // Handle optional path parameter + // Use current workspace folder as default if path is not provided + const workspaceFolders = vscode.workspace.workspaceFolders + this.logger.info(`Using default workspace folder: ${workspaceFolders?.length}`) + if (workspaceFolders && workspaceFolders.length !== 0) { + searchLocation = workspaceFolders[0].uri.fsPath + this.logger.debug(`Using default workspace folder: ${searchLocation}`) + } + } + return searchLocation + } + + private async executeRipgrep(updates?: Writable): Promise { + return new Promise(async (resolve, reject) => { + const args: string[] = [] + + // Add search options + if (!this.caseSensitive) { + args.push('-i') // Case insensitive search + } + args.push('--line-number') // Show line numbers + + // No heading (don't group matches by file) + args.push('--no-heading') + + // Don't use color in output + args.push('--color', 'never') + + // Add include/exclude patterns + if (this.includePattern) { + // Support multiple include patterns + const patterns = this.includePattern.split(',') + for (const pattern of patterns) { + args.push('--glob', pattern.trim()) + } + } + + if (this.excludePattern) { + // Support multiple exclude patterns + const patterns = this.excludePattern.split(',') + for (const pattern of patterns) { + args.push('--glob', `!${pattern.trim()}`) + } + } + + // Add search pattern and path + args.push(this.query, this.path) + + this.logger.debug(`Executing ripgrep with args: ${args.join(' ')}`) + + const options: ChildProcessOptions = { + collect: true, + logging: 'yes', + rejectOnErrorCode: (code) => { + if (code !== 0 && code !== 1) { + this.logger.error(`Ripgrep process exited with code ${code}`) + return new Error(`Ripgrep process exited with code ${code}`) + } + // For exit codes 0 and 1, don't reject + return false as unknown as Error + }, + } + + try { + const rg = new ChildProcess(rgPath, args, options) + const result = await rg.run() + this.logger.info(`Executing ripgrep with exitCode: ${result.exitCode}`) + // Process the output to format with file URLs and remove matched content + const { sanitizedOutput, totalMatchCount } = this.processRipgrepOutput(result.stdout) + + // If updates is provided, write the processed output + if (updates) { + updates.write(sanitizedOutput) + } + + this.logger.info(`Processed ripgrep result: ${totalMatchCount} matches found`) + resolve(sanitizedOutput) + } catch (err) { + reject(err) + } + }) + } + + /** + * Process ripgrep output to: + * 1. Group results by file + * 2. Format as collapsible sections + * 3. Add file URLs for clickable links + * @returns An object containing the processed output and total match count + */ + private processRipgrepOutput(output: string): { sanitizedOutput: string; totalMatchCount: number } { + if (!output || output.trim() === '') { + return { sanitizedOutput: 'No matches found.', totalMatchCount: 0 } + } + + const lines = output.split('\n') + + // Group by file path + const fileGroups: Record = {} + let totalMatchCount = 0 + + for (const line of lines) { + if (!line || line.trim() === '') { + continue + } + + // Extract file path and line number + const parts = line.split(':') + if (parts.length < 2) { + continue + } + + const filePath = parts[0] + const lineNumber = parts[1] + // Don't include match content + + if (!fileGroups[filePath]) { + fileGroups[filePath] = [] + } + + // Create a clickable link with line number using VS Code's Uri.with() method + const uri = vscode.Uri.file(filePath) + // Use the with() method to add the line number as a fragment + const uriWithLine = uri.with({ fragment: `L${lineNumber}` }) + fileGroups[filePath].push(`- [Line ${lineNumber}](${uriWithLine.toString(true)})`) + totalMatchCount++ + } + + // Sort files by match count (most matches first) + const sortedFiles = Object.entries(fileGroups).sort((a, b) => b[1].length - a[1].length) + + // Generate a title for the results + let sanitizedOutput = `## Grepped result - ${totalMatchCount} matches found: \n\n` + + // Format as simple list of files with links to all matches + sanitizedOutput += sortedFiles + .map(([filePath, _matches]) => { + const fileName = path.basename(filePath) + + // Create a URL to show all matches in the file with search term highlighted + const uri = vscode.Uri.file(filePath) + // Use the query parameter to highlight all matches in the file + const uriWithSearch = uri.with({ + query: `search=${encodeURIComponent(this.query)}`, + }) + const allMatchesUrl = uriWithSearch.toString(true) + + // Just show the filename with link to all matches + return `- [${fileName}](${allMatchesUrl})` + }) + .join('\n') + + return { sanitizedOutput, totalMatchCount } + } + + private createOutput(content: string): InvokeOutput { + return { + output: { + kind: OutputKind.Text, + content: content || 'No matches found.', + }, + } + } +} diff --git a/packages/core/src/codewhispererChat/tools/toolUtils.ts b/packages/core/src/codewhispererChat/tools/toolUtils.ts index 300d357a9fa..6fa08df39ed 100644 --- a/packages/core/src/codewhispererChat/tools/toolUtils.ts +++ b/packages/core/src/codewhispererChat/tools/toolUtils.ts @@ -2,7 +2,6 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 */ -import * as vscode from 'vscode' import { Writable } from 'stream' import { FsRead, FsReadParams } from './fsRead' import { FsWrite, FsWriteParams } from './fsWrite' @@ -15,12 +14,17 @@ import { fsReadToolResponseSize, } from './toolShared' import { ListDirectory, ListDirectoryParams } from './listDirectory' +import { GrepSearch, GrepSearchParams } from './grepSearch' +import * as vscode from 'vscode' +import { FileSearch, FileSearchParams } from './fileSearch' export enum ToolType { FsRead = 'fsRead', FsWrite = 'fsWrite', ExecuteBash = 'executeBash', ListDirectory = 'listDirectory', + GrepSearch = 'grepSearch', + FileSearch = 'fileSearch', } export type Tool = @@ -28,6 +32,8 @@ export type Tool = | { type: ToolType.FsWrite; tool: FsWrite } | { type: ToolType.ExecuteBash; tool: ExecuteBash } | { type: ToolType.ListDirectory; tool: ListDirectory } + | { type: ToolType.GrepSearch; tool: GrepSearch } + | { type: ToolType.FileSearch; tool: FileSearch } export class ToolUtils { static displayName(tool: Tool): string { @@ -40,6 +46,10 @@ export class ToolUtils { return 'Execute shell command' case ToolType.ListDirectory: return 'List directory from filesystem' + case ToolType.GrepSearch: + return 'Run Fast text-based regex search' + case ToolType.FileSearch: + return `Search for files in a directory` } } @@ -53,6 +63,10 @@ export class ToolUtils { return tool.tool.requiresAcceptance() case ToolType.ListDirectory: return tool.tool.requiresAcceptance() + case ToolType.GrepSearch: + return { requiresAcceptance: false } + case ToolType.FileSearch: + return tool.tool.requiresAcceptance() } } @@ -75,6 +89,10 @@ export class ToolUtils { return tool.tool.invoke(updates ?? undefined, cancellationToken) case ToolType.ListDirectory: return tool.tool.invoke(updates) + case ToolType.GrepSearch: + return tool.tool.invoke(updates) + case ToolType.FileSearch: + return tool.tool.invoke(updates) } } @@ -109,6 +127,12 @@ export class ToolUtils { case ToolType.ListDirectory: tool.tool.queueDescription(updates, requiresAcceptance) break + case ToolType.GrepSearch: + tool.tool.queueDescription(updates) + break + case ToolType.FileSearch: + tool.tool.queueDescription(updates) + break } } @@ -122,6 +146,10 @@ export class ToolUtils { return tool.tool.validate() case ToolType.ListDirectory: return tool.tool.validate() + case ToolType.GrepSearch: + return tool.tool.validate() + case ToolType.FileSearch: + return tool.tool.validate() } } @@ -159,6 +187,16 @@ export class ToolUtils { type: ToolType.ListDirectory, tool: new ListDirectory(value.input as unknown as ListDirectoryParams), } + case ToolType.GrepSearch: + return { + type: ToolType.GrepSearch, + tool: new GrepSearch(value.input as unknown as GrepSearchParams), + } + case ToolType.FileSearch: + return { + type: ToolType.FileSearch, + tool: new FileSearch(value.input as unknown as FileSearchParams), + } default: return { toolUseId: value.toolUseId, diff --git a/packages/core/src/shared/logger/logger.ts b/packages/core/src/shared/logger/logger.ts index 5786799ae99..0986173fb6e 100644 --- a/packages/core/src/shared/logger/logger.ts +++ b/packages/core/src/shared/logger/logger.ts @@ -19,7 +19,9 @@ export type LogTopic = | 'fsRead' | 'fsWrite' | 'executeBash' + | 'grepSearch' | 'listDirectory' + | 'fileSearch' | 'chatStream' | 'chatHistoryDb' | 'unknown' diff --git a/packages/core/src/test/codewhispererChat/tools/fileSearch.test.ts b/packages/core/src/test/codewhispererChat/tools/fileSearch.test.ts new file mode 100644 index 00000000000..084c8c631e9 --- /dev/null +++ b/packages/core/src/test/codewhispererChat/tools/fileSearch.test.ts @@ -0,0 +1,351 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from 'assert' +import sinon from 'sinon' +import * as vscode from 'vscode' +import { FileSearch, FileSearchParams } from '../../../codewhispererChat/tools/fileSearch' +import { Writable } from 'stream' +import { OutputKind } from '../../../codewhispererChat/tools/toolShared' +import fs from '../../../shared/fs/fs' +import * as workspaceUtils from '../../../shared/utilities/workspaceUtils' +import * as filesystemUtilities from '../../../shared/filesystemUtilities' + +describe('FileSearch', function () { + let sandbox: sinon.SinonSandbox + let mockUpdates: Writable + const mockWorkspacePath = '/mock/workspace' + + beforeEach(function () { + sandbox = sinon.createSandbox() + + // Create a mock Writable stream for updates + mockUpdates = new Writable({ + write: (chunk, encoding, callback) => { + callback() + }, + }) + sandbox.spy(mockUpdates, 'write') + sandbox.spy(mockUpdates, 'end') + + // Mock workspace folders + sandbox.stub(vscode.workspace, 'workspaceFolders').value([ + { + uri: { fsPath: mockWorkspacePath } as vscode.Uri, + name: 'mockWorkspace', + index: 0, + }, + ]) + + // Mock fs.existsDir to always return true + sandbox.stub(fs, 'existsDir').resolves(true) + + // Mock readDirectoryRecursively + sandbox + .stub(workspaceUtils, 'readDirectoryRecursively') + .resolves([ + 'F /mock/workspace/file1.ts', + 'F /mock/workspace/file2.js', + 'F /mock/workspace/subfolder/file3.ts', + 'F /mock/workspace/subfolder/file4.json', + 'D /mock/workspace/subfolder', + ]) + + // Mock isInDirectory + sandbox.stub(filesystemUtilities, 'isInDirectory').returns(true) + }) + + afterEach(function () { + sandbox.restore() + }) + + describe('constructor', function () { + it('should initialize with provided values', function () { + const params: FileSearchParams = { + path: '/test/path', + pattern: '.*\\.ts$', + maxDepth: 2, + caseSensitive: true, + } + + const fileSearch = new FileSearch(params) + + assert.strictEqual((fileSearch as any).fsPath, '/test/path') + assert.strictEqual((fileSearch as any).pattern.source, '.*\\.ts$') + assert.strictEqual((fileSearch as any).maxDepth, 2) + // Check that the RegExp was created with case sensitivity + assert.strictEqual((fileSearch as any).pattern.flags, '') + }) + + it('should initialize with case insensitive pattern by default', function () { + const params: FileSearchParams = { + path: '/test/path', + pattern: '.*\\.ts$', + } + + const fileSearch = new FileSearch(params) + + assert.strictEqual((fileSearch as any).pattern.flags, 'i') + }) + }) + + describe('validate', function () { + it('should throw an error if path is empty', async function () { + const fileSearch = new FileSearch({ path: '', pattern: '.*\\.ts$' }) + await assert.rejects(async () => await fileSearch.validate(), /Path cannot be empty/) + }) + + it('should throw an error if path is only whitespace', async function () { + const fileSearch = new FileSearch({ path: ' ', pattern: '.*\\.ts$' }) + await assert.rejects(async () => await fileSearch.validate(), /Path cannot be empty/) + }) + + it('should throw an error if maxDepth is negative', async function () { + const fileSearch = new FileSearch({ + path: '/test/path', + pattern: '.*\\.ts$', + maxDepth: -1, + }) + await assert.rejects(async () => await fileSearch.validate(), /MaxDepth cannot be negative/) + }) + + it('should throw an error if path does not exist', async function () { + sandbox.restore() + sandbox = sinon.createSandbox() + sandbox.stub(fs, 'existsDir').resolves(false) + + const fileSearch = new FileSearch({ + path: '/non/existent/path', + pattern: '.*\\.ts$', + }) + + await assert.rejects( + async () => await fileSearch.validate(), + /Path: "\/non\/existent\/path" does not exist or cannot be accessed/ + ) + }) + + it('should pass validation with valid path and pattern', async function () { + const fileSearch = new FileSearch({ + path: '/valid/path', + pattern: '.*\\.ts$', + }) + await assert.doesNotReject(async () => await fileSearch.validate()) + }) + }) + + describe('queueDescription', function () { + it('should write description for recursive search', function () { + const fileSearch = new FileSearch({ + path: '/test/path', + pattern: '.*\\.ts$', + }) + + fileSearch.queueDescription(mockUpdates) + + // eslint-disable-next-line @typescript-eslint/unbound-method + sinon.assert.calledWith( + // eslint-disable-next-line @typescript-eslint/unbound-method + mockUpdates.write as sinon.SinonSpy, + `Searching for files matching pattern: /.*\\.ts$/i in path recursively` + ) + // eslint-disable-next-line @typescript-eslint/unbound-method + sinon.assert.calledOnce(mockUpdates.end as sinon.SinonSpy) + }) + + it('should write description for current directory only', function () { + const fileSearch = new FileSearch({ + path: '/test/path', + pattern: '.*\\.ts$', + maxDepth: 0, + }) + + fileSearch.queueDescription(mockUpdates) + + // eslint-disable-next-line @typescript-eslint/unbound-method + sinon.assert.calledWith( + // eslint-disable-next-line @typescript-eslint/unbound-method + mockUpdates.write as sinon.SinonSpy, + `Searching for files matching pattern: /.*\\.ts$/i in path` + ) + }) + + it('should write description for limited depth search', function () { + const fileSearch = new FileSearch({ + path: '/test/path', + pattern: '.*\\.ts$', + maxDepth: 1, + }) + + fileSearch.queueDescription(mockUpdates) + + // eslint-disable-next-line @typescript-eslint/unbound-method + sinon.assert.calledWith( + // eslint-disable-next-line @typescript-eslint/unbound-method + mockUpdates.write as sinon.SinonSpy, + `Searching for files matching pattern: /.*\\.ts$/i in path limited to 1 subfolder level` + ) + }) + + it('should use plural form for multiple levels', function () { + const fileSearch = new FileSearch({ + path: '/test/path', + pattern: '.*\\.ts$', + maxDepth: 3, + }) + + fileSearch.queueDescription(mockUpdates) + + // eslint-disable-next-line @typescript-eslint/unbound-method + sinon.assert.calledWith( + // eslint-disable-next-line @typescript-eslint/unbound-method + mockUpdates.write as sinon.SinonSpy, + `Searching for files matching pattern: /.*\\.ts$/i in path limited to 3 subfolder levels` + ) + }) + }) + + describe('requiresAcceptance', function () { + it('should require acceptance when no workspace folders exist', function () { + sandbox.restore() + sandbox = sinon.createSandbox() + sandbox.stub(vscode.workspace, 'workspaceFolders').value(undefined) + + const fileSearch = new FileSearch({ + path: '/test/path', + pattern: '.*\\.ts$', + }) + + const result = fileSearch.requiresAcceptance() + assert.strictEqual(result.requiresAcceptance, true) + }) + + it('should require acceptance when path is outside workspace', function () { + sandbox.restore() + sandbox = sinon.createSandbox() + sandbox.stub(vscode.workspace, 'workspaceFolders').value([ + { + uri: { fsPath: '/workspace' } as vscode.Uri, + name: 'workspace', + index: 0, + }, + ]) + sandbox.stub(filesystemUtilities, 'isInDirectory').returns(false) + + const fileSearch = new FileSearch({ + path: '/outside/workspace', + pattern: '.*\\.ts$', + }) + + const result = fileSearch.requiresAcceptance() + assert.strictEqual(result.requiresAcceptance, true) + }) + + it('should not require acceptance when path is inside workspace', function () { + const fileSearch = new FileSearch({ + path: '/mock/workspace/subfolder', + pattern: '.*\\.ts$', + }) + + const result = fileSearch.requiresAcceptance() + assert.strictEqual(result.requiresAcceptance, false) + }) + }) + + describe('invoke', function () { + let fileSearch: FileSearch + + beforeEach(async function () { + fileSearch = new FileSearch({ + path: '/test/path', + pattern: '.*\\.ts$', + }) + await fileSearch.validate() + }) + + it('should filter files by regex pattern', async function () { + const result = await fileSearch.invoke() + + // Should only include .ts files + assert.deepStrictEqual(result, { + output: { + kind: OutputKind.Text, + content: 'F /mock/workspace/file1.ts\nF /mock/workspace/subfolder/file3.ts', + }, + }) + }) + + it('should handle case sensitivity correctly', async function () { + // Create a case-sensitive search for .TS (uppercase) + fileSearch = new FileSearch({ + path: '/test/path', + pattern: '.*\\.TS$', + caseSensitive: true, + }) + await fileSearch.validate() + + // Should not match any files since our mock files use lowercase .ts + const result = await fileSearch.invoke() + assert.deepStrictEqual(result, { + output: { + kind: OutputKind.Text, + content: '', + }, + }) + }) + + it('should throw an error if file search fails', async function () { + sandbox.restore() + sandbox = sinon.createSandbox() + // Make readDirectoryRecursively throw an error + sandbox.stub(workspaceUtils, 'readDirectoryRecursively').rejects(new Error('Access denied')) + sandbox.stub(fs, 'existsDir').resolves(true) + + fileSearch = new FileSearch({ + path: '/test/path', + pattern: '.*\\.ts$', + }) + + await assert.rejects( + async () => await fileSearch.invoke(), + /Failed to search files in "\/test\/path": Access denied/ + ) + }) + }) + + describe('createOutput', function () { + it('should create output with content', function () { + const fileSearch = new FileSearch({ + path: '/test/path', + pattern: '.*\\.ts$', + }) + + const output = (fileSearch as any).createOutput('test content') + + assert.deepStrictEqual(output, { + output: { + kind: OutputKind.Text, + content: 'test content', + }, + }) + }) + + it('should create output with empty content', function () { + const fileSearch = new FileSearch({ + path: '/test/path', + pattern: '.*\\.ts$', + }) + + const output = (fileSearch as any).createOutput('') + + assert.deepStrictEqual(output, { + output: { + kind: OutputKind.Text, + content: '', + }, + }) + }) + }) +}) diff --git a/packages/core/src/test/codewhispererChat/tools/grepSearch.test.ts b/packages/core/src/test/codewhispererChat/tools/grepSearch.test.ts new file mode 100644 index 00000000000..3538b973f95 --- /dev/null +++ b/packages/core/src/test/codewhispererChat/tools/grepSearch.test.ts @@ -0,0 +1,242 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from 'assert' +import sinon from 'sinon' +import * as vscode from 'vscode' +import { GrepSearch, GrepSearchParams } from '../../../codewhispererChat/tools/grepSearch' +import { ChildProcess } from '../../../shared/utilities/processUtils' +import { Writable } from 'stream' +import { OutputKind } from '../../../codewhispererChat/tools/toolShared' +import fs from '../../../shared/fs/fs' + +describe('GrepSearch', function () { + let sandbox: sinon.SinonSandbox + let mockUpdates: Writable + const mockWorkspacePath = '/mock/workspace' + + beforeEach(function () { + sandbox = sinon.createSandbox() + + // Create a mock Writable stream for updates + mockUpdates = new Writable({ + write: (chunk, encoding, callback) => { + callback() + }, + }) + sandbox.spy(mockUpdates, 'write') + sandbox.spy(mockUpdates, 'end') + + // Mock workspace folders + sandbox.stub(vscode.workspace, 'workspaceFolders').value([ + { + uri: { fsPath: mockWorkspacePath } as vscode.Uri, + name: 'mockWorkspace', + index: 0, + }, + ]) + + // Mock fs.existsDir to always return true + sandbox.stub(fs, 'existsDir').resolves(true) + }) + + afterEach(function () { + sandbox.restore() + }) + + describe('constructor', function () { + it('should initialize with default values', function () { + const params: GrepSearchParams = { + query: 'test-query', + } + const grepSearch = new GrepSearch(params) + + assert.strictEqual((grepSearch as any).query, 'test-query') + assert.strictEqual((grepSearch as any).caseSensitive, false) + assert.strictEqual((grepSearch as any).excludePattern, undefined) + assert.strictEqual((grepSearch as any).includePattern, undefined) + assert.strictEqual((grepSearch as any).path, mockWorkspacePath) + }) + + it('should initialize with provided values', function () { + const params: GrepSearchParams = { + query: 'test-query', + caseSensitive: true, + excludePattern: '*.log', + includePattern: '*.ts', + path: '/custom/path', + } + + const grepSearch = new GrepSearch(params) + + assert.strictEqual((grepSearch as any).query, 'test-query') + assert.strictEqual((grepSearch as any).caseSensitive, true) + assert.strictEqual((grepSearch as any).excludePattern, '*.log') + assert.strictEqual((grepSearch as any).includePattern, '*.ts') + assert.strictEqual((grepSearch as any).path, '/custom/path') + }) + }) + + describe('getSearchDirectory', function () { + it('should use provided path when available', function () { + const grepSearch = new GrepSearch({ + query: 'test-query', + path: '/custom/path', + }) + + const result = (grepSearch as any).getSearchDirectory('/custom/path') + assert.strictEqual(result, '/custom/path') + }) + + it('should use workspace folder when path is not provided', function () { + const grepSearch = new GrepSearch({ + query: 'test-query', + }) + + const result = (grepSearch as any).getSearchDirectory() + assert.strictEqual(result, mockWorkspacePath) + }) + }) + + describe('validate', function () { + it('should throw an error if query is empty', async function () { + const grepSearch = new GrepSearch({ query: '' }) + await assert.rejects(async () => await grepSearch.validate(), /Grep search query cannot be empty/) + }) + + it('should throw an error if query is only whitespace', async function () { + const grepSearch = new GrepSearch({ query: ' ' }) + + await assert.rejects(async () => await grepSearch.validate(), /Grep search query cannot be empty/) + }) + + it('should throw an error if path does not exist', async function () { + sandbox.restore() + sandbox = sinon.createSandbox() + sandbox.stub(fs, 'existsDir').resolves(false) + + const grepSearch = new GrepSearch({ + query: 'test-query', + path: '/non/existent/path', + }) + + await assert.rejects( + async () => await grepSearch.validate(), + /Path: "\/non\/existent\/path" does not exist or cannot be accessed/ + ) + }) + + it('should pass validation with valid query and path', async function () { + const grepSearch = new GrepSearch({ + query: 'test-query', + path: '/valid/path', + }) + await assert.doesNotReject(async () => await grepSearch.validate()) + }) + }) + + describe('queueDescription', function () { + it('should write description to updates stream', function () { + const grepSearch = new GrepSearch({ + query: 'test-query', + path: '/test/path', + }) + + grepSearch.queueDescription(mockUpdates) + + // eslint-disable-next-line @typescript-eslint/unbound-method + sinon.assert.calledWith( + // eslint-disable-next-line @typescript-eslint/unbound-method + mockUpdates.write as sinon.SinonSpy, + `Grepping for "test-query" in directory: /test/path` + ) + // eslint-disable-next-line @typescript-eslint/unbound-method + sinon.assert.calledOnce(mockUpdates.end as sinon.SinonSpy) + }) + }) + + describe('invoke', function () { + let grepSearch: GrepSearch + + beforeEach(async function () { + grepSearch = new GrepSearch({ + query: 'test-query', + path: '/test/path', + }) + await grepSearch.validate() + // Setup ChildProcess run method + const mockRun = sandbox.stub() + mockRun.resolves({ stdout: 'search-results', stderr: '', exitCode: 0 }) + sandbox.stub(ChildProcess.prototype, 'run').callsFake(mockRun) + + // Mock processRipgrepOutput + sandbox.stub(grepSearch as any, 'processRipgrepOutput').returns({ + sanitizedOutput: 'processed-results', + totalMatchCount: 5, + }) + }) + + it('should execute ripgrep and return results', async function () { + const result = await grepSearch.invoke() + assert.deepStrictEqual(result, { + output: { + kind: OutputKind.Text, + content: 'processed-results', + }, + }) + }) + + it('should write updates to the provided stream', async function () { + await grepSearch.invoke(mockUpdates) + + // eslint-disable-next-line @typescript-eslint/unbound-method + sinon.assert.calledWith(mockUpdates.write as sinon.SinonSpy, 'processed-results') + }) + + it('should throw an error if ripgrep execution fails', async function () { + sandbox.restore() + sandbox = sinon.createSandbox() + // Make ChildProcess.run throw an error + sandbox.stub(ChildProcess.prototype, 'run').rejects(new Error('Command failed')) + grepSearch = new GrepSearch({ + query: 'test-query', + path: '/test/path', + }) + await assert.rejects(async () => await grepSearch.invoke(), /Failed to search/) + }) + }) + + describe('createOutput', function () { + it('should create output with content', function () { + const grepSearch = new GrepSearch({ + query: 'test-query', + }) + + const output = (grepSearch as any).createOutput('test content') + + assert.deepStrictEqual(output, { + output: { + kind: OutputKind.Text, + content: 'test content', + }, + }) + }) + + it('should create output with default message when content is empty', function () { + const grepSearch = new GrepSearch({ + query: 'test-query', + }) + + const output = (grepSearch as any).createOutput('') + + assert.deepStrictEqual(output, { + output: { + kind: OutputKind.Text, + content: 'No matches found.', + }, + }) + }) + }) +})