diff --git a/package-lock.json b/package-lock.json index 1159518ee5b..5483dbf3366 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7791,9 +7791,9 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, "node_modules/color-string": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.6.0.tgz", - "integrity": "sha512-c/hGS+kRWJutUBEngKKmk4iH3sD59MBkoxVapS/0wgpCz2u7XsNloxknyvBhzwEs1IbV36D9PwqLPJ2DTu3vMA==", + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", "dependencies": { "color-name": "^1.0.0", "simple-swizzle": "^0.2.2" @@ -11935,6 +11935,14 @@ "node": ">= 0.6.0" } }, + "node_modules/jose": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/jose/-/jose-5.4.1.tgz", + "integrity": "sha512-U6QajmpV/nhL9SyfAewo000fkiRQ+Yd2H0lBxJJ9apjpOgkOcBQJWOrMo917lxLptdS/n/o/xPzMkXhF46K8hQ==", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, "node_modules/joycon": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/joycon/-/joycon-3.0.1.tgz", @@ -18779,6 +18787,7 @@ "highlight.js": "^11.9.0", "i18n-ts": "^1.0.5", "immutable": "^4.3.0", + "jose": "5.4.1", "js-yaml": "^4.1.0", "jsonc-parser": "^3.2.0", "lodash": "^4.17.21", diff --git a/packages/amazonq/.changes/next-release/Feature-69f0d3bf-3e57-4ad6-93ea-48de3c890331.json b/packages/amazonq/.changes/next-release/Feature-69f0d3bf-3e57-4ad6-93ea-48de3c890331.json new file mode 100644 index 00000000000..1fa650f128c --- /dev/null +++ b/packages/amazonq/.changes/next-release/Feature-69f0d3bf-3e57-4ad6-93ea-48de3c890331.json @@ -0,0 +1,4 @@ +{ + "type": "Feature", + "description": "Add support for [Amazon Q Chat Workspace Context](https://docs.aws.amazon.com/amazonq/latest/qdeveloper-ug/workspace-context.html). Customers can use @workspace to ask questions regarding local workspace." +} diff --git a/packages/amazonq/package.json b/packages/amazonq/package.json index 2c7d899ee46..baa39468d23 100644 --- a/packages/amazonq/package.json +++ b/packages/amazonq/package.json @@ -125,6 +125,30 @@ "markdownDescription": "%AWS.configuration.description.amazonq.shareContentWithAWS%", "default": true, "scope": "application" + }, + "amazonQ.workspaceIndex": { + "type": "boolean", + "markdownDescription": "%AWS.configuration.description.amazonq.workspaceIndex%", + "default": false, + "scope": "application" + }, + "amazonQ.workspaceIndexWorkerThreads": { + "type": "number", + "markdownDescription": "%AWS.configuration.description.amazonq.workspaceIndexWorkerThreads%", + "default": 0, + "scope": "application" + }, + "amazonQ.workspaceIndexUseGPU": { + "type": "boolean", + "markdownDescription": "%AWS.configuration.description.amazonq.workspaceIndexUseGPU%", + "default": false, + "scope": "application" + }, + "amazonQ.workspaceIndexMaxSize": { + "type": "number", + "markdownDescription": "%AWS.configuration.description.amazonq.workspaceIndexMaxSize%", + "default": 250, + "scope": "application" } } }, diff --git a/packages/amazonq/test/unit/amazonq/lsp/lspClient.test.ts b/packages/amazonq/test/unit/amazonq/lsp/lspClient.test.ts new file mode 100644 index 00000000000..68de6c0ddd5 --- /dev/null +++ b/packages/amazonq/test/unit/amazonq/lsp/lspClient.test.ts @@ -0,0 +1,51 @@ +import * as sinon from 'sinon' +import assert from 'assert' +import { globals } from 'aws-core-vscode/shared' +import { LspClient } from 'aws-core-vscode/amazonq' + +describe('Amazon Q LSP client', function () { + let lspClient: LspClient + let encryptFunc: sinon.SinonSpy + + beforeEach(async function () { + sinon.stub(globals, 'isWeb').returns(false) + lspClient = new LspClient() + encryptFunc = sinon.spy(lspClient, 'encrypt') + }) + + it('encrypts payload of query ', async () => { + await lspClient.query('mock_input') + assert.ok(encryptFunc.calledOnce) + assert.ok(encryptFunc.calledWith(JSON.stringify({ query: 'mock_input' }))) + const value = await encryptFunc.returnValues[0] + // verifies JWT encryption header + assert.ok(value.startsWith(`eyJhbGciOiJkaXIiLCJlbmMiOiJBMjU2R0NNIn0`)) + }) + + it('encrypts payload of index files ', async () => { + await lspClient.indexFiles(['fileA'], 'path', false) + assert.ok(encryptFunc.calledOnce) + assert.ok( + encryptFunc.calledWith( + JSON.stringify({ + filePaths: ['fileA'], + rootPath: 'path', + refresh: false, + }) + ) + ) + const value = await encryptFunc.returnValues[0] + // verifies JWT encryption header + assert.ok(value.startsWith(`eyJhbGciOiJkaXIiLCJlbmMiOiJBMjU2R0NNIn0`)) + }) + + it('encrypt removes readable information', async () => { + const sample = 'hello' + const encryptedSample = await lspClient.encrypt(sample) + assert.ok(!encryptedSample.includes('hello')) + }) + + afterEach(() => { + sinon.restore() + }) +}) diff --git a/packages/amazonq/test/unit/amazonq/lsp/lspController.test.ts b/packages/amazonq/test/unit/amazonq/lsp/lspController.test.ts new file mode 100644 index 00000000000..d54551e433f --- /dev/null +++ b/packages/amazonq/test/unit/amazonq/lsp/lspController.test.ts @@ -0,0 +1,51 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +import assert from 'assert' +import sinon from 'sinon' +import { Content, LspController } from 'aws-core-vscode/amazonq' +import { createTestFile } from 'aws-core-vscode/test' +import { fs } from 'aws-core-vscode/shared' + +describe('Amazon Q LSP controller', function () { + it('Download mechanism checks against hash, when hash matches', async function () { + const content = { + filename: 'qserver-linux-x64.zip', + url: 'https://x/0.0.6/qserver-linux-x64.zip', + hashes: [ + 'sha384:768412320f7b0aa5812fce428dc4706b3cae50e02a64caa16a782249bfe8efc4b7ef1ccb126255d196047dfedf17a0a9', + ], + bytes: 512, + } as Content + const lspController = new LspController() + sinon.stub(lspController, '_download') + const mockFileName = 'test_case_1.zip' + const mockDownloadFile = await createTestFile(mockFileName) + await fs.writeFile(mockDownloadFile.fsPath, 'test') + const result = await lspController.downloadAndCheckHash(mockDownloadFile.fsPath, content) + assert.strictEqual(result, true) + }) + + it('Download mechanism checks against hash, when hash does not match', async function () { + const content = { + filename: 'qserver-linux-x64.zip', + url: 'https://x/0.0.6/qserver-linux-x64.zip', + hashes: [ + 'sha384:38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b', + ], + bytes: 512, + } as Content + const lspController = new LspController() + sinon.stub(lspController, '_download') + const mockFileName = 'test_case_2.zip' + const mockDownloadFile = await createTestFile(mockFileName) + await fs.writeFile(mockDownloadFile.fsPath, 'file_content') + const result = await lspController.downloadAndCheckHash(mockDownloadFile.fsPath, content) + assert.strictEqual(result, false) + }) + + afterEach(() => { + sinon.restore() + }) +}) diff --git a/packages/core/package.json b/packages/core/package.json index c3c765c9a8e..4e7d7912c9c 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -284,6 +284,30 @@ "markdownDescription": "%AWS.configuration.description.amazonq.shareContentWithAWS%", "default": true, "scope": "application" + }, + "amazonQ.workspaceIndex": { + "type": "boolean", + "markdownDescription": "%AWS.configuration.description.amazonq.workspaceIndex%", + "default": false, + "scope": "application" + }, + "amazonQ.workspaceIndexWorkerThreads": { + "type": "number", + "markdownDescription": "%AWS.configuration.description.amazonq.workspaceIndexWorkerThreads%", + "default": 0, + "scope": "application" + }, + "amazonQ.workspaceIndexUseGPU": { + "type": "boolean", + "markdownDescription": "%AWS.configuration.description.amazonq.workspaceIndexUseGPU%", + "default": false, + "scope": "application" + }, + "amazonQ.workspaceIndexMaxSize": { + "type": "number", + "markdownDescription": "%AWS.configuration.description.amazonq.workspaceIndexMaxSize%", + "default": 250, + "scope": "application" } } }, @@ -4118,6 +4142,7 @@ "immutable": "^4.3.0", "js-yaml": "^4.1.0", "jsonc-parser": "^3.2.0", + "jose": "5.4.1", "lodash": "^4.17.21", "markdown-it": "^13.0.2", "mime-types": "^2.1.32", diff --git a/packages/core/package.nls.json b/packages/core/package.nls.json index a360cbfc6b2..da748a1b9cd 100644 --- a/packages/core/package.nls.json +++ b/packages/core/package.nls.json @@ -73,6 +73,10 @@ "AWS.configuration.description.amazonq": "Amazon Q creates a code reference when you insert a code suggestion from Amazon Q that is similar to training data. When unchecked, Amazon Q will not show code suggestions that have code references. If you authenticate through IAM Identity Center, this setting is controlled by your Amazon Q administrator. [Learn More](https://docs.aws.amazon.com/amazonq/latest/qdeveloper-ug/code-reference.html)", "AWS.configuration.description.amazonq.shareContentWithAWS": "When checked, your content processed by Amazon Q may be used for service improvement (except for content processed by the Amazon Q Enterprise service tier). Unchecking this box will cause AWS to delete any of your content used for that purpose. The information used to provide the Amazon Q service to you will not be affected. See the [Service Terms](https://aws.amazon.com/service-terms) for more detail.", "AWS.configuration.description.amazonq.importRecommendation": "Amazon Q will add import statements with inline code suggestions when necessary.", + "AWS.configuration.description.amazonq.workspaceIndex": "This feature is in BETA. When you add @workspace to your question in Amazon Q chat, Amazon Q will index your open workspace files locally to use as context for its response. Extra CPU usage is expected while indexing a workspace. This will not impact Amazon Q features or your IDE, but you may manage CPU usage by setting the number of local threads in 'Local Workspace Index Threads'.", + "AWS.configuration.description.amazonq.workspaceIndexWorkerThreads": "Number of worker threads of Amazon Q local index process. '0' will use the system default worker threads for balance performance. You may increase this number to more quickly index your workspace, but only up to your hardware's number of CPU cores. Please restart VS Code or reload the VS Code window after changing worker threads.", + "AWS.configuration.description.amazonq.workspaceIndexUseGPU": "Enable GPU to help index your local workspace files. Only applies to Linux and Windows.", + "AWS.configuration.description.amazonq.workspaceIndexMaxSize": "The maximum size of local workspace files to be indexed in MB", "AWS.command.apig.copyUrl": "Copy URL", "AWS.command.apig.invokeRemoteRestApi": "Invoke on AWS", "AWS.command.apig.invokeRemoteRestApi.cn": "Invoke on Amazon", diff --git a/packages/core/src/amazonq/activation.ts b/packages/core/src/amazonq/activation.ts index c44bedb148b..a2188da01f7 100644 --- a/packages/core/src/amazonq/activation.ts +++ b/packages/core/src/amazonq/activation.ts @@ -22,8 +22,11 @@ import { listCodeWhispererCommandsWalkthrough } from '../codewhisperer/ui/status import { Commands, placeholder } from '../shared/vscode/commands2' import { focusAmazonQPanel, focusAmazonQPanelKeybinding } from '../codewhispererChat/commands/registerCommands' import { TryChatCodeLensProvider, tryChatCodeLensCommand } from '../codewhispererChat/editor/codelens' +import { LspController } from './lsp/lspController' import { Auth } from '../auth' import { telemetry } from '../shared/telemetry' +import { CodeWhispererSettings } from '../codewhisperer/util/codewhispererSettings' +import { debounce } from '../shared/utilities/functionUtils' export async function activate(context: ExtensionContext) { const appInitContext = DefaultAmazonQAppInitContext.instance @@ -39,6 +42,10 @@ export async function activate(context: ExtensionContext) { await TryChatCodeLensProvider.register(appInitContext.onDidChangeAmazonQVisibility.event) + const setupLsp = debounce(async () => { + void LspController.instance.trySetupLsp(context) + }, 5000) + context.subscriptions.push( window.registerWebviewViewProvider(AmazonQChatViewProvider.viewType, provider, { webviewOptions: { @@ -52,7 +59,14 @@ export async function activate(context: ExtensionContext) { listCodeWhispererCommandsWalkthrough.register(), focusAmazonQPanel.register(), focusAmazonQPanelKeybinding.register(), - tryChatCodeLensCommand.register() + tryChatCodeLensCommand.register(), + vscode.workspace.onDidChangeConfiguration(async configurationChangeEvent => { + if (configurationChangeEvent.affectsConfiguration('amazonQ.workspaceIndex')) { + if (CodeWhispererSettings.instance.isLocalIndexEnabled()) { + void setupLsp() + } + } + }) ) Commands.register('aws.amazonq.learnMore', () => { @@ -60,6 +74,7 @@ export async function activate(context: ExtensionContext) { }) await activateBadge() + void setupLsp() void setupAuthNotification() } diff --git a/packages/core/src/amazonq/index.ts b/packages/core/src/amazonq/index.ts index 1f12f69da67..f6a148ec755 100644 --- a/packages/core/src/amazonq/index.ts +++ b/packages/core/src/amazonq/index.ts @@ -11,7 +11,8 @@ export { MessageListener } from './messages/messageListener' export { AuthController } from './auth/controller' export { showAmazonQWalkthroughOnce } from './onboardingPage/walkthrough' export { openAmazonQWalkthrough } from './onboardingPage/walkthrough' - +export { LspController, Content } from './lsp/lspController' +export { LspClient } from './lsp/lspClient' /** * main from createMynahUI is a purely browser dependency. Due to this * we need to create a wrapper function that will dynamically execute it diff --git a/packages/core/src/amazonq/lsp/lspClient.ts b/packages/core/src/amazonq/lsp/lspClient.ts new file mode 100644 index 00000000000..47aa38c75af --- /dev/null +++ b/packages/core/src/amazonq/lsp/lspClient.ts @@ -0,0 +1,219 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/*! + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + */ +import * as vscode from 'vscode' +import * as path from 'path' +import * as nls from 'vscode-nls' +import * as cp from 'child_process' +import * as crypto from 'crypto' +import * as jose from 'jose' + +import { Disposable, ExtensionContext } from 'vscode' + +import { LanguageClient, LanguageClientOptions, ServerOptions, TransportKind } from 'vscode-languageclient' +import { GetUsageRequestType, IndexRequestType, QueryRequestType, UpdateIndexRequestType, Usage } from './types' +import { Writable } from 'stream' +import { CodeWhispererSettings } from '../../codewhisperer/util/codewhispererSettings' +import { getLogger } from '../../shared' + +const localize = nls.loadMessageBundle() + +const key = crypto.randomBytes(32) + +/** + * Sends a json payload to the language server, who is waiting to know what the encryption key is. + * Code reference: https://github.com/aws/language-servers/blob/7da212185a5da75a72ce49a1a7982983f438651a/client/vscode/src/credentialsActivation.ts#L77 + */ +export function writeEncryptionInit(stream: Writable): void { + const request = { + version: '1.0', + mode: 'JWT', + key: key.toString('base64'), + } + stream.write(JSON.stringify(request)) + stream.write('\n') +} +/** + * LspClient manages the API call between VS Code extension and LSP server + * It encryptes the payload of API call. + */ +export class LspClient { + static #instance: LspClient + client: LanguageClient | undefined + + public static get instance() { + return (this.#instance ??= new this()) + } + + constructor() { + this.client = undefined + } + + async encrypt(payload: string) { + return await new jose.CompactEncrypt(new TextEncoder().encode(payload)) + .setProtectedHeader({ alg: 'dir', enc: 'A256GCM' }) + .encrypt(key) + } + + async indexFiles(request: string[], rootPath: string, refresh: boolean) { + try { + const encryptedRequest = await this.encrypt( + JSON.stringify({ + filePaths: request, + rootPath: rootPath, + refresh: refresh, + }) + ) + const resp = await this.client?.sendRequest(IndexRequestType, encryptedRequest) + return resp + } catch (e) { + getLogger().error(`LspClient: indexFiles error: ${e}`) + return undefined + } + } + + async query(request: string) { + try { + const encryptedRequest = await this.encrypt( + JSON.stringify({ + query: request, + }) + ) + const resp = await this.client?.sendRequest(QueryRequestType, encryptedRequest) + return resp + } catch (e) { + getLogger().error(`LspClient: query error: ${e}`) + return [] + } + } + + async getLspServerUsage(): Promise { + if (this.client) { + return (await this.client.sendRequest(GetUsageRequestType, '')) as Usage + } + } + + async updateIndex(filePath: string) { + try { + const encryptedRequest = await this.encrypt( + JSON.stringify({ + filePath: filePath, + }) + ) + const resp = await this.client?.sendRequest(UpdateIndexRequestType, encryptedRequest) + return resp + } catch (e) { + getLogger().error(`LspClient: updateIndex error: ${e}`) + return undefined + } + } +} +/** + * Activates the language server, this will start LSP server running over IPC protocol. + * It will create a output channel named Amazon Q Language Server. + * This function assumes the LSP server has already been downloaded. + */ +export async function activate(extensionContext: ExtensionContext) { + LspClient.instance + const toDispose = extensionContext.subscriptions + + let rangeFormatting: Disposable | undefined + // The server is implemented in node + const serverModule = path.join(extensionContext.extensionPath, 'resources/qserver/lspServer.js') + // The debug options for the server + // --inspect=6009: runs the server in Node's Inspector mode so VS Code can attach to the server for debugging + const debugOptions = { execArgv: ['--nolazy', '--preserve-symlinks', '--stdio'] } + + const workerThreads = CodeWhispererSettings.instance.getIndexWorkerThreads() + const gpu = CodeWhispererSettings.instance.isLocalIndexGPUEnabled() + + if (gpu) { + process.env.Q_ENABLE_GPU = 'true' + } else { + delete process.env.Q_ENABLE_GPU + } + if (workerThreads > 0 && workerThreads < 100) { + process.env.Q_WORKER_THREADS = workerThreads.toString() + } else { + delete process.env.Q_WORKER_THREADS + } + + const nodename = process.platform === 'win32' ? 'node.exe' : 'node' + + const child = cp.spawn(extensionContext.asAbsolutePath(path.join('resources', nodename)), [ + serverModule, + ...debugOptions.execArgv, + ]) + // share an encryption key using stdin + // follow same practice of DEXP LSP server + writeEncryptionInit(child.stdin) + + // If the extension is launch in debug mode the debug server options are use + // Otherwise the run options are used + let serverOptions: ServerOptions = { + run: { module: serverModule, transport: TransportKind.ipc }, + debug: { module: serverModule, transport: TransportKind.ipc, options: debugOptions }, + } + + serverOptions = () => Promise.resolve(child!) + + const documentSelector = [{ scheme: 'file', language: '*' }] + + // Options to control the language client + const clientOptions: LanguageClientOptions = { + // Register the server for json documents + documentSelector, + initializationOptions: { + handledSchemaProtocols: ['file', 'untitled'], // language server only loads file-URI. Fetching schemas with other protocols ('http'...) are made on the client. + provideFormatter: false, // tell the server to not provide formatting capability and ignore the `aws.stepfunctions.asl.format.enable` setting. + extensionPath: extensionContext.extensionPath, + }, + } + + // Create the language client and start the client. + LspClient.instance.client = new LanguageClient( + 'amazonq', + localize('amazonq.server.name', 'Amazon Q Language Server'), + serverOptions, + clientOptions + ) + LspClient.instance.client.registerProposedFeatures() + + const disposable = LspClient.instance.client.start() + toDispose.push(disposable) + + let savedDocument: vscode.Uri | undefined = undefined + + toDispose.push( + vscode.workspace.onDidSaveTextDocument(document => { + if (document.uri.scheme !== 'file') { + return + } + savedDocument = document.uri + }) + ) + toDispose.push( + vscode.window.onDidChangeActiveTextEditor(editor => { + if (savedDocument && editor && editor.document.uri.fsPath !== savedDocument.fsPath) { + void LspClient.instance.updateIndex(savedDocument.fsPath) + } + }) + ) + return LspClient.instance.client.onReady().then(() => { + const disposableFunc = { dispose: () => rangeFormatting?.dispose() as void } + toDispose.push(disposableFunc) + }) +} + +export async function deactivate(): Promise { + if (!LspClient.instance.client) { + return undefined + } + return LspClient.instance.client.stop() +} diff --git a/packages/core/src/amazonq/lsp/lspController.ts b/packages/core/src/amazonq/lsp/lspController.ts new file mode 100644 index 00000000000..97777012daf --- /dev/null +++ b/packages/core/src/amazonq/lsp/lspController.ts @@ -0,0 +1,405 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import * as vscode from 'vscode' +import * as path from 'path' +import * as fs from 'fs-extra' +import * as crypto from 'crypto' +import { getLogger } from '../../shared/logger/logger' +import { CurrentWsFolders, collectFilesForIndex } from '../../shared/utilities/workspaceUtils' +import fetch from 'node-fetch' +import request from '../../common/request' +import { LspClient } from './lspClient' +import AdmZip from 'adm-zip' +import { RelevantTextDocument } from '@amzn/codewhisperer-streaming' +import { makeTemporaryToolkitFolder, tryRemoveFolder } from '../../shared/filesystemUtilities' +import { CodeWhispererSettings } from '../../codewhisperer/util/codewhispererSettings' +import { activate as activateLsp } from './lspClient' +import { telemetry } from '../../shared/telemetry' +import { isCloud9 } from '../../shared/extensionUtilities' +import { globals, ToolkitError } from '../../shared' +import { AuthUtil } from '../../codewhisperer' +import { isWeb } from '../../shared/extensionGlobals' +import { getUserAgent } from '../../shared/telemetry/util' + +function getProjectPaths() { + const workspaceFolders = vscode.workspace.workspaceFolders + if (!workspaceFolders || workspaceFolders.length === 0) { + throw new ToolkitError('No workspace folders found') + } + return workspaceFolders.map(folder => folder.uri.fsPath) +} + +export interface Chunk { + readonly filePath: string + readonly content: string + readonly context?: string + readonly relativePath?: string + readonly programmingLanguage?: string +} + +export interface Content { + filename: string + url: string + hashes: string[] + bytes: number + serverVersion?: string +} + +export interface Target { + platform: string + arch: string + contents: Content[] +} + +export interface Manifest { + manifestSchemaVersion: string + artifactId: string + artifactDescription: string + isManifestDeprecated: boolean + versions: { + serverVersion: string + isDelisted: boolean + targets: Target[] + }[] +} +const manifestUrl = 'https://aws-toolkit-language-servers.amazonaws.com/q-context/manifest.json' +// this LSP client in Q extension is only going to work with these LSP server versions +const supportedLspServerVersions = ['0.1.2'] + +const nodeBinName = process.platform === 'win32' ? 'node.exe' : 'node' +/* + * LSP Controller manages the status of Amazon Q LSP: + * 1. Downloading, verifying and installing LSP using DEXP LSP manifest and CDN. + * 2. Managing the LSP states. There are a couple of possible LSP states: + * Not installed. Installed. Running. Indexing. Indexing Done. + * LSP Controller converts the input and output of LSP APIs. + * The IDE extension code should invoke LSP API via this controller. + * 3. It perform pre-process and post process of LSP APIs + * Pre-process the input to Index Files API + * Post-process the output from Query API + */ +export class LspController { + static #instance: LspController + private _isIndexingInProgress = false + + public static get instance() { + return (this.#instance ??= new this()) + } + constructor() {} + + isIndexingInProgress() { + return this._isIndexingInProgress + } + + async _download(localFile: string, remoteUrl: string) { + const res = await fetch(remoteUrl, { + headers: { + 'User-Agent': getUserAgent({ includePlatform: true, includeClientId: true }), + }, + }) + if (!res.ok) { + throw new ToolkitError(`Failed to download. Error: ${JSON.stringify(res)}`) + } + return new Promise((resolve, reject) => { + const file = fs.createWriteStream(localFile) + res.body.pipe(file) + res.body.on('error', err => { + reject(err) + }) + file.on('finish', () => { + file.close(resolve) + }) + }) + } + + async fetchManifest() { + try { + const resp = await request.fetch('GET', manifestUrl, { + headers: { + 'User-Agent': getUserAgent({ includePlatform: true, includeClientId: true }), + }, + }).response + if (!resp.ok) { + throw new ToolkitError(`Failed to fetch manifest. Error: ${resp.statusText}`) + } + return resp.json() + } catch (e: any) { + throw new ToolkitError(`Failed to fetch manifest. Error: ${JSON.stringify(e)}`) + } + } + + async getFileSha384(filePath: string): Promise { + const fileBuffer = await fs.promises.readFile(filePath) + const hash = crypto.createHash('sha384') + hash.update(fileBuffer) + return hash.digest('hex') + } + + isLspInstalled(context: vscode.ExtensionContext) { + const localQServer = context.asAbsolutePath(path.join('resources', 'qserver')) + const localNodeRuntime = context.asAbsolutePath(path.join('resources', nodeBinName)) + return fs.existsSync(localQServer) && fs.existsSync(localNodeRuntime) + } + + getQserverFromManifest(manifest: Manifest): Content | undefined { + if (manifest.isManifestDeprecated) { + return undefined + } + for (const version of manifest.versions) { + if (version.isDelisted) { + continue + } + if (!supportedLspServerVersions.includes(version.serverVersion)) { + continue + } + for (const t of version.targets) { + if ( + (t.platform === process.platform || (t.platform === 'windows' && process.platform === 'win32')) && + t.arch === process.arch + ) { + for (const content of t.contents) { + if (content.filename.startsWith('qserver') && content.hashes.length > 0) { + content.serverVersion = version.serverVersion + return content + } + } + } + } + } + return undefined + } + + getNodeRuntimeFromManifest(manifest: Manifest): Content | undefined { + if (manifest.isManifestDeprecated) { + return undefined + } + for (const version of manifest.versions) { + if (version.isDelisted) { + continue + } + if (!supportedLspServerVersions.includes(version.serverVersion)) { + continue + } + for (const t of version.targets) { + if ( + (t.platform === process.platform || (t.platform === 'windows' && process.platform === 'win32')) && + t.arch === process.arch + ) { + for (const content of t.contents) { + if (content.filename.startsWith('node') && content.hashes.length > 0) { + content.serverVersion = version.serverVersion + return content + } + } + } + } + } + return undefined + } + + private async hashMatch(filePath: string, content: Content) { + const sha384 = await this.getFileSha384(filePath) + if ('sha384:' + sha384 !== content.hashes[0]) { + getLogger().error( + `LspController: Downloaded file sha ${sha384} does not match manifest ${content.hashes[0]}.` + ) + fs.removeSync(filePath) + return false + } + return true + } + + async downloadAndCheckHash(filePath: string, content: Content) { + await this._download(filePath, content.url) + const match = await this.hashMatch(filePath, content) + if (!match) { + return false + } + return true + } + + async tryInstallLsp(context: vscode.ExtensionContext): Promise { + let tempFolder = undefined + try { + if (this.isLspInstalled(context)) { + getLogger().info(`LspController: LSP already installed`) + return true + } + // clean up previous downloaded LSP + const qserverPath = context.asAbsolutePath(path.join('resources', 'qserver')) + if (fs.existsSync(qserverPath)) { + await tryRemoveFolder(qserverPath) + } + // clean up previous downloaded node runtime + const nodeRuntimePath = context.asAbsolutePath(path.join('resources', nodeBinName)) + if (fs.existsSync(nodeRuntimePath)) { + fs.rmSync(nodeRuntimePath) + } + // fetch download url for qserver and node runtime + const manifest: Manifest = (await this.fetchManifest()) as Manifest + const qserverContent = this.getQserverFromManifest(manifest) + const nodeRuntimeContent = this.getNodeRuntimeFromManifest(manifest) + if (!qserverContent || !nodeRuntimeContent) { + getLogger().info(`LspController: Did not find LSP URL for ${process.platform} ${process.arch}`) + return false + } + + tempFolder = await makeTemporaryToolkitFolder() + + // download lsp to temp folder + const qserverZipTempPath = path.join(tempFolder, 'qserver.zip') + const downloadOk = await this.downloadAndCheckHash(qserverZipTempPath, qserverContent) + if (!downloadOk) { + return false + } + const zip = new AdmZip(qserverZipTempPath) + zip.extractAllTo(tempFolder) + fs.moveSync(path.join(tempFolder, 'qserver'), qserverPath) + + // download node runtime to temp folder + const nodeRuntimeTempPath = path.join(tempFolder, nodeBinName) + const downloadNodeOk = await this.downloadAndCheckHash(nodeRuntimeTempPath, nodeRuntimeContent) + if (!downloadNodeOk) { + return false + } + fs.chmodSync(nodeRuntimeTempPath, 0o755) + fs.moveSync(nodeRuntimeTempPath, nodeRuntimePath) + return true + } catch (e) { + getLogger().error(`LspController: Failed to setup LSP server ${e}`) + return false + } finally { + // clean up temp folder + if (tempFolder) { + await tryRemoveFolder(tempFolder) + } + } + } + + async query(s: string): Promise { + const chunks: Chunk[] | undefined = await LspClient.instance.query(s) + const resp: RelevantTextDocument[] = [] + chunks?.forEach(chunk => { + const text = chunk.context ? chunk.context : chunk.content + if (chunk.programmingLanguage) { + resp.push({ + text: text, + relativeFilePath: chunk.relativePath ? chunk.relativePath : path.basename(chunk.filePath), + programmingLanguage: { + languageName: chunk.programmingLanguage, + }, + }) + } else { + resp.push({ + text: text, + relativeFilePath: chunk.relativePath ? chunk.relativePath : path.basename(chunk.filePath), + }) + } + }) + return resp + } + + async buildIndex() { + getLogger().info(`LspController: Starting to build vector index of project`) + const start = performance.now() + const projPaths = getProjectPaths() + projPaths.sort() + try { + if (projPaths.length === 0) { + throw Error('No project') + } + this._isIndexingInProgress = true + const projRoot = projPaths[0] + const files = await collectFilesForIndex( + projPaths, + vscode.workspace.workspaceFolders as CurrentWsFolders, + true, + CodeWhispererSettings.instance.getMaxIndexSize() * 1024 * 1024 + ) + const totalSizeBytes = files.reduce( + (accumulator, currentFile) => accumulator + currentFile.fileSizeBytes, + 0 + ) + getLogger().info(`LspController: Found ${files.length} files in current project ${getProjectPaths()}`) + const resp = await LspClient.instance.indexFiles( + files.map(f => f.fileUri.fsPath), + projRoot, + false + ) + if (resp) { + getLogger().debug(`LspController: Finish building vector index of project`) + const usage = await LspClient.instance.getLspServerUsage() + telemetry.amazonq_indexWorkspace.emit({ + duration: performance.now() - start, + result: 'Succeeded', + amazonqIndexFileCount: files.length, + amazonqIndexMemoryUsageInMB: usage ? usage.memoryUsage / (1024 * 1024) : undefined, + amazonqIndexCpuUsagePercentage: usage ? usage.cpuUsage : undefined, + amazonqIndexFileSizeInMB: totalSizeBytes / (1024 * 1024), + credentialStartUrl: AuthUtil.instance.startUrl, + }) + } else { + getLogger().error(`LspController: Failed to build vector index of project`) + telemetry.amazonq_indexWorkspace.emit({ + duration: performance.now() - start, + result: 'Failed', + amazonqIndexFileCount: 0, + amazonqIndexFileSizeInMB: 0, + }) + } + } catch (e) { + getLogger().error(`LspController: Failed to build vector index of project`) + telemetry.amazonq_indexWorkspace.emit({ + duration: performance.now() - start, + result: 'Failed', + amazonqIndexFileCount: 0, + amazonqIndexFileSizeInMB: 0, + }) + } finally { + this._isIndexingInProgress = false + } + } + + async trySetupLsp(context: vscode.ExtensionContext) { + if (isCloud9() || isWeb()) { + // do not do anything if in Cloud9 or Web mode. + return + } + setImmediate(async () => { + if (!CodeWhispererSettings.instance.isLocalIndexEnabled()) { + // only download LSP for users who did not turn on this feature + // do not start LSP server + await LspController.instance.tryInstallLsp(context) + return + } + const ok = await LspController.instance.tryInstallLsp(context) + if (!ok) { + return + } + try { + await activateLsp(context) + getLogger().info('LspController: LSP activated') + void LspController.instance.buildIndex() + // log the LSP server CPU and Memory usage per 30 minutes. + globals.clock.setInterval( + async () => { + const usage = await LspClient.instance.getLspServerUsage() + if (usage) { + getLogger().info( + `LspController: LSP server CPU ${usage.cpuUsage}%, LSP server Memory ${ + usage.memoryUsage / (1024 * 1024) + }MB ` + ) + } + }, + 30 * 60 * 1000 + ) + } catch (e) { + getLogger().error(`LspController: LSP failed to activate ${e}`) + } + }) + } +} diff --git a/packages/core/src/amazonq/lsp/types.ts b/packages/core/src/amazonq/lsp/types.ts new file mode 100644 index 00000000000..bc6e37a75d9 --- /dev/null +++ b/packages/core/src/amazonq/lsp/types.ts @@ -0,0 +1,37 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import { RequestType } from 'vscode-languageserver' + +export type IndexRequestPayload = { + filePaths: string[] + rootPath: string + refresh: boolean +} + +export type IndexRequest = string + +export const IndexRequestType: RequestType = new RequestType('lsp/index') + +export type ClearRequest = string + +export const ClearRequestType: RequestType = new RequestType('lsp/clear') + +export type QueryRequest = string + +export const QueryRequestType: RequestType = new RequestType('lsp/query') + +export type UpdateIndexRequest = string + +export const UpdateIndexRequestType: RequestType = new RequestType('lsp/updateIndex') + +export type GetUsageRequest = string + +export const GetUsageRequestType: RequestType = new RequestType('lsp/getUsage') + +export interface Usage { + memoryUsage: number + cpuUsage: number +} diff --git a/packages/core/src/amazonq/webview/ui/apps/cwChatConnector.ts b/packages/core/src/amazonq/webview/ui/apps/cwChatConnector.ts index 06549e3a9ee..a94744c37d8 100644 --- a/packages/core/src/amazonq/webview/ui/apps/cwChatConnector.ts +++ b/packages/core/src/amazonq/webview/ui/apps/cwChatConnector.ts @@ -21,6 +21,7 @@ export interface ConnectorProps { onCWCContextCommandMessage: (message: ChatItem, command?: string) => string | undefined onError: (tabID: string, message: string, title: string) => void onWarning: (tabID: string, message: string, title: string) => void + onOpenSettingsMessage: (tabID: string) => void tabsStorage: TabsStorage } @@ -30,6 +31,7 @@ export class Connector { private readonly onWarning private readonly onChatAnswerReceived private readonly onCWCContextCommandMessage + private readonly onOpenSettingsMessage private readonly followUpGenerator: FollowUpGenerator constructor(props: ConnectorProps) { @@ -38,6 +40,7 @@ export class Connector { this.onWarning = props.onWarning this.onError = props.onError this.onCWCContextCommandMessage = props.onCWCContextCommandMessage + this.onOpenSettingsMessage = props.onOpenSettingsMessage this.followUpGenerator = new FollowUpGenerator() } @@ -324,6 +327,10 @@ export class Connector { return } + private processOpenSettingsMessage = async (messageData: any): Promise => { + this.onOpenSettingsMessage(messageData.tabID) + } + handleMessageReceive = async (messageData: any): Promise => { if (messageData.type === 'errorMessage') { this.onError(messageData.tabID, messageData.message, messageData.title) @@ -348,5 +355,10 @@ export class Connector { await this.processAuthNeededException(messageData) return } + + if (messageData.type === 'openSettingsMessage') { + await this.processOpenSettingsMessage(messageData) + return + } } } diff --git a/packages/core/src/amazonq/webview/ui/commands.ts b/packages/core/src/amazonq/webview/ui/commands.ts index 70f8f052a34..dafeca8a9fc 100644 --- a/packages/core/src/amazonq/webview/ui/commands.ts +++ b/packages/core/src/amazonq/webview/ui/commands.ts @@ -30,5 +30,6 @@ type MessageCommand = | 'footer-info-link-click' | 'file-click' | 'form-action-click' + | 'open-settings' export type ExtensionMessage = Record & { command: MessageCommand } diff --git a/packages/core/src/amazonq/webview/ui/connector.ts b/packages/core/src/amazonq/webview/ui/connector.ts index ec72d104984..0c366e0bd4e 100644 --- a/packages/core/src/amazonq/webview/ui/connector.ts +++ b/packages/core/src/amazonq/webview/ui/connector.ts @@ -38,6 +38,7 @@ export interface ConnectorProps { onAsyncEventProgress: (tabID: string, inProgress: boolean, message: string | undefined) => void onQuickHandlerCommand: (tabID: string, command?: string, eventId?: string) => void onCWCContextCommandMessage: (message: ChatItem, command?: string) => string | undefined + onOpenSettingsMessage: (tabID: string) => void onError: (tabID: string, message: string, title: string) => void onWarning: (tabID: string, message: string, title: string) => void onFileComponentUpdate: ( @@ -405,6 +406,14 @@ export class Connector { case 'gumby': this.gumbyChatConnector.onCustomFormAction(tabId, action) break + case 'cwc': + if (action.id === `open-settings`) { + this.sendMessageToExtension({ + command: 'open-settings', + type: '', + tabType: 'cwc', + }) + } } } } diff --git a/packages/core/src/amazonq/webview/ui/main.ts b/packages/core/src/amazonq/webview/ui/main.ts index be95c2deac3..3812c40e447 100644 --- a/packages/core/src/amazonq/webview/ui/main.ts +++ b/packages/core/src/amazonq/webview/ui/main.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ import { Connector } from './connector' -import { ChatItem, ChatItemType, MynahUI, MynahUIDataModel, NotificationType } from '@aws/mynah-ui' +import { ChatItem, ChatItemType, MynahIcons, MynahUI, MynahUIDataModel, NotificationType } from '@aws/mynah-ui' import { ChatPrompt } from '@aws/mynah-ui/dist/static' import { TabsStorage, TabType } from './storages/tabsStorage' import { WelcomeFollowupType } from './apps/amazonqCommonsConnector' @@ -317,6 +317,27 @@ export const createMynahUI = (ideApi: any, amazonQEnabled: boolean) => { mynahUI.updateStore(newTabID, tabDataGenerator.getTabData(tabType, true)) }, + onOpenSettingsMessage(tabId: string) { + mynahUI.addChatItem(tabId, { + type: ChatItemType.ANSWER, + body: `To add your workspace as context, enable local indexing in your IDE settings. After enabling, add @workspace to your question, and I'll generate a response using your workspace as context.`, + buttons: [ + { + id: 'open-settings', + text: 'Open settings', + icon: MynahIcons.EXTERNAL, + keepCardAfterClick: false, + status: 'info', + }, + ], + }) + tabsStorage.updateTabStatus(tabId, 'free') + mynahUI.updateStore(tabId, { + loadingChat: false, + promptInputDisabledState: tabsStorage.isTabDead(tabId), + }) + return + }, }) mynahUI = new MynahUI({ diff --git a/packages/core/src/amazonq/webview/ui/tabs/constants.ts b/packages/core/src/amazonq/webview/ui/tabs/constants.ts index d5ca79fd524..e6b20dbfa7a 100644 --- a/packages/core/src/amazonq/webview/ui/tabs/constants.ts +++ b/packages/core/src/amazonq/webview/ui/tabs/constants.ts @@ -15,7 +15,7 @@ const commonTabData: TabTypeData = { placeholder: 'Ask a question or enter "/" for quick actions', welcome: `Hi, I'm Amazon Q. I can answer your software development questions. Ask me to explain, debug, or optimize your code. - You can enter \`/\` to see a list of quick actions.`, + You can enter \`/\` to see a list of quick actions. Add @workspace to beginning of your message to include your entire workspace as context.`, } export const TabTypeDataMap: Record = { diff --git a/packages/core/src/amazonq/webview/ui/tabs/generator.ts b/packages/core/src/amazonq/webview/ui/tabs/generator.ts index 44fdff57dab..b91e180da4e 100644 --- a/packages/core/src/amazonq/webview/ui/tabs/generator.ts +++ b/packages/core/src/amazonq/webview/ui/tabs/generator.ts @@ -33,6 +33,17 @@ export class TabDataGenerator { 'Use of Amazon Q is subject to the [AWS Responsible AI Policy](https://aws.amazon.com/machine-learning/responsible-ai/policy/).', quickActionCommands: this.quickActionsGenerator.generateForTab(tabType), promptInputPlaceholder: TabTypeDataMap[tabType].placeholder, + contextCommands: [ + { + groupName: 'Mention code', + commands: [ + { + command: '@workspace', + description: '(BETA) Reference all code in workspace.', + }, + ], + }, + ], chatItems: needWelcomeMessages ? [ { diff --git a/packages/core/src/codewhisperer/client/user-service-2.json b/packages/core/src/codewhisperer/client/user-service-2.json index e183c5ab3aa..c8f42b18ff5 100644 --- a/packages/core/src/codewhisperer/client/user-service-2.json +++ b/packages/core/src/codewhisperer/client/user-service-2.json @@ -408,7 +408,8 @@ "fullResponselatency": { "shape": "Double" }, "requestLength": { "shape": "Integer" }, "responseLength": { "shape": "Integer" }, - "numberOfCodeBlocks": { "shape": "Integer" } + "numberOfCodeBlocks": { "shape": "Integer" }, + "hasProjectLevelContext": { "shape": "Boolean" } } }, "ChatHistory": { @@ -428,7 +429,8 @@ "interactionTarget": { "shape": "ChatInteractWithMessageEventInteractionTargetString" }, "acceptedCharacterCount": { "shape": "Integer" }, "acceptedLineCount": { "shape": "Integer" }, - "acceptedSnippetHasReference": { "shape": "Boolean" } + "acceptedSnippetHasReference": { "shape": "Boolean" }, + "hasProjectLevelContext": { "shape": "Boolean" } } }, "ChatInteractWithMessageEventInteractionTargetString": { @@ -471,7 +473,8 @@ "conversationId": { "shape": "ConversationId" }, "messageId": { "shape": "MessageId" }, "programmingLanguage": { "shape": "ProgrammingLanguage" }, - "modificationPercentage": { "shape": "Double" } + "modificationPercentage": { "shape": "Double" }, + "hasProjectLevelContext": { "shape": "Boolean" } } }, "CodeAnalysisFindingsSchema": { diff --git a/packages/core/src/codewhisperer/util/codewhispererSettings.ts b/packages/core/src/codewhisperer/util/codewhispererSettings.ts index 43e241c5310..9d4f6b126d4 100644 --- a/packages/core/src/codewhisperer/util/codewhispererSettings.ts +++ b/packages/core/src/codewhisperer/util/codewhispererSettings.ts @@ -8,6 +8,10 @@ const description = { showInlineCodeSuggestionsWithCodeReferences: Boolean, // eslint-disable-line id-length importRecommendationForInlineCodeSuggestions: Boolean, // eslint-disable-line id-length shareContentWithAWS: Boolean, + workspaceIndex: Boolean, + workspaceIndexWorkerThreads: Number, + workspaceIndexUseGPU: Boolean, + workspaceIndexMaxSize: Number, } export class CodeWhispererSettings extends fromExtensionManifest('amazonQ', description) { @@ -38,6 +42,23 @@ export class CodeWhispererSettings extends fromExtensionManifest('amazonQ', desc const value = this.get('shareContentWithAWS', true) return !value } + public isLocalIndexEnabled(): boolean { + return this.get('workspaceIndex', false) + } + + public isLocalIndexGPUEnabled(): boolean { + return this.get('workspaceIndexUseGPU', false) + } + + public getIndexWorkerThreads(): number { + // minimal 0 threads + return Math.max(this.get('workspaceIndexWorkerThreads', 0), 0) + } + + public getMaxIndexSize(): number { + // minimal 1MB + return Math.max(this.get('workspaceIndexMaxSize', 250), 1) + } static #instance: CodeWhispererSettings diff --git a/packages/core/src/codewhispererChat/controllers/chat/chatRequest/converter.ts b/packages/core/src/codewhispererChat/controllers/chat/chatRequest/converter.ts index 144953dcf82..e2210c1662e 100644 --- a/packages/core/src/codewhispererChat/controllers/chat/chatRequest/converter.ts +++ b/packages/core/src/codewhispererChat/controllers/chat/chatRequest/converter.ts @@ -7,6 +7,7 @@ import { CursorState, DocumentSymbol, GenerateAssistantResponseRequest, + RelevantTextDocument, SymbolType, TextDocument, } from '@amzn/codewhisperer-streaming' @@ -91,6 +92,9 @@ export function triggerPayloadToChatRequest(triggerPayload: TriggerPayload): Gen } } + const relevantDocuments: RelevantTextDocument[] = triggerPayload.relevantTextDocuments + ? triggerPayload.relevantTextDocuments + : [] // service will throw validation exception if string is empty const customizationArn: string | undefined = undefinedIfEmpty(triggerPayload.customization.arn) @@ -105,6 +109,7 @@ export function triggerPayloadToChatRequest(triggerPayload: TriggerPayload): Gen editorState: { document, cursorState, + relevantDocuments, }, }, userIntent: triggerPayload.userIntent, diff --git a/packages/core/src/codewhispererChat/controllers/chat/controller.ts b/packages/core/src/codewhispererChat/controllers/chat/controller.ts index 3f67d52e92b..cffd1dccfd9 100644 --- a/packages/core/src/codewhispererChat/controllers/chat/controller.ts +++ b/packages/core/src/codewhispererChat/controllers/chat/controller.ts @@ -2,7 +2,6 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 */ - import { Event as VSCodeEvent, Uri } from 'vscode' import { EditorContextExtractor } from '../../editor/context/extractor' import { ChatSessionStorage } from '../../storages/chatSession' @@ -45,6 +44,8 @@ import { triggerPayloadToChatRequest } from './chatRequest/converter' import { AuthUtil } from '../../../codewhisperer/util/authUtil' import { openUrl } from '../../../shared/utilities/vsCodeUtils' import { randomUUID } from '../../../common/crypto' +import { LspController } from '../../../amazonq/lsp/lspController' +import { CodeWhispererSettings } from '../../../codewhisperer/util/codewhispererSettings' import { getSelectedCustomization } from '../../../codewhisperer/util/customizationUtil' import { getHttpStatusCode } from '../../../shared/errors' @@ -404,7 +405,6 @@ export class ChatController { this.messenger.sendErrorMessage('chatMessage should be set', message.tabID, undefined) return } - try { switch (message.command) { case 'follow-up-was-clicked': @@ -542,8 +542,10 @@ export class ChatController { this.messenger.sendStaticTextResponse(responseType, triggerID, tabID) } - private async generateResponse(triggerPayload: TriggerPayload, triggerID: string) { - // Loop while we waiting for tabID to be set + private async generateResponse( + triggerPayload: TriggerPayload & { projectContextQueryLatencyMs?: number }, + triggerID: string + ) { const triggerEvent = this.triggerEventsStorage.getTriggerEvent(triggerID) if (triggerEvent === undefined) { return @@ -572,6 +574,25 @@ export class ChatController { await this.messenger.sendAuthNeededExceptionMessage(credentialsState, tabID, triggerID) return } + if (triggerPayload.message) { + const userIntentEnableProjectContext = triggerPayload.message.includes(`@workspace`) + if (userIntentEnableProjectContext) { + triggerPayload.message = triggerPayload.message.replace(/@workspace/g, '') + if (CodeWhispererSettings.instance.isLocalIndexEnabled()) { + const start = performance.now() + triggerPayload.relevantTextDocuments = await LspController.instance.query(triggerPayload.message) + triggerPayload.relevantTextDocuments.forEach(doc => { + getLogger().info( + `amazonq: Using workspace files ${doc.relativeFilePath}, content(partial): ${doc.text?.substring(0, 200)}` + ) + }) + triggerPayload.projectContextQueryLatencyMs = performance.now() - start + } else { + this.messenger.sendOpenSettingsMessage(triggerID, tabID) + return + } + } + } const request = triggerPayloadToChatRequest(triggerPayload) const session = this.sessionStorage.getSession(tabID) diff --git a/packages/core/src/codewhispererChat/controllers/chat/messenger/messenger.ts b/packages/core/src/codewhispererChat/controllers/chat/messenger/messenger.ts index ae2fcdcd7ca..3c4e4137907 100644 --- a/packages/core/src/codewhispererChat/controllers/chat/messenger/messenger.ts +++ b/packages/core/src/codewhispererChat/controllers/chat/messenger/messenger.ts @@ -9,6 +9,7 @@ import { AuthNeededException, CodeReference, EditorContextCommandMessage, + OpenSettingsMessage, QuickActionMessage, } from '../../../view/connector/connector' import { EditorContextCommandType } from '../../../commands/registerCommands' @@ -31,6 +32,7 @@ import { userGuideURL } from '../../../../amazonq/webview/ui/texts/constants' import { CodeScanIssue } from '../../../../codewhisperer/models/model' import { marked } from 'marked' import { JSDOM } from 'jsdom' +import { LspController } from '../../../../amazonq/lsp/lspController' export type StaticTextResponseType = 'quick-action-help' | 'onboarding-help' | 'transform' | 'help' @@ -126,6 +128,9 @@ export class Messenger { ) } this.telemetryHelper.setResponseStreamStartTime(tabID) + if (triggerPayload.relevantTextDocuments && triggerPayload.relevantTextDocuments.length > 0) { + this.telemetryHelper.setResponseFromProjectContext(messageID) + } const eventCounts = new Map() waitUntil( @@ -234,6 +239,29 @@ export class Messenger { this.telemetryHelper.recordMessageResponseError(triggerPayload, tabID, statusCode ?? 0) }) .finally(async () => { + if ( + triggerPayload.relevantTextDocuments && + triggerPayload.relevantTextDocuments.length > 0 && + LspController.instance.isIndexingInProgress() + ) { + this.dispatcher.sendChatMessage( + new ChatMessage( + { + message: + message + + ` \n\nBy the way, I'm still indexing this project for full context from your workspace. I may have a better response in a few minutes when it's complete if you'd like to try again then.`, + messageType: 'answer-part', + followUps: undefined, + followUpsHeader: undefined, + relatedSuggestions: undefined, + triggerID, + messageID, + }, + tabID + ) + ) + } + if (relatedSuggestions.length !== 0) { this.dispatcher.sendChatMessage( new ChatMessage( @@ -460,4 +488,8 @@ export class Messenger { new ErrorMessage('An error occurred while processing your request.', message.trimEnd().trimStart(), tabID) ) } + + public sendOpenSettingsMessage(triggerId: string, tabID: string) { + this.dispatcher.sendOpenSettingsMessage(new OpenSettingsMessage(tabID)) + } } diff --git a/packages/core/src/codewhispererChat/controllers/chat/model.ts b/packages/core/src/codewhispererChat/controllers/chat/model.ts index 091a61e4b4c..d2ed9a277e6 100644 --- a/packages/core/src/codewhispererChat/controllers/chat/model.ts +++ b/packages/core/src/codewhispererChat/controllers/chat/model.ts @@ -4,7 +4,7 @@ */ import * as vscode from 'vscode' -import { UserIntent } from '@amzn/codewhisperer-streaming' +import { RelevantTextDocument, UserIntent } from '@amzn/codewhisperer-streaming' import { MatchPolicy, CodeQuery } from '../../clients/chat/v0/model' import { Selection } from 'vscode' import { TabOpenType } from '../../../amazonq/webview/ui/storages/tabsStorage' @@ -136,11 +136,12 @@ export interface TriggerPayload { readonly fileText: string | undefined readonly fileLanguage: string | undefined readonly filePath: string | undefined - readonly message: string | undefined + message: string | undefined readonly matchPolicy: MatchPolicy | undefined readonly codeQuery: CodeQuery | undefined readonly userIntent: UserIntent | undefined readonly customization: Customization + relevantTextDocuments?: RelevantTextDocument[] } export interface InsertedCode { diff --git a/packages/core/src/codewhispererChat/controllers/chat/telemetryHelper.ts b/packages/core/src/codewhispererChat/controllers/chat/telemetryHelper.ts index 62dbe4d39fc..8ebde4fb3bf 100644 --- a/packages/core/src/codewhispererChat/controllers/chat/telemetryHelper.ts +++ b/packages/core/src/codewhispererChat/controllers/chat/telemetryHelper.ts @@ -61,6 +61,7 @@ export class CWCTelemetryHelper { private responseStreamStartTime: Map = new Map() private responseStreamTotalTime: Map = new Map() private responseStreamTimeForChunks: Map = new Map() + private responseWithProjectContext: Map = new Map() constructor(sessionStorage: ChatSessionStorage, triggerEventsStorage: TriggerEventsStorage) { this.sessionStorage = sessionStorage @@ -162,6 +163,7 @@ export class CWCTelemetryHelper { cwsprChatHasReference: message.codeReference && message.codeReference.length > 0, cwsprChatCodeBlockIndex: message.codeBlockIndex, cwsprChatTotalCodeBlocks: message.totalCodeBlocks, + cwsprChatHasProjectContext: this.responseWithProjectContext.get(message.messageId), } break case 'code_was_copied_to_clipboard': @@ -177,6 +179,7 @@ export class CWCTelemetryHelper { cwsprChatHasReference: message.codeReference && message.codeReference.length > 0, cwsprChatCodeBlockIndex: message.codeBlockIndex, cwsprChatTotalCodeBlocks: message.totalCodeBlocks, + cwsprChatHasProjectContext: this.responseWithProjectContext.get(message.messageId), } break case 'follow-up-was-clicked': @@ -187,6 +190,7 @@ export class CWCTelemetryHelper { credentialStartUrl: AuthUtil.instance.startUrl, cwsprChatMessageId: message.messageId, cwsprChatInteractionType: 'clickFollowUp', + cwsprChatHasProjectContext: this.responseWithProjectContext.get(message.messageId), } break case 'chat-item-voted': @@ -197,6 +201,7 @@ export class CWCTelemetryHelper { cwsprChatConversationId: conversationId ?? '', credentialStartUrl: AuthUtil.instance.startUrl, cwsprChatInteractionType: message.vote, + cwsprChatHasProjectContext: this.responseWithProjectContext.get(message.messageId), } break case 'source-link-click': @@ -208,6 +213,7 @@ export class CWCTelemetryHelper { credentialStartUrl: AuthUtil.instance.startUrl, cwsprChatInteractionType: 'clickLink', cwsprChatInteractionTarget: message.link, + cwsprChatHasProjectContext: this.responseWithProjectContext.get(message.messageId), } break case 'response-body-link-click': @@ -219,6 +225,7 @@ export class CWCTelemetryHelper { credentialStartUrl: AuthUtil.instance.startUrl, cwsprChatInteractionType: 'clickBodyLink', cwsprChatInteractionTarget: message.link, + cwsprChatHasProjectContext: this.responseWithProjectContext.get(message.messageId), } break case 'footer-info-link-click': @@ -237,7 +244,6 @@ export class CWCTelemetryHelper { if (!event) { return } - telemetry.amazonq_interactWithMessage.emit(event) codeWhispererClient @@ -251,6 +257,7 @@ export class CWCTelemetryHelper { acceptedCharacterCount: event.cwsprChatAcceptedCharactersLength, acceptedLineCount: event.cwsprChatAcceptedNumberOfLines, acceptedSnippetHasReference: false, + hasProjectLevelContext: this.responseWithProjectContext.get(event.cwsprChatMessageId), }, }, }) @@ -290,7 +297,10 @@ export class CWCTelemetryHelper { } } - public recordStartConversation(triggerEvent: TriggerEvent, triggerPayload: TriggerPayload) { + public recordStartConversation( + triggerEvent: TriggerEvent, + triggerPayload: TriggerPayload & { projectContextQueryLatencyMs?: number } + ) { if (triggerEvent.tabID === undefined) { return } @@ -310,6 +320,10 @@ export class CWCTelemetryHelper { cwsprChatHasCodeSnippet: triggerPayload.codeSelection && !triggerPayload.codeSelection.isEmpty, cwsprChatProgrammingLanguage: triggerPayload.fileLanguage, credentialStartUrl: AuthUtil.instance.startUrl, + cwsprChatHasProjectContext: triggerPayload.relevantTextDocuments + ? triggerPayload.relevantTextDocuments.length > 0 + : false, + cwsprChatProjectContextQueryMs: triggerPayload.projectContextQueryLatencyMs, }) } @@ -339,6 +353,9 @@ export class CWCTelemetryHelper { cwsprChatConversationType: 'Chat', credentialStartUrl: AuthUtil.instance.startUrl, codewhispererCustomizationArn: triggerPayload.customization.arn, + cwsprChatHasProjectContext: triggerPayload.relevantTextDocuments + ? triggerPayload.relevantTextDocuments.length > 0 + : false, } telemetry.amazonq_addMessage.emit(event) @@ -360,6 +377,9 @@ export class CWCTelemetryHelper { requestLength: event.cwsprChatRequestLength, responseLength: event.cwsprChatResponseLength, numberOfCodeBlocks: event.cwsprChatResponseCodeSnippetCount, + hasProjectLevelContext: triggerPayload.relevantTextDocuments + ? triggerPayload.relevantTextDocuments.length > 0 + : false, }, }, }) @@ -416,6 +436,10 @@ export class CWCTelemetryHelper { this.responseStreamTimeForChunks.set(tabID, [...chunkTimes, performance.now()]) } + public setResponseFromProjectContext(messageId: string) { + this.responseWithProjectContext.set(messageId, true) + } + private getResponseStreamTimeToFirstChunk(tabID: string): number { const chunkTimes = this.responseStreamTimeForChunks.get(tabID) ?? [0, 0] if (chunkTimes.length === 1) { diff --git a/packages/core/src/codewhispererChat/view/connector/connector.ts b/packages/core/src/codewhispererChat/view/connector/connector.ts index 755371848af..094c2b2c6d9 100644 --- a/packages/core/src/codewhispererChat/view/connector/connector.ts +++ b/packages/core/src/codewhispererChat/view/connector/connector.ts @@ -128,6 +128,10 @@ export class AuthNeededException extends UiMessage { } } +export class OpenSettingsMessage extends UiMessage { + override type = 'openSettingsMessage' +} + export interface ChatMessageProps { readonly message: string | undefined readonly messageType: ChatMessageType @@ -229,4 +233,8 @@ export class AppToWebViewMessageDispatcher { public sendAuthNeededExceptionMessage(message: AuthNeededException) { this.appsToWebViewMessagePublisher.publish(message) } + + public sendOpenSettingsMessage(message: OpenSettingsMessage) { + this.appsToWebViewMessagePublisher.publish(message) + } } diff --git a/packages/core/src/codewhispererChat/view/messages/messageListener.ts b/packages/core/src/codewhispererChat/view/messages/messageListener.ts index 3286fa90b55..4e47bae3836 100644 --- a/packages/core/src/codewhispererChat/view/messages/messageListener.ts +++ b/packages/core/src/codewhispererChat/view/messages/messageListener.ts @@ -2,13 +2,13 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 */ - import { MessageListener } from '../../../amazonq/messages/messageListener' import { ExtensionMessage } from '../../../amazonq/webview/ui/commands' import { AuthController } from '../../../amazonq/auth/controller' import { ChatControllerMessagePublishers } from '../../controllers/chat/controller' import { ReferenceLogController } from './referenceLogController' import { getLogger } from '../../../shared/logger' +import { openSettingsId } from '../../../shared/settings' export interface UIMessageListenerProps { readonly chatControllerMessagePublishers: ChatControllerMessagePublishers @@ -95,9 +95,15 @@ export class UIMessageListener { case 'footer-info-link-click': this.processFooterInfoLinkClick(msg) break + case 'open-settings': + this.processOpenSettings(msg) } } + private processOpenSettings(msg: any) { + void openSettingsId(`amazonQ.workspaceIndex`) + } + private processAuthFollowUpWasClicked(msg: any) { this.authController.handleAuth(msg.authType) } diff --git a/packages/core/src/shared/telemetry/vscodeTelemetry.json b/packages/core/src/shared/telemetry/vscodeTelemetry.json index c24318804be..378c6cca26e 100644 --- a/packages/core/src/shared/telemetry/vscodeTelemetry.json +++ b/packages/core/src/shared/telemetry/vscodeTelemetry.json @@ -154,6 +154,36 @@ "type": "boolean", "description": "true if user has selected code snippet, false otherwise." }, + { + "name": "cwsprChatHasProjectContext", + "type": "boolean", + "description": "true if query has project level context, false otherwise." + }, + { + "name": "cwsprChatProjectContextQueryMs", + "type": "int", + "description": "Query latency in ms for local project context" + }, + { + "name": "amazonqIndexFileSizeInMB", + "type": "int", + "description": "The sum of file sizes that were indexed in MB" + }, + { + "name": "amazonqIndexFileCount", + "type": "int", + "description": "Number of files indexed" + }, + { + "name": "amazonqIndexMemoryUsageInMB", + "type": "int", + "description": "Memory usage of LSP server" + }, + { + "name": "amazonqIndexCpuUsagePercentage", + "type": "int", + "description": "CPU used by LSP server as a percentage of all available CPUs on the system" + }, { "name": "cwsprChatProgrammingLanguage", "type": "string", @@ -717,6 +747,44 @@ }, { "type": "cwsprChatConversationType" + }, + { + "type": "cwsprChatHasProjectContext", + "required": false + }, + { + "type": "cwsprChatProjectContextQueryMs", + "required": false + } + ] + }, + { + "name": "amazonq_indexWorkspace", + "description": "Indexing of local workspace", + "metadata": [ + { + "type": "duration" + }, + { + "type": "result" + }, + { + "type": "amazonqIndexFileSizeInMB" + }, + { + "type": "amazonqIndexFileCount" + }, + { + "type": "amazonqIndexMemoryUsageInMB", + "required": false + }, + { + "type": "amazonqIndexCpuUsagePercentage", + "required": false + }, + { + "type": "credentialStartUrl", + "required": false } ] }, @@ -795,6 +863,10 @@ { "type": "cwsprChatConversationType" }, + { + "type": "cwsprChatHasProjectContext", + "required": false + }, { "type": "codewhispererCustomizationArn", "required": false @@ -891,6 +963,10 @@ { "type": "cwsprChatTotalCodeBlocks", "required": false + }, + { + "type": "cwsprChatHasProjectContext", + "required": false } ] }, diff --git a/packages/core/src/shared/utilities/workspaceUtils.ts b/packages/core/src/shared/utilities/workspaceUtils.ts index 0db10c5fa02..4f02d3739ed 100644 --- a/packages/core/src/shared/utilities/workspaceUtils.ts +++ b/packages/core/src/shared/utilities/workspaceUtils.ts @@ -5,6 +5,7 @@ import * as vscode from 'vscode' import * as path from 'path' +import * as os from 'os' import * as pathutils from '../../shared/utilities/pathUtils' import { getLogger } from '../logger' import { isInDirectory } from '../filesystemUtilities' @@ -360,6 +361,7 @@ export async function collectFiles( } const fileContent = await readFile(file) + if (fileContent === undefined) { continue } @@ -510,3 +512,93 @@ export function getWorkspaceFoldersByPrefixes( return results } + +/** + * Collects all files that are suitable for local indexing. + * 1. Must be a supported programming language + * 2. Must not be auto generated code + * 3. Must not be within gitignore + * 4. Ranked by priority. + * 5. Select files within maxSize limit. + * This function do not read the actual file content or compress them into a zip. + * TODO: Move this to LSP + * @param sourcePaths the paths where collection starts + * @param workspaceFolders the current workspace folders opened + * @param respectGitIgnore whether to respect gitignore file + * @returns all matched files + */ +export async function collectFilesForIndex( + sourcePaths: string[], + workspaceFolders: CurrentWsFolders, + respectGitIgnore: boolean = true, + maxSize = 250 * 1024 * 1024 // 250 MB, + // make this configurable, so we can test it +): Promise< + { + workspaceFolder: vscode.WorkspaceFolder + relativeFilePath: string + fileUri: vscode.Uri + fileSizeBytes: number + }[] +> { + const storage: Awaited> = [] + + const isLanguageSupported = (filename: string) => { + const k = + /\.(js|ts|java|py|rb|cpp|tsx|jsx|cc|c|h|html|json|css|md|php|swift|rs|scala|yaml|tf|sql|sh|go|yml|kt|smithy|config|kts|gradle|cfg|xml|vue)$/i + return k.test(filename) || filename.endsWith('Config') + } + + const isBuildOrBin = (filePath: string) => { + const k = /[/\\](bin|build|node_modules|env|\.idea)[/\\]/i + return k.test(filePath) + } + + let totalSizeBytes = 0 + for (const rootPath of sourcePaths) { + const allFiles = await vscode.workspace.findFiles( + new vscode.RelativePattern(rootPath, '**'), + getExcludePattern() + ) + const files = respectGitIgnore ? await filterOutGitignoredFiles(rootPath, allFiles) : allFiles + + for (const file of files) { + if (!isLanguageSupported(file.fsPath)) { + continue + } + if (isBuildOrBin(file.fsPath)) { + continue + } + const relativePath = getWorkspaceRelativePath(file.fsPath, { workspaceFolders }) + if (!relativePath) { + continue + } + + const fileStat = await vscode.workspace.fs.stat(file) + // ignore single file over 10 MB + if (fileStat.size > 10 * 1024 * 1024) { + continue + } + storage.push({ + workspaceFolder: relativePath.workspaceFolder, + relativeFilePath: relativePath.relativePath, + fileUri: file, + fileSizeBytes: fileStat.size, + }) + } + } + // prioritize upper level files + storage.sort((a, b) => a.fileUri.fsPath.length - b.fileUri.fsPath.length) + + const maxSizeBytes = Math.min(maxSize, os.freemem() / 2) + + let i = 0 + for (i = 0; i < storage.length; i += 1) { + totalSizeBytes += storage[i].fileSizeBytes + if (totalSizeBytes >= maxSizeBytes) { + break + } + } + // pick top 100k files below size limit + return storage.slice(0, Math.min(100000, i)) +}