Merge master into feature/emr

Author: aws-toolkit-automation

packages/amazonq/.changes/next-release/Feature-294f0da4-1fb4-43c9-90f2-e066d7351c3c.json

@@ -0,0 +1,4 @@
+{
+	"type": "Feature",
+	"description": "Automatically pause and resume @workspace indexing when OS CPU load is high"
+}

packages/amazonq/.changes/next-release/Feature-88ac445a-7a5a-4127-97f7-eaccd41629e6.json

@@ -0,0 +1,4 @@
+{
+	"type": "Feature",
+	"description": "Inline completion for more json files, and all yaml files"
+}

packages/amazonq/test/unit/codewhisperer/util/commonUtil.test.ts

@@ -4,7 +4,11 @@
  */
 
 import assert from 'assert'
-import { checkLeftContextKeywordsForJsonAndYaml, getPrefixSuffixOverlap } from 'aws-core-vscode/codewhisperer'
+import {
+    JsonConfigFileNamingConvention,
+    checkLeftContextKeywordsForJson,
+    getPrefixSuffixOverlap,
+} from 'aws-core-vscode/codewhisperer'
 
 describe('commonUtil', function () {
     describe('getPrefixSuffixOverlap', function () {
@@ -31,29 +35,47 @@ describe('commonUtil', function () {
         })
     })
 
-    describe('checkLeftContextKeywordsForJsonAndYaml', function () {
+    describe('checkLeftContextKeywordsForJson', function () {
         it('Should return true for valid left context keywords', async function () {
             assert.strictEqual(
-                checkLeftContextKeywordsForJsonAndYaml('Create an S3 Bucket named CodeWhisperer', 'json'),
-                true
-            )
-            assert.strictEqual(
-                checkLeftContextKeywordsForJsonAndYaml('Create an S3 Bucket named CodeWhisperer', 'yaml'),
+                checkLeftContextKeywordsForJson('foo.json', 'Create an S3 Bucket named CodeWhisperer', 'json'),
                 true
             )
         })
         it('Should return false for invalid left context keywords', async function () {
             assert.strictEqual(
-                checkLeftContextKeywordsForJsonAndYaml('Create an S3 Bucket named CodeWhisperer in cfn', 'yaml'),
-                false
-            )
-            assert.strictEqual(
-                checkLeftContextKeywordsForJsonAndYaml(
+                checkLeftContextKeywordsForJson(
+                    'foo.json',
                     'Create an S3 Bucket named CodeWhisperer in Cloudformation',
                     'json'
                 ),
                 false
             )
         })
+
+        for (const jsonConfigFile of JsonConfigFileNamingConvention) {
+            it(`should evalute by filename ${jsonConfigFile}`, function () {
+                assert.strictEqual(checkLeftContextKeywordsForJson(jsonConfigFile, 'foo', 'json'), false)
+
+                assert.strictEqual(checkLeftContextKeywordsForJson(jsonConfigFile.toUpperCase(), 'bar', 'json'), false)
+
+                assert.strictEqual(checkLeftContextKeywordsForJson(jsonConfigFile.toUpperCase(), 'baz', 'json'), false)
+            })
+
+            const upperCaseFilename = jsonConfigFile.toUpperCase()
+            it(`should evalute by filename and case insensitive ${upperCaseFilename}`, function () {
+                assert.strictEqual(checkLeftContextKeywordsForJson(upperCaseFilename, 'foo', 'json'), false)
+
+                assert.strictEqual(
+                    checkLeftContextKeywordsForJson(upperCaseFilename.toUpperCase(), 'bar', 'json'),
+                    false
+                )
+
+                assert.strictEqual(
+                    checkLeftContextKeywordsForJson(upperCaseFilename.toUpperCase(), 'baz', 'json'),
+                    false
+                )
+            })
+        }
     })
 })

packages/amazonq/test/unit/codewhisperer/util/crossFileContextUtil.test.ts

@@ -7,7 +7,7 @@ import assert from 'assert'
 import * as vscode from 'vscode'
 import * as sinon from 'sinon'
 import * as crossFile from 'aws-core-vscode/codewhisperer'
-import { createMockTextEditor } from 'aws-core-vscode/test'
+import { aStringWithLineCount, createMockTextEditor } from 'aws-core-vscode/test'
 import { crossFileContextConfig } from 'aws-core-vscode/codewhisperer'
 import {
     assertTabCount,
@@ -35,18 +35,18 @@ describe('crossFileContextUtil', function () {
             tempFolder = (await createTestWorkspaceFolder()).uri.fsPath
         })
 
-        it('should fetch 3 chunks and each chunk should contains 10 lines', async function () {
-            await toTextEditor(sampleFileOf60Lines, 'CrossFile.java', tempFolder, { preview: false })
+        it('should fetch 3 chunks and each chunk should contains 50 lines', async function () {
+            await toTextEditor(aStringWithLineCount(200), 'CrossFile.java', tempFolder, { preview: false })
             const myCurrentEditor = await toTextEditor('', 'TargetFile.java', tempFolder, {
                 preview: false,
             })
             const actual = await crossFile.fetchSupplementalContextForSrc(myCurrentEditor, fakeCancellationToken)
             assert.ok(actual)
             assert.ok(actual.supplementalContextItems.length === 3)
 
-            assert.strictEqual(actual.supplementalContextItems[0].content.split('\n').length, 10)
-            assert.strictEqual(actual.supplementalContextItems[1].content.split('\n').length, 10)
-            assert.strictEqual(actual.supplementalContextItems[2].content.split('\n').length, 10)
+            assert.strictEqual(actual.supplementalContextItems[0].content.split('\n').length, 50)
+            assert.strictEqual(actual.supplementalContextItems[1].content.split('\n').length, 50)
+            assert.strictEqual(actual.supplementalContextItems[2].content.split('\n').length, 50)
         })
     })
 
@@ -250,73 +250,59 @@ describe('crossFileContextUtil', function () {
             assert.strictEqual(chunks[1].content, 'line_6\nline_7')
         })
 
-        it('codewhisperer crossfile config should use 10 lines', async function () {
+        it('codewhisperer crossfile config should use 50 lines', async function () {
             const filePath = path.join(tempFolder, 'file.txt')
-            await toFile(sampleFileOf60Lines, filePath)
+            await toFile(aStringWithLineCount(210), filePath)
 
             const chunks = await crossFile.splitFileToChunks(filePath, crossFileContextConfig.numberOfLinesEachChunk)
-            assert.strictEqual(chunks.length, 6)
+
+            // (210 / 50) + 1
+            assert.strictEqual(chunks.length, 5)
+            // line0 -> line49
+            assert.strictEqual(chunks[0].content, aStringWithLineCount(50, 0))
+            // line50 -> line99
+            assert.strictEqual(chunks[1].content, aStringWithLineCount(50, 50))
+            // line100 -> line149
+            assert.strictEqual(chunks[2].content, aStringWithLineCount(50, 100))
+            // line150 -> line199
+            assert.strictEqual(chunks[3].content, aStringWithLineCount(50, 150))
+            // line 200 -> line209
+            assert.strictEqual(chunks[4].content, aStringWithLineCount(10, 200))
+        })
+
+        it('linkChunks should add another chunk which will link to the first chunk and chunk.nextContent should reflect correct value', async function () {
+            const filePath = path.join(tempFolder, 'file.txt')
+            await toFile(aStringWithLineCount(210), filePath)
+
+            const chunks = await crossFile.splitFileToChunks(filePath, crossFileContextConfig.numberOfLinesEachChunk)
+            const linkedChunks = crossFile.linkChunks(chunks)
+
+            // 210 / 50 + 2
+            assert.strictEqual(linkedChunks.length, 6)
+
+            // 0th
+            assert.strictEqual(linkedChunks[0].content, aStringWithLineCount(3, 0))
+            assert.strictEqual(linkedChunks[0].nextContent, aStringWithLineCount(50, 0))
+
+            // 1st
+            assert.strictEqual(linkedChunks[1].content, aStringWithLineCount(50, 0))
+            assert.strictEqual(linkedChunks[1].nextContent, aStringWithLineCount(50, 50))
+
+            // 2nd
+            assert.strictEqual(linkedChunks[2].content, aStringWithLineCount(50, 50))
+            assert.strictEqual(linkedChunks[2].nextContent, aStringWithLineCount(50, 100))
+
+            // 3rd
+            assert.strictEqual(linkedChunks[3].content, aStringWithLineCount(50, 100))
+            assert.strictEqual(linkedChunks[3].nextContent, aStringWithLineCount(50, 150))
+
+            // 4th
+            assert.strictEqual(linkedChunks[4].content, aStringWithLineCount(50, 150))
+            assert.strictEqual(linkedChunks[4].nextContent, aStringWithLineCount(10, 200))
+
+            // 5th
+            assert.strictEqual(linkedChunks[5].content, aStringWithLineCount(10, 200))
+            assert.strictEqual(linkedChunks[5].nextContent, aStringWithLineCount(10, 200))
         })
     })
 })
-
-const sampleFileOf60Lines = `import java.util.List;
-// we need this comment on purpose because chunk will be trimed right, adding this to avoid trimRight and make assertion easier
-/**
- * 
- * 
- * 
- * 
- * 
- **/
-class Main {
-    public static void main(String[] args) {
-        Calculator calculator = new Calculator();
-        calculator.add(1, 2);
-        calculator.subtract(1, 2);
-        calculator.multiply(1, 2);
-        calculator.divide(1, 2);
-        calculator.remainder(1, 2);
-    }
-}
-//
-class Calculator {
-    public Calculator() {
-        System.out.println("constructor");
-    }
-//
-    public add(int num1, int num2) {
-        System.out.println("add");
-        return num1 + num2;
-    }
-//
-    public subtract(int num1, int num2) {
-        System.out.println("subtract");
-        return num1 - num2;
-    }
-//
-    public multiply(int num1, int num2) {
-        System.out.println("multiply");
-        return num1 * num2;    
-    }
-//
-    public divide(int num1, int num2) {
-        System.out.println("divide");
-        return num1 / num2;
-    }
-//
-    public remainder(int num1, int num2) {
-        System.out.println("remainder");
-        return num1 % num2;
-    }
-//
-    public power(int num1, int num2) {
-        System.out.println("power");
-        return (int) Math.pow(num1, num2);
-    }
-//
-    public squareRoot(int num1) {
-        System.out.println("squareRoot");
-        return (int) Math.sqrt(num1);
-    }
-}`

packages/core/src/amazonq/lsp/lspController.ts

@@ -68,7 +68,7 @@ export interface Manifest {
 }
 const manifestUrl = 'https://aws-toolkit-language-servers.amazonaws.com/q-context/manifest.json'
 // this LSP client in Q extension is only going to work with these LSP server versions
-const supportedLspServerVersions = ['0.1.9']
+const supportedLspServerVersions = ['0.1.13']
 
 const nodeBinName = process.platform === 'win32' ? 'node.exe' : 'node'
 /*
@@ -285,7 +285,7 @@ export class LspController {
         const resp: RelevantTextDocument[] = []
         chunks?.forEach((chunk) => {
             const text = chunk.context ? chunk.context : chunk.content
-            if (chunk.programmingLanguage) {
+            if (chunk.programmingLanguage && chunk.programmingLanguage !== 'unknown') {
                 resp.push({
                     text: text,
                     relativeFilePath: chunk.relativePath ? chunk.relativePath : path.basename(chunk.filePath),

packages/core/src/codewhisperer/models/constants.ts

@@ -22,6 +22,20 @@ export const AWSTemplateKeyWords = ['AWSTemplateFormatVersion', 'Resources', 'AW
 
 export const AWSTemplateCaseInsensitiveKeyWords = ['cloudformation', 'cfn', 'template', 'description']
 
+export const JsonConfigFileNamingConvention = new Set([
+    'app.json',
+    'appsettings.json',
+    'bower.json',
+    'composer.json',
+    'db.json',
+    'manifest.json',
+    'package.json',
+    'schema.json',
+    'settings.json',
+    'tsconfig.json',
+    'vcpkg.json',
+])
+
 export const normalTextChangeRegex = /[A-Za-z0-9]/g
 
 export const autoSuggestionConfig = {
@@ -697,7 +711,7 @@ export const runSecurityScanButtonTitle = 'Run security scan'
 export const crossFileContextConfig = {
     numberOfChunkToFetch: 60,
     topK: 3,
-    numberOfLinesEachChunk: 10,
+    numberOfLinesEachChunk: 50,
 }
 
 export const utgConfig = {

packages/core/src/codewhisperer/util/commonUtil.ts

@@ -7,7 +7,11 @@ import * as vscode from 'vscode'
 import * as semver from 'semver'
 import { isCloud9 } from '../../shared/extensionUtilities'
 import { getInlineSuggestEnabled } from '../../shared/utilities/editorUtilities'
-import { AWSTemplateCaseInsensitiveKeyWords, AWSTemplateKeyWords } from '../models/constants'
+import {
+    AWSTemplateCaseInsensitiveKeyWords,
+    AWSTemplateKeyWords,
+    JsonConfigFileNamingConvention,
+} from '../models/constants'
 
 export function getLocalDatetime() {
     const timezone = Intl.DateTimeFormat().resolvedOptions().timeZone
@@ -61,11 +65,12 @@ export function getPrefixSuffixOverlap(firstString: string, secondString: string
     return secondString.slice(0, i)
 }
 
-export function checkLeftContextKeywordsForJsonAndYaml(leftFileContent: string, language: string): boolean {
+export function checkLeftContextKeywordsForJson(fileName: string, leftFileContent: string, language: string): boolean {
     if (
-        (language === 'json' || language === 'yaml') &&
+        language === 'json' &&
         !AWSTemplateKeyWords.some((substring) => leftFileContent.includes(substring)) &&
-        !AWSTemplateCaseInsensitiveKeyWords.some((substring) => leftFileContent.toLowerCase().includes(substring))
+        !AWSTemplateCaseInsensitiveKeyWords.some((substring) => leftFileContent.toLowerCase().includes(substring)) &&
+        !JsonConfigFileNamingConvention.has(fileName.toLowerCase())
     ) {
         return true
     }

packages/core/src/codewhisperer/util/editorContext.ts

@@ -14,7 +14,7 @@ import { fetchSupplementalContext } from './supplementalContext/supplementalCont
 import { supplementalContextTimeoutInMs } from '../models/constants'
 import { getSelectedCustomization } from './customizationUtil'
 import { selectFrom } from '../../shared/utilities/tsUtils'
-import { checkLeftContextKeywordsForJsonAndYaml } from './commonUtil'
+import { checkLeftContextKeywordsForJson } from './commonUtil'
 import { CodeWhispererSupplementalContext } from '../models/model'
 import { getOptOutPreference } from '../../shared/telemetry/util'
 
@@ -39,7 +39,7 @@ export function extractContextForCodeWhisperer(editor: vscode.TextEditor): codew
         )
     )
     let languageName = 'plaintext'
-    if (!checkLeftContextKeywordsForJsonAndYaml(caretLeftFileContext, editor.document.languageId)) {
+    if (!checkLeftContextKeywordsForJson(document.fileName, caretLeftFileContext, editor.document.languageId)) {
         languageName =
             runtimeLanguageContext.normalizeLanguage(editor.document.languageId) ?? editor.document.languageId
     }

packages/core/src/codewhisperer/util/supplementalContext/crossFileContextUtil.ts

@@ -164,7 +164,7 @@ function shouldFetchCrossFileContext(languageId: vscode.TextDocument['languageId
  * when a given chunk context passes the match in BM25.
  * Special handling is needed for last(its next points to its own) and first chunk
  */
-function linkChunks(chunks: Chunk[]) {
+export function linkChunks(chunks: Chunk[]) {
     const updatedChunks: Chunk[] = []
 
     // This additional chunk is needed to create a next pointer to chunk 0.

packages/core/src/test/codewhisperer/testUtil.ts

@@ -207,3 +207,12 @@ export function createMockDirentFile(fileName: string): Dirent {
     dirent.name = fileName
     return dirent
 }
+
+export function aStringWithLineCount(lineCount: number, start: number = 0): string {
+    let s = ''
+    for (let i = start; i < start + lineCount; i++) {
+        s += `line${i}\n`
+    }
+
+    return s.trimEnd()
+}