Merge master into feature/q-inline-crossfile

.eslintrc.js

@@ -180,6 +180,10 @@ module.exports = {
                         message:
                             'Avoid fs-extra, use shared/fs/fs.ts. Notify the Toolkit team if your required functionality is not available.',
                     },
+                    {
+                        name: 'fs',
+                        message: 'Avoid node:fs and use shared/fs/fs.ts when possible.',
+                    },
                 ],
             },
         ],

.github/workflows/jscpd.json

@@ -0,0 +1,7 @@
+{
+    "pattern": "packages/**/*.ts",
+    "ignore": ["**node_modules**", "**dist**"],
+    "gitignore": true,
+    "threshold": 1.34,
+    "minLines": 15
+}

.github/workflows/node.js.yml

@@ -41,7 +41,7 @@ jobs:
         runs-on: ubuntu-latest
         strategy:
             matrix:
-                node-version: [16.x]
+                node-version: [18.x]
                 vscode-version: [stable]
         env:
             NODE_OPTIONS: '--max-old-space-size=8192'
@@ -55,14 +55,32 @@ jobs:
             - run: npm run testCompile
             - run: npm run lint
 
+    jscpd:
+        needs: lint-commits
+        runs-on: ubuntu-latest
+        strategy:
+            matrix:
+                node-version: [18.x]
+        env:
+            NODE_OPTIONS: '--max-old-space-size=8192'
+        steps:
+            - uses: actions/checkout@v4
+            - name: Use Node.js ${{ matrix.node-version }}
+              uses: actions/setup-node@v4
+              with:
+                  node-version: ${{ matrix.node-version }}
+            - run: npm install jscpd
+            - name: Run jscpd
+              run: npx jscpd --config "$GITHUB_WORKSPACE/.github/workflows/jscpd.json"
+
     macos:
         needs: lint-commits
         name: test macOS
         runs-on: macos-latest
         strategy:
             fail-fast: false
             matrix:
-                node-version: [16.x]
+                node-version: [18.x]
                 vscode-version: [minimum, stable, insiders]
         env:
             VSCODE_TEST_VERSION: ${{ matrix.vscode-version }}
@@ -80,17 +98,6 @@ jobs:
               uses: coactions/setup-xvfb@v1
               with:
                   run: npm test
-            - name: Code coverage (Core)
-              env:
-                  # Unset NODE_OPTIONS because of https://github.com/codecov/uploader/issues/475
-                  NODE_OPTIONS: ''
-              if: ${{ github.repository == 'aws/aws-toolkit-vscode' && github.ref == 'master' }}
-              uses: codecov/codecov-action@v4
-              with:
-                  flags: macos-core-unittests
-                  verbose: true
-                  file: ./coverage/core/lcov.info
-                  token: ${{ secrets.CODECOV_TOKEN }}
             - name: Code coverage (Toolkit)
               env:
                   # Unset NODE_OPTIONS because of https://github.com/codecov/uploader/issues/475
@@ -113,17 +120,6 @@ jobs:
                   verbose: true
                   file: ./coverage/amazonq/lcov.info
                   token: ${{ secrets.CODECOV_TOKEN }}
-            - name: Code coverage (CodeWhisperer)
-              env:
-                  # Unset NODE_OPTIONS because of https://github.com/codecov/uploader/issues/475
-                  NODE_OPTIONS: ''
-              if: ${{ github.repository == 'aws/aws-toolkit-vscode' && github.ref == 'master' }}
-              uses: codecov/codecov-action@v4
-              with:
-                  flags: codewhisperer
-                  verbose: true
-                  file: ./coverage/core/lcov.info
-                  token: ${{ secrets.CODECOV_TOKEN }}
 
     web:
         needs: lint-commits
@@ -132,7 +128,7 @@ jobs:
         strategy:
             fail-fast: true
             matrix:
-                node-version: [16.x]
+                node-version: [18.x]
                 vscode-version: [stable, insiders]
         env:
             VSCODE_TEST_VERSION: ${{ matrix.vscode-version }}
@@ -158,7 +154,7 @@ jobs:
         strategy:
             fail-fast: false
             matrix:
-                node-version: [16.x]
+                node-version: [18.x]
                 vscode-version: [stable, insiders]
         env:
             VSCODE_TEST_VERSION: ${{ matrix.vscode-version }}

.github/workflows/notification.yml

@@ -37,7 +37,7 @@ jobs:
               if: github.event_name == 'pull_request_target'
               with:
                   node-version: '20'
-            - name: Check for tests
+            - name: Comment about contribution guidelines
               uses: actions/github-script@v7
               if: github.event_name == 'pull_request_target'
               with:

.github/workflows/notify.js

@@ -6,9 +6,10 @@
 const { hasPath, dedupComment } = require('./utils')
 
 const testFilesMessage =
-    'This pull request modifies code in src/ but no tests were added/updated. Confirm whether tests should be added or ensure the PR description explains why tests are not required.'
+    '- This pull request modifies code in `src/*` but no tests were added/updated.\n    - Confirm whether tests should be added or ensure the PR description explains why tests are not required.\n'
 
-const changelogMessage = `This pull request implements a feature or fix, so it must include a changelog entry. See [CONTRIBUTING.md#changelog](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#changelog) for instructions.`
+const changelogMessage =
+    '- This pull request implements a `feat` or `fix`, so it must include a changelog entry (unless the fix is for an *unreleased* feature). Review the [changelog guidelines](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#changelog).\n    - Note: beta or "experiment" features that have active users *should* announce fixes in the changelog.\n    - If this is not a feature or fix, use an appropriate type from the [title guidelines](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#pull-request-title). For example, telemetry-only changes should use the `telemetry` type.\n'
 
 /**
  * Remind partner teams that tests are required. We don't need to remind them if:
@@ -44,12 +45,16 @@ module.exports = async ({ github, context }) => {
         issue_number: pullRequestId,
     })
 
+    let msg = ''
     if (shouldAddTestFileMessage) {
-        await dedupComment({ github, comments, owner, repo, pullRequestId, message: testFilesMessage })
+        msg += testFilesMessage
     }
-
     if (shouldAddChangelogMessage) {
-        await dedupComment({ github, comments, owner, repo, pullRequestId, message: changelogMessage })
+        msg += changelogMessage
+    }
+
+    if (shouldAddTestFileMessage || shouldAddChangelogMessage) {
+        await dedupComment({ github, comments, owner, repo, pullRequestId, message: msg })
     }
 }
 
@@ -69,7 +74,12 @@ function requiresChangelogMessage(filenames, title) {
  * Require the test files message if there are changes to source files but aren't any
  * changes to the test files
  */
-function requiresTestFilesMessage(filenames) {
+function requiresTestFilesMessage(filenames, title) {
+    if (/^\s*[mM]erge/.test(title)) {
+        console.log('"Merge" pull request')
+        return
+    }
+
     // Check if src directory changed
     if (!hasPath(filenames, 'src/')) {
         console.log('Did not find src files in the code changes')

CONTRIBUTING.md

@@ -188,11 +188,11 @@ To run a single test in VSCode, do any one of:
 
     -   Unix/macOS/POSIX shell:
         ```
-        TEST_FILE=src/test/foo.test.ts npm run test
+        TEST_FILE=../core/src/test/foo.test.ts npm run test
         ```
     -   Powershell:
         ```
-        $Env:TEST_FILE = "src/test/foo.test.ts"; npm run test
+        $Env:TEST_FILE = "../core/src/test/foo.test.ts"; npm run test
         ```
 
 -   To run all tests in a particular subdirectory, you can edit
@@ -209,16 +209,16 @@ To run tests against a specific folder in VSCode, do any one of:
 -   Run in your terminal
     -   Unix/macOS/POSIX shell:
         ```
-        TEST_DIR=src/test/foo npm run test
+        TEST_DIR=../core/src/test/foo npm run test
         ```
     -   Powershell:
         ```
-        $Env:TEST_DIR = "src/test/foo"; npm run test
+        $Env:TEST_DIR = "../core/src/test/foo"; npm run test
         ```
 
 ### Coverage report
 
-You can find the coverage report at `./coverage/amazonq/lcov-report/index.html` and `./coverage/core/lcov-report/index.html` after running the tests. Tests ran from the workspace launch config won't generate a coverage report automatically because it can break file watching.
+You can find the coverage report at `./coverage/amazonq/lcov-report/index.html` and `./coverage/toolkit/lcov-report/index.html` after running the tests. Tests ran from the workspace launch config won't generate a coverage report automatically because it can break file watching.
 
 ### CodeCatalyst Blueprints
 
@@ -345,14 +345,16 @@ The `aws.dev.forceDevMode` setting enables or disables Toolkit "dev mode". Witho
     -   Example: `getLogger().error('topic: widget failed: %O', { foo: 'bar', baz: 42 })`
 -   Log messages are written to the extension Output channel, which you can view in vscode by visiting the "Output" panel and selecting `AWS Toolkit Logs` or `Amazon Q Logs`.
 -   Use the `aws.dev.logfile` setting to set the logfile path to a fixed location, so you can follow
-    and filter logs using shell tools like `tail` and `grep`. For example in settings.json,
-    ```
-    "aws.dev.logfile": "~/awstoolkit.log",
-    ```
-    then you can tail the logfile in your terminal:
-    ```
-    tail -F ~/awstoolkit.log
-    ```
+    and filter logs using shell tools like `tail` and `grep`.
+    -   Note: this always logs at **debug log-level** (though you can temporarily override that from the `AWS Toolkit Logs` UI).
+    -   Example `settings.json`:
+        ```
+        "aws.dev.logfile": "~/awstoolkit.log",
+        ```
+        then you can tail the logfile in your terminal:
+        ```
+        tail -F ~/awstoolkit.log
+        ```
 -   Use the `AWS (Developer): Watch Logs` command to watch and filter Toolkit logs (including
     telemetry) in VSCode.
     -   Only available if you enabled "dev mode" (`aws.dev.forceDevMode` setting, see above).

buildspec/linuxE2ETests.yml

@@ -41,7 +41,7 @@ phases:
             - VCS_COMMIT_ID="${CODEBUILD_RESOLVED_SOURCE_VERSION}"
             - CI_BUILD_URL=$(echo $CODEBUILD_BUILD_URL | sed 's/#/%23/g')
             - CI_BUILD_ID="${CODEBUILD_BUILD_ID}"
-            - test -n "${CODECOV_TOKEN}" && [ "$TARGET_BRANCH" = "master" ] && ./codecov --token=${CODECOV_TOKEN} --branch=${CODEBUILD_RESOLVED_SOURCE_VERSION} --repository=${CODEBUILD_SOURCE_REPO_URL} --file=./coverage/core/lcov.info --file=./coverage/amazonq/lcov.info --file=./coverage/toolkit/lcov.info
+            - test -n "${CODECOV_TOKEN}" && [ "$TARGET_BRANCH" = "master" ] && ./codecov --token=${CODECOV_TOKEN} --branch=${CODEBUILD_RESOLVED_SOURCE_VERSION} --repository=${CODEBUILD_SOURCE_REPO_URL} --file=./coverage/amazonq/lcov.info --file=./coverage/toolkit/lcov.info
         finally:
             - rm -rf ~/.aws/sso/cache || true
 reports:

buildspec/linuxIntegrationTests.yml

@@ -21,7 +21,7 @@ phases:
     install:
         run-as: root
         runtime-versions:
-            nodejs: 16
+            nodejs: 18
             dotnet: 6.0
             java: latest
 
@@ -75,7 +75,7 @@ phases:
             - VCS_COMMIT_ID="${CODEBUILD_RESOLVED_SOURCE_VERSION}"
             - CI_BUILD_URL=$(echo $CODEBUILD_BUILD_URL | sed 's/#/%23/g')
             - CI_BUILD_ID="${CODEBUILD_BUILD_ID}"
-            - test -n "${CODECOV_TOKEN}" && [ "$TARGET_BRANCH" = "master" ] && ./codecov --token=${CODECOV_TOKEN} --branch=${CODEBUILD_RESOLVED_SOURCE_VERSION} --repository=${CODEBUILD_SOURCE_REPO_URL} --file=./coverage/core/lcov.info --file=./coverage/amazonq/lcov.info --file=./coverage/toolkit/lcov.info
+            - test -n "${CODECOV_TOKEN}" && [ "$TARGET_BRANCH" = "master" ] && ./codecov --token=${CODECOV_TOKEN} --branch=${CODEBUILD_RESOLVED_SOURCE_VERSION} --repository=${CODEBUILD_SOURCE_REPO_URL} --file=./coverage/amazonq/lcov.info --file=./coverage/toolkit/lcov.info
     post_build:
         commands:
             # Destroy .netrc to avoid leaking $GITHUB_READONLY_TOKEN.

buildspec/linuxTests.yml

@@ -18,7 +18,7 @@ phases:
     install:
         run-as: root
         runtime-versions:
-            nodejs: 16
+            nodejs: 18
         commands:
             - bash buildspec/shared/linux-install.sh
 
@@ -48,7 +48,7 @@ phases:
             - VCS_COMMIT_ID="${CODEBUILD_RESOLVED_SOURCE_VERSION}"
             - CI_BUILD_URL=$(echo $CODEBUILD_BUILD_URL | sed 's/#/%23/g') # Encode `#` in the URL because otherwise the url is clipped in the Codecov.io site
             - CI_BUILD_ID="${CODEBUILD_BUILD_ID}"
-            - test -n "${CODECOV_TOKEN}" && [ "$TARGET_BRANCH" = "master" ] && ./codecov --token=${CODECOV_TOKEN} --branch=${CODEBUILD_RESOLVED_SOURCE_VERSION} --repository=${CODEBUILD_SOURCE_REPO_URL} --file=./coverage/core/lcov.info --file=./coverage/amazonq/lcov.info --file=./coverage/toolkit/lcov.info
+            - test -n "${CODECOV_TOKEN}" && [ "$TARGET_BRANCH" = "master" ] && ./codecov --token=${CODECOV_TOKEN} --branch=${CODEBUILD_RESOLVED_SOURCE_VERSION} --repository=${CODEBUILD_SOURCE_REPO_URL} --file=./coverage/amazonq/lcov.info --file=./coverage/toolkit/lcov.info
 
 reports:
     unit-test:

buildspec/packageTestVsix.yml

@@ -13,7 +13,7 @@ phases:
     install:
         run-as: root
         runtime-versions:
-            nodejs: 16
+            nodejs: 18
         commands:
             - bash buildspec/shared/linux-install.sh
 

buildspec/shared/common.sh

@@ -9,7 +9,7 @@
 #   - "Webview is disposed": only happens on vscode "minimum" (1.68.0)
 #   - "HTTPError: Response code …": caused by github rate-limiting.
 #   - "npm WARN deprecated querystring": transitive dep of aws sdk v2 (check `npm ls querystring`), so that's blocked until we migrate to v3.
-_ignore_pat='Timed-out waiting for browser login flow\|HTTPError: Response code 403\|HTTPError: Response code 404\|npm WARN deprecated querystring'
+_ignore_pat='Timed-out waiting for browser login flow\|HTTPError: Response code 403\|HTTPError: Response code 404\|npm WARN deprecated querystring\|npm WARN deprecated'
 if [ "$VSCODE_TEST_VERSION" = 'minimum' ]; then
     _ignore_pat="$_ignore_pat"'\|Webview is disposed'
 fi

buildspec/shared/linux-pre_build.sh

@@ -22,5 +22,5 @@ if [ "$TOOLKITS_CODEARTIFACT_DOMAIN" ] && [ "$TOOLKITS_CODEARTIFACT_REPO" ] && [
 fi
 
 # TODO: move this to the "install" phase?
-export NODE_OPTIONS=--max-old-space-size=8192
+export NODE_OPTIONS='--max-old-space-size=8192'
 npm 2>&1 ci | run_and_report 2 'npm WARN deprecated' 'Deprecated dependencies must be updated.'

buildspec/windowsTests.yml

@@ -5,7 +5,7 @@ env:
 phases:
     install:
         runtime-versions:
-            nodejs: 16
+            nodejs: 18
         commands:
             - |
                 if(-Not($Env:CODECOV_TOKEN -eq $null)) {

docs/TESTPLAN.md

@@ -42,6 +42,11 @@ The test suite has the following categories of tests:
     -   Live in `src/testE2E`
     -   These tests are heavier than Integration tests.
     -   Some E2E tests have a more complicated architecture, described in [TEST_E2E](./TEST_E2E.md)
+-   Performance Tests: **slow** tests
+    -   Live in `src/testInteg/perf`.
+    -   A subset of integration tests focused on catching performance regressions.
+    -   Use a combination of operation counting and system usage statistics such as cpu usage, memory usage, and duration.
+    -   Each test is often repeated 10 or more times for less variant system usage statistics, then median of runs is used.
 
 ## Test files
 

docs/build.md

@@ -1,11 +1,23 @@
 # Build
 
-The AmazonQ features rely on the `codewhisperer-streaming` service, who's client
-is generated from the service's smithy models and placed in
-`src.gen/@amzn/codewhisperer-streaming` (For more
-information about this client and how it is generated, please see this
+The AmazonQ features rely on the `codewhisperer-streaming` service, to support both Sigv4 and Bearer token modes of this service,
+two clients are generated from the service's smithy models and placed in
+`src.gen/@amzn/amazon-q-developer-streaming-client` and `src.gen/@amzn/codewhisperer-streaming` respectively (For more
+information about these clients and how they are generated, please see this
 [quip document](https://quip-amazon.com/2dAWAvTIYXXr/Build-instructions-for-AWS-CodeWhisperer-Streaming-Typescript-client)).
 
+## @amzn/amazon-q-developer-streaming client
+
+This client is a standalone npm project in typescript, and it is added to
+the project as a workspace in the project's root `package.json` with the line `"workspaces": [ ..., "src.gen/@amzn/amazon-q-developer-streaming" ]`.
+The client may be manually built using `npm run build -w @amzn/amazon-q-developer-streaming"`.
+The `generateClients` run script ensures that this dependency is
+built before the toolkit project itself. Workspaces are automatically ready to
+be imported in the root toolkit project by their declared package.json name,
+(`@amzn/amazon-q-developer-streaming` in this case).
+
+## @amzn/codewhisperer-streaming client
+
 This client is a standalone npm project in typescript, and it is added to
 the project as a workspace in the project's root `package.json` with the line `"workspaces": [ ..., "src.gen/@amzn/codewhisperer-streaming" ]`.
 The client may be manually built using `npm run build -w @amzn/codewhisperer-streaming"`.