Merge branch 'master' into pomParse

Author: dhasani23

.github/workflows/node.js.yml

@@ -21,6 +21,8 @@ jobs:
         env:
             VSCODE_TEST_VERSION: ${{ matrix.vscode-version }}
             NODE_OPTIONS: '--max-old-space-size=8192'
+            AWS_TOOLKIT_TEST_CACHE_DIR: '/tmp/.vscode-test/'
+            AWS_TOOLKIT_TEST_USER_DIR: '/tmp/.vscode-test/user-data/'
         steps:
             - uses: actions/checkout@v4
             - name: Use Node.js ${{ matrix.node-version }}
@@ -32,16 +34,38 @@ jobs:
               uses: coactions/setup-xvfb@v1
               with:
                   run: npm test
-            - name: Code coverage
+            - name: Code coverage (Core)
               env:
                   # Unset NODE_OPTIONS because of https://github.com/codecov/uploader/issues/475
                   NODE_OPTIONS: ''
               if: ${{ github.repository == 'aws/aws-toolkit-vscode' && ( github.ref == 'master' || github.event_name == 'pull_request' ) }}
               uses: codecov/codecov-action@v4
               with:
-                  flags: macos-unittests
+                  flags: macos-core-unittests
                   verbose: true
-                  file: ./coverage/lcov.info
+                  file: ./coverage/core/lcov.info
+                  token: ${{ secrets.CODECOV_TOKEN }}
+            - name: Code coverage (Toolkit)
+              env:
+                  # Unset NODE_OPTIONS because of https://github.com/codecov/uploader/issues/475
+                  NODE_OPTIONS: ''
+              if: ${{ github.repository == 'aws/aws-toolkit-vscode' && ( github.ref == 'master' || github.event_name == 'pull_request' ) }}
+              uses: codecov/codecov-action@v4
+              with:
+                  flags: macos-toolkit-unittests
+                  verbose: true
+                  file: ./coverage/toolkit/lcov.info
+                  token: ${{ secrets.CODECOV_TOKEN }}
+            - name: Code coverage (Amazon Q)
+              env:
+                  # Unset NODE_OPTIONS because of https://github.com/codecov/uploader/issues/475
+                  NODE_OPTIONS: ''
+              if: ${{ github.repository == 'aws/aws-toolkit-vscode' && ( github.ref == 'master' || github.event_name == 'pull_request' ) }}
+              uses: codecov/codecov-action@v4
+              with:
+                  flags: macos-amazonq-unittests
+                  verbose: true
+                  file: ./coverage/amazonq/lcov.info
                   token: ${{ secrets.CODECOV_TOKEN }}
             - name: Code coverage (CodeWhisperer)
               env:
@@ -52,7 +76,7 @@ jobs:
               with:
                   flags: codewhisperer
                   verbose: true
-                  file: ./coverage/lcov.info
+                  file: ./coverage/core/lcov.info
                   token: ${{ secrets.CODECOV_TOKEN }}
 
     web:
@@ -66,6 +90,8 @@ jobs:
         env:
             VSCODE_TEST_VERSION: ${{ matrix.vscode-version }}
             NODE_OPTIONS: '--max-old-space-size=8192'
+            AWS_TOOLKIT_TEST_CACHE_DIR: '/tmp/.vscode-test/'
+            AWS_TOOLKIT_TEST_USER_DIR: '/tmp/.vscode-test/user-data/'
         steps:
             - uses: actions/checkout@v4
             - name: Use Node.js ${{ matrix.node-version }}

.gitignore

@@ -20,11 +20,10 @@ __pycache__
 
 # Auto generated definitions
 packages/*/src/**/*.gen.ts
+!packages/core/src/**/awsSamDebugConfiguration.gen.ts
+!packages/core/src/shared/settings-*.gen.ts
 src.gen/*
 
-# Telemetry definition for testing adding telemetry
-packages/core/src/shared/telemetry/vscodeTelemetry.json
-
 # Test reports
 .test-reports
 

CONTRIBUTING.md

@@ -132,7 +132,7 @@ You can also use these NPM tasks (see `npm run` for the full list):
 
     1. Declare a global unhandledRejection handler.
         ```ts
-        process.on('unhandledRejection', e => {
+        process.on('unhandledRejection', (e) => {
             getLogger('channel').error(
                 localize(
                     'AWS.channel.aws.toolkit.activation.error',
@@ -218,12 +218,7 @@ To run tests against a specific folder in VSCode, do any one of:
 
 ### Coverage report
 
-You can find the coverage report at `./coverage/index.html` after running the tests. Tests ran from the workspace launch config won't generate a coverage report automatically because it can break file watching. A few manual steps are needed instead:
-
--   Run the command `Tasks: Run Build Task` if not already active
--   Instrument built code with `npm run instrument`
--   Exercise the code (`Extension Tests`, `Integration Tests`, etc.)
--   Generate a report with `npm run report`
+You can find the coverage report at `./coverage/amazonq/lcov-report/index.html` and `./coverage/core/lcov-report/index.html` after running the tests. Tests ran from the workspace launch config won't generate a coverage report automatically because it can break file watching.
 
 ### CodeCatalyst Blueprints
 
@@ -554,7 +549,7 @@ For extensions to contribute their own codicons, VSCode requires a font file as
 As a simple example, let's say I wanted to add a new icon for CloudWatch log streams. I would do the following:
 
 1. Place the icon in `resources/icons/aws/cloudwatch`. I'l name the icon `log-stream.svg`.
-1. Use `npm run generatePackage` to update `package.json`. Commit this change with the new icon.
+1. Use `npm run generateIcons` to update `package.json`. Commit this change with the new icon.
 1. You can now use the icon in the Toolkit:
 
     ```ts

buildspec/linuxE2ETests.yml

@@ -41,7 +41,7 @@ phases:
             - VCS_COMMIT_ID="${CODEBUILD_RESOLVED_SOURCE_VERSION}"
             - CI_BUILD_URL=$(echo $CODEBUILD_BUILD_URL | sed 's/#/%23/g')
             - CI_BUILD_ID="${CODEBUILD_BUILD_ID}"
-            - test -n "${CODECOV_TOKEN}" && ./codecov --token=${CODECOV_TOKEN} --branch=${CODEBUILD_RESOLVED_SOURCE_VERSION} --repository=${CODEBUILD_SOURCE_REPO_URL} --file=./coverage/lcov.info
+            - test -n "${CODECOV_TOKEN}" && ./codecov --token=${CODECOV_TOKEN} --branch=${CODEBUILD_RESOLVED_SOURCE_VERSION} --repository=${CODEBUILD_SOURCE_REPO_URL} --file=./coverage/core/lcov.info --file=./coverage/amazonq/lcov.info --file=./coverage/toolkit/lcov.info
         finally:
             - rm -rf ~/.aws/sso/cache || true
 reports:

buildspec/linuxIntegrationTests.yml

@@ -75,7 +75,7 @@ phases:
             - VCS_COMMIT_ID="${CODEBUILD_RESOLVED_SOURCE_VERSION}"
             - CI_BUILD_URL=$(echo $CODEBUILD_BUILD_URL | sed 's/#/%23/g')
             - CI_BUILD_ID="${CODEBUILD_BUILD_ID}"
-            - test -n "${CODECOV_TOKEN}" && ./codecov --token=${CODECOV_TOKEN} --branch=${CODEBUILD_RESOLVED_SOURCE_VERSION} --repository=${CODEBUILD_SOURCE_REPO_URL} --file=./coverage/lcov.info
+            - test -n "${CODECOV_TOKEN}" && ./codecov --token=${CODECOV_TOKEN} --branch=${CODEBUILD_RESOLVED_SOURCE_VERSION} --repository=${CODEBUILD_SOURCE_REPO_URL} --file=./coverage/core/lcov.info --file=./coverage/amazonq/lcov.info --file=./coverage/toolkit/lcov.info
     post_build:
         commands:
             # Destroy .netrc to avoid leaking $GITHUB_READONLY_TOKEN.

buildspec/linuxTests.yml

@@ -48,7 +48,7 @@ phases:
             - VCS_COMMIT_ID="${CODEBUILD_RESOLVED_SOURCE_VERSION}"
             - CI_BUILD_URL=$(echo $CODEBUILD_BUILD_URL | sed 's/#/%23/g') # Encode `#` in the URL because otherwise the url is clipped in the Codecov.io site
             - CI_BUILD_ID="${CODEBUILD_BUILD_ID}"
-            - test -n "${CODECOV_TOKEN}" && ./codecov --token=${CODECOV_TOKEN} --branch=${CODEBUILD_RESOLVED_SOURCE_VERSION} --repository=${CODEBUILD_SOURCE_REPO_URL} --file=./coverage/lcov.info
+            - test -n "${CODECOV_TOKEN}" && ./codecov --token=${CODECOV_TOKEN} --branch=${CODEBUILD_RESOLVED_SOURCE_VERSION} --repository=${CODEBUILD_SOURCE_REPO_URL} --file=./coverage/core/lcov.info --file=./coverage/amazonq/lcov.info --file=./coverage/toolkit/lcov.info
 
 reports:
     unit-test:

codecov.yml

@@ -18,15 +18,30 @@ coverage:
             codewhisperer:
                 target: 70%
                 paths:
-                    - src/codewhisperer/*
+                    - packages/core/src/codewhisperer/*
                 flags:
                     - 'codewhisperer'
             amazonqFeatureDev:
                 target: 70%
                 paths:
-                    - src/amazonqFeatureDev/*
+                    - packages/core/src/amazonqFeatureDev/*
                 flags:
                     - 'amazonqFeatureDev'
+            amazonqGumby:
+                paths:
+                    - packages/core/src/amazonqGumby/*
+            codewhispererChat:
+                paths:
+                    - packages/core/src/codewhispererChat/*
+            applicationcomposer:
+                paths:
+                    - packages/core/src/applicationcomposer/*
+            stepFunctions:
+                paths:
+                    - packages/core/src/stepFunctions/*
+            threatComposer:
+                paths:
+                    - packages/core/src/threatComposer/*
         patch: no
         changes: no
 

docs/TESTPLAN.md

@@ -104,7 +104,7 @@ Checking the state works well if user interactions are not required by the code
 To handle this, test code can register event handler that listen for when a certain type of UI element is shown. For example, if we wanted to always accept the first item of a quick pick we can do this:
 
 ```ts
-getTestWindow().onDidShowQuickPick(async picker => {
+getTestWindow().onDidShowQuickPick(async (picker) => {
     // Some pickers load items asychronously
     // Wait until the picker is not busy before accepting an item
     await picker.untilReady()
@@ -121,3 +121,9 @@ const secondPicker = await pickers.next()
 ```
 
 Exceptions thrown within one of these handlers will cause the current test to fail. This allows you to make assertions within the callback without worrying about causing the test to hang.
+
+## Common issues
+
+### Stubbing VSCode outside of core
+
+-   Stubbing VSCode imports (like executeCommand) does not work outside of core. For now you will need to put any tests that require spying/stubbing VSCode imports in core until we move more source files into the amazon q package

docs/arch_develop.md

@@ -37,8 +37,7 @@ Current quirks of the current monorepo status that should be resolved/evaluated
     -   This package contains shortcuts to some of the `npm` scripts found in the subproject(s).
     -   `createRelease` and `newChange` run at the subproject level only, e.g. from root level, try npm run createRelease -w packages/toolkit
     -   To run a script not present in the root `package.json`, use `npm run -w packages/toolkit <script>`
--   `coverage/`, `.test-reports/`, `node_modules/` are hoisted to the project root. As more subprojects are added,
-    we will need to evaluate how to merge and publish coverage reports.
+-   `coverage/`, `.test-reports/`, `node_modules/` are hoisted to the project root.
     -   `dist/` however remains at the subproject level, along with a local `node_modules/`. See [`npm workspaces`](https://docs.npmjs.com/cli/v8/using-npm/workspaces)
         for more info on how `node_modules/` hoisting works.
     -   Because of `node_modules/` hoisting, references to this folder in code access the root project modules folder. This may be
@@ -72,39 +71,37 @@ Some components of the core library depend on the `package.json`s of the extensi
     -   Does not restore, it is a superset of what exists in `packages/core` for `configuration.properties`.
     -   To develop for the Amazon Q extension: add all changes to `packages/amazonq/package.json`, EXCEPT for settings that are references by code in the core library, or settings that already exist in the core `package.json`
 
-## Shared vs Common names
+If you are modifying or registering new debuggers in VS Code via the `debuggers` contribution point, you may need to regenerate the [definitions file](../packages/core/src/shared/sam/debugger/awsSamDebugConfiguration.gen.ts). After updating ['toolkit/package.json'](../packages/toolkit/package.json), run `npm run generateConfigurationAttributes -w packages/toolkit`
 
-In this repo, the keywords **"shared"** and **"common"** have specific meanings in the context of file/folder names.
+## `web`, `node`, `common`, `shared` naming conventions
 
-### "common"
+This project can run in different environments, eg Web mode (in the browser with no compute backend), or in Node.js on your desktop (the most common way).
+A problem arises when we use code that is exclusive to one environment, an example being Node.js' Filesystem module which will fail in Web mode.
 
-Code within a folder/file that has the "common" keyword implies that it can run in any environment. Examples of environments are: Web mode, or Node.js (local desktop)
+To ensure developers use compatible code for their environment we have subfolders in each topic which contains environment specific code in a single place.
 
-We need this distinction since not all code can run in any environment. A common example is filesystem code, where the actual implementation used could work in Node.js but not in Web mode.
+Using this file tree as reference, here are the rules:
 
-### "shared"
-
-Code within a folder/file that has the "shared" keyword implies that it is intended to be reused wherever it can be. This is generalized code that "Feature A" or "Feature B" could use if it works for their use case.
-
-An example is the `waitUntil()` function which continuously invokes an arbitrary callback function until it succeeds.
-
-> NOTE: Something that is "shared" does not mean it is "common", as it could be reused in different places but only work in Node.js for example.
+```
+src/
+├── myTopic/
+│   ├── {file}.ts
+│   ├── node/
+│   │   └── {file}.ts
+│   └── web/
+│       └── {file}.ts
+└── shared/
+```
 
-### How to apply this
+-   `myTopic/` is the general name of the folder, eg `request` for http requests.
+-   `myTopic/{file}.ts` is for code that works in any environment, we refer to this as `"common"` code.
+-   `node/{file}.ts` is for code that works exclusively in Node.js.
+-   `web/{file}.ts` is for code that works exclusively in Web mode.
+-   `shared/` is for code that is intended to be reused, i.e general purpose utils.
+    -   Note environment specific code should be place in to a `web/` or `node/` subfolder.
+    -   If the code is not in a subfolder then it is considered `shared common` code.
 
--   Aim to make code compatible with "common" from the beginning.
--   In a "topic" folder, if you have common code, create a subfolder named "common" and add your common code to there.
-    ```
-    src/
-      |
-      myTopic/
-        |
-        common/
-        nonCommon.ts
-    ```
--   See if yours, or existing code can be moved in to a "shared" folder. Maybe it can be easily modified to become "shared".
--   If there is no "shared" or "common" naming used for the file/folder, then assume it only works in Node.js.
--   In the rare case your code only works in Web mode, create a `web` subfolder for that code.
+> IMPORTANT: The current codebase does not fully follow this convention yet, the transition is being done incrementally. Due to this, code that is `"common"` may not actually be common yet. If you run in to this, please move that code to the appropriate subfolder.
 
 ## Commands
 
@@ -293,22 +290,22 @@ Commands and events are defined on the backend via sub-classes of `VueWebview`.
     ```ts
     client
         .foo()
-        .then(response => console.log(response))
-        .catch(err => console.log(err))
+        .then((response) => console.log(response))
+        .catch((err) => console.log(err))
     ```
 
     The backend protocol is allowed to throw errors. These result in rejected Promises on the frontend.
 
 -   Registering for events:
 
     ```ts
-    client.onBar(num => console.log(num))
+    client.onBar((num) => console.log(num))
     ```
 
 -   Methods called `init` will only return data on the initial webview load:
 
     ```ts
-    client.init(data => (this.data = data ?? this.data))
+    client.init((data) => (this.data = data ?? this.data))
     ```
 
 ## Webviews (non Vue)
@@ -494,8 +491,8 @@ class ExampleWizard extends Wizard<ExampleState> {
             { label: '1', data: 1 },
             { label: '2', data: 2 },
         ]
-        this.form.bar.bindPrompter(state => createQuickPick(items, { title: `Select a number (${state.foo})` }), {
-            showWhen: state => state.foo?.length > 5,
+        this.form.bar.bindPrompter((state) => createQuickPick(items, { title: `Select a number (${state.foo})` }), {
+            showWhen: (state) => state.foo?.length > 5,
         })
     }
 }

docs/icons.md

@@ -1,15 +1,17 @@
 # Icons
 
-All icons that are used in the Toolkit can be found in `resources/icons`.
+All icons that are used in the extensions can be found in `resources/icons`.
 
-A [build script](../../scripts/build/generateIcons.ts) generates Toolkit artifacts:
+A [build script](../scripts/generateIcons.ts) generates extension artifacts in [core/](../packages/core/):
 
 -   `resources/icons/cloud9/generated`
 -   `resources/fonts/aws-toolkit-icons.woff`
 -   `resources/css/icons.css`
--   `contributes.icons` in [package.json](../../package.json)
+-   `contributes.icons` in [amazonq package.json](../packages/amazonq/package.json) and [toolkit package.json](../packages/toolkit/package.json)
 
-This script should be ran using `npm run generatePackage` after making updates. Any changes made to `package.json` should be committed with the relevant icons.
+This script should be ran using `npm run generateIcons` after making updates. Any changes made to `package.json` should be committed with the relevant icons. Type checking in `core/` relies on the entries in `core/package.json`. However, the individual extensions require entries in their `package.json`s as well. Currently, resources (including icons) are shared between `core/` and the individual extensions. If `contributes.icons` in each of the extensions does not match the entry in `core/`, then CI will fail.
+
+To sync the icons to the individual extensions, run `npm run copyFiles && npm run generateIcons` for each extension.
 
 ## Fonts