fix: Code coverage report for amazon q (aws#5322)

jpinkney-aws · web-flow · commit 6dbff72de26e · 2024-07-17T12:07:59.000-04:00
* fix: code coverage not reporting for amazon q

Problem:
- Codecoverage for amazon q wasn't actually generating any lcov.info information since the c8rc.json lived in core and amazonq couldn't find it

Solution:
- Make a c8rc file inside of q for configuring c8
    - note: tried hoisting the c8rc from core into the root and sharing it between the two projects but it seems to break file paths
- Add code coverage report for q
- Change code coverage directory for toolkit
- Update codecov codewhisperer/featureDev paths
diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml
@@ -32,16 +32,38 @@ jobs:
               uses: coactions/setup-xvfb@v1
               with:
                   run: npm test
-            - name: Code coverage
+            - name: Code coverage (Core)
               env:
                   # Unset NODE_OPTIONS because of https://github.com/codecov/uploader/issues/475
                   NODE_OPTIONS: ''
               if: ${{ github.repository == 'aws/aws-toolkit-vscode' && ( github.ref == 'master' || github.event_name == 'pull_request' ) }}
               uses: codecov/codecov-action@v4
               with:
-                  flags: macos-unittests
+                  flags: macos-core-unittests
                   verbose: true
-                  file: ./coverage/lcov.info
+                  file: ./coverage/core/lcov.info
+                  token: ${{ secrets.CODECOV_TOKEN }}
+            - name: Code coverage (Toolkit)
+              env:
+                  # Unset NODE_OPTIONS because of https://github.com/codecov/uploader/issues/475
+                  NODE_OPTIONS: ''
+              if: ${{ github.repository == 'aws/aws-toolkit-vscode' && ( github.ref == 'master' || github.event_name == 'pull_request' ) }}
+              uses: codecov/codecov-action@v4
+              with:
+                  flags: macos-toolkit-unittests
+                  verbose: true
+                  file: ./coverage/toolkit/lcov.info
+                  token: ${{ secrets.CODECOV_TOKEN }}
+            - name: Code coverage (Amazon Q)
+              env:
+                  # Unset NODE_OPTIONS because of https://github.com/codecov/uploader/issues/475
+                  NODE_OPTIONS: ''
+              if: ${{ github.repository == 'aws/aws-toolkit-vscode' && ( github.ref == 'master' || github.event_name == 'pull_request' ) }}
+              uses: codecov/codecov-action@v4
+              with:
+                  flags: macos-amazonq-unittests
+                  verbose: true
+                  file: ./coverage/amazonq/lcov.info
                   token: ${{ secrets.CODECOV_TOKEN }}
             - name: Code coverage (CodeWhisperer)
               env:
@@ -52,7 +74,7 @@ jobs:
               with:
                   flags: codewhisperer
                   verbose: true
-                  file: ./coverage/lcov.info
+                  file: ./coverage/core/lcov.info
                   token: ${{ secrets.CODECOV_TOKEN }}
 
     web:
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -218,12 +218,7 @@ To run tests against a specific folder in VSCode, do any one of:
 
 ### Coverage report
 
-You can find the coverage report at `./coverage/index.html` after running the tests. Tests ran from the workspace launch config won't generate a coverage report automatically because it can break file watching. A few manual steps are needed instead:
-
--   Run the command `Tasks: Run Build Task` if not already active
--   Instrument built code with `npm run instrument`
--   Exercise the code (`Extension Tests`, `Integration Tests`, etc.)
--   Generate a report with `npm run report`
+You can find the coverage report at `./coverage/amazonq/lcov-report/index.html` and `./coverage/core/lcov-report/index.html` after running the tests. Tests ran from the workspace launch config won't generate a coverage report automatically because it can break file watching.
 
 ### CodeCatalyst Blueprints
 
diff --git a/buildspec/linuxE2ETests.yml b/buildspec/linuxE2ETests.yml
@@ -41,7 +41,7 @@ phases:
             - VCS_COMMIT_ID="${CODEBUILD_RESOLVED_SOURCE_VERSION}"
             - CI_BUILD_URL=$(echo $CODEBUILD_BUILD_URL | sed 's/#/%23/g')
             - CI_BUILD_ID="${CODEBUILD_BUILD_ID}"
-            - test -n "${CODECOV_TOKEN}" && ./codecov --token=${CODECOV_TOKEN} --branch=${CODEBUILD_RESOLVED_SOURCE_VERSION} --repository=${CODEBUILD_SOURCE_REPO_URL} --file=./coverage/lcov.info
+            - test -n "${CODECOV_TOKEN}" && ./codecov --token=${CODECOV_TOKEN} --branch=${CODEBUILD_RESOLVED_SOURCE_VERSION} --repository=${CODEBUILD_SOURCE_REPO_URL} --file=./coverage/core/lcov.info --file=./coverage/amazonq/lcov.info --file=./coverage/toolkit/lcov.info
         finally:
             - rm -rf ~/.aws/sso/cache || true
 reports:
diff --git a/buildspec/linuxIntegrationTests.yml b/buildspec/linuxIntegrationTests.yml
@@ -75,7 +75,7 @@ phases:
             - VCS_COMMIT_ID="${CODEBUILD_RESOLVED_SOURCE_VERSION}"
             - CI_BUILD_URL=$(echo $CODEBUILD_BUILD_URL | sed 's/#/%23/g')
             - CI_BUILD_ID="${CODEBUILD_BUILD_ID}"
-            - test -n "${CODECOV_TOKEN}" && ./codecov --token=${CODECOV_TOKEN} --branch=${CODEBUILD_RESOLVED_SOURCE_VERSION} --repository=${CODEBUILD_SOURCE_REPO_URL} --file=./coverage/lcov.info
+            - test -n "${CODECOV_TOKEN}" && ./codecov --token=${CODECOV_TOKEN} --branch=${CODEBUILD_RESOLVED_SOURCE_VERSION} --repository=${CODEBUILD_SOURCE_REPO_URL} --file=./coverage/core/lcov.info --file=./coverage/amazonq/lcov.info --file=./coverage/toolkit/lcov.info
     post_build:
         commands:
             # Destroy .netrc to avoid leaking $GITHUB_READONLY_TOKEN.
diff --git a/buildspec/linuxTests.yml b/buildspec/linuxTests.yml
@@ -48,7 +48,7 @@ phases:
             - VCS_COMMIT_ID="${CODEBUILD_RESOLVED_SOURCE_VERSION}"
             - CI_BUILD_URL=$(echo $CODEBUILD_BUILD_URL | sed 's/#/%23/g') # Encode `#` in the URL because otherwise the url is clipped in the Codecov.io site
             - CI_BUILD_ID="${CODEBUILD_BUILD_ID}"
-            - test -n "${CODECOV_TOKEN}" && ./codecov --token=${CODECOV_TOKEN} --branch=${CODEBUILD_RESOLVED_SOURCE_VERSION} --repository=${CODEBUILD_SOURCE_REPO_URL} --file=./coverage/lcov.info
+            - test -n "${CODECOV_TOKEN}" && ./codecov --token=${CODECOV_TOKEN} --branch=${CODEBUILD_RESOLVED_SOURCE_VERSION} --repository=${CODEBUILD_SOURCE_REPO_URL} --file=./coverage/core/lcov.info --file=./coverage/amazonq/lcov.info --file=./coverage/toolkit/lcov.info
 
 reports:
     unit-test:
diff --git a/codecov.yml b/codecov.yml
@@ -18,13 +18,13 @@ coverage:
             codewhisperer:
                 target: 70%
                 paths:
-                    - src/codewhisperer/*
+                    - packages/core/src/codewhisperer/*
                 flags:
                     - 'codewhisperer'
             amazonqFeatureDev:
                 target: 70%
                 paths:
-                    - src/amazonqFeatureDev/*
+                    - packages/core/src/amazonqFeatureDev/*
                 flags:
                     - 'amazonqFeatureDev'
         patch: no
diff --git a/docs/arch_develop.md b/docs/arch_develop.md
@@ -37,8 +37,7 @@ Current quirks of the current monorepo status that should be resolved/evaluated
     -   This package contains shortcuts to some of the `npm` scripts found in the subproject(s).
     -   `createRelease` and `newChange` run at the subproject level only, e.g. from root level, try npm run createRelease -w packages/toolkit
     -   To run a script not present in the root `package.json`, use `npm run -w packages/toolkit <script>`
--   `coverage/`, `.test-reports/`, `node_modules/` are hoisted to the project root. As more subprojects are added,
-    we will need to evaluate how to merge and publish coverage reports.
+-   `coverage/`, `.test-reports/`, `node_modules/` are hoisted to the project root.
     -   `dist/` however remains at the subproject level, along with a local `node_modules/`. See [`npm workspaces`](https://docs.npmjs.com/cli/v8/using-npm/workspaces)
         for more info on how `node_modules/` hoisting works.
     -   Because of `node_modules/` hoisting, references to this folder in code access the root project modules folder. This may be
diff --git a/packages/amazonq/.c8rc.json b/packages/amazonq/.c8rc.json
@@ -0,0 +1,7 @@
+{
+    "report-dir": "../../coverage/amazonq",
+    "reporter": ["lcov"],
+    "all": true,
+    "include": ["src/**", "dist/src/**"],
+    "exclude": ["dist/test/**", "src/test/**"]
+}
diff --git a/packages/core/.c8rc.json b/packages/core/.c8rc.json
@@ -1,5 +1,5 @@
 {
-    "report-dir": "../../coverage",
+    "report-dir": "../../coverage/core",
     "reporter": ["lcov"],
     "all": true,
     "include": ["src/**", "dist/src/**"],

Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,5 @@`
`1`	`1`	`{`
`2`		`- "report-dir": "../../coverage",`
	`2`	`+ "report-dir": "../../coverage/core",`
`3`	`3`	`"reporter": ["lcov"],`
`4`	`4`	`"all": true,`
`5`	`5`	`"include": ["src/", "dist/src/"],`